From ebd32cc3cd6d14753d02d87a209062b0d9fee304 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch>
Date: Tue, 15 Aug 2023 16:02:59 +0200
Subject: [PATCH 01/29] [uss_qualifier/netrid/dss_interoperability] Add check
 for DSS0210 requirement (DSS test instances)

---
 .../configurations/dev/faa/uft/uft.yaml       |  1 +
 .../dev/library/environment.yaml              |  5 +++
 .../dev/non_docker/resources.yaml             |  4 ++
 .../uss_qualifier/resources/astm/f3411/dss.py | 20 +++++++++-
 .../netrid/common/dss_interoperability.py     | 37 +++++++++++++++++++
 .../scenarios/astm/netrid/dss_wrapper.py      |  8 ++++
 .../astm/netrid/v19/dss_interoperability.md   | 14 +++++++
 .../astm/netrid/v22a/dss_interoperability.md  | 13 +++++++
 8 files changed, 100 insertions(+), 2 deletions(-)

diff --git a/monitoring/uss_qualifier/configurations/dev/faa/uft/uft.yaml b/monitoring/uss_qualifier/configurations/dev/faa/uft/uft.yaml
index e40aeb6255..53e2ced887 100644
--- a/monitoring/uss_qualifier/configurations/dev/faa/uft/uft.yaml
+++ b/monitoring/uss_qualifier/configurations/dev/faa/uft/uft.yaml
@@ -73,6 +73,7 @@ v1:
                             - participant_id: uss8082
                               rid_version: F3411-19
                               base_url: http://host.docker.internal:8082
+                              has_private_address: true
 
         action:
             test_suite:
diff --git a/monitoring/uss_qualifier/configurations/dev/library/environment.yaml b/monitoring/uss_qualifier/configurations/dev/library/environment.yaml
index 65c95ad7ca..d230912044 100644
--- a/monitoring/uss_qualifier/configurations/dev/library/environment.yaml
+++ b/monitoring/uss_qualifier/configurations/dev/library/environment.yaml
@@ -54,9 +54,11 @@ net_rid:
                 - participant_id: uss1
                   rid_version: F3411-19
                   base_url: http://host.docker.internal:8082
+                  has_private_address: true
                 - participant_id: uss2
                   rid_version: F3411-19
                   base_url: http://host.docker.internal:8082
+                  has_private_address: true
     netrid_dss_instances_v22a:
         $content_schema: monitoring/uss_qualifier/resources/definitions/ResourceDeclaration.json
         resource_type: resources.astm.f3411.DSSInstancesResource
@@ -67,9 +69,11 @@ net_rid:
                 - participant_id: uss1
                   rid_version: F3411-22a
                   base_url: http://host.docker.internal:8082/rid/v2/
+                  has_private_address: true
                 - participant_id: uss2
                   rid_version: F3411-22a
                   base_url: http://host.docker.internal:8082/rid/v2/
+                  has_private_address: true
 
 f3548:
     flight_planners:
@@ -99,6 +103,7 @@ f3548:
         specification:
             participant_id: uss1
             base_url: http://host.docker.internal:8082
+            has_private_address: true
 
 f3548_single_scenario:
     uss1:
diff --git a/monitoring/uss_qualifier/configurations/dev/non_docker/resources.yaml b/monitoring/uss_qualifier/configurations/dev/non_docker/resources.yaml
index 03d7d17fee..7d26ffd46d 100644
--- a/monitoring/uss_qualifier/configurations/dev/non_docker/resources.yaml
+++ b/monitoring/uss_qualifier/configurations/dev/non_docker/resources.yaml
@@ -54,9 +54,11 @@ net_rid:
               - participant_id: uss1
                 rid_version: F3411-19
                 base_url: http://localhost:8082
+                has_private_address: true
               - participant_id: uss2
                 rid_version: F3411-19
                 base_url: http://localhost:8082
+                has_private_address: true
   netrid_dss_instances_v22a:
       resource_type: resources.astm.f3411.DSSInstancesResource
       dependencies:
@@ -66,9 +68,11 @@ net_rid:
               - participant_id: uss1
                 rid_version: F3411-22a
                 base_url: http://localhost:8082/rid/v2/
+                has_private_address: true
               - participant_id: uss2
                 rid_version: F3411-22a
                 base_url: http://localhost:8082/rid/v2/
+                has_private_address: true
 net_rid_sims:
   adjacent_circular_flights_data:
     resource_type: resources.netrid.FlightDataResource
diff --git a/monitoring/uss_qualifier/resources/astm/f3411/dss.py b/monitoring/uss_qualifier/resources/astm/f3411/dss.py
index 9e285301da..748eb83f18 100644
--- a/monitoring/uss_qualifier/resources/astm/f3411/dss.py
+++ b/monitoring/uss_qualifier/resources/astm/f3411/dss.py
@@ -1,5 +1,5 @@
 from __future__ import annotations
-from typing import List
+from typing import List, Optional
 from urllib.parse import urlparse
 
 from implicitdict import ImplicitDict
@@ -12,6 +12,7 @@
 
 
 class DSSInstanceSpecification(ImplicitDict):
+
     participant_id: ParticipantID
     """ID of the USS responsible for this DSS instance"""
 
@@ -21,6 +22,9 @@ class DSSInstanceSpecification(ImplicitDict):
     base_url: str
     """Base URL for the DSS instance according to the ASTM F3411 API appropriate to the specified rid_version"""
 
+    has_private_address: Optional[bool]
+    """Whether this DSS instance is expected to have a private address that is not publicly addressable."""
+
     def __init__(self, *args, **kwargs):
         super().__init__(**kwargs)
         try:
@@ -32,19 +36,26 @@ def __init__(self, *args, **kwargs):
 class DSSInstance(object):
     participant_id: ParticipantID
     rid_version: RIDVersion
+    base_url: str
+    has_private_address: bool = False
     client: infrastructure.UTMClientSession
 
     def __init__(
         self,
         participant_id: ParticipantID,
         base_url: str,
+        has_private_address: Optional[bool],
         rid_version: RIDVersion,
         auth_adapter: infrastructure.AuthAdapter,
     ):
         self.participant_id = participant_id
+        self.base_url = base_url
         self.rid_version = rid_version
         self.client = infrastructure.UTMClientSession(base_url, auth_adapter)
 
+        if has_private_address is not None:
+            self.has_private_address = has_private_address
+
 
 class DSSInstanceResource(Resource[DSSInstanceSpecification]):
     dss_instance: DSSInstance
@@ -55,6 +66,7 @@ def __init__(
         self.dss_instance = DSSInstance(
             specification.participant_id,
             specification.base_url,
+            specification.has_private_address,
             specification.rid_version,
             auth_adapter.adapter,
         )
@@ -80,7 +92,11 @@ def __init__(
     ):
         self.dss_instances = [
             DSSInstance(
-                s.participant_id, s.base_url, s.rid_version, auth_adapter.adapter
+                s.participant_id,
+                s.base_url,
+                s.has_private_address,
+                s.rid_version,
+                auth_adapter.adapter,
             )
             for s in specification.dss_instances
         ]
diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss_interoperability.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss_interoperability.py
index 473cd59f06..6a8e7c7c5c 100644
--- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss_interoperability.py
+++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss_interoperability.py
@@ -1,11 +1,15 @@
+import ipaddress
+import socket
 import time
 import uuid
 from dataclasses import dataclass
 import datetime
 from enum import Enum
 from typing import List, Dict, Optional
+from urllib.parse import urlparse
 
 import s2sphere
+
 from monitoring.uss_qualifier.common_data_definitions import Severity
 from monitoring.uss_qualifier.resources.astm.f3411.dss import (
     DSSInstancesResource,
@@ -83,6 +87,14 @@ def _get_entities_by_prefix(self, prefix: str) -> Dict[str, TestEntity]:
     def run(self):
         self.begin_test_scenario()
 
+        self.begin_test_case("Prerequisites")
+
+        self.begin_test_step("Test environment requirements")
+        self._test_env_reqs()
+        self.end_test_step()
+
+        self.end_test_case()
+
         if self._dss_others:
             self.begin_test_case("Interoperability sequence")
 
@@ -96,6 +108,31 @@ def run(self):
 
         self.end_test_scenario()
 
+    def _test_env_reqs(self):
+        for dss in [self._dss_primary] + self._dss_others:
+            with self.check(
+                "DSS instance is publicly addressable", [dss.participant_id]
+            ) as check:
+                parsed_url = urlparse(dss.base_url)
+                ip_addr = socket.gethostbyname(parsed_url.hostname)
+
+                if dss.has_private_address:
+                    self.record_note(
+                        f"{dss.participant_id}_private_address",
+                        f"DSS instance (URL: {dss.base_url}, netloc: {parsed_url.netloc}, resolved IP: {ip_addr}) is declared as explicitly having a private address, skipping check",
+                    )
+                elif ipaddress.ip_address(ip_addr).is_private:
+                    check.record_failed(
+                        summary=f"DSS host {parsed_url.netloc} is not publicly addressable",
+                        severity=Severity.Medium,
+                        participants=[dss.participant_id],
+                        details=f"DSS (URL: {dss.base_url}, netloc: {parsed_url.netloc}, resolved IP: {ip_addr}) is not publicly addressable",
+                    )
+
+            with self.check("DSS instance is reachable", [dss.participant_id]) as check:
+                # dummy search query
+                dss.search_subs(check, VERTICES)
+
     def step1(self):
         """Create ISA in Primary DSS with 10 min TTL."""
 
diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py b/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py
index d83c0f00f4..146a991b07 100644
--- a/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py
+++ b/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py
@@ -39,6 +39,14 @@ def __init__(
     def participant_id(self) -> str:
         return self._dss.participant_id
 
+    @property
+    def base_url(self) -> str:
+        return self._dss.base_url
+
+    @property
+    def has_private_address(self) -> bool:
+        return self._dss.has_private_address
+
     def _handle_query_error(
         self,
         check: PendingCheck,
diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss_interoperability.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss_interoperability.md
index 7c713f97ed..70e1f3ad8c 100644
--- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss_interoperability.md
+++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss_interoperability.md
@@ -36,6 +36,20 @@ A resources.astm.f3411.DSSInstancesResource containing at least two DSS instance
 
 ![Sequence diagram for interoperability test](../common/dss_interoperability_test.png)
 
+## Prerequisites test case
+
+### Test environment requirements test step
+
+#### DSS instance is publicly addressable check
+As per **[astm.f3411.v19.DSS0210](../../../../requirements/astm/f3411/v19.md)** the DSS instance should be publicly addressable.
+As such, this check will fail if the resolved IP of the DSS host is a private IP address, unless that is explicitly
+expected.
+
+#### DSS instance is reachable check
+As per **[astm.f3411.v19.DSS0210](../../../../requirements/astm/f3411/v19.md)** the DSS instance should be publicly addressable.
+As such, this check will fail if the DSS is not reachable with a dummy query,
+
+
 ## Interoperability sequence test case
 
 ### S1 test step
diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss_interoperability.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss_interoperability.md
index d6bc7480cc..0177430e14 100644
--- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss_interoperability.md
+++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss_interoperability.md
@@ -36,6 +36,19 @@ the note to wait >D seconds from a particular time
 
 ![Sequence diagram for interoperability test](../common/dss_interoperability_test.png)
 
+## Prerequisites test case
+
+### Test environment requirements test step
+
+#### DSS instance is publicly addressable check
+As per **[astm.f3411.v22a.DSS0210](../../../../requirements/astm/f3411/v22a.md)** the DSS instance should be publicly addressable.
+As such, this check will fail if the resolved IP of the DSS host is a private IP address, unless that is explicitly
+expected.
+
+#### DSS instance is reachable check
+As per **[astm.f3411.v22a.DSS0210](../../../../requirements/astm/f3411/v22a.md)** the DSS instance should be publicly addressable.
+As such, this check will fail if the DSS is not reachable with a dummy query,
+
 ## Interoperability sequence test case
 
 ### S1 test step

From 3fcaaaab6690b35dfb4b3f66c232b6c22adaee77 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch>
Date: Thu, 17 Aug 2023 09:11:11 +0200
Subject: [PATCH 02/29] debug attempt for CI

---
 .github/workflows/monitoring-test.yml | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/.github/workflows/monitoring-test.yml b/.github/workflows/monitoring-test.yml
index a2f9e77f2c..df73414403 100644
--- a/.github/workflows/monitoring-test.yml
+++ b/.github/workflows/monitoring-test.yml
@@ -17,6 +17,11 @@ jobs:
     runs-on: ubuntu-latest
     name: ${{ inputs.name }} test
     steps:
+    # todo: remove me
+    - run: |
+        sudo tcpdump -nn -i any -w sntp.cap &
+        sudo sh -c 'while true; do ss -s ; date ; sleep 10; done' &
+        sleep 1
     - name: Job information
       run: |
         echo "Job information"
@@ -47,3 +52,20 @@ jobs:
         path: |
           monitoring/uss_qualifier/output
           monitoring/prober/output
+    # todo: remove me
+    - name: Prepare capture
+      if: always()
+      run: |
+        sleep 1
+        sudo kill -2 $(pgrep tcpdump)
+        sudo kill -2 $(pgrep ss)
+        sleep 1
+        tar -czvf sntp.cap.tar.gz sntp.cap
+    # todo: remove me
+    - name: Upload capture
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: capture-${{ inputs.name }}
+        path: |
+          sntp.cap.tar.gz

From 2746872aa3cf8bfd1301102e3ce86a8a7a8e818a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch>
Date: Thu, 17 Aug 2023 09:18:02 +0200
Subject: [PATCH 03/29] another debug attempt CI

---
 .github/workflows/ci.yml              |  4 ++++
 .github/workflows/monitoring-test.yml | 18 +++++++-----------
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 4d567d6fad..c52b9cfcd4 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -54,6 +54,10 @@ jobs:
     with:
       name: uss_qualifier_F3411-22a
       script: |
+        sudo tcpdump -nn -i any -w /tmp/sntp.cap &
+        sudo sh -c 'while true; do ss -s ; date ; sleep 10; done' &
+        sleep 1
+
         export RID_VERSION=F3411-22a \
           CONFIG_NAME="" \
           USS_QUALIFIER_STOP_FAST=true
diff --git a/.github/workflows/monitoring-test.yml b/.github/workflows/monitoring-test.yml
index df73414403..331e347b69 100644
--- a/.github/workflows/monitoring-test.yml
+++ b/.github/workflows/monitoring-test.yml
@@ -17,11 +17,6 @@ jobs:
     runs-on: ubuntu-latest
     name: ${{ inputs.name }} test
     steps:
-    # todo: remove me
-    - run: |
-        sudo tcpdump -nn -i any -w sntp.cap &
-        sudo sh -c 'while true; do ss -s ; date ; sleep 10; done' &
-        sleep 1
     - name: Job information
       run: |
         echo "Job information"
@@ -56,11 +51,12 @@ jobs:
     - name: Prepare capture
       if: always()
       run: |
-        sleep 1
-        sudo kill -2 $(pgrep tcpdump)
-        sudo kill -2 $(pgrep ss)
-        sleep 1
-        tar -czvf sntp.cap.tar.gz sntp.cap
+        if [ -f /tmp/sntp.cap ]; then
+          sleep 1
+          sudo kill -2 $(pgrep tcpdump)
+          sleep 1
+          tar -czvf /tmp/sntp.cap.tar.gz /tmp/sntp.cap
+        fi
     # todo: remove me
     - name: Upload capture
       if: always()
@@ -68,4 +64,4 @@ jobs:
       with:
         name: capture-${{ inputs.name }}
         path: |
-          sntp.cap.tar.gz
+          /tmp/sntp.cap.tar.gz

From b59240f055ce372ea0b93a1c0a78ce673730c4c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch>
Date: Thu, 17 Aug 2023 10:29:39 +0200
Subject: [PATCH 04/29] more data loggued

---
 .github/workflows/ci.yml              | 7 ++++++-
 .github/workflows/monitoring-test.yml | 3 +--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c52b9cfcd4..ec3f002bfa 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -54,8 +54,13 @@ jobs:
     with:
       name: uss_qualifier_F3411-22a
       script: |
+        sudo ulimit -n
+        sudo cat /proc/sys/net/netfilter/nf_conntrack_max
+        sudo sysctl net.ipv4.ip_local_port_range
+        sudo sysctl net.ipv4.tcp_fin_timeout
         sudo tcpdump -nn -i any -w /tmp/sntp.cap &
-        sudo sh -c 'while true; do ss -s ; date ; sleep 10; done' &
+        sudo sh -c 'while true; do ss -s ; sysctl fs.file-nr ; date ; sleep 10; done' &
+        sudo sh -c 'while true; lsof -n | wc -l ; date ; sleep 30; done' &
         sleep 1
 
         export RID_VERSION=F3411-22a \
diff --git a/.github/workflows/monitoring-test.yml b/.github/workflows/monitoring-test.yml
index 331e347b69..7acb0ead8a 100644
--- a/.github/workflows/monitoring-test.yml
+++ b/.github/workflows/monitoring-test.yml
@@ -55,7 +55,6 @@ jobs:
           sleep 1
           sudo kill -2 $(pgrep tcpdump)
           sleep 1
-          tar -czvf /tmp/sntp.cap.tar.gz /tmp/sntp.cap
         fi
     # todo: remove me
     - name: Upload capture
@@ -64,4 +63,4 @@ jobs:
       with:
         name: capture-${{ inputs.name }}
         path: |
-          /tmp/sntp.cap.tar.gz
+          /tmp/sntp.cap

From 05382bc3ef0b35bba66938f2894c0cc5dbca3dfc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch>
Date: Thu, 17 Aug 2023 11:16:10 +0200
Subject: [PATCH 05/29] fix

---
 .github/workflows/ci.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index ec3f002bfa..79f16351dd 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -54,7 +54,6 @@ jobs:
     with:
       name: uss_qualifier_F3411-22a
       script: |
-        sudo ulimit -n
         sudo cat /proc/sys/net/netfilter/nf_conntrack_max
         sudo sysctl net.ipv4.ip_local_port_range
         sudo sysctl net.ipv4.tcp_fin_timeout

From 7f33a2264a1ec6071ba156da789501932b798230 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch>
Date: Thu, 17 Aug 2023 11:17:28 +0200
Subject: [PATCH 06/29] fix

---
 .github/workflows/ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 79f16351dd..4684535602 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -58,8 +58,8 @@ jobs:
         sudo sysctl net.ipv4.ip_local_port_range
         sudo sysctl net.ipv4.tcp_fin_timeout
         sudo tcpdump -nn -i any -w /tmp/sntp.cap &
-        sudo sh -c 'while true; do ss -s ; sysctl fs.file-nr ; date ; sleep 10; done' &
-        sudo sh -c 'while true; lsof -n | wc -l ; date ; sleep 30; done' &
+        sudo sh -c 'while true; do ss -s ; sysctl fs.file-nr ; netstat -s ; date ; sleep 10; done' &
+        sudo sh -c 'while true; do lsof -n | wc -l ; date ; sleep 30; done' &
         sleep 1
 
         export RID_VERSION=F3411-22a \

From a39c52052c51628255a70f358b3b74e5501111c5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch>
Date: Thu, 17 Aug 2023 11:18:35 +0200
Subject: [PATCH 07/29] fix

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 4684535602..9438c068a1 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -58,7 +58,7 @@ jobs:
         sudo sysctl net.ipv4.ip_local_port_range
         sudo sysctl net.ipv4.tcp_fin_timeout
         sudo tcpdump -nn -i any -w /tmp/sntp.cap &
-        sudo sh -c 'while true; do ss -s ; sysctl fs.file-nr ; netstat -s ; date ; sleep 10; done' &
+        sudo sh -c 'while true; do ss -s ; ss -ti ; sysctl fs.file-nr ; netstat -s ; date ; sleep 10; done' &
         sudo sh -c 'while true; do lsof -n | wc -l ; date ; sleep 30; done' &
         sleep 1
 

From 5f09a29070de994bf235e1a9fd985a92bfd9fe1b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch>
Date: Thu, 17 Aug 2023 12:16:25 +0200
Subject: [PATCH 08/29] add data

---
 .github/workflows/ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 9438c068a1..244dd9b4f5 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -55,6 +55,7 @@ jobs:
       name: uss_qualifier_F3411-22a
       script: |
         sudo cat /proc/sys/net/netfilter/nf_conntrack_max
+        sudo cat /proc/sys/net/core/somaxconn
         sudo sysctl net.ipv4.ip_local_port_range
         sudo sysctl net.ipv4.tcp_fin_timeout
         sudo tcpdump -nn -i any -w /tmp/sntp.cap &

From eb6fe9f5f40e40aceff738151b1c7c3f5fbaecd4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch>
Date: Thu, 17 Aug 2023 12:26:29 +0200
Subject: [PATCH 09/29] try prev. version

---
 .github/workflows/monitoring-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/monitoring-test.yml b/.github/workflows/monitoring-test.yml
index 7acb0ead8a..d7f3269da7 100644
--- a/.github/workflows/monitoring-test.yml
+++ b/.github/workflows/monitoring-test.yml
@@ -14,7 +14,7 @@ on:
 
 jobs:
   monitoring-test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     name: ${{ inputs.name }} test
     steps:
     - name: Job information

From 7aa074a42540ce182f754cfc7337a5e58e11308b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch>
Date: Thu, 17 Aug 2023 13:10:19 +0200
Subject: [PATCH 10/29] update a limit

---
 .github/workflows/ci.yml              | 3 +++
 .github/workflows/monitoring-test.yml | 3 ++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 244dd9b4f5..2560cd6065 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -56,8 +56,11 @@ jobs:
       script: |
         sudo cat /proc/sys/net/netfilter/nf_conntrack_max
         sudo cat /proc/sys/net/core/somaxconn
+        sudo cat /proc/sys/net/ipv4/tcp_max_syn_backlog
         sudo sysctl net.ipv4.ip_local_port_range
         sudo sysctl net.ipv4.tcp_fin_timeout
+        sudo sysctl net.ipv4.tcp_max_syn_backlog=16384
+        sudo cat /proc/sys/net/ipv4/tcp_max_syn_backlog
         sudo tcpdump -nn -i any -w /tmp/sntp.cap &
         sudo sh -c 'while true; do ss -s ; ss -ti ; sysctl fs.file-nr ; netstat -s ; date ; sleep 10; done' &
         sudo sh -c 'while true; do lsof -n | wc -l ; date ; sleep 30; done' &
diff --git a/.github/workflows/monitoring-test.yml b/.github/workflows/monitoring-test.yml
index d7f3269da7..a1bc9b8177 100644
--- a/.github/workflows/monitoring-test.yml
+++ b/.github/workflows/monitoring-test.yml
@@ -14,7 +14,8 @@ on:
 
 jobs:
   monitoring-test:
-    runs-on: ubuntu-20.04
+#    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     name: ${{ inputs.name }} test
     steps:
     - name: Job information

From 1be5741a4bb7e6a964be6569a5f850886aac40f4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch>
Date: Thu, 17 Aug 2023 14:01:30 +0200
Subject: [PATCH 11/29] attempt ulimits

---
 build/dev/docker-compose.yaml                     | 8 ++++++++
 monitoring/mock_uss/run_locally_atproxy_client.sh | 1 +
 monitoring/mock_uss/run_locally_scdsc.sh          | 1 +
 monitoring/mock_uss/run_locally_tracer.sh         | 1 +
 monitoring/uss_qualifier/run_locally.sh           | 1 +
 5 files changed, 12 insertions(+)

diff --git a/build/dev/docker-compose.yaml b/build/dev/docker-compose.yaml
index 7220e2e947..81fe399de3 100644
--- a/build/dev/docker-compose.yaml
+++ b/build/dev/docker-compose.yaml
@@ -17,6 +17,10 @@ services:
     restart: always
     networks:
       - dss_internal_network
+    ulimits:
+      nofile:
+        soft: 10000
+        hard: 10000
 
   rid_bootstrapper:
     image: interuss/dss:v0.8.0-rc2
@@ -57,6 +61,10 @@ services:
     networks:
       - dss_internal_network
       - interop_ecosystem_network
+    ulimits:
+      nofile:
+        soft: 10000
+        hard: 10000
 
   oauth:
     hostname: oauth
diff --git a/monitoring/mock_uss/run_locally_atproxy_client.sh b/monitoring/mock_uss/run_locally_atproxy_client.sh
index 16c5cd345f..470aca0e80 100755
--- a/monitoring/mock_uss/run_locally_atproxy_client.sh
+++ b/monitoring/mock_uss/run_locally_atproxy_client.sh
@@ -28,6 +28,7 @@ docker container rm -f ${container_name} || echo "No pre-existing ${container_na
 
 # shellcheck disable=SC2086
 docker run ${docker_args} --name ${container_name} \
+  --ulimit nofile=10000 \
   -e MOCK_USS_AUTH_SPEC="${AUTH}" \
   -e MOCK_USS_DSS_URL="${DSS}" \
   -e MOCK_USS_PUBLIC_KEY="${PUBLIC_KEY}" \
diff --git a/monitoring/mock_uss/run_locally_scdsc.sh b/monitoring/mock_uss/run_locally_scdsc.sh
index 136b0e0f53..9142f90cab 100755
--- a/monitoring/mock_uss/run_locally_scdsc.sh
+++ b/monitoring/mock_uss/run_locally_scdsc.sh
@@ -24,6 +24,7 @@ docker container rm -f "${CONTAINER_NAME}" || echo "No pre-existing ${CONTAINER_
 
 # shellcheck disable=SC2086
 docker run ${docker_args} --name "${CONTAINER_NAME}" \
+  --ulimit nofile=10000 \
   -e MOCK_USS_AUTH_SPEC="${AUTH}" \
   -e MOCK_USS_DSS_URL="${DSS}" \
   -e MOCK_USS_PUBLIC_KEY="${PUBLIC_KEY}" \
diff --git a/monitoring/mock_uss/run_locally_tracer.sh b/monitoring/mock_uss/run_locally_tracer.sh
index 10054e87dd..f4ec0bf084 100755
--- a/monitoring/mock_uss/run_locally_tracer.sh
+++ b/monitoring/mock_uss/run_locally_tracer.sh
@@ -33,6 +33,7 @@ docker container rm -f "${CONTAINER_NAME}" || echo "No pre-existing ${CONTAINER_
 
 # shellcheck disable=SC2086
 docker run ${docker_args} --name "${CONTAINER_NAME}" \
+  --ulimit nofile=10000 \
   -u "$(id -u):$(id -g)" \
   -e MOCK_USS_AUTH_SPEC="${AUTH}" \
   -e MOCK_USS_DSS_URL="${DSS}" \
diff --git a/monitoring/uss_qualifier/run_locally.sh b/monitoring/uss_qualifier/run_locally.sh
index 0f8e5ec1e3..3cde517ef1 100755
--- a/monitoring/uss_qualifier/run_locally.sh
+++ b/monitoring/uss_qualifier/run_locally.sh
@@ -56,6 +56,7 @@ else
   echo "========== Running uss_qualifier for configuration ${CONFIG_NAME} =========="
   # shellcheck disable=SC2086
   docker run ${docker_args} --name uss_qualifier \
+    --ulimit nofile=10000 \
     --rm \
     -u "$(id -u):$(id -g)" \
     -e PYTHONBUFFERED=1 \

From 7b27ee74298e58b3837493a0287c50eedfb16f9a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch>
Date: Thu, 17 Aug 2023 14:20:06 +0200
Subject: [PATCH 12/29] other

---
 .github/workflows/ci.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 2560cd6065..798310d2af 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -61,6 +61,10 @@ jobs:
         sudo sysctl net.ipv4.tcp_fin_timeout
         sudo sysctl net.ipv4.tcp_max_syn_backlog=16384
         sudo cat /proc/sys/net/ipv4/tcp_max_syn_backlog
+        sudo ulimit -n 1048576
+        sudo ulimit -u 1048576
+        sudo sysctl -w net.ipv4.neigh.default.gc_thresh3=4096
+        sudo sysctl fs.inotify.max_user_instances=1048576
         sudo tcpdump -nn -i any -w /tmp/sntp.cap &
         sudo sh -c 'while true; do ss -s ; ss -ti ; sysctl fs.file-nr ; netstat -s ; date ; sleep 10; done' &
         sudo sh -c 'while true; do lsof -n | wc -l ; date ; sleep 30; done' &

From 8129f5ee174e01c09524faedba3074b16dce0001 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch>
Date: Thu, 17 Aug 2023 14:25:46 +0200
Subject: [PATCH 13/29] limit

---
 .github/workflows/ci.yml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 798310d2af..ea6f9045f1 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -61,10 +61,13 @@ jobs:
         sudo sysctl net.ipv4.tcp_fin_timeout
         sudo sysctl net.ipv4.tcp_max_syn_backlog=16384
         sudo cat /proc/sys/net/ipv4/tcp_max_syn_backlog
-        sudo ulimit -n 1048576
-        sudo ulimit -u 1048576
+        ulimit -n 1048576
+        ulimit -u 1048576
         sudo sysctl -w net.ipv4.neigh.default.gc_thresh3=4096
         sudo sysctl fs.inotify.max_user_instances=1048576
+        sudo prlimit --pid $$ --nofile=1048576:1048576
+        sudo sysctl fs.inotify.max_user_instances=1280
+        sudo sysctl fs.inotify.max_user_watches=655360
         sudo tcpdump -nn -i any -w /tmp/sntp.cap &
         sudo sh -c 'while true; do ss -s ; ss -ti ; sysctl fs.file-nr ; netstat -s ; date ; sleep 10; done' &
         sudo sh -c 'while true; do lsof -n | wc -l ; date ; sleep 30; done' &

From ed2bc957bea8b766d9a11c520c92cd43822b092b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch>
Date: Thu, 17 Aug 2023 14:32:32 +0200
Subject: [PATCH 14/29] value

---
 .github/workflows/ci.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index ea6f9045f1..5168d1722b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -61,8 +61,6 @@ jobs:
         sudo sysctl net.ipv4.tcp_fin_timeout
         sudo sysctl net.ipv4.tcp_max_syn_backlog=16384
         sudo cat /proc/sys/net/ipv4/tcp_max_syn_backlog
-        ulimit -n 1048576
-        ulimit -u 1048576
         sudo sysctl -w net.ipv4.neigh.default.gc_thresh3=4096
         sudo sysctl fs.inotify.max_user_instances=1048576
         sudo prlimit --pid $$ --nofile=1048576:1048576

From cf3b516a225c69979d05bff4e7cfae35e1deb170 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch>
Date: Thu, 17 Aug 2023 14:43:32 +0200
Subject: [PATCH 15/29] add docker logs

---
 .github/workflows/monitoring-test.yml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/.github/workflows/monitoring-test.yml b/.github/workflows/monitoring-test.yml
index a1bc9b8177..681f0f4248 100644
--- a/.github/workflows/monitoring-test.yml
+++ b/.github/workflows/monitoring-test.yml
@@ -55,6 +55,7 @@ jobs:
         if [ -f /tmp/sntp.cap ]; then
           sleep 1
           sudo kill -2 $(pgrep tcpdump)
+          journalctl -xu docker.service > /tmp/dockerlogs
           sleep 1
         fi
     # todo: remove me
@@ -65,3 +66,12 @@ jobs:
         name: capture-${{ inputs.name }}
         path: |
           /tmp/sntp.cap
+    # todo: docker logs
+    - name: Upload capture
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: capture-${{ inputs.name }}
+        path: |
+          /tmp/sntp.cap
+          /tmp/dockerlogs

From 7c901f6d08ad2a795a587c7dd771fb2d76a57aca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch>
Date: Thu, 17 Aug 2023 14:56:44 +0200
Subject: [PATCH 16/29] otherlimit

---
 .github/workflows/ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 5168d1722b..48bf35a86f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -66,6 +66,7 @@ jobs:
         sudo prlimit --pid $$ --nofile=1048576:1048576
         sudo sysctl fs.inotify.max_user_instances=1280
         sudo sysctl fs.inotify.max_user_watches=655360
+        sudo sysctl net.core.netdev_max_backlog=65536
         sudo tcpdump -nn -i any -w /tmp/sntp.cap &
         sudo sh -c 'while true; do ss -s ; ss -ti ; sysctl fs.file-nr ; netstat -s ; date ; sleep 10; done' &
         sudo sh -c 'while true; do lsof -n | wc -l ; date ; sleep 30; done' &

From b740109030f1bb8df8d764535e25cbe358f34bc0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch>
Date: Thu, 17 Aug 2023 14:58:22 +0200
Subject: [PATCH 17/29] fix

---
 .github/workflows/monitoring-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/monitoring-test.yml b/.github/workflows/monitoring-test.yml
index 681f0f4248..7b188a62c1 100644
--- a/.github/workflows/monitoring-test.yml
+++ b/.github/workflows/monitoring-test.yml
@@ -71,7 +71,7 @@ jobs:
       if: always()
       uses: actions/upload-artifact@v3
       with:
-        name: capture-${{ inputs.name }}
+        name: capture-${{ inputs.name }}-dockerlogs
         path: |
           /tmp/sntp.cap
           /tmp/dockerlogs

From 0cf3b637595632134ca6c843dd5c04ef2085b877 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch>
Date: Thu, 17 Aug 2023 15:03:34 +0200
Subject: [PATCH 18/29] logs

---
 .github/workflows/monitoring-test.yml | 14 +++-----------
 1 file changed, 3 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/monitoring-test.yml b/.github/workflows/monitoring-test.yml
index 7b188a62c1..f22f9203e6 100644
--- a/.github/workflows/monitoring-test.yml
+++ b/.github/workflows/monitoring-test.yml
@@ -55,23 +55,15 @@ jobs:
         if [ -f /tmp/sntp.cap ]; then
           sleep 1
           sudo kill -2 $(pgrep tcpdump)
-          journalctl -xu docker.service > /tmp/dockerlogs
+          journalctl -x > /tmp/alllogs
           sleep 1
         fi
-    # todo: remove me
-    - name: Upload capture
-      if: always()
-      uses: actions/upload-artifact@v3
-      with:
-        name: capture-${{ inputs.name }}
-        path: |
-          /tmp/sntp.cap
     # todo: docker logs
     - name: Upload capture
       if: always()
       uses: actions/upload-artifact@v3
       with:
-        name: capture-${{ inputs.name }}-dockerlogs
+        name: capture-${{ inputs.name }}
         path: |
           /tmp/sntp.cap
-          /tmp/dockerlogs
+          /tmp/alllogs

From 623ba8c1d0d25b1e86fa8dc142b7e85698be0878 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch>
Date: Tue, 10 Oct 2023 13:14:16 +0200
Subject: [PATCH 19/29] format

---
 monitoring/uss_qualifier/resources/astm/f3411/dss.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/monitoring/uss_qualifier/resources/astm/f3411/dss.py b/monitoring/uss_qualifier/resources/astm/f3411/dss.py
index 5980dde7bb..1595864d70 100644
--- a/monitoring/uss_qualifier/resources/astm/f3411/dss.py
+++ b/monitoring/uss_qualifier/resources/astm/f3411/dss.py
@@ -25,7 +25,6 @@ class DSSInstanceSpecification(ImplicitDict):
     has_private_address: Optional[bool]
     """Whether this DSS instance is expected to have a private address that is not publicly addressable."""
 
-
     local_debug: Optional[bool]
     """Whether this DSS instance is running locally for debugging or development purposes. Mostly used for relaxing
     constraints around encryption.

From a65ad9eea8599ec6451994b20f09854ff9b4fd50 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch>
Date: Fri, 13 Oct 2023 10:56:21 +0200
Subject: [PATCH 20/29] try installing modules

---
 .github/workflows/ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 2dc0859192..ea9d0da616 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -54,6 +54,7 @@ jobs:
     with:
       name: uss_qualifier
       script: |
+        sudo apt install linux-modules-extra-$(uname -r)
         sudo cat /proc/sys/net/netfilter/nf_conntrack_max
         sudo cat /proc/sys/net/core/somaxconn
         sudo cat /proc/sys/net/ipv4/tcp_max_syn_backlog

From ab05552b1b94441d6eb6ff992feeb06a88f4a335 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch>
Date: Fri, 13 Oct 2023 11:37:51 +0200
Subject: [PATCH 21/29] disable ipv6

---
 .github/workflows/ci.yml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index ea9d0da616..db2b169a7d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -58,6 +58,11 @@ jobs:
         sudo cat /proc/sys/net/netfilter/nf_conntrack_max
         sudo cat /proc/sys/net/core/somaxconn
         sudo cat /proc/sys/net/ipv4/tcp_max_syn_backlog
+
+        sudo sysctl net.ipv6.conf.all.disable_ipv6=1
+        sudo sysctl net.ipv6.conf.default.disable_ipv6=1
+        sudo sysctl net.ipv6.conf.lo.disable_ipv6=1
+
         sudo sysctl net.ipv4.ip_local_port_range
         sudo sysctl net.ipv4.tcp_fin_timeout
         sudo sysctl net.ipv4.tcp_max_syn_backlog=16384
@@ -68,6 +73,9 @@ jobs:
         sudo sysctl fs.inotify.max_user_instances=1280
         sudo sysctl fs.inotify.max_user_watches=655360
         sudo sysctl net.core.netdev_max_backlog=65536
+
+        sudo sysctl -p
+
         sudo tcpdump -nn -i any -w /tmp/sntp.cap &
         sudo sh -c 'while true; do ss -s ; ss -ti ; sysctl fs.file-nr ; netstat -s ; date ; sleep 10; done' &
         sudo sh -c 'while true; do lsof -n | wc -l ; date ; sleep 30; done' &

From aca5af73bf6e8d5684f7313d18ac5283b6e6b164 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch>
Date: Fri, 13 Oct 2023 12:27:43 +0200
Subject: [PATCH 22/29] merge run configs

---
 monitoring/uss_qualifier/main.py        |  78 ++++++++++++----
 monitoring/uss_qualifier/run_locally.sh | 117 ++++++++++++------------
 2 files changed, 117 insertions(+), 78 deletions(-)

diff --git a/monitoring/uss_qualifier/main.py b/monitoring/uss_qualifier/main.py
index 240435f433..50436d0197 100644
--- a/monitoring/uss_qualifier/main.py
+++ b/monitoring/uss_qualifier/main.py
@@ -4,6 +4,7 @@
 import json
 import os
 import sys
+from typing import Optional
 
 from implicitdict import ImplicitDict
 from loguru import logger
@@ -110,12 +111,16 @@ def execute_test_run(
     )
 
 
-def main() -> int:
-    args = parseArgs()
-
-    config_src = load_dict_with_references(args.config)
+def run_config(
+    config_name: str,
+    config_output: str,
+    report_path: str,
+    skip_validation: bool,
+    exit_before_execution: bool,
+):
+    config_src = load_dict_with_references(config_name)
 
-    if not args.skip_validation:
+    if not skip_validation:
         logger.info("Validating configuration...")
         validation_errors = validate_config(config_src)
         if validation_errors:
@@ -127,33 +132,33 @@ def main() -> int:
 
     whole_config = ImplicitDict.parse(config_src, USSQualifierConfiguration)
 
-    if args.config_output:
-        logger.info("Writing flattened configuration to {}", args.config_output)
-        if args.config_output.lower().endswith(".json"):
-            with open(args.config_output, "w") as f:
+    if config_output:
+        logger.info("Writing flattened configuration to {}", config_output)
+        if config_output.lower().endswith(".json"):
+            with open(config_output, "w") as f:
                 json.dump(whole_config, f, indent=2, sort_keys=True)
-        elif args.config_output.lower().endswith(".yaml"):
-            with open(args.config_output, "w") as f:
+        elif config_output.lower().endswith(".yaml"):
+            with open(config_output, "w") as f:
                 yaml.dump(json.loads(json.dumps(whole_config)), f, sort_keys=True)
         else:
             raise ValueError(
                 "Unsupported extension for --config-output; only .json or .yaml file paths may be specified"
             )
 
-    if args.exit_before_execution:
+    if exit_before_execution:
         logger.info("Exiting because --exit-before-execution specified.")
-        return os.EX_OK
+        return
 
     config = whole_config.v1
-    if args.report:
+    if report_path:
         if not config.artifacts:
             config.artifacts = ArtifactsConfiguration(
-                ReportConfiguration(report_path=args.report)
+                ReportConfiguration(report_path=report_path)
             )
         elif not config.artifacts.report:
-            config.artifacts.report = ReportConfiguration(report_path=args.report)
+            config.artifacts.report = ReportConfiguration(report_path=report_path)
         else:
-            config.artifacts.report.report_path = args.report
+            config.artifacts.report.report_path = report_path
 
     do_not_save_report = False
     if config.test_run:
@@ -204,6 +209,45 @@ def main() -> int:
             logger.info(f"Writing tested requirements view to {path}")
             generate_tested_requirements(report, config.artifacts.tested_requirements)
 
+
+def main() -> int:
+    args = parseArgs()
+
+    config_names = str(args.config).split(",")
+
+    if args.config_output:
+        config_outputs = str(args.config_output).split(",")
+        if len(config_outputs) != len(config_names):
+            raise ValueError(
+                f"Need matching number of config_output, expected {len(config_names)}, got {len(config_outputs)}"
+            )
+    else:
+        config_outputs = ["" for _ in config_names]
+
+    if args.report:
+        report_paths = str(args.report).split(",")
+        if len(report_paths) != len(config_names):
+            raise ValueError(
+                f"Need matching number of report, expected {len(config_names)}, got {len(report_paths)}"
+            )
+    else:
+        report_paths = ["" for _ in config_names]
+
+    for idx, config_name in enumerate(config_names):
+        logger.info(
+            f"========== Running uss_qualifier for configuration {config_name} =========="
+        )
+        run_config(
+            config_name,
+            config_outputs[idx],
+            report_paths[idx],
+            args.skip_validation,
+            args.exit_before_execution,
+        )
+        logger.info(
+            f"========== Completed uss_qualifier for configuration {config_name} =========="
+        )
+
     return os.EX_OK
 
 
diff --git a/monitoring/uss_qualifier/run_locally.sh b/monitoring/uss_qualifier/run_locally.sh
index 325b49e8a7..74893f93af 100755
--- a/monitoring/uss_qualifier/run_locally.sh
+++ b/monitoring/uss_qualifier/run_locally.sh
@@ -24,72 +24,67 @@ CONFIG_NAME="${1:-ALL}"
 OTHER_ARGS=${@:2}
 
 if [ "$CONFIG_NAME" == "ALL" ]; then
-  declare -a all_configurations=( \
-    "configurations.dev.noop" \
-    "configurations.dev.dss_probing" \
-    "configurations.dev.geoawareness_cis" \
-    "configurations.dev.generate_rid_test_data" \
-    "configurations.dev.geospatial_comprehension" \
-    "configurations.dev.general_flight_auth" \
-    "configurations.dev.f3548" \
-    "configurations.dev.f3548_self_contained" \
-    "configurations.dev.netrid_v22a" \
-    "configurations.dev.uspace" \
-  )
-  # TODO: Add configurations.dev.netrid_v19
-  echo "Running configurations: ${all_configurations[*]}"
-  for configuration_name in "${all_configurations[@]}"; do
-    monitoring/uss_qualifier/run_locally.sh "$configuration_name"
-  done
-else
-  CONFIG_FLAG="--config ${CONFIG_NAME}"
+  CONFIG_NAME="\
+configurations.dev.noop,\
+configurations.dev.dss_probing,\
+configurations.dev.geoawareness_cis,\
+configurations.dev.generate_rid_test_data,\
+configurations.dev.geospatial_comprehension,\
+configurations.dev.general_flight_auth,\
+configurations.dev.f3548,\
+configurations.dev.f3548_self_contained,\
+configurations.dev.netrid_v22a,\
+configurations.dev.uspace"
+fi
+# TODO: Add configurations.dev.netrid_v19
 
-  AUTH_SPEC='DummyOAuth(http://oauth.authority.localutm:8085/token,uss_qualifier)'
+echo "Running configuration(s): ${CONFIG_NAME}"
 
-  QUALIFIER_OPTIONS="$CONFIG_FLAG $OTHER_ARGS"
+CONFIG_FLAG="--config ${CONFIG_NAME}"
 
-  OUTPUT_DIR="monitoring/uss_qualifier/output"
-  mkdir -p "$OUTPUT_DIR"
+AUTH_SPEC='DummyOAuth(http://oauth.authority.localutm:8085/token,uss_qualifier)'
 
-  CACHE_DIR="monitoring/uss_qualifier/.templates_cache"
-  mkdir -p "$CACHE_DIR"
+QUALIFIER_OPTIONS="$CONFIG_FLAG $OTHER_ARGS"
 
-  if [ "$CI" == "true" ]; then
-    docker_args="--add-host host.docker.internal:host-gateway" # Required to reach other containers in Ubuntu (used for Github Actions)
-  else
-    docker_args="-it"
-  fi
+OUTPUT_DIR="monitoring/uss_qualifier/output"
+mkdir -p "$OUTPUT_DIR"
+
+CACHE_DIR="monitoring/uss_qualifier/.templates_cache"
+mkdir -p "$CACHE_DIR"
 
-  start_time=$(date +%Y-%m-%dT%H:%M:%S)
-  echo "========== Running uss_qualifier for configuration ${CONFIG_NAME} =========="
-  # shellcheck disable=SC2086
-  docker run ${docker_args} --name uss_qualifier \
-    --ulimit nofile=10000 \
-    --rm \
-    --network interop_ecosystem_network \
-    -u "$(id -u):$(id -g)" \
-    -e PYTHONBUFFERED=1 \
-    -e AUTH_SPEC=${AUTH_SPEC} \
-    -e USS_QUALIFIER_STOP_FAST=${USS_QUALIFIER_STOP_FAST:-} \
-    -e MONITORING_GITHUB_ROOT=${MONITORING_GITHUB_ROOT:-} \
-    -v "$(pwd)/$OUTPUT_DIR:/app/$OUTPUT_DIR" \
-    -v "$(pwd)/$CACHE_DIR:/app/$CACHE_DIR" \
-    -w /app/monitoring/uss_qualifier \
-    interuss/monitoring \
-    python main.py $QUALIFIER_OPTIONS
-  echo "========== Completed uss_qualifier for configuration ${CONFIG_NAME} =========="
-
-  # Set return code according to whether the test run was fully successful
-  reports_generated=$(find ./monitoring/uss_qualifier/output/report*.json -newermt "$start_time")
-  # shellcheck disable=SC2068
-  for REPORT in ${reports_generated[@]}; do
-    successful=$(python build/dev/extract_json_field.py report.*.successful "$REPORT")
-    if echo "${successful}" | grep -iqF true; then
-      echo "Full success indicated by $REPORT"
-    else
-      echo "Could not establish that all uss_qualifier tests passed in $REPORT"
-      exit 1
-    fi
-  done
+if [ "$CI" == "true" ]; then
+  docker_args="--add-host host.docker.internal:host-gateway" # Required to reach other containers in Ubuntu (used for Github Actions)
+else
+  docker_args="-it"
 fi
 
+start_time=$(date +%Y-%m-%dT%H:%M:%S)
+# shellcheck disable=SC2086
+docker run ${docker_args} --name uss_qualifier \
+  --ulimit nofile=10000 \
+  --rm \
+  --network interop_ecosystem_network \
+  -u "$(id -u):$(id -g)" \
+  -e PYTHONBUFFERED=1 \
+  -e AUTH_SPEC=${AUTH_SPEC} \
+  -e USS_QUALIFIER_STOP_FAST=${USS_QUALIFIER_STOP_FAST:-} \
+  -e MONITORING_GITHUB_ROOT=${MONITORING_GITHUB_ROOT:-} \
+  -v "$(pwd)/$OUTPUT_DIR:/app/$OUTPUT_DIR" \
+  -v "$(pwd)/$CACHE_DIR:/app/$CACHE_DIR" \
+  -w /app/monitoring/uss_qualifier \
+  interuss/monitoring \
+  python main.py $QUALIFIER_OPTIONS
+
+# Set return code according to whether the test run was fully successful
+reports_generated=$(find ./monitoring/uss_qualifier/output/report*.json -newermt "$start_time")
+# shellcheck disable=SC2068
+for REPORT in ${reports_generated[@]}; do
+  successful=$(python build/dev/extract_json_field.py report.*.successful "$REPORT")
+  if echo "${successful}" | grep -iqF true; then
+    echo "Full success indicated by $REPORT"
+  else
+    echo "Could not establish that all uss_qualifier tests passed in $REPORT"
+    exit 1
+  fi
+done
+

From a03f2a04cba69a5ee33f1602074301bb97563a2f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch>
Date: Fri, 13 Oct 2023 17:38:41 +0200
Subject: [PATCH 23/29] timeout++

---
 monitoring/monitorlib/fetch/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/monitoring/monitorlib/fetch/__init__.py b/monitoring/monitorlib/fetch/__init__.py
index f9b1cadcdf..8aab9dbe41 100644
--- a/monitoring/monitorlib/fetch/__init__.py
+++ b/monitoring/monitorlib/fetch/__init__.py
@@ -18,7 +18,7 @@
 from monitoring.monitorlib import infrastructure
 from monitoring.monitorlib.rid import RIDVersion
 
-TIMEOUTS = (5, 5)  # Timeouts of `connect` and `read` in seconds
+TIMEOUTS = (5, 25)  # Timeouts of `connect` and `read` in seconds
 ATTEMPTS = (
     2  # Number of attempts to query when experiencing a retryable error like a timeout
 )

From 0cda9c898c40abcba58ee3f5b5aa134632d1d095 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch>
Date: Fri, 13 Oct 2023 18:30:21 +0200
Subject: [PATCH 24/29] increase gunicorn log level

---
 monitoring/mock_uss/gunicorn.conf.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/monitoring/mock_uss/gunicorn.conf.py b/monitoring/mock_uss/gunicorn.conf.py
index fbb3b140fd..a7f7214221 100644
--- a/monitoring/mock_uss/gunicorn.conf.py
+++ b/monitoring/mock_uss/gunicorn.conf.py
@@ -8,6 +8,7 @@
 
 from monitoring.mock_uss import webapp
 
+loglevel = "debug"
 
 def on_starting(server: Arbiter):
     """gunicorn server hook called just before master process is initialized."""

From f9b1b2013188936df5cd3ec4f5353fef24879a09 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch>
Date: Fri, 13 Oct 2023 18:32:57 +0200
Subject: [PATCH 25/29] format

---
 monitoring/mock_uss/gunicorn.conf.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/monitoring/mock_uss/gunicorn.conf.py b/monitoring/mock_uss/gunicorn.conf.py
index a7f7214221..635bdcc4f5 100644
--- a/monitoring/mock_uss/gunicorn.conf.py
+++ b/monitoring/mock_uss/gunicorn.conf.py
@@ -10,6 +10,7 @@
 
 loglevel = "debug"
 
+
 def on_starting(server: Arbiter):
     """gunicorn server hook called just before master process is initialized."""
     logger.debug("on_starting")

From 72cb679b79483dcd62f9221625879672495fe302 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch>
Date: Fri, 13 Oct 2023 18:36:24 +0200
Subject: [PATCH 26/29] increase threads

---
 monitoring/mock_uss/gunicorn.conf.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/monitoring/mock_uss/gunicorn.conf.py b/monitoring/mock_uss/gunicorn.conf.py
index 635bdcc4f5..918ee2f5bc 100644
--- a/monitoring/mock_uss/gunicorn.conf.py
+++ b/monitoring/mock_uss/gunicorn.conf.py
@@ -11,6 +11,9 @@
 loglevel = "debug"
 
 
+threads = 2
+
+
 def on_starting(server: Arbiter):
     """gunicorn server hook called just before master process is initialized."""
     logger.debug("on_starting")

From 96b45a6efbdfa47c4cd255ac25f002c0da73e032 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch>
Date: Fri, 13 Oct 2023 18:42:30 +0200
Subject: [PATCH 27/29] better gunicorn?

---
 monitoring/mock_uss/gunicorn.conf.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/monitoring/mock_uss/gunicorn.conf.py b/monitoring/mock_uss/gunicorn.conf.py
index 918ee2f5bc..089d41d19c 100644
--- a/monitoring/mock_uss/gunicorn.conf.py
+++ b/monitoring/mock_uss/gunicorn.conf.py
@@ -11,7 +11,16 @@
 loglevel = "debug"
 
 
-threads = 2
+workers = 2
+
+
+threads = 4
+
+
+worker_tmp_dir = "/dev/shm"
+
+
+worker_class = "gthread"
 
 
 def on_starting(server: Arbiter):

From 94a4f940875ee8d34f54872cec64509d1f926758 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch>
Date: Mon, 16 Oct 2023 11:21:08 +0200
Subject: [PATCH 28/29] try log multiprocessing

---
 monitoring/mock_uss/__init__.py      | 5 +++++
 monitoring/mock_uss/gunicorn.conf.py | 3 +++
 2 files changed, 8 insertions(+)

diff --git a/monitoring/mock_uss/__init__.py b/monitoring/mock_uss/__init__.py
index 94587f3b96..366706f8a1 100644
--- a/monitoring/mock_uss/__init__.py
+++ b/monitoring/mock_uss/__init__.py
@@ -1,4 +1,6 @@
 import inspect
+import logging
+import multiprocessing
 import os
 from typing import Any, Optional, Callable
 from loguru import logger
@@ -17,6 +19,9 @@
 webapp = MockUSS(__name__)
 enabled_services = set()
 
+mp_logger = multiprocessing.log_to_stderr()
+mp_logger.setLevel(logging.DEBUG)
+
 
 def import_environment_variable(
     var_name: str,
diff --git a/monitoring/mock_uss/gunicorn.conf.py b/monitoring/mock_uss/gunicorn.conf.py
index 089d41d19c..af0abf8d78 100644
--- a/monitoring/mock_uss/gunicorn.conf.py
+++ b/monitoring/mock_uss/gunicorn.conf.py
@@ -23,6 +23,9 @@
 worker_class = "gthread"
 
 
+preload_app = True
+
+
 def on_starting(server: Arbiter):
     """gunicorn server hook called just before master process is initialized."""
     logger.debug("on_starting")

From cfff3482d2355ea1d23c739e44f35d9012035fca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch>
Date: Mon, 16 Oct 2023 12:11:50 +0200
Subject: [PATCH 29/29] try out gevent

---
 monitoring/mock_uss/gunicorn.conf.py | 2 +-
 requirements.txt                     | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/monitoring/mock_uss/gunicorn.conf.py b/monitoring/mock_uss/gunicorn.conf.py
index af0abf8d78..1f61ba6cbf 100644
--- a/monitoring/mock_uss/gunicorn.conf.py
+++ b/monitoring/mock_uss/gunicorn.conf.py
@@ -20,7 +20,7 @@
 worker_tmp_dir = "/dev/shm"
 
 
-worker_class = "gthread"
+worker_class = "gevent"
 
 
 preload_app = True
diff --git a/requirements.txt b/requirements.txt
index cee4d53f0b..e19ce1cded 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -12,6 +12,7 @@ faker===8.1.0  # uss_qualifier
 flask==1.1.2
 Flask-HTTPAuth==4.7.0  # atproxy
 geojson===2.5.0  # uss_qualifier
+gevent==22.10.2
 google-auth==1.6.3
 graphviz==0.20.1  # uss_qualifier
 gunicorn==20.1.0