From 424bdb66c383f87400e3616a47105eaa2b487f3c Mon Sep 17 00:00:00 2001 From: cjen1-msft Date: Wed, 22 Oct 2025 13:26:47 +0100 Subject: [PATCH 1/7] see-branch-name --- tests/e2e_operations.py | 327 ++++++++++++++++++++-------------------- 1 file changed, 164 insertions(+), 163 deletions(-) diff --git a/tests/e2e_operations.py b/tests/e2e_operations.py index 623aa01741e..b8049fb07ab 100644 --- a/tests/e2e_operations.py +++ b/tests/e2e_operations.py @@ -1095,54 +1095,54 @@ def run_initial_uvm_descriptor_checks(args): LOG.info("Start a recovery network and stop it") current_ledger_dir, committed_ledger_dirs = primary.get_ledger() - recovered_network = infra.network.Network( + with infra.network.network( args.nodes, args.binary_dir, args.debug_nodes, existing_network=network, - ) - - args.previous_service_identity_file = os.path.join( - old_common, "service_cert.pem" - ) - recovered_network.start_in_recovery( - args, - ledger_dir=current_ledger_dir, - committed_ledger_dirs=committed_ledger_dirs, - snapshots_dir=snapshots_dir, - ) - recovered_primary, _ = recovered_network.find_primary() - LOG.info("Check that the UVM descriptor is present in the recovery tx") - recovery_seqno = None - with recovered_primary.client() as c: - r = c.get("/node/network").body.json() - recovery_seqno = int(r["current_service_create_txid"].split(".")[1]) - network.stop_all_nodes() - ledger = ccf.ledger.Ledger( - recovered_primary.remote.ledger_paths(), - committed_only=False, - read_recovery_files=True, - ) - for chunk in ledger: - _, chunk_end_seqno = chunk.get_seqnos() - if chunk_end_seqno < recovery_seqno: - continue - for tx in chunk: - tables = tx.get_public_domain().get_tables() - seqno = tx.get_public_domain().get_seqno() - if seqno < recovery_seqno: - continue - else: - tables = tx.get_public_domain().get_tables() - endorsements = tables["public:ccf.gov.nodes.snp.uvm_endorsements"] - assert len(endorsements) == 1, endorsements - (key,) = endorsements.keys() - assert key.startswith(b"did:x509:"), key - LOG.info( - f"Recovery UVM endorsement found in ledger: {endorsements[key]}" - ) - return - assert False, "No UVM endorsement found in recovery ledger" + ) as recovered_network: + + args.previous_service_identity_file = os.path.join( + old_common, "service_cert.pem" + ) + recovered_network.start_in_recovery( + args, + ledger_dir=current_ledger_dir, + committed_ledger_dirs=committed_ledger_dirs, + snapshots_dir=snapshots_dir, + ) + recovered_primary, _ = recovered_network.find_primary() + LOG.info("Check that the UVM descriptor is present in the recovery tx") + recovery_seqno = None + with recovered_primary.client() as c: + r = c.get("/node/network").body.json() + recovery_seqno = int(r["current_service_create_txid"].split(".")[1]) + network.stop_all_nodes() + ledger = ccf.ledger.Ledger( + recovered_primary.remote.ledger_paths(), + committed_only=False, + read_recovery_files=True, + ) + for chunk in ledger: + _, chunk_end_seqno = chunk.get_seqnos() + if chunk_end_seqno < recovery_seqno: + continue + for tx in chunk: + tables = tx.get_public_domain().get_tables() + seqno = tx.get_public_domain().get_seqno() + if seqno < recovery_seqno: + continue + else: + tables = tx.get_public_domain().get_tables() + endorsements = tables["public:ccf.gov.nodes.snp.uvm_endorsements"] + assert len(endorsements) == 1, endorsements + (key,) = endorsements.keys() + assert key.startswith(b"did:x509:"), key + LOG.info( + f"Recovery UVM endorsement found in ledger: {endorsements[key]}" + ) + return + assert False, "No UVM endorsement found in recovery ledger" def run_initial_tcb_version_checks(args): @@ -1171,49 +1171,50 @@ def run_initial_tcb_version_checks(args): LOG.info("Start a recovery network and stop it") current_ledger_dir, committed_ledger_dirs = primary.get_ledger() - recovered_network = infra.network.Network( + + with infra.network.network( args.nodes, args.binary_dir, args.debug_nodes, existing_network=network, - ) - args.previous_service_identity_file = os.path.join( - old_common, "service_cert.pem" - ) - recovered_network.start_in_recovery( - args, - ledger_dir=current_ledger_dir, - committed_ledger_dirs=committed_ledger_dirs, - snapshots_dir=snapshots_dir, - ) - recovered_primary, _ = recovered_network.find_primary() - LOG.info("Check that the TCB_version is present in the recovery tx") - recovery_seqno = None - with recovered_primary.client() as c: - r = c.get("/node/network").body.json() - recovery_seqno = int(r["current_service_create_txid"].split(".")[1]) - network.stop_all_nodes() - ledger = ccf.ledger.Ledger( - recovered_primary.remote.ledger_paths(), - committed_only=False, - read_recovery_files=True, - ) - for chunk in ledger: - _, chunk_end_seqno = chunk.get_seqnos() - if chunk_end_seqno < recovery_seqno: - continue - for tx in chunk: - tables = tx.get_public_domain().get_tables() - seqno = tx.get_public_domain().get_seqno() - if seqno < recovery_seqno: - continue - else: - tables = tx.get_public_domain().get_tables() - tcb_versions = tables["public:ccf.gov.nodes.snp.tcb_versions"] - assert len(tcb_versions) == 1, tcb_versions - LOG.info(f"Recovery TCB_version found in ledger: {tcb_versions}") - return - assert False, "No TCB_version found in recovery ledger" + ) as recovered_network: + args.previous_service_identity_file = os.path.join( + old_common, "service_cert.pem" + ) + recovered_network.start_in_recovery( + args, + ledger_dir=current_ledger_dir, + committed_ledger_dirs=committed_ledger_dirs, + snapshots_dir=snapshots_dir, + ) + recovered_primary, _ = recovered_network.find_primary() + LOG.info("Check that the TCB_version is present in the recovery tx") + recovery_seqno = None + with recovered_primary.client() as c: + r = c.get("/node/network").body.json() + recovery_seqno = int(r["current_service_create_txid"].split(".")[1]) + network.stop_all_nodes() + ledger = ccf.ledger.Ledger( + recovered_primary.remote.ledger_paths(), + committed_only=False, + read_recovery_files=True, + ) + for chunk in ledger: + _, chunk_end_seqno = chunk.get_seqnos() + if chunk_end_seqno < recovery_seqno: + continue + for tx in chunk: + tables = tx.get_public_domain().get_tables() + seqno = tx.get_public_domain().get_seqno() + if seqno < recovery_seqno: + continue + else: + tables = tx.get_public_domain().get_tables() + tcb_versions = tables["public:ccf.gov.nodes.snp.tcb_versions"] + assert len(tcb_versions) == 1, tcb_versions + LOG.info(f"Recovery TCB_version found in ledger: {tcb_versions}") + return + assert False, "No TCB_version found in recovery ledger" def run_recovery_local_unsealing( @@ -1249,29 +1250,29 @@ def run_recovery_local_unsealing( recovery_network_args.previous_sealed_ledger_secret_location = ( node_secret_map[node.local_node_id] ) - recovery_network = infra.network.Network( + with infra.network.network( recovery_network_args.nodes, recovery_network_args.binary_dir, next_node_id=prev_network.next_node_id, - ) + ) as recovery_network: - # Reset consortium and users to prevent issues with hosts from existing_network - recovery_network.consortium = prev_network.consortium - recovery_network.users = prev_network.users - recovery_network.txs = prev_network.txs - recovery_network.jwt_issuer = prev_network.jwt_issuer - - current_ledger_dir, committed_ledger_dirs = node.get_ledger() - recovery_network.start_in_recovery( - recovery_network_args, - ledger_dir=current_ledger_dir, - committed_ledger_dirs=committed_ledger_dirs, - ) + # Reset consortium and users to prevent issues with hosts from existing_network + recovery_network.consortium = prev_network.consortium + recovery_network.users = prev_network.users + recovery_network.txs = prev_network.txs + recovery_network.jwt_issuer = prev_network.jwt_issuer - recovery_network.recover(recovery_network_args, via_local_sealing=True) + current_ledger_dir, committed_ledger_dirs = node.get_ledger() + recovery_network.start_in_recovery( + recovery_network_args, + ledger_dir=current_ledger_dir, + committed_ledger_dirs=committed_ledger_dirs, + ) - recovery_network.stop_all_nodes() - prev_network = recovery_network + recovery_network.recover(recovery_network_args, via_local_sealing=True) + + recovery_network.stop_all_nodes() + prev_network = recovery_network def run_recovery_unsealing_validate_audit(const_args): @@ -1305,43 +1306,43 @@ def run_recovery_unsealing_validate_audit(const_args): recovery_network_args.previous_sealed_ledger_secret_location = ( node0_secrets ) - recovery_network = infra.network.Network( + with infra.network.network( recovery_network_args.nodes, recovery_network_args.binary_dir, next_node_id=prev_network.next_node_id, - ) + ) as recovery_network: - # Reset consortium and users to prevent issues with hosts from existing_network - recovery_network.consortium = prev_network.consortium - recovery_network.users = prev_network.users - recovery_network.txs = prev_network.txs - recovery_network.jwt_issuer = prev_network.jwt_issuer - - current_ledger_dir, committed_ledger_dirs = network.nodes[0].get_ledger() - recovery_network.start_in_recovery( - recovery_network_args, - ledger_dir=current_ledger_dir, - committed_ledger_dirs=committed_ledger_dirs, - ) + # Reset consortium and users to prevent issues with hosts from existing_network + recovery_network.consortium = prev_network.consortium + recovery_network.users = prev_network.users + recovery_network.txs = prev_network.txs + recovery_network.jwt_issuer = prev_network.jwt_issuer - recovery_network.recover( - recovery_network_args, via_local_sealing=via_local_unsealing - ) + current_ledger_dir, committed_ledger_dirs = network.nodes[0].get_ledger() + recovery_network.start_in_recovery( + recovery_network_args, + ledger_dir=current_ledger_dir, + committed_ledger_dirs=committed_ledger_dirs, + ) - latest_public_tables, _ = recovery_network.get_latest_ledger_public_state() - recovery_type = latest_public_tables[ - "public:ccf.internal.last_recovery_type" - ][b"\x00\x00\x00\x00\x00\x00\x00\x00"].decode("utf-8") - expected_recovery_type = ( - '"LOCAL_UNSEALING"' if via_local_unsealing else '"RECOVERY_SHARES"' - ) - assert ( - recovery_type == expected_recovery_type - ), f"Network recovery type was {recovery_type} instead of {expected_recovery_type}" + recovery_network.recover( + recovery_network_args, via_local_sealing=via_local_unsealing + ) + + latest_public_tables, _ = recovery_network.get_latest_ledger_public_state() + recovery_type = latest_public_tables[ + "public:ccf.internal.last_recovery_type" + ][b"\x00\x00\x00\x00\x00\x00\x00\x00"].decode("utf-8") + expected_recovery_type = ( + '"LOCAL_UNSEALING"' if via_local_unsealing else '"RECOVERY_SHARES"' + ) + assert ( + recovery_type == expected_recovery_type + ), f"Network recovery type was {recovery_type} instead of {expected_recovery_type}" - recovery_network.stop_all_nodes() + recovery_network.stop_all_nodes() - prev_network = recovery_network + prev_network = recovery_network def run_recovery_unsealing_corrupt(const_args, recovery_f=0): @@ -1447,43 +1448,43 @@ def run(self, src_dir, dst_dir): recovery_network_args.previous_sealed_ledger_secret_location = ( corrupt_ledger_secret ) - recovery_network = infra.network.Network( + with infra.network.network( recovery_network_args.nodes, recovery_network_args.binary_dir, next_node_id=prev_network.next_node_id, - ) - - # Reset consortium and users to prevent issues with hosts from existing_network - recovery_network.consortium = prev_network.consortium - recovery_network.users = prev_network.users - recovery_network.txs = prev_network.txs - recovery_network.jwt_issuer = prev_network.jwt_issuer - - current_ledger_dir, committed_ledger_dirs = node.get_ledger() - exception_thrown = None - try: - recovery_network.start_in_recovery( - recovery_network_args, - ledger_dir=current_ledger_dir, - committed_ledger_dirs=committed_ledger_dirs, - ) - - recovery_network.recover(recovery_network_args, via_local_sealing=True) - except Exception as e: - exception_thrown = e - pass - - if corruption.expected_exception: - assert ( - exception_thrown is not None - ), f"Expected exception to be thrown for {corruption.tag} corruption" - else: - assert ( - exception_thrown is None - ), f"Expected no exception to be thrown for {corruption.tag} corruption" - - recovery_network.stop_all_nodes() - prev_network = recovery_network + ) as recovery_network: + + # Reset consortium and users to prevent issues with hosts from existing_network + recovery_network.consortium = prev_network.consortium + recovery_network.users = prev_network.users + recovery_network.txs = prev_network.txs + recovery_network.jwt_issuer = prev_network.jwt_issuer + + current_ledger_dir, committed_ledger_dirs = node.get_ledger() + exception_thrown = None + try: + recovery_network.start_in_recovery( + recovery_network_args, + ledger_dir=current_ledger_dir, + committed_ledger_dirs=committed_ledger_dirs, + ) + + recovery_network.recover(recovery_network_args, via_local_sealing=True) + except Exception as e: + exception_thrown = e + pass + + if corruption.expected_exception: + assert ( + exception_thrown is not None + ), f"Expected exception to be thrown for {corruption.tag} corruption" + else: + assert ( + exception_thrown is None + ), f"Expected no exception to be thrown for {corruption.tag} corruption" + + recovery_network.stop_all_nodes() + prev_network = recovery_network def run_read_ledger_on_testdata(args): From af4b4253ac56720e17f5eb7ae3c2576a53e101bc Mon Sep 17 00:00:00 2001 From: cjen1-msft Date: Wed, 22 Oct 2025 13:55:40 +0100 Subject: [PATCH 2/7] Ensure all networks have an approiate label --- tests/e2e_operations.py | 354 ++++++++++++++++++++++------------------ 1 file changed, 192 insertions(+), 162 deletions(-) diff --git a/tests/e2e_operations.py b/tests/e2e_operations.py index b8049fb07ab..791b526e4ef 100644 --- a/tests/e2e_operations.py +++ b/tests/e2e_operations.py @@ -1067,7 +1067,9 @@ def run_empty_ledger_dir_check(args): ) -def run_initial_uvm_descriptor_checks(args): +def run_initial_uvm_descriptor_checks(const_args): + args = copy.deepcopy(const_args) + args.label += "_uvm_descriptor" with infra.network.network( args.nodes, args.binary_dir, @@ -1095,57 +1097,64 @@ def run_initial_uvm_descriptor_checks(args): LOG.info("Start a recovery network and stop it") current_ledger_dir, committed_ledger_dirs = primary.get_ledger() + recovered_network_args = copy.deepcopy(args) + recovered_network_args.label += "_recovery" + with infra.network.network( - args.nodes, - args.binary_dir, - args.debug_nodes, + recovered_network_args.nodes, + recovered_network_args.binary_dir, + recovered_network_args.debug_nodes, existing_network=network, ) as recovered_network: - args.previous_service_identity_file = os.path.join( - old_common, "service_cert.pem" - ) - recovered_network.start_in_recovery( - args, - ledger_dir=current_ledger_dir, - committed_ledger_dirs=committed_ledger_dirs, - snapshots_dir=snapshots_dir, - ) - recovered_primary, _ = recovered_network.find_primary() - LOG.info("Check that the UVM descriptor is present in the recovery tx") - recovery_seqno = None - with recovered_primary.client() as c: - r = c.get("/node/network").body.json() - recovery_seqno = int(r["current_service_create_txid"].split(".")[1]) - network.stop_all_nodes() - ledger = ccf.ledger.Ledger( - recovered_primary.remote.ledger_paths(), - committed_only=False, - read_recovery_files=True, - ) - for chunk in ledger: - _, chunk_end_seqno = chunk.get_seqnos() - if chunk_end_seqno < recovery_seqno: - continue - for tx in chunk: - tables = tx.get_public_domain().get_tables() - seqno = tx.get_public_domain().get_seqno() - if seqno < recovery_seqno: - continue - else: - tables = tx.get_public_domain().get_tables() - endorsements = tables["public:ccf.gov.nodes.snp.uvm_endorsements"] - assert len(endorsements) == 1, endorsements - (key,) = endorsements.keys() - assert key.startswith(b"did:x509:"), key - LOG.info( - f"Recovery UVM endorsement found in ledger: {endorsements[key]}" - ) - return - assert False, "No UVM endorsement found in recovery ledger" - - -def run_initial_tcb_version_checks(args): + recovered_network_args.previous_service_identity_file = os.path.join( + old_common, "service_cert.pem" + ) + recovered_network.start_in_recovery( + recovered_network_args, + ledger_dir=current_ledger_dir, + committed_ledger_dirs=committed_ledger_dirs, + snapshots_dir=snapshots_dir, + ) + recovered_primary, _ = recovered_network.find_primary() + LOG.info("Check that the UVM descriptor is present in the recovery tx") + recovery_seqno = None + with recovered_primary.client() as c: + r = c.get("/node/network").body.json() + recovery_seqno = int(r["current_service_create_txid"].split(".")[1]) + network.stop_all_nodes() + ledger = ccf.ledger.Ledger( + recovered_primary.remote.ledger_paths(), + committed_only=False, + read_recovery_files=True, + ) + for chunk in ledger: + _, chunk_end_seqno = chunk.get_seqnos() + if chunk_end_seqno < recovery_seqno: + continue + for tx in chunk: + tables = tx.get_public_domain().get_tables() + seqno = tx.get_public_domain().get_seqno() + if seqno < recovery_seqno: + continue + else: + tables = tx.get_public_domain().get_tables() + endorsements = tables[ + "public:ccf.gov.nodes.snp.uvm_endorsements" + ] + assert len(endorsements) == 1, endorsements + (key,) = endorsements.keys() + assert key.startswith(b"did:x509:"), key + LOG.info( + f"Recovery UVM endorsement found in ledger: {endorsements[key]}" + ) + return + assert False, "No UVM endorsement found in recovery ledger" + + +def run_initial_tcb_version_checks(const_args): + args = copy.deepcopy(const_args) + args.label += "_tcb_version" with infra.network.network( args.nodes, args.binary_dir, @@ -1172,49 +1181,53 @@ def run_initial_tcb_version_checks(args): LOG.info("Start a recovery network and stop it") current_ledger_dir, committed_ledger_dirs = primary.get_ledger() + recovered_network_args = copy.deepcopy(args) + recovered_network_args.previous_service_identity_file = os.path.join( + old_common, "service_cert.pem" + ) + recovered_network_args.label += "_recovery" with infra.network.network( - args.nodes, - args.binary_dir, - args.debug_nodes, + recovered_network_args.nodes, + recovered_network_args.binary_dir, + recovered_network_args.debug_nodes, existing_network=network, ) as recovered_network: - args.previous_service_identity_file = os.path.join( - old_common, "service_cert.pem" - ) - recovered_network.start_in_recovery( - args, - ledger_dir=current_ledger_dir, - committed_ledger_dirs=committed_ledger_dirs, - snapshots_dir=snapshots_dir, - ) - recovered_primary, _ = recovered_network.find_primary() - LOG.info("Check that the TCB_version is present in the recovery tx") - recovery_seqno = None - with recovered_primary.client() as c: - r = c.get("/node/network").body.json() - recovery_seqno = int(r["current_service_create_txid"].split(".")[1]) - network.stop_all_nodes() - ledger = ccf.ledger.Ledger( - recovered_primary.remote.ledger_paths(), - committed_only=False, - read_recovery_files=True, - ) - for chunk in ledger: - _, chunk_end_seqno = chunk.get_seqnos() - if chunk_end_seqno < recovery_seqno: - continue - for tx in chunk: - tables = tx.get_public_domain().get_tables() - seqno = tx.get_public_domain().get_seqno() - if seqno < recovery_seqno: - continue - else: - tables = tx.get_public_domain().get_tables() - tcb_versions = tables["public:ccf.gov.nodes.snp.tcb_versions"] - assert len(tcb_versions) == 1, tcb_versions - LOG.info(f"Recovery TCB_version found in ledger: {tcb_versions}") - return - assert False, "No TCB_version found in recovery ledger" + recovered_network.start_in_recovery( + recovered_network_args, + ledger_dir=current_ledger_dir, + committed_ledger_dirs=committed_ledger_dirs, + snapshots_dir=snapshots_dir, + ) + recovered_primary, _ = recovered_network.find_primary() + LOG.info("Check that the TCB_version is present in the recovery tx") + recovery_seqno = None + with recovered_primary.client() as c: + r = c.get("/node/network").body.json() + recovery_seqno = int(r["current_service_create_txid"].split(".")[1]) + network.stop_all_nodes() + ledger = ccf.ledger.Ledger( + recovered_primary.remote.ledger_paths(), + committed_only=False, + read_recovery_files=True, + ) + for chunk in ledger: + _, chunk_end_seqno = chunk.get_seqnos() + if chunk_end_seqno < recovery_seqno: + continue + for tx in chunk: + tables = tx.get_public_domain().get_tables() + seqno = tx.get_public_domain().get_seqno() + if seqno < recovery_seqno: + continue + else: + tables = tx.get_public_domain().get_tables() + tcb_versions = tables["public:ccf.gov.nodes.snp.tcb_versions"] + assert len(tcb_versions) == 1, tcb_versions + LOG.info( + f"Recovery TCB_version found in ledger: {tcb_versions}" + ) + return + assert False, "No TCB_version found in recovery ledger" def run_recovery_local_unsealing( @@ -1224,6 +1237,7 @@ def run_recovery_local_unsealing( args = copy.deepcopy(const_args) args.nodes = infra.e2e_args.min_nodes(args, f=1) args.enable_local_sealing = True + args.label += "_unsealing" with infra.network.network(args.nodes, args.binary_dir) as network: network.start_and_open(args) @@ -1250,29 +1264,31 @@ def run_recovery_local_unsealing( recovery_network_args.previous_sealed_ledger_secret_location = ( node_secret_map[node.local_node_id] ) + recovery_network_args.label += f"_recovery_from_node_{node.local_node_id}" + with infra.network.network( recovery_network_args.nodes, recovery_network_args.binary_dir, next_node_id=prev_network.next_node_id, ) as recovery_network: - # Reset consortium and users to prevent issues with hosts from existing_network - recovery_network.consortium = prev_network.consortium - recovery_network.users = prev_network.users - recovery_network.txs = prev_network.txs - recovery_network.jwt_issuer = prev_network.jwt_issuer - - current_ledger_dir, committed_ledger_dirs = node.get_ledger() - recovery_network.start_in_recovery( - recovery_network_args, - ledger_dir=current_ledger_dir, - committed_ledger_dirs=committed_ledger_dirs, - ) + # Reset consortium and users to prevent issues with hosts from existing_network + recovery_network.consortium = prev_network.consortium + recovery_network.users = prev_network.users + recovery_network.txs = prev_network.txs + recovery_network.jwt_issuer = prev_network.jwt_issuer + + current_ledger_dir, committed_ledger_dirs = node.get_ledger() + recovery_network.start_in_recovery( + recovery_network_args, + ledger_dir=current_ledger_dir, + committed_ledger_dirs=committed_ledger_dirs, + ) - recovery_network.recover(recovery_network_args, via_local_sealing=True) + recovery_network.recover(recovery_network_args, via_local_sealing=True) - recovery_network.stop_all_nodes() - prev_network = recovery_network + recovery_network.stop_all_nodes() + prev_network = recovery_network def run_recovery_unsealing_validate_audit(const_args): @@ -1280,6 +1296,7 @@ def run_recovery_unsealing_validate_audit(const_args): args = copy.deepcopy(const_args) args.nodes = infra.e2e_args.min_nodes(args, f=1) args.enable_local_sealing = True + args.label += "_unsealing_audit" with infra.network.network(args.nodes, args.binary_dir) as network: network.start_and_open(args) @@ -1302,6 +1319,11 @@ def run_recovery_unsealing_validate_audit(const_args): for via_local_unsealing in [True, False]: recovery_network_args = copy.deepcopy(args) recovery_network_args.nodes = infra.e2e_args.min_nodes(args, f=0) + if via_local_unsealing: + recovery_network_args.label += "_via_local_unsealing" + else: + recovery_network_args.label += "_via_recovery_shares" + if via_local_unsealing: recovery_network_args.previous_sealed_ledger_secret_location = ( node0_secrets @@ -1312,37 +1334,41 @@ def run_recovery_unsealing_validate_audit(const_args): next_node_id=prev_network.next_node_id, ) as recovery_network: - # Reset consortium and users to prevent issues with hosts from existing_network - recovery_network.consortium = prev_network.consortium - recovery_network.users = prev_network.users - recovery_network.txs = prev_network.txs - recovery_network.jwt_issuer = prev_network.jwt_issuer - - current_ledger_dir, committed_ledger_dirs = network.nodes[0].get_ledger() - recovery_network.start_in_recovery( - recovery_network_args, - ledger_dir=current_ledger_dir, - committed_ledger_dirs=committed_ledger_dirs, - ) + # Reset consortium and users to prevent issues with hosts from existing_network + recovery_network.consortium = prev_network.consortium + recovery_network.users = prev_network.users + recovery_network.txs = prev_network.txs + recovery_network.jwt_issuer = prev_network.jwt_issuer + + current_ledger_dir, committed_ledger_dirs = network.nodes[ + 0 + ].get_ledger() + recovery_network.start_in_recovery( + recovery_network_args, + ledger_dir=current_ledger_dir, + committed_ledger_dirs=committed_ledger_dirs, + ) - recovery_network.recover( - recovery_network_args, via_local_sealing=via_local_unsealing - ) + recovery_network.recover( + recovery_network_args, via_local_sealing=via_local_unsealing + ) - latest_public_tables, _ = recovery_network.get_latest_ledger_public_state() - recovery_type = latest_public_tables[ - "public:ccf.internal.last_recovery_type" - ][b"\x00\x00\x00\x00\x00\x00\x00\x00"].decode("utf-8") - expected_recovery_type = ( - '"LOCAL_UNSEALING"' if via_local_unsealing else '"RECOVERY_SHARES"' - ) - assert ( - recovery_type == expected_recovery_type - ), f"Network recovery type was {recovery_type} instead of {expected_recovery_type}" + latest_public_tables, _ = ( + recovery_network.get_latest_ledger_public_state() + ) + recovery_type = latest_public_tables[ + "public:ccf.internal.last_recovery_type" + ][b"\x00\x00\x00\x00\x00\x00\x00\x00"].decode("utf-8") + expected_recovery_type = ( + '"LOCAL_UNSEALING"' if via_local_unsealing else '"RECOVERY_SHARES"' + ) + assert ( + recovery_type == expected_recovery_type + ), f"Network recovery type was {recovery_type} instead of {expected_recovery_type}" - recovery_network.stop_all_nodes() + recovery_network.stop_all_nodes() - prev_network = recovery_network + prev_network = recovery_network def run_recovery_unsealing_corrupt(const_args, recovery_f=0): @@ -1350,6 +1376,7 @@ def run_recovery_unsealing_corrupt(const_args, recovery_f=0): args = copy.deepcopy(const_args) args.nodes = infra.e2e_args.min_nodes(args, f=1) args.enable_local_sealing = True + args.label += "_recovery_unsealing_corrupt" with infra.network.network(args.nodes, args.binary_dir) as network: network.start_and_open(args) @@ -1444,47 +1471,50 @@ def run(self, src_dir, dst_dir): corruption.run(ledger_secret, corrupt_ledger_secret) recovery_network_args = copy.deepcopy(args) - recovery_network_args.nodes = infra.e2e_args.min_nodes(args, f=recovery_f) + recovery_network_args.nodes = infra.e2e_args.min_nodes(recovery_network_args, f=recovery_f) recovery_network_args.previous_sealed_ledger_secret_location = ( corrupt_ledger_secret ) + recovery_network_args.label += f"_{corruption.tag}" with infra.network.network( recovery_network_args.nodes, recovery_network_args.binary_dir, next_node_id=prev_network.next_node_id, ) as recovery_network: - # Reset consortium and users to prevent issues with hosts from existing_network - recovery_network.consortium = prev_network.consortium - recovery_network.users = prev_network.users - recovery_network.txs = prev_network.txs - recovery_network.jwt_issuer = prev_network.jwt_issuer - - current_ledger_dir, committed_ledger_dirs = node.get_ledger() - exception_thrown = None - try: - recovery_network.start_in_recovery( - recovery_network_args, - ledger_dir=current_ledger_dir, - committed_ledger_dirs=committed_ledger_dirs, - ) - - recovery_network.recover(recovery_network_args, via_local_sealing=True) - except Exception as e: - exception_thrown = e - pass - - if corruption.expected_exception: - assert ( - exception_thrown is not None - ), f"Expected exception to be thrown for {corruption.tag} corruption" - else: - assert ( - exception_thrown is None - ), f"Expected no exception to be thrown for {corruption.tag} corruption" - - recovery_network.stop_all_nodes() - prev_network = recovery_network + # Reset consortium and users to prevent issues with hosts from existing_network + recovery_network.consortium = prev_network.consortium + recovery_network.users = prev_network.users + recovery_network.txs = prev_network.txs + recovery_network.jwt_issuer = prev_network.jwt_issuer + + current_ledger_dir, committed_ledger_dirs = node.get_ledger() + exception_thrown = None + try: + recovery_network.start_in_recovery( + recovery_network_args, + ledger_dir=current_ledger_dir, + committed_ledger_dirs=committed_ledger_dirs, + ) + + recovery_network.recover( + recovery_network_args, via_local_sealing=True + ) + except Exception as e: + exception_thrown = e + pass + + if corruption.expected_exception: + assert ( + exception_thrown is not None + ), f"Expected exception to be thrown for {corruption.tag} corruption" + else: + assert ( + exception_thrown is None + ), f"Expected no exception to be thrown for {corruption.tag} corruption" + + recovery_network.stop_all_nodes() + prev_network = recovery_network def run_read_ledger_on_testdata(args): From 38717a3fb30e0cd36588adc3a1a136554807f2c6 Mon Sep 17 00:00:00 2001 From: cjen1-msft Date: Wed, 22 Oct 2025 15:00:37 +0100 Subject: [PATCH 3/7] fmt --- tests/e2e_operations.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/e2e_operations.py b/tests/e2e_operations.py index 791b526e4ef..7655f0aead3 100644 --- a/tests/e2e_operations.py +++ b/tests/e2e_operations.py @@ -1471,7 +1471,9 @@ def run(self, src_dir, dst_dir): corruption.run(ledger_secret, corrupt_ledger_secret) recovery_network_args = copy.deepcopy(args) - recovery_network_args.nodes = infra.e2e_args.min_nodes(recovery_network_args, f=recovery_f) + recovery_network_args.nodes = infra.e2e_args.min_nodes( + recovery_network_args, f=recovery_f + ) recovery_network_args.previous_sealed_ledger_secret_location = ( corrupt_ledger_secret ) From af05e1a84da665cf2d58c0ee7e8b085fa0f4afa9 Mon Sep 17 00:00:00 2001 From: cjen1-msft Date: Thu, 23 Oct 2025 10:36:55 +0100 Subject: [PATCH 4/7] Forward kwargs network context --- tests/infra/network.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/infra/network.py b/tests/infra/network.py index 3d67a9f1df2..d69c4528738 100644 --- a/tests/infra/network.py +++ b/tests/infra/network.py @@ -1789,6 +1789,7 @@ def network( version=None, service_load=None, node_data_json_file=None, + **kwargs, ): """ Context manager for Network class. @@ -1798,6 +1799,7 @@ def network( :param dbg_nodes: default: []. List of node id's that will not start (user is prompted to start them manually) :param pdb: default: False. Debugger. :param txs: default: None. Transactions committed on that network. + :param kwargs. Parameters to forward to the Network constructor :return: a Network instance that can be used to create/access nodes, handle the genesis state (add members, create node.json), and stop all the nodes that belong to the network """ @@ -1815,6 +1817,7 @@ def network( version=version, service_load=service_load, node_data_json_file=node_data_json_file, + **kwargs, ) try: yield net From 204f57cde5e5cd692bebb970c3946c7c44b9ff5b Mon Sep 17 00:00:00 2001 From: cjen1-msft Date: Mon, 27 Oct 2025 13:39:06 +0000 Subject: [PATCH 5/7] Use pre-existing common for recovery networks --- tests/e2e_operations.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/e2e_operations.py b/tests/e2e_operations.py index 6cdb3585113..b266ddb2757 100644 --- a/tests/e2e_operations.py +++ b/tests/e2e_operations.py @@ -1110,6 +1110,7 @@ def run_initial_uvm_descriptor_checks(const_args): ) recovered_network.start_in_recovery( recovered_network_args, + common_dir=old_common, ledger_dir=current_ledger_dir, committed_ledger_dirs=committed_ledger_dirs, snapshots_dir=snapshots_dir, @@ -1192,6 +1193,7 @@ def run_initial_tcb_version_checks(const_args): ) as recovered_network: recovered_network.start_in_recovery( recovered_network_args, + common_dir=old_common, ledger_dir=current_ledger_dir, committed_ledger_dirs=committed_ledger_dirs, snapshots_dir=snapshots_dir, @@ -1279,6 +1281,7 @@ def run_recovery_local_unsealing( current_ledger_dir, committed_ledger_dirs = node.get_ledger() recovery_network.start_in_recovery( recovery_network_args, + common_dir = infra.network.get_common_folder_name(args.workspace, args.label), ledger_dir=current_ledger_dir, committed_ledger_dirs=committed_ledger_dirs, ) @@ -1493,6 +1496,7 @@ def run(self, src_dir, dst_dir): try: recovery_network.start_in_recovery( recovery_network_args, + common_dir = infra.network.get_common_folder_name(args.workspace, args.label), ledger_dir=current_ledger_dir, committed_ledger_dirs=committed_ledger_dirs, ) From 569a0201e79ecd6308313abf3469559cba628a7f Mon Sep 17 00:00:00 2001 From: cjen1-msft Date: Mon, 27 Oct 2025 13:39:59 +0000 Subject: [PATCH 6/7] fmt --- tests/e2e_operations.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/e2e_operations.py b/tests/e2e_operations.py index b266ddb2757..b42c6fcd20e 100644 --- a/tests/e2e_operations.py +++ b/tests/e2e_operations.py @@ -1281,7 +1281,9 @@ def run_recovery_local_unsealing( current_ledger_dir, committed_ledger_dirs = node.get_ledger() recovery_network.start_in_recovery( recovery_network_args, - common_dir = infra.network.get_common_folder_name(args.workspace, args.label), + common_dir=infra.network.get_common_folder_name( + args.workspace, args.label + ), ledger_dir=current_ledger_dir, committed_ledger_dirs=committed_ledger_dirs, ) @@ -1496,7 +1498,9 @@ def run(self, src_dir, dst_dir): try: recovery_network.start_in_recovery( recovery_network_args, - common_dir = infra.network.get_common_folder_name(args.workspace, args.label), + common_dir=infra.network.get_common_folder_name( + args.workspace, args.label + ), ledger_dir=current_ledger_dir, committed_ledger_dirs=committed_ledger_dirs, ) From 39526bf9806f0178cae8302f652a4dd3a327b5d1 Mon Sep 17 00:00:00 2001 From: cjen1-msft Date: Tue, 28 Oct 2025 10:53:42 +0000 Subject: [PATCH 7/7] snag --- tests/e2e_operations.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/e2e_operations.py b/tests/e2e_operations.py index b42c6fcd20e..6247838be3d 100644 --- a/tests/e2e_operations.py +++ b/tests/e2e_operations.py @@ -1348,6 +1348,9 @@ def run_recovery_unsealing_validate_audit(const_args): ].get_ledger() recovery_network.start_in_recovery( recovery_network_args, + common_dir=infra.network.get_common_folder_name( + args.workspace, args.label + ), ledger_dir=current_ledger_dir, committed_ledger_dirs=committed_ledger_dirs, )