From 1dd3f32b5ef389c2f6825a4e9c54e7da1c56ff5d Mon Sep 17 00:00:00 2001 From: Olivier Mehani Date: Mon, 5 May 2025 17:18:13 +1000 Subject: [PATCH 01/10] wip --- git_hg_sync/repo_synchronizer.py | 38 +++++++++++++++++++++++------ tests/test_repo_synchronizer.py | 41 ++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 7 deletions(-) diff --git a/git_hg_sync/repo_synchronizer.py b/git_hg_sync/repo_synchronizer.py index eea0a10..efc7edd 100644 --- a/git_hg_sync/repo_synchronizer.py +++ b/git_hg_sync/repo_synchronizer.py @@ -61,10 +61,10 @@ def sync( logger.info(f"Syncing {operations} to {destination_url} ...") try: repo = self.get_clone_repo() - except PermissionError as exc: + except PermissionError as e: raise PermissionError( f"Failed to create local clone from {destination_url}" - ) from exc + ) from e destination_remote = f"hg::{destination_url}" @@ -83,12 +83,36 @@ def sync( op for op in operations if isinstance(op, SyncBranchOperation) ] for branch_operation in branch_ops: + destination_ref = ( + f"refs/heads/branches/{branch_operation.destination_branch}/tip" + ) + # If the source_commit is already an ancestor of the destination branch, we + # silently skip it, as it should be a NoOp, but would fail as the push would + # not be a fast-forward. try: - push_args.append( - f"{branch_operation.source_commit}:refs/heads/branches/{branch_operation.destination_branch}/tip" - ) - except Exception as e: - raise RepoSyncError(branch_operation, e) from e + breakpoint() + if repo.git.execute( + [ + "git", + "merge-base", + "--is-ancestor", + branch_operation.source_commit, + destination_ref, + ] + ): + logger.warning( + f"Source commit {branch_operation.source_commit} is already an ancestor of the current tip at {destination_url}, skipping ..." + ) + continue + except exc.GitCommandError as e: + if f"Not a valid object name {destination_ref}" in e.stderr: + # The target branch doesn't exist yet, we'll create it soon. + pass + else: + raise RepoSyncError( + f"Error checking ancestry of {branch_operation.source_commit} to {destination_ref}" + ) from e + push_args.append(f"{branch_operation.source_commit}:{destination_ref}") os.environ[REQUEST_USER_ENV_VAR] = request_user logger.debug(f"{REQUEST_USER_ENV_VAR} set to {request_user}") diff --git a/tests/test_repo_synchronizer.py b/tests/test_repo_synchronizer.py index 7dac181..b9f356a 100644 --- a/tests/test_repo_synchronizer.py +++ b/tests/test_repo_synchronizer.py @@ -104,6 +104,47 @@ def test_sync_process_( assert hg_rev(hg_destination, branch) in tag_log +def test_sync_process_ancestor( + git_source: Repo, + hg_destination: Path, + tmp_path: Path, +) -> None: + branch = "bar" + + repo = Repo(git_source) + + # Create a new commit on git repo + bar_path = git_source / "bar.txt" + bar_path.write_text("BAR CONTENT") + repo.index.add([bar_path]) + git_commit_sha1 = repo.index.commit("add bar.txt").hexsha + + baz_path = git_source / "baz.txt" + baz_path.write_text("BAZ CONTENT") + repo.index.add([baz_path]) + git_commit_sha2 = repo.index.commit("add baz.txt").hexsha + + # Sync new commit with mercurial repository + git_local_repo_path = tmp_path / "clones" / "myrepo" + syncrepos = RepoSynchronizer(git_local_repo_path, str(git_source)) + operations: list[SyncBranchOperation | SyncTagOperation] = [ + SyncBranchOperation(source_commit=git_commit_sha2, destination_branch=branch), + ] + + request_user = "request_user@example.com" + syncrepos.sync(str(hg_destination), operations, request_user) + + # Sync an earlier commit. + operations: list[SyncBranchOperation | SyncTagOperation] = [ + SyncBranchOperation(source_commit=git_commit_sha1, destination_branch=branch), + ] + syncrepos.sync(str(hg_destination), operations, request_user) + + # test + assert "BAR CONTENT" in hg_cat(hg_destination, "bar.txt", branch) + assert "BAZ CONTENT" in hg_cat(hg_destination, "baz.txt", branch) + + def test_sync_process_duplicate_tags( git_source: Repo, hg_destination: Path, From 28481ef392de28cb3a0ec67b6e0298358ccc93de Mon Sep 17 00:00:00 2001 From: Olivier Mehani Date: Fri, 9 May 2025 15:02:07 +1000 Subject: [PATCH 02/10] repo_synchroniser: note about NOT using refs/cinnabar --- git_hg_sync/repo_synchronizer.py | 34 ++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/git_hg_sync/repo_synchronizer.py b/git_hg_sync/repo_synchronizer.py index efc7edd..c0521c8 100644 --- a/git_hg_sync/repo_synchronizer.py +++ b/git_hg_sync/repo_synchronizer.py @@ -90,24 +90,34 @@ def sync( # silently skip it, as it should be a NoOp, but would fail as the push would # not be a fast-forward. try: - breakpoint() - if repo.git.execute( + cinnabar_destination_ref = f"refs/cinnabar/{destination_ref}" + destination_exists = destination_ref in repo.git.execute( + ["git", "ls-remote", destination_remote, destination_ref], + stdout_as_string=True, + ) + retval, _, _ = repo.git.execute( [ "git", "merge-base", "--is-ancestor", branch_operation.source_commit, - destination_ref, - ] - ): + cinnabar_destination_ref, + ], + with_extended_output=True, + with_exceptions=False, + ) + commit_is_ancestor = retval == 0 + if destination_exists and commit_is_ancestor: logger.warning( f"Source commit {branch_operation.source_commit} is already an ancestor of the current tip at {destination_url}, skipping ..." ) continue except exc.GitCommandError as e: - if f"Not a valid object name {destination_ref}" in e.stderr: + if f"Not a valid object name {cinnabar_destination_ref}" in e.stderr: # The target branch doesn't exist yet, we'll create it soon. - pass + logger.info( + f"Destination {destination_ref} doesn't exist on {destination_url} yet, it will be created" + ) else: raise RepoSyncError( f"Error checking ancestry of {branch_operation.source_commit} to {destination_ref}" @@ -121,7 +131,7 @@ def sync( # Some of these commits could be tagged in the same synchronization and # tagging can only be done on a commit that already have mercurial # metadata - if branch_ops: + if len(push_args) > 1: retry( "adding mercurial metadata to git commits", lambda: repo.git.execute( @@ -141,7 +151,7 @@ def sync( for tag_operation in tag_ops: tag_branch = tag_operation.tags_destination_branch remote_tag_ref = f"refs/heads/branches/{tag_branch}/tip" - if repo.git.execute( + if remote_tag_ref in repo.git.execute( ["git", "ls-remote", destination_remote, remote_tag_ref], stdout_as_string=True, ): @@ -227,7 +237,11 @@ def _ensure_cinnabar_metadata(self, repo: Repo, destination_remote: str) -> None # This is needed only on first initialisation of the repository, as subsequent # pushes update the metadata locally. - + # + # WARNING: While we make a direct reference to `refs/cinnabar` here, it MUST NOT + # be used explicitely in subsequent git operations. This set of references get + # updated on every `fetch`, and is therefore not stable enough to be trusted. + # # Repo.git_dir is a PathLike union which is either a str, or a smarter thing. We # assume the less smart one. cinnabar_metadata_dir = Path(repo.git_dir) / "refs/cinnabar/metadata" From 6b601a19fd096d8303774763b4be7f0fde49e362 Mon Sep 17 00:00:00 2001 From: Olivier Mehani Date: Fri, 9 May 2025 15:16:42 +1000 Subject: [PATCH 03/10] repo_synchroniser: don't push commits to tip to avoid non fast-forward issues when reprocessing old messages --- git_hg_sync/repo_synchronizer.py | 72 ++++++++++++++++---------------- 1 file changed, 37 insertions(+), 35 deletions(-) diff --git a/git_hg_sync/repo_synchronizer.py b/git_hg_sync/repo_synchronizer.py index c0521c8..3c6cb23 100644 --- a/git_hg_sync/repo_synchronizer.py +++ b/git_hg_sync/repo_synchronizer.py @@ -83,45 +83,47 @@ def sync( op for op in operations if isinstance(op, SyncBranchOperation) ] for branch_operation in branch_ops: - destination_ref = ( - f"refs/heads/branches/{branch_operation.destination_branch}/tip" - ) - # If the source_commit is already an ancestor of the destination branch, we - # silently skip it, as it should be a NoOp, but would fail as the push would - # not be a fast-forward. - try: - cinnabar_destination_ref = f"refs/cinnabar/{destination_ref}" - destination_exists = destination_ref in repo.git.execute( - ["git", "ls-remote", destination_remote, destination_ref], - stdout_as_string=True, - ) - retval, _, _ = repo.git.execute( + # Here, we use `/` rather than `tip` to work around inherent + # limitations in the mapping between Git and Hg references. + # + # We could use `/tip`, which would work in most cases. However, when + # reprocessing old messages (as is sometimes necessary to recover from + # issues), we may find ourselves processing a push for a commit which is now + # an ancestor of the current `tip`. In this situation, git would refuse to + # push, claiming it's not a fast-forward. + # + # To handle this case, we push each commit to a separate reference matching + # their own SHA1. Those references only exist on the git side, so their name + # doesn't impact what gets created on the Mercurial side. [The name of the + # branch matters with `cinnabar.experiments=branch`, but not the name of the + # final reference.] + # + # Mercurial maintains `tip` automatically to be the latest new commit (and + # we only allow single heads on pushable repositories, which guarantees it's + # the furthest from the root). + # + destination_ref = f"refs/heads/branches/{branch_operation.destination_branch}/${branch_operation.source_commit}" + # We only push the commit if it's not already present, because Mercurial + # refuses pushes which don't change anything. + if self._commit_has_mercurial_metadata( + repo, branch_operation.source_commit + ): + # Resolving the HG SHA is not sufficient, because we may know it from + # another repository, so we need to make sure it's not already present here. + hg_sha = self._git2hg(repo, branch_operation.source_commit) + if hg_sha not in repo.git.execute( [ "git", - "merge-base", - "--is-ancestor", - branch_operation.source_commit, - cinnabar_destination_ref, + "ls-remote", + destination_remote, + f"refs/heads/branches/{branch_operation.destination_branch}/{hg_sha}", ], - with_extended_output=True, - with_exceptions=False, - ) - commit_is_ancestor = retval == 0 - if destination_exists and commit_is_ancestor: - logger.warning( - f"Source commit {branch_operation.source_commit} is already an ancestor of the current tip at {destination_url}, skipping ..." - ) - continue - except exc.GitCommandError as e: - if f"Not a valid object name {cinnabar_destination_ref}" in e.stderr: - # The target branch doesn't exist yet, we'll create it soon. + stdout_as_string=True, + ): logger.info( - f"Destination {destination_ref} doesn't exist on {destination_url} yet, it will be created" + f"Commit {branch_operation.source_commit} is already present on {destination_remote}, skipping ..." ) - else: - raise RepoSyncError( - f"Error checking ancestry of {branch_operation.source_commit} to {destination_ref}" - ) from e + continue push_args.append(f"{branch_operation.source_commit}:{destination_ref}") os.environ[REQUEST_USER_ENV_VAR] = request_user @@ -130,7 +132,7 @@ def sync( # Add mercurial metadata to new commits from synced branches # Some of these commits could be tagged in the same synchronization and # tagging can only be done on a commit that already have mercurial - # metadata + # metadata. if len(push_args) > 1: retry( "adding mercurial metadata to git commits", From 472fd126c76cd6d3f8542f04de915da9316fe6bf Mon Sep 17 00:00:00 2001 From: Olivier Mehani Date: Fri, 9 May 2025 15:51:39 +1000 Subject: [PATCH 04/10] config: update the development branch tags --- config-development.toml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/config-development.toml b/config-development.toml index 97e194d..eaca559 100644 --- a/config-development.toml +++ b/config-development.toml @@ -42,7 +42,7 @@ destination_branch = "\\1" source_url = "https://github.com/mozilla-conduit/ff-test" tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(BETA|NIGHTLY)_(\\d+)_(BASE|END)$" destination_url = "ssh://hg.mozilla.org/conduit-testing/ff-test-dev" -tags_destination_branch = "dev-tags-bug1963805" +tags_destination_branch = "tags-dev" # Default #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD" @@ -50,7 +50,7 @@ tags_destination_branch = "dev-tags-bug1963805" source_url = "https://github.com/mozilla-conduit/ff-test" tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(\\d+(_\\d+)+)b\\d+_(BUILD\\d+|RELEASE)$" destination_url = "ssh://hg.mozilla.org/conduit-testing/ff-test-dev" -tags_destination_branch = "dev-tags-bug1963805" +tags_destination_branch = "tags-dev" # Default #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD" @@ -58,7 +58,7 @@ tags_destination_branch = "dev-tags-bug1963805" source_url = "https://github.com/mozilla-conduit/ff-test" tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(\\d+)(_\\d+)+esr_(BUILD\\d+|RELEASE)$" destination_url = "ssh://hg.mozilla.org/conduit-testing/ff-test-dev" -tags_destination_branch = "dev-tags-bug1963805" +tags_destination_branch = "tags-dev" # Default #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD" @@ -66,7 +66,7 @@ tags_destination_branch = "dev-tags-bug1963805" source_url = "https://github.com/mozilla-conduit/ff-test" tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(\\d+(_\\d+)+)_(BUILD\\d+|RELEASE)$" destination_url = "ssh://hg.mozilla.org/conduit-testing/ff-test-dev" -tags_destination_branch = "dev-tags-bug1963805" +tags_destination_branch = "tags-dev" # Default #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD" @@ -74,7 +74,7 @@ tags_destination_branch = "dev-tags-bug1963805" source_url = "https://github.com/mozilla-conduit/ff-test" tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(\\d+(_\\d+)+)b\\d+_(BUILD\\d+|RELEASE)$" destination_url = "ssh://hg.mozilla.org/conduit-testing/ff-test-dev" -tags_destination_branch = "dev-tags-bug1963805" +tags_destination_branch = "tags-dev" # Default #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD" @@ -82,7 +82,7 @@ tags_destination_branch = "dev-tags-bug1963805" source_url = "https://github.com/mozilla-conduit/ff-test" tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_RELEASE_(\\d+)+_(BASE|END)$" destination_url = "ssh://hg.mozilla.org/conduit-testing/ff-test-dev" -tags_destination_branch = "dev-tags-bug1963805" +tags_destination_branch = "tags-dev" # Default #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD" @@ -101,6 +101,6 @@ destination_branch = "default" source_url = "https://github.com/mozilla-conduit/test-repo" tag_pattern = "^(DEV)_(BETA|NIGHTLY)_(\\d+)_(BASE|END)$" destination_url = "ssh://hg.mozilla.org/conduit-testing/test-repo-github-dev" -tags_destination_branch = "dev-tags-bug1963805" +tags_destination_branch = "tags-dev" # Default #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD" From 785bb948e21b23cd480359a4b2724d17ee715fcb Mon Sep 17 00:00:00 2001 From: Olivier Mehani Date: Mon, 12 May 2025 11:05:27 +1000 Subject: [PATCH 05/10] config: reenable tags in prod --- config-production.toml | 118 ++++++++++++++++++++--------------------- 1 file changed, 59 insertions(+), 59 deletions(-) diff --git a/config-production.toml b/config-production.toml index 2b1b2dd..979e9d7 100644 --- a/config-production.toml +++ b/config-production.toml @@ -36,7 +36,7 @@ url = "https://github.com/mozilla-firefox/firefox.git" source_url = "https://github.com/mozilla-firefox/firefox.git" branch_pattern = "THIS_SHOULD_MATCH_NOTHING" destination_url = "https://hg.mozilla.org/mozilla-unified/" -destination_branch = "default" +destination_branch = "NOT_A_VALID_BRANCH" # @@ -58,30 +58,30 @@ branch_pattern = "beta" destination_url = "ssh://hg.mozilla.org/releases/mozilla-beta/" destination_branch = "default" -# [[tag_mappings]] -# source_url = "https://github.com/mozilla-firefox/firefox.git" -# # _(_

...)b BUILD and RELEASE tags to mozilla-beta -# tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(\\d+(_\\d+)+)b\\d+_(BUILD\\d+|RELEASE)$" -# destination_url = "ssh://hg.mozilla.org/releases/mozilla-beta/" -# tags_destination_branch = "tags-beta" -# # Default -# #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD" -# -# [[tag_mappings]] -# source_url = "https://github.com/mozilla-firefox/firefox.git" -# # BETA_ BASE and END tags to mozilla-beta -# tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_BETA_(\\d+)+_(BASE|END)$" -# destination_url = "ssh://hg.mozilla.org/releases/mozilla-beta/" -# tags_destination_branch = "tags-beta" -# # Default -# #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD" -# -# [[tag_mappings]] -# source_url = "https://github.com/mozilla-firefox/firefox.git" -# # RELEASE_ BASE tags to mozilla-beta -# tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_RELEASE_(\\d+)+_BASE$" -# destination_url = "ssh://hg.mozilla.org/releases/mozilla-beta/" -# tags_destination_branch = "tags-beta" +[[tag_mappings]] +source_url = "https://github.com/mozilla-firefox/firefox.git" +# _(_

...)b BUILD and RELEASE tags to mozilla-beta +tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(\\d+(_\\d+)+)b\\d+_(BUILD\\d+|RELEASE)$" +destination_url = "ssh://hg.mozilla.org/releases/mozilla-beta/" +tags_destination_branch = "tags-beta" +# Default +#tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD" + +[[tag_mappings]] +source_url = "https://github.com/mozilla-firefox/firefox.git" +# BETA_ BASE and END tags to mozilla-beta +tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_BETA_(\\d+)+_(BASE|END)$" +destination_url = "ssh://hg.mozilla.org/releases/mozilla-beta/" +tags_destination_branch = "tags-beta" +# Default +#tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD" + +[[tag_mappings]] +source_url = "https://github.com/mozilla-firefox/firefox.git" +# RELEASE_ BASE tags to mozilla-beta +tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_RELEASE_(\\d+)+_BASE$" +destination_url = "ssh://hg.mozilla.org/releases/mozilla-beta/" +tags_destination_branch = "tags-beta" # Default #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD" @@ -96,12 +96,12 @@ branch_pattern = "^(esr\\d+)$" destination_url = "ssh://hg.mozilla.org/releases/mozilla-\\1/" destination_branch = "default" -# [[tag_mappings]] -# source_url = "https://github.com/mozilla-firefox/firefox.git" -# # _(_

...)esr BUILD and RELEASE tags to mozilla-esr -# tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(\\d+)(_\\d+)+esr_(BUILD\\d+|RELEASE)$" -# destination_url = "ssh://hg.mozilla.org/releases/mozilla-esr\\2/" -# tags_destination_branch = "tags-\\1" +[[tag_mappings]] +source_url = "https://github.com/mozilla-firefox/firefox.git" +# _(_

...)esr BUILD and RELEASE tags to mozilla-esr +tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(\\d+)(_\\d+)+esr_(BUILD\\d+|RELEASE)$" +destination_url = "ssh://hg.mozilla.org/releases/mozilla-esr\\2/" +tags_destination_branch = "tags-esr\\2" # Default #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD" @@ -125,12 +125,12 @@ branch_pattern = "main" destination_url = "ssh://hg.mozilla.org/mozilla-central/" destination_branch = "default" -# [[tag_mappings]] -# source_url = "https://github.com/mozilla-firefox/firefox.git" -# # BETA_ and NIGHTLY_ tags to m-c -# tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(BETA|NIGHTLY)_(\\d+)_(BASE|END)$" -# destination_url = "ssh://hg.mozilla.org/mozilla-central/" -# tags_destination_branch = "tags-main" +[[tag_mappings]] +source_url = "https://github.com/mozilla-firefox/firefox.git" +# BETA_ and NIGHTLY_ tags to m-c +tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(BETA|NIGHTLY)_(\\d+)_(BASE|END)$" +destination_url = "ssh://hg.mozilla.org/mozilla-central/" +tags_destination_branch = "tags-main" # Default #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD" @@ -144,21 +144,21 @@ branch_pattern = "release" destination_url = "ssh://hg.mozilla.org/releases/mozilla-release/" destination_branch = "default" -# [[tag_mappings]] -# source_url = "https://github.com/mozilla-firefox/firefox.git" -# # _(_

...) BUILD and RELEASE tags to mozilla-release -# tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(\\d+(_\\d+)+)_(BUILD\\d+|RELEASE)$" -# destination_url = "ssh://hg.mozilla.org/releases/mozilla-release/" -# tags_destination_branch = "tags-release" +[[tag_mappings]] +source_url = "https://github.com/mozilla-firefox/firefox.git" +# _(_

...) BUILD and RELEASE tags to mozilla-release +tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(\\d+(_\\d+)+)_(BUILD\\d+|RELEASE)$" +destination_url = "ssh://hg.mozilla.org/releases/mozilla-release/" +tags_destination_branch = "tags-release" # # Default # #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD" # -# [[tag_mappings]] -# source_url = "https://github.com/mozilla-firefox/firefox.git" -# # RELEASE_ BASE and END tags to mozilla-release -# tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_RELEASE_(\\d+)+_(BASE|END)$" -# destination_url = "ssh://hg.mozilla.org/releases/mozilla-release/" -# tags_destination_branch = "tags-release" +[[tag_mappings]] +source_url = "https://github.com/mozilla-firefox/firefox.git" +# RELEASE_ BASE and END tags to mozilla-release +tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_RELEASE_(\\d+)+_(BASE|END)$" +destination_url = "ssh://hg.mozilla.org/releases/mozilla-release/" +tags_destination_branch = "tags-release" # Default #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD" @@ -202,11 +202,11 @@ branch_pattern = "autoland" destination_url = "ssh://hg.mozilla.org/conduit-testing/infra-testing/" destination_branch = "default" -# [[tag_mappings]] -# source_url = "https://github.com/mozilla-firefox/infra-testing.git" -# tag_pattern = "TAGTESTING.*" -# destination_url = "ssh://hg.mozilla.org/conduit-testing/infra-testing/" -# tags_destination_branch = "tags-autoland" +[[tag_mappings]] +source_url = "https://github.com/mozilla-firefox/infra-testing.git" +tag_pattern = ".+" +destination_url = "ssh://hg.mozilla.org/conduit-testing/infra-testing/" +tags_destination_branch = "tags-testing" # Default #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD" @@ -225,10 +225,10 @@ branch_pattern = ".+" destination_url = "ssh://hg.mozilla.org/conduit-testing/review/" destination_branch = "default" -# [[tag_mappings]] -# source_url = "https://github.com/mozilla-conduit/review.git" -# tag_pattern = ".+" -# destination_url = "ssh://hg.mozilla.org/conduit-testing/review/" -# tags_destination_branch = "tags" +[[tag_mappings]] +source_url = "https://github.com/mozilla-conduit/review.git" +tag_pattern = ".+" +destination_url = "ssh://hg.mozilla.org/conduit-testing/review/" +tags_destination_branch = "tags" # Default #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD" From e46531770ef8dbbb130adb3b902a739743c9ad8b Mon Sep 17 00:00:00 2001 From: Olivier Mehani Date: Mon, 12 May 2025 11:41:22 +1000 Subject: [PATCH 06/10] fixup! config: reenable tags in prod --- config-production.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config-production.toml b/config-production.toml index 979e9d7..0c52907 100644 --- a/config-production.toml +++ b/config-production.toml @@ -194,7 +194,7 @@ url = "https://github.com/mozilla-firefox/infra-testing.git" source_url = "https://github.com/mozilla-firefox/infra-testing.git" branch_pattern = "THIS_SHOULD_MATCH_NOTHING" destination_url = "https://hg.mozilla.org/mozilla-unified/" -destination_branch = "default" +destination_branch = "NOT_A_VALID_BRANCH" [[branch_mappings]] source_url = "https://github.com/mozilla-firefox/infra-testing.git" From c890f7881dffcc6261baf0dfc66e46d112db3387 Mon Sep 17 00:00:00 2001 From: Olivier Mehani Date: Tue, 13 May 2025 14:15:11 +1000 Subject: [PATCH 07/10] fixup! repo_synchroniser: don't push commits to tip to avoid non fast-forward issues when reprocessing old messages --- git_hg_sync/repo_synchronizer.py | 2 +- tests/test_repo_synchronizer.py | 73 +++++++++++++++++++++++++++++--- 2 files changed, 67 insertions(+), 8 deletions(-) diff --git a/git_hg_sync/repo_synchronizer.py b/git_hg_sync/repo_synchronizer.py index 3c6cb23..cdc0a5a 100644 --- a/git_hg_sync/repo_synchronizer.py +++ b/git_hg_sync/repo_synchronizer.py @@ -111,7 +111,7 @@ def sync( # Resolving the HG SHA is not sufficient, because we may know it from # another repository, so we need to make sure it's not already present here. hg_sha = self._git2hg(repo, branch_operation.source_commit) - if hg_sha not in repo.git.execute( + if hg_sha in repo.git.execute( [ "git", "ls-remote", diff --git a/tests/test_repo_synchronizer.py b/tests/test_repo_synchronizer.py index b9f356a..4df5ad2 100644 --- a/tests/test_repo_synchronizer.py +++ b/tests/test_repo_synchronizer.py @@ -23,7 +23,16 @@ def tracked_repositories() -> list[TrackedRepository]: @pytest.fixture def hg_destination(tmp_path: Path) -> Path: - hg_remote_repo_path = tmp_path / "hg-remotes" / "myrepo" + return _hg_destination(tmp_path) + + +@pytest.fixture +def hg_destination_other(tmp_path: Path) -> Path: + return _hg_destination(tmp_path, "_other") + + +def _hg_destination(tmp_path: Path, repo_suffix: str = "") -> Path: + hg_remote_repo_path = tmp_path / "hg-remotes" / f"myrepo{repo_suffix}" hg_remote_repo_path.mkdir(parents=True) subprocess.run(["hg", "init"], cwd=hg_remote_repo_path, check=True) @@ -180,9 +189,59 @@ def test_sync_process_duplicate_tags( assert tag in tag_log -def test_get_connection_and_queue(pulse_config: PulseConfig) -> None: - connection = get_connection(pulse_config) - queue = get_queue(pulse_config) - assert connection.userid == pulse_config.userid - assert connection.host == f"{pulse_config.host}:{pulse_config.port}" - assert queue.name == pulse_config.queue +def test_sync_process_different_destination( + git_source: Repo, + hg_destination: Path, + hg_destination_other: Path, + tmp_path: Path, +) -> None: + branch = "bar" + tag_branch = "tags" + tag = "mytag" + tag_suffix = "some suffix" + + repo = Repo(git_source) + + # Create a new commit on git repo + bar_path = git_source / "bar.txt" + bar_path.write_text("BAR CONTENT") + repo.index.add([bar_path]) + git_commit_sha = repo.index.commit("add bar.txt").hexsha + + # Sync new commit with mercurial repository + git_local_repo_path = tmp_path / "clones" / "myrepo" + syncrepos = RepoSynchronizer(git_local_repo_path, str(git_source)) + + request_user = "request_user@example.com" + + operations: list[SyncBranchOperation | SyncTagOperation] = [ + SyncBranchOperation(source_commit=git_commit_sha, destination_branch=branch), + SyncTagOperation( + source_commit=git_commit_sha, + tag=tag, + tags_destination_branch=tag_branch, + tag_message_suffix=tag_suffix, + ), + ] + syncrepos.sync(str(hg_destination), operations, request_user) + + syncrepos.sync(str(hg_destination_other), operations, request_user) + + # test + assert "BAR CONTENT" in hg_cat(hg_destination_other, "bar.txt", branch) + + # XXX: In the current state, cinnabar refuses to re-create a tag which already + # exists anywhere in its working state. This means we can't duplicate an existing tag to a + # different destination. This is probably sane, as we should instead have a more + # controlled way of graduating tags to repos (e.g., to m-c). + # + # Leaving this here for later reference (and to support the discussion above). + # + # assert "BAR CONTENT" in hg_cat(hg_destination_other, "bar.txt", tag) + # + # test tag commit message + # tag_log = hg_log(hg_destination_other, tag_branch, ["-T", "{desc}"]) + # assert "No bug - Tagging" in tag_log + # assert tag_suffix in tag_log + # assert tag in tag_log + # assert hg_rev(hg_destination_other, branch) in tag_log From ab43d36aa1a4715e28f5751751cab5ba4ab083e5 Mon Sep 17 00:00:00 2001 From: Olivier Mehani Date: Tue, 13 May 2025 15:10:39 +1000 Subject: [PATCH 08/10] fixup! fixup! repo_synchroniser: don't push commits to tip to avoid non fast-forward issues when reprocessing old messages --- git_hg_sync/repo_synchronizer.py | 1 + tests/test_repo_synchronizer.py | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/git_hg_sync/repo_synchronizer.py b/git_hg_sync/repo_synchronizer.py index cdc0a5a..6746551 100644 --- a/git_hg_sync/repo_synchronizer.py +++ b/git_hg_sync/repo_synchronizer.py @@ -108,6 +108,7 @@ def sync( if self._commit_has_mercurial_metadata( repo, branch_operation.source_commit ): + breakpoint() # Resolving the HG SHA is not sufficient, because we may know it from # another repository, so we need to make sure it's not already present here. hg_sha = self._git2hg(repo, branch_operation.source_commit) diff --git a/tests/test_repo_synchronizer.py b/tests/test_repo_synchronizer.py index 4df5ad2..1c6ac79 100644 --- a/tests/test_repo_synchronizer.py +++ b/tests/test_repo_synchronizer.py @@ -245,3 +245,11 @@ def test_sync_process_different_destination( # assert tag_suffix in tag_log # assert tag in tag_log # assert hg_rev(hg_destination_other, branch) in tag_log + + +def test_get_connection_and_queue(pulse_config: PulseConfig) -> None: + connection = get_connection(pulse_config) + queue = get_queue(pulse_config) + assert connection.userid == pulse_config.userid + assert connection.host == f"{pulse_config.host}:{pulse_config.port}" + assert queue.name == pulse_config.queue From 788d4e9e66f393dd69960b60879957e6692eea4a Mon Sep 17 00:00:00 2001 From: Olivier Mehani Date: Wed, 14 May 2025 10:55:51 +1000 Subject: [PATCH 09/10] fixup! fixup! fixup! repo_synchroniser: don't push commits to tip to avoid non fast-forward issues when reprocessing old messages --- git_hg_sync/repo_synchronizer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/git_hg_sync/repo_synchronizer.py b/git_hg_sync/repo_synchronizer.py index 6746551..cdc0a5a 100644 --- a/git_hg_sync/repo_synchronizer.py +++ b/git_hg_sync/repo_synchronizer.py @@ -108,7 +108,6 @@ def sync( if self._commit_has_mercurial_metadata( repo, branch_operation.source_commit ): - breakpoint() # Resolving the HG SHA is not sufficient, because we may know it from # another repository, so we need to make sure it's not already present here. hg_sha = self._git2hg(repo, branch_operation.source_commit) From 7e6828dc0769e87648b6734f1fcf75692d02f519 Mon Sep 17 00:00:00 2001 From: Olivier Mehani Date: Wed, 28 May 2025 09:28:01 +1000 Subject: [PATCH 10/10] Update git_hg_sync/repo_synchronizer.py Co-authored-by: Zeid <2043828+zzzeid@users.noreply.github.com> --- git_hg_sync/repo_synchronizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git_hg_sync/repo_synchronizer.py b/git_hg_sync/repo_synchronizer.py index cdc0a5a..8fb3d52 100644 --- a/git_hg_sync/repo_synchronizer.py +++ b/git_hg_sync/repo_synchronizer.py @@ -241,7 +241,7 @@ def _ensure_cinnabar_metadata(self, repo: Repo, destination_remote: str) -> None # pushes update the metadata locally. # # WARNING: While we make a direct reference to `refs/cinnabar` here, it MUST NOT - # be used explicitely in subsequent git operations. This set of references get + # be used explicitly in subsequent git operations. This set of references get # updated on every `fetch`, and is therefore not stable enough to be trusted. # # Repo.git_dir is a PathLike union which is either a str, or a smarter thing. We