diff --git a/config-development.toml b/config-development.toml index 97e194d..eaca559 100644 --- a/config-development.toml +++ b/config-development.toml @@ -42,7 +42,7 @@ destination_branch = "\\1" source_url = "https://github.com/mozilla-conduit/ff-test" tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(BETA|NIGHTLY)_(\\d+)_(BASE|END)$" destination_url = "ssh://hg.mozilla.org/conduit-testing/ff-test-dev" -tags_destination_branch = "dev-tags-bug1963805" +tags_destination_branch = "tags-dev" # Default #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD" @@ -50,7 +50,7 @@ tags_destination_branch = "dev-tags-bug1963805" source_url = "https://github.com/mozilla-conduit/ff-test" tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(\\d+(_\\d+)+)b\\d+_(BUILD\\d+|RELEASE)$" destination_url = "ssh://hg.mozilla.org/conduit-testing/ff-test-dev" -tags_destination_branch = "dev-tags-bug1963805" +tags_destination_branch = "tags-dev" # Default #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD" @@ -58,7 +58,7 @@ tags_destination_branch = "dev-tags-bug1963805" source_url = "https://github.com/mozilla-conduit/ff-test" tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(\\d+)(_\\d+)+esr_(BUILD\\d+|RELEASE)$" destination_url = "ssh://hg.mozilla.org/conduit-testing/ff-test-dev" -tags_destination_branch = "dev-tags-bug1963805" +tags_destination_branch = "tags-dev" # Default #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD" @@ -66,7 +66,7 @@ tags_destination_branch = "dev-tags-bug1963805" source_url = "https://github.com/mozilla-conduit/ff-test" tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(\\d+(_\\d+)+)_(BUILD\\d+|RELEASE)$" destination_url = "ssh://hg.mozilla.org/conduit-testing/ff-test-dev" -tags_destination_branch = "dev-tags-bug1963805" +tags_destination_branch = "tags-dev" # Default #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD" @@ -74,7 +74,7 @@ tags_destination_branch = "dev-tags-bug1963805" source_url = "https://github.com/mozilla-conduit/ff-test" tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(\\d+(_\\d+)+)b\\d+_(BUILD\\d+|RELEASE)$" destination_url = "ssh://hg.mozilla.org/conduit-testing/ff-test-dev" -tags_destination_branch = "dev-tags-bug1963805" +tags_destination_branch = "tags-dev" # Default #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD" @@ -82,7 +82,7 @@ tags_destination_branch = "dev-tags-bug1963805" source_url = "https://github.com/mozilla-conduit/ff-test" tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_RELEASE_(\\d+)+_(BASE|END)$" destination_url = "ssh://hg.mozilla.org/conduit-testing/ff-test-dev" -tags_destination_branch = "dev-tags-bug1963805" +tags_destination_branch = "tags-dev" # Default #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD" @@ -101,6 +101,6 @@ destination_branch = "default" source_url = "https://github.com/mozilla-conduit/test-repo" tag_pattern = "^(DEV)_(BETA|NIGHTLY)_(\\d+)_(BASE|END)$" destination_url = "ssh://hg.mozilla.org/conduit-testing/test-repo-github-dev" -tags_destination_branch = "dev-tags-bug1963805" +tags_destination_branch = "tags-dev" # Default #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD" diff --git a/config-production.toml b/config-production.toml index 2b1b2dd..0c52907 100644 --- a/config-production.toml +++ b/config-production.toml @@ -36,7 +36,7 @@ url = "https://github.com/mozilla-firefox/firefox.git" source_url = "https://github.com/mozilla-firefox/firefox.git" branch_pattern = "THIS_SHOULD_MATCH_NOTHING" destination_url = "https://hg.mozilla.org/mozilla-unified/" -destination_branch = "default" +destination_branch = "NOT_A_VALID_BRANCH" # @@ -58,30 +58,30 @@ branch_pattern = "beta" destination_url = "ssh://hg.mozilla.org/releases/mozilla-beta/" destination_branch = "default" -# [[tag_mappings]] -# source_url = "https://github.com/mozilla-firefox/firefox.git" -# # _(_

...)b BUILD and RELEASE tags to mozilla-beta -# tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(\\d+(_\\d+)+)b\\d+_(BUILD\\d+|RELEASE)$" -# destination_url = "ssh://hg.mozilla.org/releases/mozilla-beta/" -# tags_destination_branch = "tags-beta" -# # Default -# #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD" -# -# [[tag_mappings]] -# source_url = "https://github.com/mozilla-firefox/firefox.git" -# # BETA_ BASE and END tags to mozilla-beta -# tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_BETA_(\\d+)+_(BASE|END)$" -# destination_url = "ssh://hg.mozilla.org/releases/mozilla-beta/" -# tags_destination_branch = "tags-beta" -# # Default -# #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD" -# -# [[tag_mappings]] -# source_url = "https://github.com/mozilla-firefox/firefox.git" -# # RELEASE_ BASE tags to mozilla-beta -# tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_RELEASE_(\\d+)+_BASE$" -# destination_url = "ssh://hg.mozilla.org/releases/mozilla-beta/" -# tags_destination_branch = "tags-beta" +[[tag_mappings]] +source_url = "https://github.com/mozilla-firefox/firefox.git" +# _(_

...)b BUILD and RELEASE tags to mozilla-beta +tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(\\d+(_\\d+)+)b\\d+_(BUILD\\d+|RELEASE)$" +destination_url = "ssh://hg.mozilla.org/releases/mozilla-beta/" +tags_destination_branch = "tags-beta" +# Default +#tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD" + +[[tag_mappings]] +source_url = "https://github.com/mozilla-firefox/firefox.git" +# BETA_ BASE and END tags to mozilla-beta +tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_BETA_(\\d+)+_(BASE|END)$" +destination_url = "ssh://hg.mozilla.org/releases/mozilla-beta/" +tags_destination_branch = "tags-beta" +# Default +#tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD" + +[[tag_mappings]] +source_url = "https://github.com/mozilla-firefox/firefox.git" +# RELEASE_ BASE tags to mozilla-beta +tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_RELEASE_(\\d+)+_BASE$" +destination_url = "ssh://hg.mozilla.org/releases/mozilla-beta/" +tags_destination_branch = "tags-beta" # Default #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD" @@ -96,12 +96,12 @@ branch_pattern = "^(esr\\d+)$" destination_url = "ssh://hg.mozilla.org/releases/mozilla-\\1/" destination_branch = "default" -# [[tag_mappings]] -# source_url = "https://github.com/mozilla-firefox/firefox.git" -# # _(_

...)esr BUILD and RELEASE tags to mozilla-esr -# tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(\\d+)(_\\d+)+esr_(BUILD\\d+|RELEASE)$" -# destination_url = "ssh://hg.mozilla.org/releases/mozilla-esr\\2/" -# tags_destination_branch = "tags-\\1" +[[tag_mappings]] +source_url = "https://github.com/mozilla-firefox/firefox.git" +# _(_

...)esr BUILD and RELEASE tags to mozilla-esr +tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(\\d+)(_\\d+)+esr_(BUILD\\d+|RELEASE)$" +destination_url = "ssh://hg.mozilla.org/releases/mozilla-esr\\2/" +tags_destination_branch = "tags-esr\\2" # Default #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD" @@ -125,12 +125,12 @@ branch_pattern = "main" destination_url = "ssh://hg.mozilla.org/mozilla-central/" destination_branch = "default" -# [[tag_mappings]] -# source_url = "https://github.com/mozilla-firefox/firefox.git" -# # BETA_ and NIGHTLY_ tags to m-c -# tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(BETA|NIGHTLY)_(\\d+)_(BASE|END)$" -# destination_url = "ssh://hg.mozilla.org/mozilla-central/" -# tags_destination_branch = "tags-main" +[[tag_mappings]] +source_url = "https://github.com/mozilla-firefox/firefox.git" +# BETA_ and NIGHTLY_ tags to m-c +tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(BETA|NIGHTLY)_(\\d+)_(BASE|END)$" +destination_url = "ssh://hg.mozilla.org/mozilla-central/" +tags_destination_branch = "tags-main" # Default #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD" @@ -144,21 +144,21 @@ branch_pattern = "release" destination_url = "ssh://hg.mozilla.org/releases/mozilla-release/" destination_branch = "default" -# [[tag_mappings]] -# source_url = "https://github.com/mozilla-firefox/firefox.git" -# # _(_

...) BUILD and RELEASE tags to mozilla-release -# tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(\\d+(_\\d+)+)_(BUILD\\d+|RELEASE)$" -# destination_url = "ssh://hg.mozilla.org/releases/mozilla-release/" -# tags_destination_branch = "tags-release" +[[tag_mappings]] +source_url = "https://github.com/mozilla-firefox/firefox.git" +# _(_

...) BUILD and RELEASE tags to mozilla-release +tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(\\d+(_\\d+)+)_(BUILD\\d+|RELEASE)$" +destination_url = "ssh://hg.mozilla.org/releases/mozilla-release/" +tags_destination_branch = "tags-release" # # Default # #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD" # -# [[tag_mappings]] -# source_url = "https://github.com/mozilla-firefox/firefox.git" -# # RELEASE_ BASE and END tags to mozilla-release -# tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_RELEASE_(\\d+)+_(BASE|END)$" -# destination_url = "ssh://hg.mozilla.org/releases/mozilla-release/" -# tags_destination_branch = "tags-release" +[[tag_mappings]] +source_url = "https://github.com/mozilla-firefox/firefox.git" +# RELEASE_ BASE and END tags to mozilla-release +tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_RELEASE_(\\d+)+_(BASE|END)$" +destination_url = "ssh://hg.mozilla.org/releases/mozilla-release/" +tags_destination_branch = "tags-release" # Default #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD" @@ -194,7 +194,7 @@ url = "https://github.com/mozilla-firefox/infra-testing.git" source_url = "https://github.com/mozilla-firefox/infra-testing.git" branch_pattern = "THIS_SHOULD_MATCH_NOTHING" destination_url = "https://hg.mozilla.org/mozilla-unified/" -destination_branch = "default" +destination_branch = "NOT_A_VALID_BRANCH" [[branch_mappings]] source_url = "https://github.com/mozilla-firefox/infra-testing.git" @@ -202,11 +202,11 @@ branch_pattern = "autoland" destination_url = "ssh://hg.mozilla.org/conduit-testing/infra-testing/" destination_branch = "default" -# [[tag_mappings]] -# source_url = "https://github.com/mozilla-firefox/infra-testing.git" -# tag_pattern = "TAGTESTING.*" -# destination_url = "ssh://hg.mozilla.org/conduit-testing/infra-testing/" -# tags_destination_branch = "tags-autoland" +[[tag_mappings]] +source_url = "https://github.com/mozilla-firefox/infra-testing.git" +tag_pattern = ".+" +destination_url = "ssh://hg.mozilla.org/conduit-testing/infra-testing/" +tags_destination_branch = "tags-testing" # Default #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD" @@ -225,10 +225,10 @@ branch_pattern = ".+" destination_url = "ssh://hg.mozilla.org/conduit-testing/review/" destination_branch = "default" -# [[tag_mappings]] -# source_url = "https://github.com/mozilla-conduit/review.git" -# tag_pattern = ".+" -# destination_url = "ssh://hg.mozilla.org/conduit-testing/review/" -# tags_destination_branch = "tags" +[[tag_mappings]] +source_url = "https://github.com/mozilla-conduit/review.git" +tag_pattern = ".+" +destination_url = "ssh://hg.mozilla.org/conduit-testing/review/" +tags_destination_branch = "tags" # Default #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD" diff --git a/git_hg_sync/repo_synchronizer.py b/git_hg_sync/repo_synchronizer.py index eea0a10..8fb3d52 100644 --- a/git_hg_sync/repo_synchronizer.py +++ b/git_hg_sync/repo_synchronizer.py @@ -61,10 +61,10 @@ def sync( logger.info(f"Syncing {operations} to {destination_url} ...") try: repo = self.get_clone_repo() - except PermissionError as exc: + except PermissionError as e: raise PermissionError( f"Failed to create local clone from {destination_url}" - ) from exc + ) from e destination_remote = f"hg::{destination_url}" @@ -83,12 +83,48 @@ def sync( op for op in operations if isinstance(op, SyncBranchOperation) ] for branch_operation in branch_ops: - try: - push_args.append( - f"{branch_operation.source_commit}:refs/heads/branches/{branch_operation.destination_branch}/tip" - ) - except Exception as e: - raise RepoSyncError(branch_operation, e) from e + # Here, we use `/` rather than `tip` to work around inherent + # limitations in the mapping between Git and Hg references. + # + # We could use `/tip`, which would work in most cases. However, when + # reprocessing old messages (as is sometimes necessary to recover from + # issues), we may find ourselves processing a push for a commit which is now + # an ancestor of the current `tip`. In this situation, git would refuse to + # push, claiming it's not a fast-forward. + # + # To handle this case, we push each commit to a separate reference matching + # their own SHA1. Those references only exist on the git side, so their name + # doesn't impact what gets created on the Mercurial side. [The name of the + # branch matters with `cinnabar.experiments=branch`, but not the name of the + # final reference.] + # + # Mercurial maintains `tip` automatically to be the latest new commit (and + # we only allow single heads on pushable repositories, which guarantees it's + # the furthest from the root). + # + destination_ref = f"refs/heads/branches/{branch_operation.destination_branch}/${branch_operation.source_commit}" + # We only push the commit if it's not already present, because Mercurial + # refuses pushes which don't change anything. + if self._commit_has_mercurial_metadata( + repo, branch_operation.source_commit + ): + # Resolving the HG SHA is not sufficient, because we may know it from + # another repository, so we need to make sure it's not already present here. + hg_sha = self._git2hg(repo, branch_operation.source_commit) + if hg_sha in repo.git.execute( + [ + "git", + "ls-remote", + destination_remote, + f"refs/heads/branches/{branch_operation.destination_branch}/{hg_sha}", + ], + stdout_as_string=True, + ): + logger.info( + f"Commit {branch_operation.source_commit} is already present on {destination_remote}, skipping ..." + ) + continue + push_args.append(f"{branch_operation.source_commit}:{destination_ref}") os.environ[REQUEST_USER_ENV_VAR] = request_user logger.debug(f"{REQUEST_USER_ENV_VAR} set to {request_user}") @@ -96,8 +132,8 @@ def sync( # Add mercurial metadata to new commits from synced branches # Some of these commits could be tagged in the same synchronization and # tagging can only be done on a commit that already have mercurial - # metadata - if branch_ops: + # metadata. + if len(push_args) > 1: retry( "adding mercurial metadata to git commits", lambda: repo.git.execute( @@ -117,7 +153,7 @@ def sync( for tag_operation in tag_ops: tag_branch = tag_operation.tags_destination_branch remote_tag_ref = f"refs/heads/branches/{tag_branch}/tip" - if repo.git.execute( + if remote_tag_ref in repo.git.execute( ["git", "ls-remote", destination_remote, remote_tag_ref], stdout_as_string=True, ): @@ -203,7 +239,11 @@ def _ensure_cinnabar_metadata(self, repo: Repo, destination_remote: str) -> None # This is needed only on first initialisation of the repository, as subsequent # pushes update the metadata locally. - + # + # WARNING: While we make a direct reference to `refs/cinnabar` here, it MUST NOT + # be used explicitly in subsequent git operations. This set of references get + # updated on every `fetch`, and is therefore not stable enough to be trusted. + # # Repo.git_dir is a PathLike union which is either a str, or a smarter thing. We # assume the less smart one. cinnabar_metadata_dir = Path(repo.git_dir) / "refs/cinnabar/metadata" diff --git a/tests/test_repo_synchronizer.py b/tests/test_repo_synchronizer.py index 7dac181..1c6ac79 100644 --- a/tests/test_repo_synchronizer.py +++ b/tests/test_repo_synchronizer.py @@ -23,7 +23,16 @@ def tracked_repositories() -> list[TrackedRepository]: @pytest.fixture def hg_destination(tmp_path: Path) -> Path: - hg_remote_repo_path = tmp_path / "hg-remotes" / "myrepo" + return _hg_destination(tmp_path) + + +@pytest.fixture +def hg_destination_other(tmp_path: Path) -> Path: + return _hg_destination(tmp_path, "_other") + + +def _hg_destination(tmp_path: Path, repo_suffix: str = "") -> Path: + hg_remote_repo_path = tmp_path / "hg-remotes" / f"myrepo{repo_suffix}" hg_remote_repo_path.mkdir(parents=True) subprocess.run(["hg", "init"], cwd=hg_remote_repo_path, check=True) @@ -104,6 +113,47 @@ def test_sync_process_( assert hg_rev(hg_destination, branch) in tag_log +def test_sync_process_ancestor( + git_source: Repo, + hg_destination: Path, + tmp_path: Path, +) -> None: + branch = "bar" + + repo = Repo(git_source) + + # Create a new commit on git repo + bar_path = git_source / "bar.txt" + bar_path.write_text("BAR CONTENT") + repo.index.add([bar_path]) + git_commit_sha1 = repo.index.commit("add bar.txt").hexsha + + baz_path = git_source / "baz.txt" + baz_path.write_text("BAZ CONTENT") + repo.index.add([baz_path]) + git_commit_sha2 = repo.index.commit("add baz.txt").hexsha + + # Sync new commit with mercurial repository + git_local_repo_path = tmp_path / "clones" / "myrepo" + syncrepos = RepoSynchronizer(git_local_repo_path, str(git_source)) + operations: list[SyncBranchOperation | SyncTagOperation] = [ + SyncBranchOperation(source_commit=git_commit_sha2, destination_branch=branch), + ] + + request_user = "request_user@example.com" + syncrepos.sync(str(hg_destination), operations, request_user) + + # Sync an earlier commit. + operations: list[SyncBranchOperation | SyncTagOperation] = [ + SyncBranchOperation(source_commit=git_commit_sha1, destination_branch=branch), + ] + syncrepos.sync(str(hg_destination), operations, request_user) + + # test + assert "BAR CONTENT" in hg_cat(hg_destination, "bar.txt", branch) + assert "BAZ CONTENT" in hg_cat(hg_destination, "baz.txt", branch) + + def test_sync_process_duplicate_tags( git_source: Repo, hg_destination: Path, @@ -139,6 +189,64 @@ def test_sync_process_duplicate_tags( assert tag in tag_log +def test_sync_process_different_destination( + git_source: Repo, + hg_destination: Path, + hg_destination_other: Path, + tmp_path: Path, +) -> None: + branch = "bar" + tag_branch = "tags" + tag = "mytag" + tag_suffix = "some suffix" + + repo = Repo(git_source) + + # Create a new commit on git repo + bar_path = git_source / "bar.txt" + bar_path.write_text("BAR CONTENT") + repo.index.add([bar_path]) + git_commit_sha = repo.index.commit("add bar.txt").hexsha + + # Sync new commit with mercurial repository + git_local_repo_path = tmp_path / "clones" / "myrepo" + syncrepos = RepoSynchronizer(git_local_repo_path, str(git_source)) + + request_user = "request_user@example.com" + + operations: list[SyncBranchOperation | SyncTagOperation] = [ + SyncBranchOperation(source_commit=git_commit_sha, destination_branch=branch), + SyncTagOperation( + source_commit=git_commit_sha, + tag=tag, + tags_destination_branch=tag_branch, + tag_message_suffix=tag_suffix, + ), + ] + syncrepos.sync(str(hg_destination), operations, request_user) + + syncrepos.sync(str(hg_destination_other), operations, request_user) + + # test + assert "BAR CONTENT" in hg_cat(hg_destination_other, "bar.txt", branch) + + # XXX: In the current state, cinnabar refuses to re-create a tag which already + # exists anywhere in its working state. This means we can't duplicate an existing tag to a + # different destination. This is probably sane, as we should instead have a more + # controlled way of graduating tags to repos (e.g., to m-c). + # + # Leaving this here for later reference (and to support the discussion above). + # + # assert "BAR CONTENT" in hg_cat(hg_destination_other, "bar.txt", tag) + # + # test tag commit message + # tag_log = hg_log(hg_destination_other, tag_branch, ["-T", "{desc}"]) + # assert "No bug - Tagging" in tag_log + # assert tag_suffix in tag_log + # assert tag in tag_log + # assert hg_rev(hg_destination_other, branch) in tag_log + + def test_get_connection_and_queue(pulse_config: PulseConfig) -> None: connection = get_connection(pulse_config) queue = get_queue(pulse_config)