repo_synchroniser: avoid non fast-forward and nothing changed issues (bug 1965991)

config-development.toml

@@ -42,47 +42,47 @@ destination_branch = "\\1"
 source_url = "https://github.com/mozilla-conduit/ff-test"
 tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(BETA|NIGHTLY)_(\\d+)_(BASE|END)$"
 destination_url = "ssh://hg.mozilla.org/conduit-testing/ff-test-dev"
-tags_destination_branch = "dev-tags-bug1963805"
+tags_destination_branch = "tags-dev"
 # Default
 #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD"
 
 [[tag_mappings]]
 source_url = "https://github.com/mozilla-conduit/ff-test"
 tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(\\d+(_\\d+)+)b\\d+_(BUILD\\d+|RELEASE)$"
 destination_url = "ssh://hg.mozilla.org/conduit-testing/ff-test-dev"
-tags_destination_branch = "dev-tags-bug1963805"
+tags_destination_branch = "tags-dev"
 # Default
 #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD"
 
 [[tag_mappings]]
 source_url = "https://github.com/mozilla-conduit/ff-test"
 tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(\\d+)(_\\d+)+esr_(BUILD\\d+|RELEASE)$"
 destination_url = "ssh://hg.mozilla.org/conduit-testing/ff-test-dev"
-tags_destination_branch = "dev-tags-bug1963805"
+tags_destination_branch = "tags-dev"
 # Default
 #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD"
 
 [[tag_mappings]]
 source_url = "https://github.com/mozilla-conduit/ff-test"
 tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(\\d+(_\\d+)+)_(BUILD\\d+|RELEASE)$"
 destination_url = "ssh://hg.mozilla.org/conduit-testing/ff-test-dev"
-tags_destination_branch = "dev-tags-bug1963805"
+tags_destination_branch = "tags-dev"
 # Default
 #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD"
 
 [[tag_mappings]]
 source_url = "https://github.com/mozilla-conduit/ff-test"
 tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(\\d+(_\\d+)+)b\\d+_(BUILD\\d+|RELEASE)$"
 destination_url = "ssh://hg.mozilla.org/conduit-testing/ff-test-dev"
-tags_destination_branch = "dev-tags-bug1963805"
+tags_destination_branch = "tags-dev"
 # Default
 #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD"
 
 [[tag_mappings]]
 source_url = "https://github.com/mozilla-conduit/ff-test"
 tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_RELEASE_(\\d+)+_(BASE|END)$"
 destination_url = "ssh://hg.mozilla.org/conduit-testing/ff-test-dev"
-tags_destination_branch = "dev-tags-bug1963805"
+tags_destination_branch = "tags-dev"
 # Default
 #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD"
 
@@ -101,6 +101,6 @@ destination_branch = "default"
 source_url = "https://github.com/mozilla-conduit/test-repo"
 tag_pattern = "^(DEV)_(BETA|NIGHTLY)_(\\d+)_(BASE|END)$"
 destination_url = "ssh://hg.mozilla.org/conduit-testing/test-repo-github-dev"
-tags_destination_branch = "dev-tags-bug1963805"
+tags_destination_branch = "tags-dev"
 # Default
 #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD"

config-production.toml

@@ -36,7 +36,7 @@ url = "https://github.com/mozilla-firefox/firefox.git"
 source_url = "https://github.com/mozilla-firefox/firefox.git"
 branch_pattern = "THIS_SHOULD_MATCH_NOTHING"
 destination_url = "https://hg.mozilla.org/mozilla-unified/"
-destination_branch = "default"
+destination_branch = "NOT_A_VALID_BRANCH"
 
 
 #
@@ -58,30 +58,30 @@ branch_pattern = "beta"
 destination_url = "ssh://hg.mozilla.org/releases/mozilla-beta/"
 destination_branch = "default"
 
-# [[tag_mappings]]
-# source_url = "https://github.com/mozilla-firefox/firefox.git"
-# # <M>_<m>(_<p>...)b<n> BUILD and RELEASE tags to mozilla-beta
-# tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(\\d+(_\\d+)+)b\\d+_(BUILD\\d+|RELEASE)$"
-# destination_url = "ssh://hg.mozilla.org/releases/mozilla-beta/"
-# tags_destination_branch = "tags-beta"
-# # Default
-# #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD"
-#
-# [[tag_mappings]]
-# source_url = "https://github.com/mozilla-firefox/firefox.git"
-# # BETA_<M> BASE and END tags to mozilla-beta
-# tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_BETA_(\\d+)+_(BASE|END)$"
-# destination_url = "ssh://hg.mozilla.org/releases/mozilla-beta/"
-# tags_destination_branch = "tags-beta"
-# # Default
-# #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD"
-#
-# [[tag_mappings]]
-# source_url = "https://github.com/mozilla-firefox/firefox.git"
-# # RELEASE_<M> BASE tags to mozilla-beta
-# tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_RELEASE_(\\d+)+_BASE$"
-# destination_url = "ssh://hg.mozilla.org/releases/mozilla-beta/"
-# tags_destination_branch = "tags-beta"
+[[tag_mappings]]
+source_url = "https://github.com/mozilla-firefox/firefox.git"
+# <M>_<m>(_<p>...)b<n> BUILD and RELEASE tags to mozilla-beta
+tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(\\d+(_\\d+)+)b\\d+_(BUILD\\d+|RELEASE)$"
+destination_url = "ssh://hg.mozilla.org/releases/mozilla-beta/"
+tags_destination_branch = "tags-beta"
+# Default
+#tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD"
+
+[[tag_mappings]]
+source_url = "https://github.com/mozilla-firefox/firefox.git"
+# BETA_<M> BASE and END tags to mozilla-beta
+tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_BETA_(\\d+)+_(BASE|END)$"
+destination_url = "ssh://hg.mozilla.org/releases/mozilla-beta/"
+tags_destination_branch = "tags-beta"
+# Default
+#tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD"
+
+[[tag_mappings]]
+source_url = "https://github.com/mozilla-firefox/firefox.git"
+# RELEASE_<M> BASE tags to mozilla-beta
+tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_RELEASE_(\\d+)+_BASE$"
+destination_url = "ssh://hg.mozilla.org/releases/mozilla-beta/"
+tags_destination_branch = "tags-beta"
 # Default
 #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD"
 
@@ -96,12 +96,12 @@ branch_pattern = "^(esr\\d+)$"
 destination_url = "ssh://hg.mozilla.org/releases/mozilla-\\1/"
 destination_branch = "default"
 
-# [[tag_mappings]]
-# source_url = "https://github.com/mozilla-firefox/firefox.git"
-# # <M>_<m>(_<p>...)esr BUILD and RELEASE tags to mozilla-esr<M>
-# tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(\\d+)(_\\d+)+esr_(BUILD\\d+|RELEASE)$"
-# destination_url = "ssh://hg.mozilla.org/releases/mozilla-esr\\2/"
-# tags_destination_branch = "tags-\\1"
+[[tag_mappings]]
+source_url = "https://github.com/mozilla-firefox/firefox.git"
+# <M>_<m>(_<p>...)esr BUILD and RELEASE tags to mozilla-esr<M>
+tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(\\d+)(_\\d+)+esr_(BUILD\\d+|RELEASE)$"
+destination_url = "ssh://hg.mozilla.org/releases/mozilla-esr\\2/"
+tags_destination_branch = "tags-esr\\2"
 # Default
 #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD"
 
@@ -125,12 +125,12 @@ branch_pattern = "main"
 destination_url = "ssh://hg.mozilla.org/mozilla-central/"
 destination_branch = "default"
 
-# [[tag_mappings]]
-# source_url = "https://github.com/mozilla-firefox/firefox.git"
-# # BETA_<M> and NIGHTLY_<M> tags to m-c
-# tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(BETA|NIGHTLY)_(\\d+)_(BASE|END)$"
-# destination_url = "ssh://hg.mozilla.org/mozilla-central/"
-# tags_destination_branch = "tags-main"
+[[tag_mappings]]
+source_url = "https://github.com/mozilla-firefox/firefox.git"
+# BETA_<M> and NIGHTLY_<M> tags to m-c
+tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(BETA|NIGHTLY)_(\\d+)_(BASE|END)$"
+destination_url = "ssh://hg.mozilla.org/mozilla-central/"
+tags_destination_branch = "tags-main"
 # Default
 #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD"
 
@@ -144,21 +144,21 @@ branch_pattern = "release"
 destination_url = "ssh://hg.mozilla.org/releases/mozilla-release/"
 destination_branch = "default"
 
-# [[tag_mappings]]
-# source_url = "https://github.com/mozilla-firefox/firefox.git"
-# # <M>_<m>(_<p>...) BUILD and RELEASE tags to mozilla-release
-# tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(\\d+(_\\d+)+)_(BUILD\\d+|RELEASE)$"
-# destination_url = "ssh://hg.mozilla.org/releases/mozilla-release/"
-# tags_destination_branch = "tags-release"
+[[tag_mappings]]
+source_url = "https://github.com/mozilla-firefox/firefox.git"
+# <M>_<m>(_<p>...) BUILD and RELEASE tags to mozilla-release
+tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_(\\d+(_\\d+)+)_(BUILD\\d+|RELEASE)$"
+destination_url = "ssh://hg.mozilla.org/releases/mozilla-release/"
+tags_destination_branch = "tags-release"
 # # Default
 # #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD"
 #
-# [[tag_mappings]]
-# source_url = "https://github.com/mozilla-firefox/firefox.git"
-# # RELEASE_<M> BASE and END tags to mozilla-release
-# tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_RELEASE_(\\d+)+_(BASE|END)$"
-# destination_url = "ssh://hg.mozilla.org/releases/mozilla-release/"
-# tags_destination_branch = "tags-release"
+[[tag_mappings]]
+source_url = "https://github.com/mozilla-firefox/firefox.git"
+# RELEASE_<M> BASE and END tags to mozilla-release
+tag_pattern = "^(FIREFOX|DEVEDITION|FIREFOX-ANDROID)_RELEASE_(\\d+)+_(BASE|END)$"
+destination_url = "ssh://hg.mozilla.org/releases/mozilla-release/"
+tags_destination_branch = "tags-release"
 # Default
 #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD"
 
@@ -194,19 +194,19 @@ url = "https://github.com/mozilla-firefox/infra-testing.git"
 source_url = "https://github.com/mozilla-firefox/infra-testing.git"
 branch_pattern = "THIS_SHOULD_MATCH_NOTHING"
 destination_url = "https://hg.mozilla.org/mozilla-unified/"
-destination_branch = "default"
+destination_branch = "NOT_A_VALID_BRANCH"
 
 [[branch_mappings]]
 source_url = "https://github.com/mozilla-firefox/infra-testing.git"
 branch_pattern = "autoland"
 destination_url = "ssh://hg.mozilla.org/conduit-testing/infra-testing/"
 destination_branch = "default"
 
-# [[tag_mappings]]
-# source_url = "https://github.com/mozilla-firefox/infra-testing.git"
-# tag_pattern = "TAGTESTING.*"
-# destination_url = "ssh://hg.mozilla.org/conduit-testing/infra-testing/"
-# tags_destination_branch = "tags-autoland"
+[[tag_mappings]]
+source_url = "https://github.com/mozilla-firefox/infra-testing.git"
+tag_pattern = ".+"
+destination_url = "ssh://hg.mozilla.org/conduit-testing/infra-testing/"
+tags_destination_branch = "tags-testing"
 # Default
 #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD"
 
@@ -225,10 +225,10 @@ branch_pattern = ".+"
 destination_url = "ssh://hg.mozilla.org/conduit-testing/review/"
 destination_branch = "default"
 
-# [[tag_mappings]]
-# source_url = "https://github.com/mozilla-conduit/review.git"
-# tag_pattern = ".+"
-# destination_url = "ssh://hg.mozilla.org/conduit-testing/review/"
-# tags_destination_branch = "tags"
+[[tag_mappings]]
+source_url = "https://github.com/mozilla-conduit/review.git"
+tag_pattern = ".+"
+destination_url = "ssh://hg.mozilla.org/conduit-testing/review/"
+tags_destination_branch = "tags"
 # Default
 #tag_message_suffix = "a=tagging CLOSED TREE DONTBUILD"

git_hg_sync/repo_synchronizer.py

@@ -61,10 +61,10 @@ def sync(
         logger.info(f"Syncing {operations} to {destination_url} ...")
         try:
             repo = self.get_clone_repo()
-        except PermissionError as exc:
+        except PermissionError as e:
             raise PermissionError(
                 f"Failed to create local clone from {destination_url}"
-            ) from exc
+            ) from e
 
         destination_remote = f"hg::{destination_url}"
 
@@ -83,21 +83,57 @@ def sync(
             op for op in operations if isinstance(op, SyncBranchOperation)
         ]
         for branch_operation in branch_ops:
-            try:
-                push_args.append(
-                    f"{branch_operation.source_commit}:refs/heads/branches/{branch_operation.destination_branch}/tip"
-                )
-            except Exception as e:
-                raise RepoSyncError(branch_operation, e) from e
+            # Here, we use `<BRANCH>/<SHA1>` rather than `tip` to work around inherent
+            # limitations in the mapping between Git and Hg references.
+            #
+            # We could use `<BRANCH>/tip`, which would work in most cases. However, when
+            # reprocessing old messages (as is sometimes necessary to recover from
+            # issues), we may find ourselves processing a push for a commit which is now
+            # an ancestor of the current `tip`. In this situation, git would refuse to
+            # push, claiming it's not a fast-forward.
+            #
+            # To handle this case, we push each commit to a separate reference matching
+            # their own SHA1. Those references only exist on the git side, so their name
+            # doesn't impact what gets created on the Mercurial side. [The name of the
+            # branch matters with `cinnabar.experiments=branch`, but not the name of the
+            # final reference.]
+            #
+            # Mercurial maintains `tip` automatically to be the latest new commit (and
+            # we only allow single heads on pushable repositories, which guarantees it's
+            # the furthest from the root).
+            #
+            destination_ref = f"refs/heads/branches/{branch_operation.destination_branch}/${branch_operation.source_commit}"
+            # We only push the commit if it's not already present, because Mercurial
+            # refuses pushes which don't change anything.
+            if self._commit_has_mercurial_metadata(
+                repo, branch_operation.source_commit
+            ):
+                # Resolving the HG SHA is not sufficient, because we may know it from
+                # another repository, so we need to make sure it's not already present here.
+                hg_sha = self._git2hg(repo, branch_operation.source_commit)
+                if hg_sha in repo.git.execute(
+                    [
+                        "git",
+                        "ls-remote",
+                        destination_remote,
+                        f"refs/heads/branches/{branch_operation.destination_branch}/{hg_sha}",
+                    ],
+                    stdout_as_string=True,
+                ):
+                    logger.info(
+                        f"Commit {branch_operation.source_commit} is already present on {destination_remote}, skipping ..."
+                    )
+                    continue
+            push_args.append(f"{branch_operation.source_commit}:{destination_ref}")
 
         os.environ[REQUEST_USER_ENV_VAR] = request_user
         logger.debug(f"{REQUEST_USER_ENV_VAR} set to {request_user}")
 
         # Add mercurial metadata to new commits from synced branches
         # Some of these commits could be tagged in the same synchronization and
         # tagging can only be done on a commit that already have mercurial
-        # metadata
-        if branch_ops:
+        # metadata.
+        if len(push_args) > 1:
             retry(
                 "adding mercurial metadata to git commits",
                 lambda: repo.git.execute(
@@ -117,7 +153,7 @@ def sync(
         for tag_operation in tag_ops:
             tag_branch = tag_operation.tags_destination_branch
             remote_tag_ref = f"refs/heads/branches/{tag_branch}/tip"
-            if repo.git.execute(
+            if remote_tag_ref in repo.git.execute(
                 ["git", "ls-remote", destination_remote, remote_tag_ref],
                 stdout_as_string=True,
             ):
@@ -203,7 +239,11 @@ def _ensure_cinnabar_metadata(self, repo: Repo, destination_remote: str) -> None
 
         # This is needed only on first initialisation of the repository, as subsequent
         # pushes update the metadata locally.
-
+        #
+        # WARNING: While we make a direct reference to `refs/cinnabar` here, it MUST NOT
+        # be used explicitly in subsequent git operations. This set of references get
+        # updated on every `fetch`, and is therefore not stable enough to be trusted.
+        #
         # Repo.git_dir is a PathLike union which is either a str, or a smarter thing. We
         # assume the less smart one.
         cinnabar_metadata_dir = Path(repo.git_dir) / "refs/cinnabar/metadata"