From 3c78dbcaef3999a0fbd16737f61091d6c914f099 Mon Sep 17 00:00:00 2001 From: Musab Shakeel Date: Mon, 27 Jan 2025 16:32:50 -0500 Subject: [PATCH] SigV4: Allow specifying Host header as config Instead of determining the Host header value from url.getHost(), allow specifying the Host to use for the SigV4 signature calculation via config. Rationale: - Assume you're using the Gerrit LFS plugin and pointing it to AWS S3 or GCS (S3 REST API) - Your Gerrit (and git clients pulling/cloning) are hosted on an on-prem k8s cluster - You download say 40TB/month of LFS content (because your CI system needs it maybe) - The network egress costs from AWS S3 / GCS for pulling all this content will easily be $2-5k+/ month - So you will want to maintain an on-prem caching proxy e.g. NGINX, Varnish etc - That is, you will set hostname in your lfs.config on Gerrit to my-varnish-proxy.internaldomain.io - my-varnish-proxy will proxy the requests to storage.googleapis.com - In order to avoid adding SigV4 logic in your proxy, it is necessary that the Gerrit LFS layer uses the S3 endpoint to calculate the SigV4 signature - This is because when GCP is verifying the signature in the presigned URL, it seems to be verifying against the Host: storage.googleapis.com and not the Host header passed in/overridden via the proxy MinIO used to support this exact use case. It provided its own S3 API (with SigV4 auth), and then it proxied the requests to GCS. However, as of 2022 MinIO deprecated support for proxying to GCS [1] [1] https://blog.min.io/deprecation-of-the-minio-gateway/ Change-Id: Id2df956c50d43b83ffec3c7955dc34504cde616b --- .../eclipse/jgit/lfs/server/s3/S3Config.java | 47 ++++++++++++++++++- .../eclipse/jgit/lfs/server/s3/SignerV4.java | 16 ++++++- 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/s3/S3Config.java b/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/s3/S3Config.java index 9b44aebe2a0..c609c28875a 100644 --- a/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/s3/S3Config.java +++ b/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/s3/S3Config.java @@ -16,6 +16,7 @@ * @since 4.3 */ public class S3Config { + private final String signatureHostname; private final String hostname; private final String region; private final String bucket; @@ -30,6 +31,8 @@ public class S3Config { * Constructor for S3Config. *

* + * @param signatureHostname + * Hostname to use for the SigV4 signature * @param hostname * S3 API host * @param region @@ -50,9 +53,10 @@ public class S3Config { * verification * @since 5.8 */ - public S3Config(String hostname, String region, String bucket, String storageClass, + public S3Config(String signatureHostname, String hostname, String region, String bucket, String storageClass, String accessKey, String secretKey, int expirationSeconds, boolean disableSslVerify) { + this.signatureHostname = signatureHostname; this.hostname = hostname; this.region = region; this.bucket = bucket; @@ -63,6 +67,37 @@ public S3Config(String hostname, String region, String bucket, String storageCla this.disableSslVerify = disableSslVerify; } + /** + *

+ * Constructor for S3Config. + *

+ * + * @param hostname + * S3 API host + * @param region + * AWS region + * @param bucket + * S3 storage bucket + * @param storageClass + * S3 storage class + * @param accessKey + * access key for authenticating to AWS + * @param secretKey + * secret key for authenticating to AWS + * @param expirationSeconds + * period in seconds after which requests signed for this bucket + * will expire + * @param disableSslVerify + * if {@code true} disable Amazon server certificate and hostname + * verification + * @since 5.8 + */ + public S3Config(String hostname, String region, String bucket, String storageClass, + String accessKey, String secretKey, int expirationSeconds, + boolean disableSslVerify) { + this(hostname, hostname, region, bucket, storageClass, accessKey, secretKey, expirationSeconds, disableSslVerify); + } + /** *

Constructor for S3Config.

* @@ -91,6 +126,16 @@ public S3Config(String region, String bucket, String storageClass, disableSslVerify); } + /** + * Get the hostname. + * + * @return Get the hostname to use for SigV4 signature calculation + * @since 5.8 + */ + public String getSignatureHostname() { + return signatureHostname; + } + /** * Get the hostname. * diff --git a/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/s3/SignerV4.java b/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/s3/SignerV4.java index d88cf13615d..a94c3f67bbf 100644 --- a/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/s3/SignerV4.java +++ b/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/s3/SignerV4.java @@ -13,6 +13,8 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.eclipse.jgit.util.HttpSupport.HDR_AUTHORIZATION; +import com.google.common.base.Strings; + import java.io.UnsupportedEncodingException; import java.net.URL; import java.net.URLEncoder; @@ -92,7 +94,12 @@ class SignerV4 { static String createAuthorizationQuery(S3Config bucketConfig, URL url, String httpMethod, Map headers, Map queryParameters, String bodyHash) { - addHostHeader(url, headers); + + if (!Strings.isNullOrEmpty(bucketConfig.getSignatureHostname())) { + headers.put("Host", bucketConfig.getSignatureHostname()); //$NON-NLS-1$ + } else { + addHostHeader(url, headers); + } queryParameters.put(X_AMZ_ALGORITHM, SCHEME + "-" + ALGORITHM); //$NON-NLS-1$ @@ -161,7 +168,12 @@ private static void appendQuery(StringBuilder s, String key, static Map createHeaderAuthorization( S3Config bucketConfig, URL url, String httpMethod, Map headers, String bodyHash) { - addHostHeader(url, headers); + + if (!Strings.isNullOrEmpty(bucketConfig.getSignatureHostname())) { + headers.put("Host", bucketConfig.getSignatureHostname()); //$NON-NLS-1$ + } else { + addHostHeader(url, headers); + } Date now = new Date(); String dateTimeStamp = dateTimeStampISO8601(now);