diff --git a/.github/workflows/java-ci.yml b/.github/workflows/java-ci.yml index e77259ecd36e..d6739f0527b2 100644 --- a/.github/workflows/java-ci.yml +++ b/.github/workflows/java-ci.yml @@ -109,7 +109,7 @@ jobs: with: distribution: zulu java-version: ${{ matrix.jvm }} - - run: ./gradlew -DallModules build -x test -x javadoc -x integrationTest + - run: ./gradlew -DallModules build -x test -x javadoc -x integrationTest -x intTest build-javadoc: runs-on: ubuntu-24.04 diff --git a/build.gradle b/build.gradle index 52d25bc33b51..bef9da972e29 100644 --- a/build.gradle +++ b/build.gradle @@ -357,9 +357,43 @@ project(':iceberg-common') { } project(':iceberg-core') { + apply plugin: 'java-test-fixtures' + apply plugin: 'jvm-test-suite' + test { useJUnitPlatform() } + + testing { + suites { + intTest(JvmTestSuite) { + useJUnitJupiter() + dependencies { + implementation project() + implementation testFixtures(project()) + } + check.dependsOn intTest + targets { + all { + testTask.configure { + shouldRunAfter(test) + mustRunAfter(test) + } + } + } + } + } + } + + configurations { + docs { + description = 'Dependencies for generating configuration documentation' + canBeResolved = true + canBeConsumed = false + visible = false + } + } + dependencies { api project(':iceberg-api') implementation project(':iceberg-common') @@ -384,11 +418,16 @@ project(':iceberg-core') { exclude group: 'org.slf4j', module: 'slf4j-log4j12' } + // OAuth2 + implementation libs.nimbus.oauth2.oidc.sdk + implementation libs.nimbus.jose.jwt + testImplementation libs.jetty.servlet testImplementation libs.jakarta.servlet testImplementation libs.jetty.server testImplementation libs.mockserver.netty testImplementation libs.mockserver.client.java + testImplementation libs.bouncycastle.bcpkix testImplementation libs.sqlite.jdbc testImplementation libs.derby.core testImplementation libs.derby.tools @@ -398,7 +437,108 @@ project(':iceberg-core') { exclude group: 'junit' } testImplementation libs.awaitility + + testFixturesApi libs.junit.jupiter + testFixturesApi libs.junit.pioneer + testFixturesApi libs.assertj.core + testFixturesApi libs.mockito.core + + testFixturesApi libs.httpcomponents.httpclient5 + testFixturesApi libs.nimbus.oauth2.oidc.sdk + testFixturesApi libs.nimbus.jose.jwt + + testFixturesApi libs.mockserver.netty + testFixturesApi libs.mockserver.client.java + + testFixturesApi libs.testcontainers + testFixturesApi libs.testcontainers.junit.jupiter + testFixturesApi libs.testcontainers.keycloak + + testFixturesApi libs.keycloak.admin.client + + testFixturesImplementation project(path: ':iceberg-bundled-guava', configuration: 'shadow') + + testFixturesAnnotationProcessor libs.immutables.value + testFixturesCompileOnly libs.immutables.value + + intTestImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') // for CatalogTests + intTestImplementation project(path: ':iceberg-core', configuration: 'testArtifacts') // for CatalogTests + intTestImplementation project(path: ':iceberg-bundled-guava', configuration: 'shadow') + + intTestImplementation libs.jetty.servlet + intTestImplementation libs.jakarta.servlet + intTestImplementation libs.jetty.server + + docs 'com.thoughtworks.qdox:qdox:2.2.0' + docs project(path: ':iceberg-bundled-guava', configuration: 'shadow') } + + sourceSets { + docs { + java { + srcDir 'src/docs/java' + } + resources { + srcDir 'src/docs/resources' + } + compileClasspath += configurations.docs + runtimeClasspath += configurations.docs + } + } + + tasks.named('processDocsResources', ProcessResources).configure { + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } + + tasks.register('generateOAuth2Docs', JavaExec) { + group = 'documentation' + description = 'Generates REST OAuth2 configuration documentation from OAuth2Config' + mainClass.set('org.apache.iceberg.rest.auth.oauth2.docs.OAuth2DocumentationGenerator') + classpath = sourceSets.docs.runtimeClasspath + + def inputFile = project.file('src/main/java/org/apache/iceberg/rest/auth/oauth2/OAuth2Config.java') + def outputFile = rootProject.file('docs/docs/oauth2-configuration.md') + + inputs.files(inputFile) + outputs.file(outputFile) + + args(inputFile.absolutePath, outputFile.absolutePath) + + doFirst { outputFile.parentFile.mkdirs() } + } + + assemble.dependsOn('generateOAuth2Docs') + + tasks.register('checkOAuth2Docs', JavaExec) { + group = 'verification' + description = 'Checks that the OAuth2 configuration documentation is up to date' + mainClass.set('org.apache.iceberg.rest.auth.oauth2.docs.OAuth2DocumentationGenerator') + classpath = sourceSets.docs.runtimeClasspath + mustRunAfter('generateOAuth2Docs') + + def inputFile = project.file('src/main/java/org/apache/iceberg/rest/auth/oauth2/OAuth2Config.java') + def committedFile = rootProject.file('docs/docs/oauth2-configuration.md') + def tempFile = project.layout.buildDirectory.file('generated-docs/oauth2-configuration.md') + + inputs.files(inputFile, committedFile) + outputs.file(tempFile) + + args(inputFile.absolutePath, tempFile.get().asFile.absolutePath) + + doFirst { tempFile.get().asFile.parentFile.mkdirs() } + + doLast { + def expected = tempFile.get().asFile.text + def actual = committedFile.text + if (expected != actual) { + throw new GradleException( + "OAuth2 configuration documentation is out of date. " + + "Please run './gradlew :iceberg-core:generateOAuth2Docs' to update it.") + } + } + } + + check.dependsOn('checkOAuth2Docs') } project(':iceberg-data') { diff --git a/core/src/docs/java/org/apache/iceberg/rest/auth/oauth2/docs/OAuth2DocumentationGenerator.java b/core/src/docs/java/org/apache/iceberg/rest/auth/oauth2/docs/OAuth2DocumentationGenerator.java new file mode 100644 index 000000000000..38e906fd3abf --- /dev/null +++ b/core/src/docs/java/org/apache/iceberg/rest/auth/oauth2/docs/OAuth2DocumentationGenerator.java @@ -0,0 +1,371 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.docs; + +import com.thoughtworks.qdox.JavaProjectBuilder; +import com.thoughtworks.qdox.model.JavaClass; +import com.thoughtworks.qdox.model.JavaField; +import com.thoughtworks.qdox.model.JavaMethod; +import java.io.BufferedWriter; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.base.Splitter; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; + +/** + * Parses the properties from the source code and generates documentation for them. + * + *

This generator is mostly intended to parse the `OAuth2Config` class. The parser relies heavily + * on conventions, such as the use of `PREFIX` fields, or fields starting with `DEFAULT_`, or the + * presence of nested classes to structure the properties into sections. + */ +@SuppressWarnings("ParameterAssignment") +public class OAuth2DocumentationGenerator { + + private static final String HEADER = + """ + + + + + # REST OAuth2 Configuration + + """; + + private static final Pattern CODE_PATTERN = Pattern.compile("\\{@code\\s(.+?)}"); + + private static final Pattern REF_PATTERN = + Pattern.compile("\\{@(?:link(?:plain)?|value)\\s+([^ }]+)( [^}]+)?}"); + + private static final Pattern EXTERNAL_LINK_PATTERN = + Pattern.compile("]*>([^<]+)"); + + private static final Pattern PRE_PATTERN = + Pattern.compile("

\\s*(?:\\{@code)?(.*?)(}\\s*)?
", Pattern.DOTALL); + + private static final Map KNOWN_REFS; + + static { + Map refs = new LinkedHashMap<>(); + refs.put("GrantType#CLIENT_CREDENTIALS", "client_credentials"); + refs.put("GrantType#REFRESH_TOKEN", "refresh_token"); + refs.put("GrantType#TOKEN_EXCHANGE", "urn:ietf:params:oauth:grant-type:token-exchange"); + refs.put("ClientAuthenticationMethod#NONE", "none"); + refs.put("ClientAuthenticationMethod#CLIENT_SECRET_BASIC", "client_secret_basic"); + refs.put("ClientAuthenticationMethod#CLIENT_SECRET_POST", "client_secret_post"); + KNOWN_REFS = Map.copyOf(refs); + } + + private static final String ROOT_CLASS_NAME = "org.apache.iceberg.rest.auth.oauth2.OAuth2Config"; + + private final Path rootConfigFile; + private final Path outputFile; + + private JavaProjectBuilder builder; + private String rootPrefix; + private Map sections; + + public static void main(String[] args) throws IOException { + String sourceFile = args[0]; + String outputFile = args[1]; + OAuth2DocumentationGenerator generator = + new OAuth2DocumentationGenerator(Path.of(sourceFile), Path.of(outputFile)); + generator.run(); + } + + public OAuth2DocumentationGenerator(Path rootConfigFile, Path outputFile) { + this.rootConfigFile = rootConfigFile; + this.outputFile = outputFile; + } + + public void run() throws IOException { + parse(); + generate(); + } + + private void parse() throws IOException { + + builder = new JavaProjectBuilder(); + builder.addSource(rootConfigFile.toFile()); + + File[] files = + rootConfigFile + .resolveSibling("config") + .toFile() + .listFiles(file -> file.getName().endsWith("Config.java")); + Preconditions.checkNotNull(files, "Failed to list config files"); + + for (File file : files) { + builder.addSource(file); + } + + JavaClass topClass = builder.getClassByName(ROOT_CLASS_NAME); + rootPrefix = topClass.getFieldByName("PREFIX").getInitializationExpression().replace("\"", ""); + + sections = new LinkedHashMap<>(); + + for (JavaMethod method : topClass.getMethods()) { + + if (method.getName().matches("\\w+Config")) { + JavaClass sectionConfigClass = (JavaClass) method.getReturnType(); + sections.put(sectionConfigClass.getFullyQualifiedName(), new Section(sectionConfigClass)); + } + } + + for (Section section : sections.values()) { + section.parseProperties(); + } + } + + private void generate() throws IOException { + + try (BufferedWriter writer = Files.newBufferedWriter(outputFile, StandardCharsets.UTF_8)) { + writer.write(HEADER); + + for (Section section : sections.values()) { + + writer.write("## " + section.name + "\n\n"); + if (section.description != null && !section.description.isEmpty()) { + writer.write(section.description); + } + + for (Property property : section.properties) { + writer.write("\n### `" + property.name + "`\n\n"); + writer.write(property.description); + } + } + } + } + + @SuppressWarnings({"UnnecessaryUnicodeEscape", "AvoidEscapedUnicodeCharacters"}) + private String sanitizeDescription(Section section, String text) { + + Matcher matcher = EXTERNAL_LINK_PATTERN.matcher(text); + StringBuilder sb = new StringBuilder(); + while (matcher.find()) { + String url = matcher.group(1); + String linkText = matcher.group(2); + matcher.appendReplacement(sb, "[" + linkText + "](" + url + ")"); + } + matcher.appendTail(sb); + text = sb.toString(); + + matcher = CODE_PATTERN.matcher(text); + sb = new StringBuilder(); + while (matcher.find()) { + String codeBlock = matcher.group(1); + matcher.appendReplacement(sb, "`" + codeBlock + "`"); + } + matcher.appendTail(sb); + text = sb.toString(); + + matcher = REF_PATTERN.matcher(text); + sb = new StringBuilder(); + while (matcher.find()) { + String fieldRef = matcher.group(1); + String refText = matcher.group(2); + String resolvedReference = resolveReference(section, fieldRef, refText); + matcher.appendReplacement(sb, resolvedReference); + } + matcher.appendTail(sb); + text = sb.toString(); + + text = cleanupHtmlTags(text); + + // Clean up extra whitespace and normalize line breaks + text = text.replaceAll("\\r\\n", "\n"); + text = text.replaceAll("\\r", "\n"); + text = text.replaceAll("\n(?![\n\\-])", " "); + text = text.replaceAll(" {2,}", " "); + text = text.replaceAll("\n ", "\n"); + text = text.replaceAll("\n{3,}", "\n\n"); + text = text.replaceAll("\u2424", "\n"); + text = text.trim() + "\n"; + + return text; + } + + private String resolveReference(Section section, String ref, String text) { + String refTarget = KNOWN_REFS.get(ref); + if (refTarget == null) { + if (ref.equals("OAuth2Config#PREFIX")) { + refTarget = rootPrefix; + } else if (ref.startsWith("#")) { + // local ref + String fieldName = ref.substring(1); + refTarget = section.refs.get(fieldName); + } else if (section != null) { + // external ref + List parts = Splitter.on('#').splitToList(ref); + String className = section.configClass.getPackageName() + "." + parts.get(0); + String fieldName = parts.get(1); + JavaClass classRef = builder.getClassByName(className); + Section refSection = sections.get(classRef.getFullyQualifiedName()); + refTarget = refSection.refs.get(fieldName); + } + } + if (text == null) { + return "`" + refTarget + "`"; + } + text = text.trim(); + return text.isEmpty() || text.equals(refTarget) + ? "`" + refTarget + "`" + : text + " (`" + refTarget + "`)"; + } + + @SuppressWarnings({"UnnecessaryUnicodeEscape", "AvoidEscapedUnicodeCharacters"}) + private static String cleanupHtmlTags(String text) { + // Convert HTML lists + text = text.replaceAll("
    *", ""); + text = text.replaceAll(" *
", ""); + text = text.replaceAll(" *
  • *", "- "); + // Convert HTML paragraphs + text = text.replaceAll("

    \\s*", "\n"); + + // Temporarily replace newlines in code blocks with a different character + // to preserve them during the next steps. Here we use 'SYMBOL FOR NEWLINE' (U+2424). + Matcher matcher = PRE_PATTERN.matcher(text); + StringBuilder sb = new StringBuilder(); + while (matcher.find()) { + String codeBlock = matcher.group(1).trim(); + codeBlock = codeBlock.replaceAll("[\r\n]", "\u2424"); + matcher.appendReplacement(sb, "\n\n```\u2424" + codeBlock + "\u2424```\n\n"); + } + matcher.appendTail(sb); + text = sb.toString(); + + return text; + } + + private class Section { + + private final JavaClass configClass; + private final String prefix; + private final String name; + private final String description; + private final Map refs = Maps.newLinkedHashMap(); + private final List properties = Lists.newArrayList(); + + private Section(JavaClass configClass) { + this.configClass = configClass; + this.name = sanitizeSectionName(configClass.getSimpleName()); + this.prefix = resolvePrefix(); + parseLocalReferences(); + this.description = sanitizeDescription(this, configClass.getComment()); + } + + private String sanitizeSectionName(String className) { + return className.replace("Config", "").replaceAll("([A-Z])", " $1").trim() + " Settings"; + } + + private String resolvePrefix() { + if (configClass.getSimpleName().equals("BasicConfig")) { + // Basic config: shares the root prefix + return rootPrefix; + } else { + JavaField prefixField = configClass.getFieldByName("PREFIX"); + return prefixField + .getInitializationExpression() + .replace("OAuth2Config.PREFIX + ", rootPrefix) + .replace("\"", ""); + } + } + + private void parseLocalReferences() { + for (JavaField field : configClass.getFields()) { + String refName = field.getName(); + if (refName.equals("PREFIX")) { + refs.put("PREFIX", prefix); + } else if (refName.startsWith("DEFAULT_")) { + String refText = field.getInitializationExpression(); + if (refText.startsWith("Duration.parse")) { + refText = refText.substring(refText.indexOf("(\"") + 2, refText.indexOf("\")")); + } else { + String asRef = refText.replace('.', '#'); + String resolved = KNOWN_REFS.get(asRef); + refText = resolved != null ? resolved : refText.replace("\"", ""); + } + refs.put(refName, refText); + } else { + String refText = + field + .getInitializationExpression() + .replaceAll("PREFIX \\+\\s*", prefix) + .replace("\"", ""); + refs.put(refName, refText); + } + } + } + + private void parseProperties() { + for (JavaMethod method : configClass.getMethods()) { + if (method.getComment() == null) { + continue; + } + method.getAnnotations().stream() + .filter(a -> a.getType().getName().equals("ConfigOption")) + .findFirst() + .ifPresent( + ann -> { + String configOption = ann.getNamedParameter("value").toString(); + String propertyName = refs.get(configOption); + String propertyDescription = sanitizeDescription(this, method.getComment()); + if (ann.getNamedParameter("prefixMap") != null) { + boolean prefixMap = + Boolean.parseBoolean((String) ann.getNamedParameter("prefixMap")); + if (prefixMap) { + propertyName = propertyName + ".*"; + } + } + properties.add(new Property(propertyName, propertyDescription)); + }); + } + } + } + + private record Property(String name, String description) {} +} diff --git a/core/src/intTest/java/org/apache/iceberg/rest/auth/oauth2/ITOAuth2RESTCatalog.java b/core/src/intTest/java/org/apache/iceberg/rest/auth/oauth2/ITOAuth2RESTCatalog.java new file mode 100644 index 000000000000..c1f7212cde4f --- /dev/null +++ b/core/src/intTest/java/org/apache/iceberg/rest/auth/oauth2/ITOAuth2RESTCatalog.java @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.nimbusds.oauth2.sdk.GrantType; +import com.nimbusds.oauth2.sdk.auth.Secret; +import com.nimbusds.oauth2.sdk.id.ClientID; +import com.nimbusds.oauth2.sdk.token.AccessToken; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.nio.file.Path; +import java.util.Map; +import java.util.function.Consumer; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.Table; +import org.apache.iceberg.catalog.CatalogTests; +import org.apache.iceberg.exceptions.NotFoundException; +import org.apache.iceberg.inmemory.InMemoryCatalog; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.rest.HTTPHeaders; +import org.apache.iceberg.rest.HTTPRequest; +import org.apache.iceberg.rest.RESTCatalog; +import org.apache.iceberg.rest.RESTCatalogAdapter; +import org.apache.iceberg.rest.RESTCatalogServlet; +import org.apache.iceberg.rest.RESTResponse; +import org.apache.iceberg.rest.auth.oauth2.client.OAuth2Client; +import org.apache.iceberg.rest.auth.oauth2.config.ConfigUtil; +import org.apache.iceberg.rest.auth.oauth2.test.ImmutableTestEnvironment; +import org.apache.iceberg.rest.auth.oauth2.test.TestEnvironment; +import org.apache.iceberg.rest.auth.oauth2.test.container.KeycloakContainer; +import org.apache.iceberg.rest.auth.oauth2.test.junit.KeycloakExtension; +import org.apache.iceberg.rest.responses.ErrorResponse; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; + +/** + * {@link CatalogTests} for {@link RESTCatalog} with {@link OAuth2Manager} and a real Keycloak. + * + *

    In these tests, OAuth2 is configured as follows: + * + *

      + *
    • Catalog session: uses client_credentials as an initial grant, and token exchange for token + * refreshes (legacy behavior). + *
    • Session context: uses token exchange as the initial grant. The subject token is obtained + * off-band with the client_credentials grant. The actor token comes from the parent catalog + * session. Token refreshes use the standard refresh_token grant. + *
    + * + * The equivalent catalog configuration would be: + * + * 
    {@code
+ * rest.auth.type=oauth2
+ * rest.auth.oauth2.issuer-url=https://
+ * rest.auth.oauth2.grant-type=client_credentials
+ * rest.auth.oauth2.client-id=
+ * rest.auth.oauth2.client-secret=
+ * rest.auth.oauth2.scope=catalog
+ * }
    + * + * The equivalent session context configuration would be: + * + * 
    {@code
+ * rest.auth.oauth2.issuer-url=https://
+ * rest.auth.oauth2.grant-type=urn:ietf:params:oauth:grant-type:token-exchange
+ * rest.auth.oauth2.client-id=
+ * rest.auth.oauth2.client-secret=
+ * rest.auth.oauth2.scope=session
+ * rest.auth.oauth2.token-exchange.subject-token=
+ * rest.auth.oauth2.token-exchange.actor-token=::parent::
+ * }
    + */ +@ExtendWith(KeycloakExtension.class) +public class ITOAuth2RESTCatalog extends CatalogTests { + + @TempDir public static Path tempDir; + + private static InMemoryCatalog backendCatalog; + private static Server httpServer; + private static TestEnvironment testEnvironment; + + private RESTCatalog restCatalog; + + @BeforeAll + static void beforeClass( + KeycloakContainer keycloakContainer, + ImmutableTestEnvironment.Builder envBuilder1, + ImmutableTestEnvironment.Builder envBuilder2) + throws Exception { + backendCatalog = new InMemoryCatalog(); + backendCatalog.initialize( + "in-memory", + ImmutableMap.of( + CatalogProperties.WAREHOUSE_LOCATION, + tempDir.resolve("warehouse").toAbsolutePath().toString())); + + ServletContextHandler servletContext = + new ServletContextHandler(ServletContextHandler.NO_SESSIONS); + servletContext.addServlet( + new ServletHolder( + new RESTCatalogServlet( + new RESTCatalogAdapter(backendCatalog) { + @Override + public T execute( + HTTPRequest request, + Class responseType, + Consumer errorHandler, + Consumer> responseHeaders) { + // validate the oauth2 token + HTTPHeaders.HTTPHeader authorization = + request.headers().firstEntry("Authorization").orElseThrow(); + String authHeader = authorization.value(); + String token = authHeader.substring("Bearer ".length()); + keycloakContainer.verifyToken(token); + return super.execute(request, responseType, errorHandler, responseHeaders); + } + })), + "/*"); + + httpServer = new Server(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0)); + httpServer.setHandler(servletContext); + httpServer.start(); + + AccessToken subjectToken; + try (TestEnvironment env = + envBuilder1 + .grantType(GrantType.CLIENT_CREDENTIALS) + .clientId(new ClientID(KeycloakExtension.CLIENT_ID2)) + .clientSecret(new Secret(KeycloakExtension.CLIENT_SECRET2)) + .build(); + OAuth2Client subjectClient = env.newOAuth2Client()) { + subjectToken = subjectClient.authenticate(); + } + + testEnvironment = + envBuilder2 + .catalogServerUrl(httpServer.getURI()) + // Catalog session + .grantType(GrantType.CLIENT_CREDENTIALS) + .clientId(new ClientID(KeycloakExtension.CLIENT_ID1)) + .clientSecret(new Secret(KeycloakExtension.CLIENT_SECRET1)) + // Contextual session + .sessionContextGrantType(GrantType.TOKEN_EXCHANGE) + .sessionContextSubjectTokenString(subjectToken.getValue()) + .sessionContextActorTokenString(ConfigUtil.PARENT_TOKEN) + .build(); + } + + @AfterAll + static void afterClass() throws Exception { + if (testEnvironment != null) { + testEnvironment.close(); + } + + if (httpServer != null) { + httpServer.stop(); + httpServer.join(); + } + + if (backendCatalog != null) { + backendCatalog.close(); + } + } + + @BeforeEach + public void before() { + restCatalog = initCatalog("oauth2-test-catalog", Map.of()); + } + + @AfterEach + public void after() throws Exception { + if (restCatalog != null) { + restCatalog.close(); + } + + if (backendCatalog != null) { + backendCatalog.close(); // clears the in-memory data structures + } + } + + @Override + @SuppressWarnings("MustBeClosedChecker") + protected RESTCatalog initCatalog(String catalogName, Map additionalProperties) { + return testEnvironment.newCatalog(additionalProperties); + } + + @Override + protected RESTCatalog catalog() { + return restCatalog; + } + + @Override + protected boolean supportsServerSideRetry() { + return true; + } + + @Override + protected boolean supportsNestedNamespaces() { + return true; + } + + @Override + protected boolean requiresNamespaceCreate() { + return true; + } + + @Test + @Override + @SuppressWarnings("resource") + public void testLoadTableWithMissingMetadataFile(@TempDir Path ignored) { + + if (requiresNamespaceCreate()) { + restCatalog.createNamespace(TBL.namespace()); + } + + restCatalog.buildTable(TBL, SCHEMA).create(); + assertThat(restCatalog.tableExists(TBL)).as("Table should exist").isTrue(); + + Table table = restCatalog.loadTable(TBL); + String metadataFileLocation = + ((HasTableOperations) table).operations().current().metadataFileLocation(); + table.io().deleteFile(metadataFileLocation); + + assertThatThrownBy(() -> restCatalog.loadTable(TBL)) + .isInstanceOf(NotFoundException.class) + .hasMessageContaining("No in-memory file found for location: " + metadataFileLocation); + } +} diff --git a/core/src/intTest/java/org/apache/iceberg/rest/auth/oauth2/client/ITOAuth2ClientKeycloak.java b/core/src/intTest/java/org/apache/iceberg/rest/auth/oauth2/client/ITOAuth2ClientKeycloak.java new file mode 100644 index 000000000000..3572f6956e72 --- /dev/null +++ b/core/src/intTest/java/org/apache/iceberg/rest/auth/oauth2/client/ITOAuth2ClientKeycloak.java @@ -0,0 +1,323 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.client; + +import static com.nimbusds.oauth2.sdk.GrantType.CLIENT_CREDENTIALS; +import static com.nimbusds.oauth2.sdk.GrantType.TOKEN_EXCHANGE; +import static com.nimbusds.oauth2.sdk.auth.ClientAuthenticationMethod.CLIENT_SECRET_BASIC; +import static com.nimbusds.oauth2.sdk.auth.ClientAuthenticationMethod.CLIENT_SECRET_POST; +import static org.apache.iceberg.rest.auth.oauth2.test.junit.KeycloakExtension.CLIENT_ID1; +import static org.apache.iceberg.rest.auth.oauth2.test.junit.KeycloakExtension.CLIENT_ID2; +import static org.apache.iceberg.rest.auth.oauth2.test.junit.KeycloakExtension.CLIENT_ID3; +import static org.apache.iceberg.rest.auth.oauth2.test.junit.KeycloakExtension.CLIENT_SECRET2; +import static org.apache.iceberg.rest.auth.oauth2.test.junit.KeycloakExtension.CLIENT_SECRET3; +import static org.apache.iceberg.rest.auth.oauth2.test.junit.KeycloakExtension.SCOPE1; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.InstanceOfAssertFactories.type; + +import com.nimbusds.jwt.JWTClaimsSet; +import com.nimbusds.oauth2.sdk.auth.Secret; +import com.nimbusds.oauth2.sdk.id.ClientID; +import com.nimbusds.oauth2.sdk.token.AccessToken; +import com.nimbusds.oauth2.sdk.token.TokenTypeURI; +import java.text.ParseException; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.TimeUnit; +import org.apache.iceberg.rest.auth.oauth2.OAuth2Exception; +import org.apache.iceberg.rest.auth.oauth2.config.ConfigUtil; +import org.apache.iceberg.rest.auth.oauth2.flow.TokensResult; +import org.apache.iceberg.rest.auth.oauth2.test.ImmutableTestEnvironment; +import org.apache.iceberg.rest.auth.oauth2.test.TestEnvironment; +import org.apache.iceberg.rest.auth.oauth2.test.container.KeycloakContainer; +import org.apache.iceberg.rest.auth.oauth2.test.junit.KeycloakExtension; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +@ExtendWith(KeycloakExtension.class) +public class ITOAuth2ClientKeycloak { + + private static KeycloakContainer keycloak; + + @BeforeAll + static void setKeycloakContainer(KeycloakContainer keycloakContainer) { + keycloak = keycloakContainer; + } + + @Test + void clientSecretBasic(ImmutableTestEnvironment.Builder envBuilder) throws Exception { + try (TestEnvironment env = envBuilder.clientAuthenticationMethod(CLIENT_SECRET_BASIC).build(); + OAuth2Client client = env.newOAuth2Client()) { + testClient(client, true); + } + } + + @Test + void clientSecretPost(ImmutableTestEnvironment.Builder envBuilder) throws Exception { + try (TestEnvironment env = envBuilder.clientAuthenticationMethod(CLIENT_SECRET_POST).build(); + OAuth2Client client = env.newOAuth2Client()) { + testClient(client, true); + } + } + + /** + * Tests a token exchange scenario with a fixed subject token obtained off-band, and no actor + * token. + */ + @Test + void impersonation( + ImmutableTestEnvironment.Builder envBuilder1, ImmutableTestEnvironment.Builder envBuilder2) + throws Exception { + AccessToken subjectToken; + try (TestEnvironment env = + envBuilder1 + .grantType(CLIENT_CREDENTIALS) + .clientId(new ClientID(CLIENT_ID2)) + .clientSecret(new Secret(CLIENT_SECRET2)) + .build(); + OAuth2Client subjectClient = env.newOAuth2Client()) { + subjectToken = subjectClient.authenticate(); + } + + try (TestEnvironment env = + envBuilder2 + .grantType(TOKEN_EXCHANGE) + .requestedTokenType(TokenTypeURI.ACCESS_TOKEN) + .subjectTokenString(subjectToken.getValue()) + .actorTokenString(Optional.empty()) + .build(); + OAuth2Client client = env.newOAuth2Client()) { + testClient(client, false); + } + } + + /** + * Tests a token exchange scenario with fixed subject and actor tokens, both obtained off-band. + */ + @Test + void delegation( + ImmutableTestEnvironment.Builder envBuilder1, + ImmutableTestEnvironment.Builder envBuilder2, + ImmutableTestEnvironment.Builder envBuilder3) + throws Exception { + AccessToken subjectToken; + try (TestEnvironment env = + envBuilder1 + .grantType(CLIENT_CREDENTIALS) + .clientId(new ClientID(CLIENT_ID2)) + .clientSecret(new Secret(CLIENT_SECRET2)) + .build(); + OAuth2Client subjectClient = env.newOAuth2Client()) { + subjectToken = subjectClient.authenticate(); + } + + AccessToken actorToken; + try (TestEnvironment env = + envBuilder2 + .grantType(CLIENT_CREDENTIALS) + .clientId(new ClientID(CLIENT_ID3)) + .clientSecret(new Secret(CLIENT_SECRET3)) + .build(); + OAuth2Client actorClient = env.newOAuth2Client()) { + actorToken = actorClient.authenticate(); + } + + try (TestEnvironment env = + envBuilder3 + .grantType(TOKEN_EXCHANGE) + .requestedTokenType(TokenTypeURI.ACCESS_TOKEN) + .subjectTokenString(subjectToken.getValue()) + .actorTokenString(actorToken.getValue()) + .build(); + OAuth2Client client = env.newOAuth2Client()) { + testClient(client, false); + } + } + + /** + * Tests a token exchange scenario with the subject token inherited from a separate, parent + * client, and no actor token. + */ + @Test + void parentSubject( + ImmutableTestEnvironment.Builder envBuilder1, ImmutableTestEnvironment.Builder envBuilder2) + throws Exception { + try (TestEnvironment parent = + envBuilder1 + .grantType(CLIENT_CREDENTIALS) + .clientId(new ClientID(CLIENT_ID2)) + .clientSecret(new Secret(CLIENT_SECRET2)) + .build(); + OAuth2Client parentClient = parent.newOAuth2Client()) { + try (TestEnvironment env = + envBuilder2 + .grantType(TOKEN_EXCHANGE) + .requestedTokenType(TokenTypeURI.ACCESS_TOKEN) + .parentClient(parentClient) + .subjectTokenString(ConfigUtil.PARENT_TOKEN) + .actorTokenString(Optional.empty()) + .build(); + OAuth2Client client = env.newOAuth2Client()) { + testClient(client, false); + } + } + } + + /** + * Tests a token exchange scenario with a fixed subject token obtained off-band, and the actor + * token inherited from a separate, parent client. + */ + @Test + void parentActor( + ImmutableTestEnvironment.Builder envBuilder1, + ImmutableTestEnvironment.Builder envBuilder2, + ImmutableTestEnvironment.Builder envBuilder3) + throws Exception { + AccessToken subjectToken; + try (TestEnvironment env = + envBuilder1 + .grantType(CLIENT_CREDENTIALS) + .clientId(new ClientID(CLIENT_ID2)) + .clientSecret(new Secret(CLIENT_SECRET2)) + .build(); + OAuth2Client subjectClient = env.newOAuth2Client()) { + subjectToken = subjectClient.authenticate(); + } + + try (TestEnvironment parent = + envBuilder2 + .grantType(CLIENT_CREDENTIALS) + .clientId(new ClientID(CLIENT_ID3)) + .clientSecret(new Secret(CLIENT_SECRET3)) + .build(); + OAuth2Client parentClient = parent.newOAuth2Client()) { + try (TestEnvironment env = + envBuilder3 + .grantType(TOKEN_EXCHANGE) + .requestedTokenType(TokenTypeURI.ACCESS_TOKEN) + .parentClient(parentClient) + .subjectTokenString(subjectToken.getValue()) + .actorTokenString(ConfigUtil.PARENT_TOKEN) + .build(); + OAuth2Client client = env.newOAuth2Client()) { + testClient(client, false); + } + } + } + + @Test + void parallelAuthenticate(ImmutableTestEnvironment.Builder envBuilder) throws Exception { + try (TestEnvironment env = envBuilder.build()) { + + CyclicBarrier barrier = new CyclicBarrier(10); + List errors = new CopyOnWriteArrayList<>(); + + Thread[] threads = new Thread[10]; + for (int i = 0; i < 10; i++) { + threads[i] = + new Thread( + () -> { + try (OAuth2Client client = env.newOAuth2Client()) { + barrier.await(5, TimeUnit.SECONDS); + testClient(client, true); + } catch (Throwable t) { + errors.add(t); + } + }); + threads[i].start(); + } + + for (Thread t : threads) { + t.join(10_000); + } + + assertThat(errors).as("No thread should have thrown an exception").isEmpty(); + } + } + + /** Tests dynamically-obtained tokens with refresh forcibly disabled. */ + @Test + void refreshDisabled(ImmutableTestEnvironment.Builder envBuilder) throws Exception { + try (TestEnvironment env = + envBuilder.grantType(CLIENT_CREDENTIALS).tokenRefreshEnabled(false).build(); + OAuth2Client client = env.newOAuth2Client()) { + // initial grant + TokensResult firstTokens = client.authenticateInternal(); + introspectToken(firstTokens.tokens().getAccessToken()); + assertThat(client).extracting("tokenRefreshFuture").isNull(); + } + } + + @Test + void unauthorizedBadClientSecret(ImmutableTestEnvironment.Builder envBuilder) { + try (TestEnvironment env = envBuilder.clientSecret(new Secret("BAD SECRET")).build(); + OAuth2Client client = env.newOAuth2Client()) { + assertThatThrownBy(client::authenticate) + .asInstanceOf(type(OAuth2Exception.class)) + .extracting(OAuth2Exception::statusCode, e -> e.error().code()) + .containsExactly(401, "unauthorized_client"); + } + } + + /** Tests copying a client before and after closing the original client. */ + @Test + void clientCopy(ImmutableTestEnvironment.Builder envBuilder) throws Exception { + try (TestEnvironment env = envBuilder.build(); + OAuth2Client client = env.newOAuth2Client()) { + // copy before close + try (OAuth2Client client2 = client.copy()) { + testClient(client2, true); + } + + client.close(); + // copy after close + try (OAuth2Client client3 = client.copy()) { + testClient(client3, true); + } + } + } + + private static void testClient(OAuth2Client client, boolean refreshPossible) throws Exception { + // fetch initial tokens + TokensResult initial = client.authenticateInternal(); + introspectToken(initial.tokens().getAccessToken()); + if (refreshPossible) { + // Note: the client is configured to use token exchange when the initial grant is + // client_credentials, and refresh_token otherwise. Keycloak is configured to support both. + TokensResult refreshed = client.refreshCurrentTokens(initial).toCompletableFuture().get(); + introspectToken(refreshed.tokens().getAccessToken()); + } else { + assertThat(initial.tokens().getRefreshToken()).isNull(); + } + // fetch new tokens + TokensResult renewed = client.fetchNewTokens().toCompletableFuture().get(); + introspectToken(renewed.tokens().getAccessToken()); + } + + private static void introspectToken(AccessToken accessToken) throws ParseException { + assertThat(accessToken).isNotNull(); + JWTClaimsSet claims = keycloak.verifyToken(accessToken.getValue()); + assertThat(claims.getStringClaim("azp")).isEqualTo(CLIENT_ID1); + assertThat(claims.getStringClaim("scope")).contains(SCOPE1); + } +} diff --git a/core/src/main/java/org/apache/iceberg/rest/ErrorHandlers.java b/core/src/main/java/org/apache/iceberg/rest/ErrorHandlers.java index 791eb732bb7c..ebb216cfb9f7 100644 --- a/core/src/main/java/org/apache/iceberg/rest/ErrorHandlers.java +++ b/core/src/main/java/org/apache/iceberg/rest/ErrorHandlers.java @@ -92,6 +92,13 @@ public static Consumer defaultErrorHandler() { return DefaultErrorHandler.INSTANCE; } + /** + * The OAuth error handler. + * + * @deprecated will be removed in 1.14.0, use {@link + * org.apache.iceberg.rest.auth.oauth2.OAuth2Manager} instead. + */ + @Deprecated public static Consumer oauthErrorHandler() { return OAuthErrorHandler.INSTANCE; } @@ -339,6 +346,11 @@ public void accept(ErrorResponse error) { } } + /** + * @deprecated will be removed in 1.14.0, use {@link + * org.apache.iceberg.rest.auth.oauth2.OAuth2Manager} instead. + */ + @Deprecated private static class OAuthErrorHandler extends ErrorHandler { private static final ErrorHandler INSTANCE = new OAuthErrorHandler(); diff --git a/core/src/main/java/org/apache/iceberg/rest/HTTPClient.java b/core/src/main/java/org/apache/iceberg/rest/HTTPClient.java index 86eceba21c95..69cb20f1c3de 100644 --- a/core/src/main/java/org/apache/iceberg/rest/HTTPClient.java +++ b/core/src/main/java/org/apache/iceberg/rest/HTTPClient.java @@ -26,6 +26,7 @@ import java.net.URI; import java.nio.charset.StandardCharsets; import java.util.Map; +import java.util.Optional; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; @@ -163,6 +164,11 @@ public HTTPClient withAuthSession(AuthSession session) { return new HTTPClient(this, session); } + @Override + public Optional unwrap(Class clazz) { + return clazz.isInstance(httpClient) ? Optional.of(clazz.cast(httpClient)) : super.unwrap(clazz); + } + private static String extractResponseBodyAsString(CloseableHttpResponse response) { try { if (response.getEntity() == null) { diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTClient.java b/core/src/main/java/org/apache/iceberg/rest/RESTClient.java index c4d1bd84a928..7967816f12f2 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTClient.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTClient.java @@ -20,6 +20,7 @@ import java.io.Closeable; import java.util.Map; +import java.util.Optional; import java.util.function.Consumer; import java.util.function.Supplier; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; @@ -226,4 +227,13 @@ T postForm( default RESTClient withAuthSession(AuthSession session) { return this; } + + /** + * Unwrap this client to the given class if possible. + * + * @return The unwrapped client, or empty if this client cannot be unwrapped to the given class. + */ + default Optional unwrap(Class clazz) { + return Optional.ofNullable(clazz.isInstance(this) ? clazz.cast(this) : null); + } } diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/AuthConfig.java b/core/src/main/java/org/apache/iceberg/rest/auth/AuthConfig.java index 16d781e43579..5e76458d8052 100644 --- a/core/src/main/java/org/apache/iceberg/rest/auth/AuthConfig.java +++ b/core/src/main/java/org/apache/iceberg/rest/auth/AuthConfig.java @@ -27,10 +27,14 @@ /** * The purpose of this class is to hold OAuth configuration options for {@link * OAuth2Util.AuthSession}. + * + * @deprecated will be removed in 1.14.0, use {@link + * org.apache.iceberg.rest.auth.oauth2.OAuth2Config} instead. */ @Value.Style(redactedMask = "****") @Value.Immutable @SuppressWarnings({"ImmutablesStyle", "SafeLoggingPropagation"}) +@Deprecated public interface AuthConfig { @Nullable @Value.Redacted diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/AuthManagers.java b/core/src/main/java/org/apache/iceberg/rest/auth/AuthManagers.java index 4e48118561f7..8f4814ce8536 100644 --- a/core/src/main/java/org/apache/iceberg/rest/auth/AuthManagers.java +++ b/core/src/main/java/org/apache/iceberg/rest/auth/AuthManagers.java @@ -105,7 +105,16 @@ public static AuthManager loadAuthManager(String name, Map prope impl = authType; } - LOG.info("Loading AuthManager implementation: {}", impl); + if (impl.equals(AuthProperties.AUTH_MANAGER_IMPL_OAUTH2_LEGACY)) { + LOG.warn( + "The AuthManager implementation {} is deprecated and will be removed in a future release. " + + "Please migrate to {}.", + AuthProperties.AUTH_MANAGER_IMPL_OAUTH2_LEGACY, + AuthProperties.AUTH_MANAGER_IMPL_OAUTH2_NEW); + } else { + LOG.info("Loading AuthManager implementation: {}", impl); + } + DynConstructors.Ctor ctor; try { ctor = diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/AuthProperties.java b/core/src/main/java/org/apache/iceberg/rest/auth/AuthProperties.java index 331c7a734a27..c287b5a54d09 100644 --- a/core/src/main/java/org/apache/iceberg/rest/auth/AuthProperties.java +++ b/core/src/main/java/org/apache/iceberg/rest/auth/AuthProperties.java @@ -34,8 +34,12 @@ private AuthProperties() {} "org.apache.iceberg.rest.auth.NoopAuthManager"; public static final String AUTH_MANAGER_IMPL_BASIC = "org.apache.iceberg.rest.auth.BasicAuthManager"; - public static final String AUTH_MANAGER_IMPL_OAUTH2 = + static final String AUTH_MANAGER_IMPL_OAUTH2_LEGACY = "org.apache.iceberg.rest.auth.OAuth2Manager"; + static final String AUTH_MANAGER_IMPL_OAUTH2_NEW = + "org.apache.iceberg.rest.auth.oauth2.OAuth2Manager"; + public static final String AUTH_MANAGER_IMPL_OAUTH2 = + AUTH_MANAGER_IMPL_OAUTH2_LEGACY; // TODO switch to new manager public static final String AUTH_MANAGER_IMPL_SIGV4 = "org.apache.iceberg.aws.RESTSigV4AuthManager"; public static final String AUTH_MANAGER_IMPL_GOOGLE = diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/AuthSessionCache.java b/core/src/main/java/org/apache/iceberg/rest/auth/AuthSessionCache.java index fe64244a8111..07bbc3c3105d 100644 --- a/core/src/main/java/org/apache/iceberg/rest/auth/AuthSessionCache.java +++ b/core/src/main/java/org/apache/iceberg/rest/auth/AuthSessionCache.java @@ -32,7 +32,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** A cache for {@link AuthSession} instances. */ +/** + * A cache for {@link AuthSession} instances. + * + * @deprecated will be removed in 1.14.0, use {@link + * org.apache.iceberg.rest.auth.oauth2.OAuth2Manager} instead. + */ +@Deprecated public class AuthSessionCache implements AutoCloseable { private static final Logger LOG = LoggerFactory.getLogger(AuthSessionCache.class); diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/BasicAuthManager.java b/core/src/main/java/org/apache/iceberg/rest/auth/BasicAuthManager.java index d0d56d3d3794..1725bcb42b30 100644 --- a/core/src/main/java/org/apache/iceberg/rest/auth/BasicAuthManager.java +++ b/core/src/main/java/org/apache/iceberg/rest/auth/BasicAuthManager.java @@ -18,8 +18,11 @@ */ package org.apache.iceberg.rest.auth; +import java.nio.charset.StandardCharsets; +import java.util.Base64; import java.util.Map; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.rest.HTTPHeaders; import org.apache.iceberg.rest.RESTClient; @@ -43,7 +46,12 @@ public AuthSession catalogSession(RESTClient sharedClient, Map p String username = properties.get(AuthProperties.BASIC_USERNAME); String password = properties.get(AuthProperties.BASIC_PASSWORD); String credentials = username + ":" + password; - return DefaultAuthSession.of(HTTPHeaders.of(OAuth2Util.basicAuthHeaders(credentials))); + // Note: RFC 7617 specifies ISO-8859-1 as the default encoding for Basic authentication + // credentials. This implementation uses UTF-8 for backwards compatibility. + String encoded = + Base64.getEncoder().encodeToString(credentials.getBytes(StandardCharsets.UTF_8)); + return DefaultAuthSession.of( + HTTPHeaders.of(ImmutableMap.of("Authorization", "Basic " + encoded))); } @Override diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/OAuth2Manager.java b/core/src/main/java/org/apache/iceberg/rest/auth/OAuth2Manager.java index d9e314c5105e..5f84d212b6b3 100644 --- a/core/src/main/java/org/apache/iceberg/rest/auth/OAuth2Manager.java +++ b/core/src/main/java/org/apache/iceberg/rest/auth/OAuth2Manager.java @@ -40,6 +40,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * @deprecated will be removed in 1.14.0, use {@link + * org.apache.iceberg.rest.auth.oauth2.OAuth2Manager} instead. + */ +@Deprecated public class OAuth2Manager implements AuthManager { private static final Logger LOG = LoggerFactory.getLogger(OAuth2Manager.class); diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/OAuth2Properties.java b/core/src/main/java/org/apache/iceberg/rest/auth/OAuth2Properties.java index cf9018ff6f95..ee18105d3c18 100644 --- a/core/src/main/java/org/apache/iceberg/rest/auth/OAuth2Properties.java +++ b/core/src/main/java/org/apache/iceberg/rest/auth/OAuth2Properties.java @@ -18,6 +18,11 @@ */ package org.apache.iceberg.rest.auth; +/** + * @deprecated will be removed in 1.14.0, use {@link + * org.apache.iceberg.rest.auth.oauth2.OAuth2Config} instead. + */ +@Deprecated public class OAuth2Properties { private OAuth2Properties() {} diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/OAuth2Util.java b/core/src/main/java/org/apache/iceberg/rest/auth/OAuth2Util.java index c2b47e6e944f..8c608f71eec9 100644 --- a/core/src/main/java/org/apache/iceberg/rest/auth/OAuth2Util.java +++ b/core/src/main/java/org/apache/iceberg/rest/auth/OAuth2Util.java @@ -55,6 +55,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * @deprecated will be removed in 1.14.0, use {@link + * org.apache.iceberg.rest.auth.oauth2.OAuth2Manager} instead. + */ +@Deprecated public class OAuth2Util { private OAuth2Util() {} diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/OAuth2Config.java b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/OAuth2Config.java new file mode 100644 index 000000000000..fea6ff2ede6a --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/OAuth2Config.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2; + +import com.nimbusds.oauth2.sdk.GrantType; +import java.util.Map; +import org.apache.iceberg.rest.auth.oauth2.config.BasicConfig; +import org.apache.iceberg.rest.auth.oauth2.config.ConfigValidator; +import org.apache.iceberg.rest.auth.oauth2.config.ImmutableTokenExchangeConfig; +import org.apache.iceberg.rest.auth.oauth2.config.ImmutableTokenRefreshConfig; +import org.apache.iceberg.rest.auth.oauth2.config.TokenExchangeConfig; +import org.apache.iceberg.rest.auth.oauth2.config.TokenRefreshConfig; +import org.immutables.value.Value; + +/** The configuration for the OAuth2 AuthManager. */ +@Value.Immutable(prehash = true) // prehash for use as cache key +public interface OAuth2Config { + + String PREFIX = "rest.auth.oauth2."; + + /** + * The basic configuration, including token endpoint, grant type, client id and client secret. + * Required. + */ + BasicConfig basicConfig(); + + /** The token refresh configuration. Optional. */ + @Value.Default + default TokenRefreshConfig tokenRefreshConfig() { + return ImmutableTokenRefreshConfig.builder().build(); + } + + /** + * The token exchange configuration. Required for the {@link GrantType#TOKEN_EXCHANGE} grant type. + */ + @Value.Default + default TokenExchangeConfig tokenExchangeConfig() { + return ImmutableTokenExchangeConfig.builder().build(); + } + + @Value.Check + default void validate() { + // At this level, we only need to validate constraints that span multiple + // configuration classes; individual configuration classes are validated + // internally in their respective validate() methods. + ConfigValidator validator = new ConfigValidator(); + GrantType grantType = basicConfig().grantType(); + + if (grantType.equals(GrantType.TOKEN_EXCHANGE)) { + validator.check( + tokenExchangeConfig().subjectTokenString().isPresent(), + TokenExchangeConfig.SUBJECT_TOKEN, + "subject token must be set if grant type is '%s'", + GrantType.TOKEN_EXCHANGE.getValue()); + } + + validator.validate(); + } + + /** Creates an {@link OAuth2Config} from the given properties map. */ + static OAuth2Config of(Map properties) { + return ImmutableOAuth2Config.builder() + .basicConfig(BasicConfig.parse(properties).build()) + .tokenRefreshConfig(TokenRefreshConfig.parse(properties).build()) + .tokenExchangeConfig(TokenExchangeConfig.parse(properties).build()) + .build(); + } +} diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/OAuth2Exception.java b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/OAuth2Exception.java new file mode 100644 index 000000000000..0583c36b3d28 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/OAuth2Exception.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2; + +import java.net.URI; +import java.util.Map; +import java.util.Optional; +import org.immutables.value.Value; + +/** An exception thrown when the OAuth2 authorization server replies with an error. */ +public final class OAuth2Exception extends RuntimeException { + + private final int statusCode; + private final OAuth2Error error; + + public OAuth2Exception(int statusCode, OAuth2Error error) { + this("OAuth2 request failed: " + error.description().orElseGet(error::code), statusCode, error); + } + + public OAuth2Exception(String message, int statusCode, OAuth2Error error) { + super(message); + this.statusCode = statusCode; + this.error = error; + } + + /** The HTTP status code of the response. */ + public int statusCode() { + return statusCode; + } + + /** The error object returned by the server. */ + public OAuth2Error error() { + return error; + } + + /** + * An OAuth2 error returned by the server. + * + * @see RFC 6749 Section + * 5.2 + * @see com.nimbusds.oauth2.sdk.ErrorObject + */ + @Value.Immutable + public interface OAuth2Error { + + /** The OAuth2 error code. */ + String code(); + + /** The OAuth2 error description, if any. */ + Optional description(); + + /** The OAuth2 error URI, if any. */ + Optional uri(); + + /** The custom parameters in the error, or an empty map if not specified. */ + Map parameters(); + } +} diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/OAuth2Manager.java b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/OAuth2Manager.java new file mode 100644 index 000000000000..db8f5710ee35 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/OAuth2Manager.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.catalog.SessionCatalog.SessionContext; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; +import org.apache.iceberg.rest.RESTClient; +import org.apache.iceberg.rest.RESTUtil; +import org.apache.iceberg.rest.auth.AuthManager; +import org.apache.iceberg.rest.auth.AuthSession; +import org.apache.iceberg.rest.auth.oauth2.config.ConfigMigrator; +import org.apache.iceberg.util.ThreadPools; + +public class OAuth2Manager implements AuthManager { + + // For legacy properties migration & sanitization + private final ConfigMigrator configMigrator = new ConfigMigrator(); + private String catalogUri; + + // Main OAuth2 sessions + private OAuth2Session initSession; + private OAuth2Session catalogSession; + + // Runtime dependencies for OAuth2 sessions + private RESTClient restClient; + + // OAuth2 session cache management + private final AtomicReference> cacheById = new AtomicReference<>(); + private final AtomicReference> cacheByConfig = + new AtomicReference<>(); + + public OAuth2Manager(String ignoredManagerName) {} + + @Override + public AuthSession initSession(RESTClient initClient, Map initProperties) { + restClient = initClient; + catalogUri = initProperties.get(CatalogProperties.URI); + OAuth2Config initConfig = configMigrator.migrateCatalogConfig(initProperties, catalogUri); + initSession = new OAuth2Session(initConfig, () -> restClient, ThreadPools.authRefreshPool()); + return initSession; + } + + @Override + public AuthSession catalogSession( + RESTClient sharedClient, Map catalogProperties) { + restClient = sharedClient; + catalogUri = catalogProperties.get(CatalogProperties.URI); + OAuth2Config catalogConfig = configMigrator.migrateCatalogConfig(catalogProperties, catalogUri); + // Copy the existing session if the config is the same as the init session + // to avoid requiring from users to log in again, for human-based flows. + catalogSession = + initSession != null && catalogConfig.equals(initSession.config()) + ? initSession.copy() + : new OAuth2Session(catalogConfig, () -> restClient, ThreadPools.authRefreshPool()); + return catalogSession; + } + + @Override + public AuthSession contextualSession(SessionContext context, AuthSession parent) { + Map contextProperties = + RESTUtil.merge( + Optional.ofNullable(context.properties()).orElseGet(Map::of), + Optional.ofNullable(context.credentials()).orElseGet(Map::of)); + if (contextProperties.isEmpty()) { + return parent; + } + + OAuth2Config parentConfig = ((OAuth2Session) parent).config(); + OAuth2Config childConfig = + configMigrator.migrateContextualConfig(parentConfig, contextProperties, catalogUri); + + if (childConfig.equals(parentConfig)) { + return parent; + } + + return getOrCreateCacheById(childConfig) + .get( + context.sessionId(), + id -> + new OAuth2Session( + (OAuth2Session) parent, + childConfig, + () -> restClient, + ThreadPools.authRefreshPool())); + } + + @Override + public AuthSession tableSession( + TableIdentifier table, Map properties, AuthSession parent) { + if (properties.isEmpty()) { + return parent; + } + + OAuth2Config parentConfig = ((OAuth2Session) parent).config(); + OAuth2Config childConfig = configMigrator.migrateTableConfig(parentConfig, properties); + + if (childConfig.equals(parentConfig)) { + return parent; + } + + return getOrCreateCacheByConfig(childConfig) + .get( + childConfig, + cfg -> + new OAuth2Session( + (OAuth2Session) parent, cfg, () -> restClient, ThreadPools.authRefreshPool())); + } + + @Override + public AuthSession tableSession(RESTClient sharedClient, Map properties) { + OAuth2Config config = + configMigrator.migrateCatalogConfig(properties, properties.get(CatalogProperties.URI)); + return getOrCreateCacheByConfig(config) + .get( + config, + cfg -> new OAuth2Session(cfg, () -> sharedClient, ThreadPools.authRefreshPool())); + } + + @Override + public void close() { + OAuth2Session init = initSession; + OAuth2Session catalog = catalogSession; + try (catalog; + init) { + invalidateCache(cacheById.getAndSet(null)); + invalidateCache(cacheByConfig.getAndSet(null)); + } finally { + initSession = null; + catalogSession = null; + restClient = null; + catalogUri = null; + } + } + + private static void invalidateCache(Cache cache) { + if (cache != null) { + cache.invalidateAll(); + cache.cleanUp(); + } + } + + private Cache getOrCreateCacheById(OAuth2Config config) { + return getOrCreateCache(cacheById, config); + } + + private Cache getOrCreateCacheByConfig(OAuth2Config config) { + return getOrCreateCache(cacheByConfig, config); + } + + private Cache getOrCreateCache( + AtomicReference> ref, OAuth2Config config) { + Cache cache = ref.get(); + if (cache == null) { + ref.compareAndSet(null, newCache(config)); + cache = ref.get(); + } + return cache; + } + + @VisibleForTesting + Cache newCache(OAuth2Config config) { + return Caffeine.newBuilder() + .executor(ThreadPools.authRefreshPool()) + .expireAfterAccess(config.basicConfig().sessionCacheTimeout()) + .removalListener( + (id, auth, cause) -> { + if (auth != null) { + auth.close(); + } + }) + .build(); + } + + @VisibleForTesting + Cache cacheById() { + return cacheById.get(); + } + + @VisibleForTesting + Cache cacheByConfig() { + return cacheByConfig.get(); + } +} diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/OAuth2Runtime.java b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/OAuth2Runtime.java new file mode 100644 index 000000000000..54f487985812 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/OAuth2Runtime.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2; + +import com.nimbusds.oauth2.sdk.http.HTTPRequest; +import com.nimbusds.oauth2.sdk.http.HTTPRequestSender; +import java.time.Clock; +import java.util.Optional; +import java.util.concurrent.ScheduledExecutorService; +import java.util.function.Supplier; +import javax.annotation.Nullable; +import org.apache.iceberg.rest.RESTClient; +import org.apache.iceberg.rest.auth.oauth2.client.OAuth2Client; +import org.apache.iceberg.rest.auth.oauth2.http.RESTClientAdapter; +import org.apache.iceberg.util.ThreadPools; +import org.immutables.value.Value; + +/** + * A runtime context for OAuth2. + * + *

    This component groups together dependencies that are not part of the OAuth2 configuration as + * provided by the user in {@link OAuth2Config}, but rather are provided by the environment. + */ +@Value.Immutable +public interface OAuth2Runtime { + + static OAuth2Runtime of( + Supplier restClientSupplier, + ScheduledExecutorService executor, + @Nullable OAuth2Client parent) { + return ImmutableOAuth2Runtime.builder() + .httpClient(new RESTClientAdapter(restClientSupplier)) + .executor(executor) + .parent(Optional.ofNullable(parent)) + .build(); + } + + /** + * The {@link HTTPRequestSender} to use for network requests. In production, this is generally an + * instance of {@link RESTClientAdapter}. + */ + @Value.Default + default HTTPRequestSender httpClient() { + return request -> { + if (request instanceof HTTPRequest req) { + return req.send(); + } + + throw new IllegalArgumentException( + "Default HTTPRequestSender only supports HTTPRequest instances, but got: " + + request.getClass().getName()); + }; + } + + /** The executor to use for token refresh operations. */ + @Value.Default + default ScheduledExecutorService executor() { + return ThreadPools.authRefreshPool(); + } + + /** + * The parent client, if any. This is used to inherit the parent's token in certain token exchange + * scenarios. + */ + Optional parent(); + + /** + * The clock to use for time-based operations. Defaults to the system clock. Mostly used for + * testing. + */ + @Value.Default + default Clock clock() { + return Clock.systemUTC(); + } +} diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/OAuth2Session.java b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/OAuth2Session.java new file mode 100644 index 000000000000..3f837d6de738 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/OAuth2Session.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2; + +import com.nimbusds.oauth2.sdk.token.AccessToken; +import java.util.concurrent.ScheduledExecutorService; +import java.util.function.Supplier; +import org.apache.iceberg.rest.HTTPHeaders; +import org.apache.iceberg.rest.HTTPHeaders.HTTPHeader; +import org.apache.iceberg.rest.HTTPRequest; +import org.apache.iceberg.rest.ImmutableHTTPRequest; +import org.apache.iceberg.rest.RESTClient; +import org.apache.iceberg.rest.auth.AuthSession; +import org.apache.iceberg.rest.auth.oauth2.client.OAuth2Client; + +public class OAuth2Session implements AuthSession { + + private final OAuth2Config config; + private final OAuth2Client client; + + public OAuth2Session( + OAuth2Config config, + Supplier restClientSupplier, + ScheduledExecutorService executor) { + this.config = config; + this.client = new OAuth2Client(config, OAuth2Runtime.of(restClientSupplier, executor, null)); + } + + public OAuth2Session( + OAuth2Session parent, + OAuth2Config config, + Supplier restClientSupplier, + ScheduledExecutorService executor) { + this.config = config; + this.client = + new OAuth2Client(config, OAuth2Runtime.of(restClientSupplier, executor, parent.client)); + } + + private OAuth2Session(OAuth2Session toCopy) { + this.config = toCopy.config; + this.client = toCopy.client.copy(); + } + + public OAuth2Config config() { + return config; + } + + /** + * Copies this session and the underlying client. This is only needed when reusing an init session + * as a catalog session. + */ + public OAuth2Session copy() { + return new OAuth2Session(this); + } + + @Override + public HTTPRequest authenticate(HTTPRequest request) { + AccessToken accessToken = client.authenticate(); + HTTPHeader authorization = HTTPHeader.of("Authorization", "Bearer " + accessToken.getValue()); + HTTPHeaders newHeaders = request.headers().putIfAbsent(HTTPHeaders.of(authorization)); + return newHeaders.equals(request.headers()) + ? request + : ImmutableHTTPRequest.builder().from(request).headers(newHeaders).build(); + } + + @Override + public void close() { + client.close(); + } +} diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/client/OAuth2Client.java b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/client/OAuth2Client.java new file mode 100644 index 000000000000..40f6e46cd78d --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/client/OAuth2Client.java @@ -0,0 +1,423 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.client; + +import com.nimbusds.oauth2.sdk.GrantType; +import com.nimbusds.oauth2.sdk.token.AccessToken; +import com.nimbusds.oauth2.sdk.token.Tokens; +import java.io.Closeable; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; +import javax.annotation.Nullable; +import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; +import org.apache.iceberg.relocated.com.google.common.base.Throwables; +import org.apache.iceberg.rest.auth.oauth2.OAuth2Config; +import org.apache.iceberg.rest.auth.oauth2.OAuth2Exception; +import org.apache.iceberg.rest.auth.oauth2.OAuth2Runtime; +import org.apache.iceberg.rest.auth.oauth2.flow.Flow; +import org.apache.iceberg.rest.auth.oauth2.flow.FlowFactory; +import org.apache.iceberg.rest.auth.oauth2.flow.TokensResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * An OAuth2 client is responsible for fetching and refreshing tokens, following the configuration + * provided by an {@link OAuth2Config} object. + */ +public final class OAuth2Client implements Closeable { + + private static final Logger LOGGER = LoggerFactory.getLogger(OAuth2Client.class); + + private static final Duration MIN_REFRESH_DELAY = Duration.ofSeconds(1); + + // Internal state + private final OAuth2Config config; + private final ScheduledExecutorService executor; + private final FlowFactory flowFactory; + private final Clock clock; + + // Lifecycle + private final AtomicBoolean closed = new AtomicBoolean(); + + // Token management & refresh scheduling + private volatile CompletableFuture currentTokensFuture; + private volatile ScheduledFuture tokenRefreshFuture; + + public OAuth2Client(OAuth2Config cfg, OAuth2Runtime runtime) { + config = cfg; + executor = runtime.executor(); + clock = runtime.clock(); + flowFactory = FlowFactory.of(cfg, runtime); + config + .basicConfig() + .token() + .ifPresentOrElse(this::initWithStaticToken, this::initWithDynamicToken); + } + + /** Copy constructor. Only accessible from the {@link #copy()} method. */ + private OAuth2Client(OAuth2Client toCopy) { + LOGGER.debug("Copying client"); + config = toCopy.config; + executor = toCopy.executor; + flowFactory = toCopy.flowFactory; + clock = toCopy.clock; + tokenRefreshFuture = null; + TokensResult currentTokens = getNow(toCopy.currentTokensFuture); + currentTokensFuture = + (currentTokens != null + ? CompletableFuture.completedFuture(currentTokens) + : CompletableFuture.supplyAsync(this::fetchNewTokens, executor) + .thenCompose(Function.identity())) + .whenComplete((tokens, error) -> scheduleNextRenewal(tokens)); + } + + /** + * Authenticates the client synchronously, waiting for the authentication to complete, and returns + * the current access token. If the authentication fails, or if the client is closed, an exception + * is thrown. + */ + public AccessToken authenticate() { + return authenticateInternal().tokens().getAccessToken(); + } + + /** + * Authenticates the client asynchronously and returns a future that completes when the + * authentication completes (either successfully or with an error). + */ + public CompletionStage authenticateAsync() { + return authenticateAsyncInternal() + .thenApply(TokensResult::tokens) + .thenApply(Tokens::getAccessToken); + } + + /** + * Creates a copy of this client. The copy will share the same config, executor and flow factory + * as the original client, as well as its current tokens, if any. If token refresh is enabled, the + * copy will create its own token refresh schedule. + */ + public OAuth2Client copy() { + return new OAuth2Client(this); + } + + /** Closes this client, releasing any resources held by it. This method is idempotent. */ + @Override + public void close() { + if (closed.compareAndSet(false, true)) { + try { + LOGGER.debug("Closing..."); + ScheduledFuture refreshFuture = tokenRefreshFuture; + if (refreshFuture != null) { + refreshFuture.cancel(true); + } + CompletableFuture tokensFuture = currentTokensFuture; + if (tokensFuture != null) { + tokensFuture.cancel(true); + } + } finally { + tokenRefreshFuture = null; + // Don't clear currentTokensFuture, we'll need it in case this client is copied. + LOGGER.debug("Closed"); + } + } + } + + /** + * Same as {@link #authenticate()} but returns the full {@link TokensResult} object, including the + * refresh token, if any. Only intended for testing. + */ + @VisibleForTesting + TokensResult authenticateInternal() { + LOGGER.debug("Authenticating synchronously"); + return currentTokens(); + } + + /** + * Same as {@link #authenticateAsync()} but returns the full {@link TokensResult} object, + * including the refresh token, if any. Only intended for testing. + */ + @VisibleForTesting + CompletionStage authenticateAsyncInternal() { + LOGGER.debug("Authenticating asynchronously"); + return currentTokensFuture; + } + + /** Returns the current tokens, waiting for them to be available if necessary. */ + private TokensResult currentTokens() { + try { + return currentTokensFuture.get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof TimeoutException) { + throw new RuntimeException("Token acquisition timed out", cause); + } else if (cause instanceof Error) { + throw (Error) cause; + } else if (cause instanceof RuntimeException) { + throw (RuntimeException) cause; + } else { + throw new RuntimeException("Token acquisition failed", cause); + } + } + } + + /** + * Initializes the client with a static initial access token. In this situation, a static access + * token has been provided in the configuration, and it will be used as-is as the first initial + * token. + */ + private void initWithStaticToken(AccessToken token) { + TokensResult currentTokens = TokensResult.of(token, clock); + currentTokensFuture = CompletableFuture.completedFuture(currentTokens); + if (refreshWithTokenExchange()) { + // A static token can only be refreshed if using the token exchange grant. + scheduleNextRenewal(currentTokens); + } + } + + /** Initializes the client with a dynamically-obtained initial access token. */ + private void initWithDynamicToken() { + currentTokensFuture = + CompletableFuture.supplyAsync(this::fetchNewTokens, executor) + .thenCompose(Function.identity()) + .whenComplete(this::logRenewal) + .whenComplete((tokens, error) -> scheduleNextRenewal(tokens)); + } + + /** + * Executes a token renewal operation, either by refreshing the current access token (if any) or + * by fetching a new access token. + * + *

    This method rotates the current tokens future; the new future is completed when the renewal + * operation completes and the next one is scheduled. + * + *

    This method is always executed as a scheduled task. + */ + @VisibleForTesting + void renewTokens() { + if (!closed.get()) { + CompletableFuture oldTokensFuture = currentTokensFuture; + TokensResult oldTokens = getNow(oldTokensFuture); + CompletableFuture newTokensFuture = + oldTokensFuture + // 1) try refreshing the current access token, if any, and if possible + .thenCompose(this::refreshCurrentTokens) + // 2) if that fails, try fetching brand-new tokens using the configured initial grant + .handle(this::handleRefreshResult) + .thenCompose(Function.identity()) + // 3) Log the result of the token renewal attempt + .whenComplete(this::logRenewal) + // 4) if the renewal attempt failed, keep the old tokens if available + .handle((newTokens, error) -> handleRenewalResult(oldTokens, newTokens, error)) + .thenCompose(Function.identity()) + // 5) schedule the next token renewal + .whenComplete((tokens, error) -> scheduleNextRenewal(tokens)); + currentTokensFuture = newTokensFuture; + if (closed.get()) { + // We raced with close(): cancel the future we just created. + newTokensFuture.cancel(true); + } + } + } + + /** Fetches new tokens using the configured initial grant. */ + @VisibleForTesting + CompletionStage fetchNewTokens() { + Flow flow = flowFactory.newInitialFlow(); + LOGGER.debug("Fetching new access token using {}", flow.grantType()); + Duration timeout = config.basicConfig().tokenAcquisitionTimeout(); + return flow.execute() + .toCompletableFuture() + .orTimeout(timeout.toMillis(), TimeUnit.MILLISECONDS); + } + + /** Refreshes the current tokens using the configured refresh grant. */ + @VisibleForTesting + CompletionStage refreshCurrentTokens(TokensResult currentTokens) { + if (currentTokens.tokens().getRefreshToken() == null) { + if (!refreshWithTokenExchange()) { + LOGGER.debug("Must fetch new tokens, refresh token is null"); + return MUST_FETCH_NEW_TOKENS_FUTURE; + } + } + + Flow flow = flowFactory.newRefreshFlow(currentTokens.tokens()); + LOGGER.debug("Refreshing tokens using {}", flow.grantType()); + Duration timeout = config.basicConfig().tokenAcquisitionTimeout(); + return flow.execute() + .toCompletableFuture() + .orTimeout(timeout.toMillis(), TimeUnit.MILLISECONDS); + } + + /** + * Whether to use the token exchange grant to refresh tokens. If unspecified in the config, the + * default is to use token exchange only if the initial grant is {@link + * GrantType#CLIENT_CREDENTIALS}, in order to maintain backward compatibility. + */ + private boolean refreshWithTokenExchange() { + return config + .tokenRefreshConfig() + .tokenExchangeEnabled() + .orElseGet(() -> config.basicConfig().grantType().equals(GrantType.CLIENT_CREDENTIALS)); + } + + /** + * Handles the result of a token refresh attempt. If the refresh wasn't successful, or if the + * refreshed access token lifespan is too short, the refreshed token is discarded and a new token + * is fetched. + * + *

    Refreshed access token lifespans can sometimes be too short to be usable, because their + * lifetime is bound by the refresh token's lifetime, and hence, by the user session expiration + * time. In such cases, it's better to fetch a new token (and thus, ask the user to + * re-authenticate) rather than keep using a too short-lived token. + */ + private CompletionStage handleRefreshResult( + @Nullable TokensResult newTokens, @Nullable Throwable error) { + if (newTokens != null) { + Instant now = clock.instant(); + Instant expirationTime = expirationTime(newTokens); + Duration safetyMargin = config.tokenRefreshConfig().safetyMargin(); + if (expirationTime.minus(safetyMargin).isAfter(now)) { + return CompletableFuture.completedFuture(newTokens); + } else { + LOGGER.debug("Refreshed access token is too short: fetching new tokens instead"); + } + } else if (error != null) { + Throwable root = Throwables.getRootCause(error); + if (!(root instanceof MustFetchNewTokensException)) { + LOGGER.debug("Refresh failed unexpectedly, fetching new tokens instead", error); + } + } + + return fetchNewTokens(); + } + + /** + * Handles the result of a token renewal attempt. If the renewal wasn't successful, the old tokens + * are kept if available; otherwise, the error is propagated. + */ + private CompletionStage handleRenewalResult( + @Nullable TokensResult oldTokens, + @Nullable TokensResult newTokens, + @Nullable Throwable error) { + return error == null + ? CompletableFuture.completedFuture(newTokens) + : oldTokens != null + ? CompletableFuture.completedFuture(oldTokens) + : CompletableFuture.failedFuture(error); + } + + private void scheduleNextRenewal(@Nullable TokensResult currentTokens) { + if (config.tokenRefreshConfig().enabled() && !closed.get()) { + Instant now = clock.instant(); + Duration delay = nextRenewal(currentTokens, now); + LOGGER.debug("Scheduling token renewal in {}", delay); + ScheduledFuture refreshFuture = + executor.schedule(this::renewTokens, delay.toMillis(), TimeUnit.MILLISECONDS); + this.tokenRefreshFuture = refreshFuture; + if (closed.get()) { + // We raced with close(): cancel the future we just created and clear the field. + refreshFuture.cancel(true); + this.tokenRefreshFuture = null; + } + } + } + + /** + * Determines when the next token renewal should be scheduled, based on the current tokens and the + * current time. + */ + private Duration nextRenewal(@Nullable TokensResult currentTokens, Instant now) { + if (currentTokens == null) { + return MIN_REFRESH_DELAY; + } + + Instant expirationTime = expirationTime(currentTokens); + Duration safetyMargin = config.tokenRefreshConfig().safetyMargin(); + Duration delay = Duration.between(now, expirationTime).minus(safetyMargin); + if (delay.compareTo(MIN_REFRESH_DELAY) < 0) { + LOGGER.debug("Next renewal delay was too short: {}", delay); + delay = MIN_REFRESH_DELAY; + } + + return delay; + } + + /** + * Returns the access token expiration time, based on {@link + * TokensResult#accessTokenExpirationTime()} if available, or by adding the default access token + * lifespan to {@link TokensResult#receivedAt()} otherwise. + */ + private Instant expirationTime(TokensResult currentTokens) { + return currentTokens + .accessTokenExpirationTime() + .orElseGet( + () -> + currentTokens.receivedAt().plus(config.tokenRefreshConfig().accessTokenLifespan())); + } + + private void logRenewal(@Nullable TokensResult newTokens, @Nullable Throwable error) { + if (!closed.get()) { + if (newTokens != null) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug( + "Successfully renewed tokens. Access token expiration time: {}", + newTokens.accessTokenExpirationTime().orElse(null)); + } + } else if (error != null) { + Throwable root = Throwables.getRootCause(error); + if (root instanceof OAuth2Exception) { + // Don't include the stack trace if the error is an OAuth2Exception, + // since it's not very useful and just clutters the logs. + LOGGER.warn("Failed to renew tokens: {}", root.toString()); + } else { + LOGGER.warn("Failed to renew tokens", root); + } + } + } + } + + @Nullable + private static T getNow(@Nullable CompletableFuture future) { + return future != null && future.isDone() && !future.isCompletedExceptionally() + ? future.getNow(null) + : null; + } + + /** + * Internal exception used solely to signal that a new token must be fetched. This exception is + * not propagated to the user and is only used to short-circuit the token refresh logic. + */ + @VisibleForTesting + static class MustFetchNewTokensException extends RuntimeException {} + + private static final CompletableFuture MUST_FETCH_NEW_TOKENS_FUTURE = + CompletableFuture.failedFuture(new MustFetchNewTokensException()); +} diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/config/BasicConfig.java b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/config/BasicConfig.java new file mode 100644 index 000000000000..b6f52668a451 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/config/BasicConfig.java @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.config; + +import com.nimbusds.oauth2.sdk.GrantType; +import com.nimbusds.oauth2.sdk.Scope; +import com.nimbusds.oauth2.sdk.auth.ClientAuthenticationMethod; +import com.nimbusds.oauth2.sdk.auth.Secret; +import com.nimbusds.oauth2.sdk.id.ClientID; +import com.nimbusds.oauth2.sdk.token.AccessToken; +import com.nimbusds.oauth2.sdk.token.BearerAccessToken; +import java.net.URI; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import org.apache.iceberg.rest.RESTUtil; +import org.apache.iceberg.rest.auth.oauth2.OAuth2Config; +import org.immutables.value.Value; + +/** + * Basic OAuth2 properties. These properties are used to configure the basic OAuth2 options such as + * the issuer URL, token endpoint, client ID, and client secret. + */ +@Value.Immutable +@Value.Style(redactedMask = "****") +@SuppressWarnings({"ImmutablesStyle", "SafeLoggingPropagation"}) +public interface BasicConfig { + + String PREFIX = OAuth2Config.PREFIX; + + String TOKEN = PREFIX + "token"; + String ISSUER_URL = PREFIX + "issuer-url"; + String TOKEN_ENDPOINT = PREFIX + "token-endpoint"; + String GRANT_TYPE = PREFIX + "grant-type"; + String CLIENT_ID = PREFIX + "client-id"; + String CLIENT_AUTH = PREFIX + "client-auth"; + String CLIENT_SECRET = PREFIX + "client-secret"; + String SCOPE = PREFIX + "scope"; + String EXTRA_PARAMS = PREFIX + "extra-params"; + String TIMEOUT = PREFIX + "timeout"; + String SESSION_CACHE_TIMEOUT = PREFIX + "session-cache.timeout"; + + GrantType DEFAULT_GRANT_TYPE = GrantType.CLIENT_CREDENTIALS; + ClientAuthenticationMethod DEFAULT_CLIENT_AUTH = ClientAuthenticationMethod.CLIENT_SECRET_BASIC; + Duration DEFAULT_TIMEOUT = Duration.parse("PT5M"); + Duration DEFAULT_SESSION_CACHE_TIMEOUT = Duration.parse("PT1H"); + + Duration MIN_TIMEOUT = Duration.parse("PT30S"); + + /** + * The initial access token to use. Optional. If this is set, the OAuth2 client will not attempt + * to fetch an initial token from the Authorization server, but will use this token instead. + * + *

    This option should be avoided as in most cases, the token cannot be refreshed. + */ + @ConfigOption(TOKEN) + @Value.Redacted + Optional token(); + + /** + * The root URL of the Authorization server, which will be used for discovering supported + * endpoints and their locations. For Keycloak, this is typically the realm URL: {@code + * https:///realms/}. + * + *

    Two "well-known" paths are supported for endpoint discovery: {@code + * .well-known/openid-configuration} and {@code .well-known/oauth-authorization-server}. The full + * metadata discovery URL will be constructed by appending these paths to the issuer URL. + * + *

    Unless a {@linkplain #TOKEN static token} is provided, either this property or {@link + * #TOKEN_ENDPOINT} must be set. + * + * @see OpenID + * Connect Discovery 1.0 + * @see RFC 8414 Section 5 + */ + @ConfigOption(ISSUER_URL) + Optional issuerUrl(); + + /** + * URL of the OAuth2 token endpoint. For Keycloak, this is typically {@code + * https:///realms//protocol/openid-connect/token}. + * + *

    Unless a {@linkplain #TOKEN static token} is provided, either this property or {@link + * #ISSUER_URL} must be set. In case it is not set, the token endpoint will be discovered from the + * {@link #ISSUER_URL issuer URL}, using the OpenID Connect Discovery metadata published by the + * issuer. + */ + @ConfigOption(TOKEN_ENDPOINT) + Optional tokenEndpoint(); + + /** + * The grant type to use when authenticating against the OAuth2 server. Valid values are: + * + *

      + *
    • {@link GrantType#CLIENT_CREDENTIALS client_credentials} + *
    • {@link GrantType#TOKEN_EXCHANGE urn:ietf:params:oauth:grant-type:token-exchange} + *
    + * + * Optional, defaults to {@link #DEFAULT_GRANT_TYPE}. + */ + @ConfigOption(GRANT_TYPE) + @Value.Default + default GrantType grantType() { + return DEFAULT_GRANT_TYPE; + } + + /** + * Client ID to use when authenticating against the OAuth2 server. Required, unless a {@linkplain + * #TOKEN static token} is provided. + */ + @ConfigOption(CLIENT_ID) + Optional clientId(); + + /** + * The OAuth2 client authentication method to use. Valid values are: + * + *
      + *
    • {@link ClientAuthenticationMethod#NONE none}: the client does not authenticate itself at + * the token endpoint, because it is a public client with no client secret or other + * authentication mechanism. + *
    • {@link ClientAuthenticationMethod#CLIENT_SECRET_BASIC client_secret_basic}: client secret + * is sent in the HTTP Basic Authorization header. + *
    • {@link ClientAuthenticationMethod#CLIENT_SECRET_POST client_secret_post}: client secret + * is sent in the request body as a form parameter. + *
    + * + * The default is {@link #DEFAULT_CLIENT_AUTH}. + */ + @ConfigOption(CLIENT_AUTH) + @Value.Default + default ClientAuthenticationMethod clientAuthenticationMethod() { + return DEFAULT_CLIENT_AUTH; + } + + /** + * Client secret to use when authenticating against the OAuth2 server. Required if the client is + * private and is authenticated using the standard "client-secret" methods. + */ + @ConfigOption(CLIENT_SECRET) + @Value.Redacted + Optional clientSecret(); + + /** + * Space-separated list of scopes to include in each request to the OAuth2 server. Optional, + * defaults to empty (no scopes). + * + *

    The scope names will not be validated by the OAuth2 client; make sure they are valid + * according to RFC 6749 + * Section 3.3. + */ + @ConfigOption(SCOPE) + Optional scope(); + + /** + * Extra parameters to include in each request to the token endpoint. This is useful for custom + * parameters that are not covered by the standard OAuth2 specification. Optional, defaults to + * empty. + * + *

    This is a prefix property, and multiple values can be set, each with a different key and + * value. The values must NOT be URL-encoded. Example: + * + * 

    {@code
+   * rest.auth.oauth2.extra-params.custom_param1=custom_value1
+   * rest.auth.oauth2.extra-params.custom_param2=custom_value2
+   * }
    + * + * For example, Auth0 requires the {@code audience} parameter to be set to the API identifier. + * This can be done by setting the following configuration: + * + * 
    {@code
+   * rest.auth.oauth2.extra-params.audience=https://iceberg-rest-catalog/api
+   * }
    + */ + @ConfigOption(value = EXTRA_PARAMS, prefixMap = true) + Map extraRequestParameters(); + + /** + * The token acquisition timeout. Optional, defaults to {@link #DEFAULT_TIMEOUT}. The default + * timeout is intentionally large, in order to accommodate for long-running flows that require + * human intervention (e.g. Authorization Code flow). + * + *

    Must be a valid ISO-8601 + * duration. + */ + @ConfigOption(TIMEOUT) + @Value.Default + default Duration tokenAcquisitionTimeout() { + return DEFAULT_TIMEOUT; + } + + /** + * The session cache timeout. Cached sessions will become eligible for eviction after this + * duration of inactivity. Defaults to 1 hour. Must be a valid ISO-8601 duration. + * + *

    This value is used for housekeeping; it does not mean that cached sessions will stop working + * after this time, but that the session cache will evict the session after this time of + * inactivity. If the context is used again, a new session will be created and cached. + * + *

    This property can only be specified at catalog session level. It is ignored if present in + * other levels. + */ + @ConfigOption(SESSION_CACHE_TIMEOUT) + @Value.Default + default Duration sessionCacheTimeout() { + return DEFAULT_SESSION_CACHE_TIMEOUT; + } + + /** + * The minimum timeout for token acquisition. + * + *

    This option is not exposed as a public configuration property, and is intended for testing + * purposes only. + */ + @Value.Default + default Duration minTokenAcquisitionTimeout() { + return MIN_TIMEOUT; + } + + @Value.Check + default void validate() { + ConfigValidator validator = new ConfigValidator(); + + if (token().isEmpty()) { + + validator.check( + ConfigUtil.SUPPORTED_INITIAL_GRANT_TYPES.contains(grantType()), + GRANT_TYPE, + "grant type must be one of: %s", + ConfigUtil.SUPPORTED_INITIAL_GRANT_TYPES.stream() + .map(GrantType::getValue) + .collect(Collectors.joining("', '", "'", "'"))); + + validator.check( + ConfigUtil.SUPPORTED_CLIENT_AUTH_METHODS.contains(clientAuthenticationMethod()), + CLIENT_AUTH, + "client authentication method must be one of: %s", + ConfigUtil.SUPPORTED_CLIENT_AUTH_METHODS.stream() + .map(ClientAuthenticationMethod::getValue) + .collect(Collectors.joining("', '", "'", "'"))); + + validator.check( + issuerUrl().isPresent() || tokenEndpoint().isPresent(), + List.of(ISSUER_URL, TOKEN_ENDPOINT), + "either issuer URL or token endpoint must be set"); + + validator.check(clientId().isPresent(), CLIENT_ID, "client ID must not be empty"); + + if (ConfigUtil.requiresClientSecret(clientAuthenticationMethod())) { + validator.check( + clientSecret().isPresent(), + List.of(CLIENT_AUTH, CLIENT_SECRET), + "client secret must not be empty when client authentication is '%s'", + clientAuthenticationMethod().getValue()); + } else if (clientAuthenticationMethod().equals(ClientAuthenticationMethod.NONE)) { + validator.check( + clientSecret().isEmpty(), + List.of(CLIENT_AUTH, CLIENT_SECRET), + "client secret must not be set when client authentication is '%s'", + ClientAuthenticationMethod.NONE.getValue()); + validator.check( + !grantType().equals(DEFAULT_GRANT_TYPE), + List.of(CLIENT_AUTH, GRANT_TYPE), + "grant type must not be '%s' when client authentication is '%s'", + DEFAULT_GRANT_TYPE.getValue(), + ClientAuthenticationMethod.NONE.getValue()); + } + } + + if (issuerUrl().isPresent()) { + validator.checkEndpoint(issuerUrl().get(), ISSUER_URL, "Issuer URL"); + } + + if (tokenEndpoint().isPresent()) { + validator.checkEndpoint(tokenEndpoint().get(), TOKEN_ENDPOINT, "Token endpoint"); + } + + validator.check( + tokenAcquisitionTimeout().compareTo(minTokenAcquisitionTimeout()) >= 0, + TIMEOUT, + "timeout must be greater than or equal to %s", + minTokenAcquisitionTimeout()); + + validator.validate(); + } + + static ImmutableBasicConfig.Builder parse(Map properties) { + List scopes = ConfigUtil.parseList(properties, SCOPE, " "); + return ImmutableBasicConfig.builder() + .token(ConfigUtil.parseOptional(properties, TOKEN, BearerAccessToken::new)) + .issuerUrl(ConfigUtil.parseOptional(properties, ISSUER_URL, URI::create)) + .tokenEndpoint(ConfigUtil.parseOptional(properties, TOKEN_ENDPOINT, URI::create)) + .grantType( + ConfigUtil.parseOptional(properties, GRANT_TYPE, GrantType::parse) + .orElse(DEFAULT_GRANT_TYPE)) + .clientAuthenticationMethod( + ConfigUtil.parseOptional(properties, CLIENT_AUTH, ClientAuthenticationMethod::parse) + .orElse(DEFAULT_CLIENT_AUTH)) + .clientId(ConfigUtil.parseOptional(properties, CLIENT_ID, ClientID::new)) + .clientSecret(ConfigUtil.parseOptional(properties, CLIENT_SECRET, Secret::new)) + .scope( + scopes.isEmpty() + ? Optional.empty() + : Optional.of(new Scope(scopes.toArray(String[]::new)))) + .extraRequestParameters(RESTUtil.extractPrefixMap(properties, EXTRA_PARAMS + '.')) + .tokenAcquisitionTimeout( + ConfigUtil.parseOptional(properties, TIMEOUT, Duration::parse).orElse(DEFAULT_TIMEOUT)) + .sessionCacheTimeout( + ConfigUtil.parseOptional(properties, SESSION_CACHE_TIMEOUT, Duration::parse) + .orElse(DEFAULT_SESSION_CACHE_TIMEOUT)); + } +} diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/config/ConfigMigrator.java b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/config/ConfigMigrator.java new file mode 100644 index 000000000000..a3e0fe4134db --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/config/ConfigMigrator.java @@ -0,0 +1,461 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.config; + +import com.nimbusds.oauth2.sdk.GrantType; +import com.nimbusds.oauth2.sdk.id.Audience; +import com.nimbusds.oauth2.sdk.id.ClientID; +import com.nimbusds.oauth2.sdk.token.TokenTypeURI; +import com.nimbusds.oauth2.sdk.token.TypelessAccessToken; +import java.net.URI; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.stream.Collectors; +import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.base.Splitter; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.rest.RESTUtil; +import org.apache.iceberg.rest.ResourcePaths; +import org.apache.iceberg.rest.auth.OAuth2Properties; +import org.apache.iceberg.rest.auth.oauth2.ImmutableOAuth2Config; +import org.apache.iceberg.rest.auth.oauth2.OAuth2Config; +import org.apache.iceberg.util.PropertyUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A component for migrating from legacy OAuth2 properties (from {@link OAuth2Properties}) to the + * new OAuth2 properties (as declared in {@link OAuth2Config}). + */ +@SuppressWarnings("deprecation") +public final class ConfigMigrator { + + /** + * The default client ID to use when no client ID is provided in the legacy {@link + * OAuth2Properties#CREDENTIAL} property. + */ + public static final ClientID DEFAULT_CLIENT_ID = new ClientID("iceberg"); + + private static final Logger LOGGER = LoggerFactory.getLogger(ConfigMigrator.class); + + private static final Splitter CREDENTIAL_SPLITTER = Splitter.on(":").limit(2).trimResults(); + + private static final Set TABLE_CONFIG_ALLOW_LIST = + Set.of( + BasicConfig.TOKEN, + TokenExchangeConfig.SUBJECT_TOKEN, + TokenExchangeConfig.SUBJECT_TOKEN_TYPE, + TokenExchangeConfig.ACTOR_TOKEN, + TokenExchangeConfig.ACTOR_TOKEN_TYPE); + + @VisibleForTesting + static final String MESSAGE_TEMPLATE_LEGACY_OPTION = + "Detected legacy OAuth2 property '{}', please use option{} {} instead."; + + @VisibleForTesting + static final String MESSAGE_TEMPLATE_NO_CLIENT_ID = + "The legacy OAuth2 property 'credential' was provided, but it did not contain a client ID; assuming '{}'."; + + @VisibleForTesting + static final String MESSAGE_TEMPLATE_MISSING_TOKEN_ENDPOINT = + "The OAuth2 configuration does not specify a token endpoint nor an issuer URL: " + + "the token endpoint URL will default to {}. " + + "This automatic fallback will be removed in a future Iceberg release. " + + "Please configure OAuth2 endpoints using the following properties: '{}' or '{}'. " + + "This warning will disappear if OAuth2 endpoints are properly configured. " + + "See https://github.com/apache/iceberg/issues/10537"; + + @VisibleForTesting + static final String MESSAGE_TEMPLATE_RELATIVE_TOKEN_ENDPOINT = + "The OAuth2 token endpoint URL is a relative URL. " + + "It will be resolved against the catalog URI, resulting in the absolute URL: '{}'. " + + "This automatic fallback will be removed in a future Iceberg release. " + + "Please configure OAuth2 endpoints using absolute URLs."; + + @VisibleForTesting + static final String MESSAGE_TEMPLATE_MERGED_CONTEXTUAL_CONFIG = + "The OAuth2 configuration property '{}' was not found in the context session, " + + "and will be inherited from the parent session. " + + "This automatic fallback will be removed in a future Iceberg release."; + + @VisibleForTesting + static final String MESSAGE_TEMPLATE_VENDED_TOKEN = + "The OAuth2 configuration property '{}' was found in the table configuration " + + "and indicates that the catalog server vended an OAuth2 token. " + + "Vended OAuth2 tokens will be disallowed in a future Iceberg release."; + + @VisibleForTesting + static final String MESSAGE_TEMPLATE_TABLE_CONFIG_NOT_ALLOWED = + "The OAuth2 configuration property '{}' is not allowed to be vended by catalog servers."; + + private final BiConsumer logConsumer; + + public ConfigMigrator() { + this(LOGGER); + } + + public ConfigMigrator(Logger logger) { + this(logger::warn); + } + + @VisibleForTesting + ConfigMigrator(BiConsumer logConsumer) { + this.logConsumer = logConsumer; + } + + /** + * Migrates catalog-level properties. + * + *

    Legacy Iceberg OAuth2 properties are migrated, and warnings are logged for each detected + * legacy property. + * + *

    The migration will further check the token endpoint for the following legacy situations: + * + *

      + *
    1. If no token endpoint is provided, a default one will be added to the migrated properties; + * its value is the catalog URI + {@link ResourcePaths#tokens()} – that is, it will point to + * the (deprecated) REST Catalog token endpoint. + *
    2. If a token endpoint is provided, but is a relative path, it will be resolved against the + * catalog URI. + *
    + * + * In both cases, a warning will be logged. + * + * @param properties The properties to migrate + * @param catalogUri The catalog URI, for extended token endpoint checks + */ + public OAuth2Config migrateCatalogConfig(Map properties, String catalogUri) { + Map migrated = migrateProperties(properties); + handleTokenEndpoint(migrated, catalogUri); + return OAuth2Config.of(migrated); + } + + /** + * Migrates session context properties. + * + *

    Legacy Iceberg OAuth2 properties are migrated, and warnings are logged for each detected + * legacy property. + * + *

    See {@link #migrateCatalogConfig(Map, String)} for details on token endpoint checks. + * + *

    Contextual configs inherit some properties from their parent config. This is legacy + * behavior, and will be removed after 1.13; a warning will be logged when this happens. + * + * @param parent The parent config + * @param properties The properties to migrate + * @param catalogUri The catalog URI, for extended token endpoint checks + */ + @SuppressWarnings("CyclomaticComplexity") + public OAuth2Config migrateContextualConfig( + OAuth2Config parent, Map properties, String catalogUri) { + Map migrated = migrateProperties(properties); + + if (!migrated.containsKey(BasicConfig.CLIENT_ID) + && parent.basicConfig().clientId().isPresent()) { + warnOnMergedContextualConfig(BasicConfig.CLIENT_ID); + migrated.put(BasicConfig.CLIENT_ID, parent.basicConfig().clientId().get().getValue()); + } + + if (!migrated.containsKey(BasicConfig.CLIENT_SECRET) + && !migrated.containsKey(BasicConfig.CLIENT_AUTH) + && parent.basicConfig().clientSecret().isPresent()) { + warnOnMergedContextualConfig(BasicConfig.CLIENT_SECRET); + migrated.put(BasicConfig.CLIENT_SECRET, parent.basicConfig().clientSecret().get().getValue()); + } + + if (!migrated.containsKey(BasicConfig.TOKEN_ENDPOINT) + && !migrated.containsKey(BasicConfig.ISSUER_URL) + && parent.basicConfig().tokenEndpoint().isPresent()) { + warnOnMergedContextualConfig(BasicConfig.TOKEN_ENDPOINT); + migrated.put( + BasicConfig.TOKEN_ENDPOINT, parent.basicConfig().tokenEndpoint().get().toString()); + } + + if (!migrated.containsKey(BasicConfig.SCOPE) && parent.basicConfig().scope().isPresent()) { + warnOnMergedContextualConfig(BasicConfig.SCOPE); + migrated.put(BasicConfig.SCOPE, parent.basicConfig().scope().get().toString()); + } + + if (!migrated.containsKey(TokenExchangeConfig.RESOURCES) + && !parent.tokenExchangeConfig().resources().isEmpty()) { + warnOnMergedContextualConfig(TokenExchangeConfig.RESOURCES); + migrated.put( + TokenExchangeConfig.RESOURCES, + parent.tokenExchangeConfig().resources().stream() + .map(URI::toString) + .collect(Collectors.joining(","))); + } + + if (!migrated.containsKey(TokenExchangeConfig.AUDIENCES) + && !parent.tokenExchangeConfig().audiences().isEmpty()) { + warnOnMergedContextualConfig(TokenExchangeConfig.AUDIENCES); + migrated.put( + TokenExchangeConfig.AUDIENCES, + parent.tokenExchangeConfig().audiences().stream() + .map(Audience::getValue) + .collect(Collectors.joining(","))); + } + + if (migrated.isEmpty()) { + return parent; + } + + handleTokenEndpoint(migrated, catalogUri); + + return OAuth2Config.of(migrated); + } + + /** + * Migrates table properties. + * + *

    Legacy Iceberg OAuth2 properties are migrated, and warnings are logged for each detected + * legacy property. + * + *

    Table configs are not allowed to contain any property that is not in the allow list; a + * warning will be logged for each detected disallowed property. + * + *

    Moreover, table configs are considered deprecated, and a warning will be logged on any + * OAuth2 property found, even if it is allowed. + * + * @param parent The parent config + * @param properties The properties to migrate + */ + public OAuth2Config migrateTableConfig(OAuth2Config parent, Map properties) { + + Map migrated = migrateProperties(properties); + Map filtered = Maps.newHashMap(); + + for (Entry entry : migrated.entrySet()) { + if (TABLE_CONFIG_ALLOW_LIST.contains(entry.getKey())) { + warnOnVendedToken(entry.getKey()); + filtered.put(entry.getKey(), entry.getValue()); + } else { + warnOnForbiddenTableConfig(entry.getKey()); + } + } + + ImmutableBasicConfig.Builder basicBuilder = + ImmutableBasicConfig.builder().from(parent.basicConfig()); + ImmutableTokenExchangeConfig.Builder tokenExchangeBuilder = + ImmutableTokenExchangeConfig.builder().from(parent.tokenExchangeConfig()); + + if (filtered.containsKey(BasicConfig.TOKEN)) { + // static vended token use case + basicBuilder.token( + ConfigUtil.parseOptional(filtered, BasicConfig.TOKEN, TypelessAccessToken::new)); + } else { + // vended token exchange use cases + if (filtered.containsKey(TokenExchangeConfig.SUBJECT_TOKEN)) { + basicBuilder.grantType(GrantType.TOKEN_EXCHANGE); + tokenExchangeBuilder.subjectTokenString( + ConfigUtil.parseOptional(filtered, TokenExchangeConfig.SUBJECT_TOKEN)); + } + + if (filtered.containsKey(TokenExchangeConfig.SUBJECT_TOKEN_TYPE)) { + tokenExchangeBuilder.subjectTokenType( + ConfigUtil.parseOptional( + filtered, TokenExchangeConfig.SUBJECT_TOKEN_TYPE, TokenTypeURI::parse) + .orElse(TokenTypeURI.ACCESS_TOKEN)); + } + + if (filtered.containsKey(TokenExchangeConfig.ACTOR_TOKEN)) { + tokenExchangeBuilder.actorTokenString( + ConfigUtil.parseOptional(filtered, TokenExchangeConfig.ACTOR_TOKEN)); + } + + if (filtered.containsKey(TokenExchangeConfig.ACTOR_TOKEN_TYPE)) { + tokenExchangeBuilder.actorTokenType( + ConfigUtil.parseOptional( + filtered, TokenExchangeConfig.ACTOR_TOKEN_TYPE, TokenTypeURI::parse) + .orElse(TokenTypeURI.ACCESS_TOKEN)); + } + } + + return ImmutableOAuth2Config.builder() + .from(parent) + .basicConfig(basicBuilder.build()) + .tokenExchangeConfig(tokenExchangeBuilder.build()) + .build(); + } + + @VisibleForTesting + Map migrateProperties(Map properties) { + Map migrated = Maps.newLinkedHashMap(); + for (Entry entry : properties.entrySet()) { + switch (entry.getKey()) { + case OAuth2Properties.TOKEN: + warnOnLegacyOption(entry.getKey(), BasicConfig.TOKEN); + migrated.put(BasicConfig.TOKEN, entry.getValue()); + break; + case OAuth2Properties.CREDENTIAL: + warnOnLegacyOption( + entry.getKey(), true, BasicConfig.CLIENT_ID, BasicConfig.CLIENT_SECRET); + List parts = CREDENTIAL_SPLITTER.splitToList(entry.getValue()); + if (parts.size() == 2) { + migrated.put(BasicConfig.CLIENT_ID, parts.get(0)); + migrated.put(BasicConfig.CLIENT_SECRET, parts.get(1)); + } else { + logConsumer.accept( + MESSAGE_TEMPLATE_NO_CLIENT_ID, new String[] {DEFAULT_CLIENT_ID.getValue()}); + migrated.put(BasicConfig.CLIENT_ID, DEFAULT_CLIENT_ID.getValue()); + migrated.put(BasicConfig.CLIENT_SECRET, parts.get(0)); + } + + break; + case OAuth2Properties.TOKEN_EXPIRES_IN_MS: + warnOnLegacyOption(entry.getKey(), TokenRefreshConfig.ACCESS_TOKEN_LIFESPAN); + Duration duration = + Duration.ofMillis( + PropertyUtil.propertyAsLong( + properties, + OAuth2Properties.TOKEN_EXPIRES_IN_MS, + OAuth2Properties.TOKEN_EXPIRES_IN_MS_DEFAULT)); + migrated.put(TokenRefreshConfig.ACCESS_TOKEN_LIFESPAN, duration.toString()); + break; + case OAuth2Properties.TOKEN_REFRESH_ENABLED: + warnOnLegacyOption(entry.getKey(), TokenRefreshConfig.ENABLED); + migrated.put( + TokenRefreshConfig.ENABLED, String.valueOf(Boolean.parseBoolean(entry.getValue()))); + break; + case OAuth2Properties.OAUTH2_SERVER_URI: + warnOnLegacyOption( + entry.getKey(), false, BasicConfig.ISSUER_URL, BasicConfig.TOKEN_ENDPOINT); + migrated.put(BasicConfig.TOKEN_ENDPOINT, entry.getValue()); + break; + case OAuth2Properties.SCOPE: + warnOnLegacyOption(entry.getKey(), BasicConfig.SCOPE); + migrated.put(BasicConfig.SCOPE, entry.getValue()); + break; + case OAuth2Properties.AUDIENCE: + warnOnLegacyOption(entry.getKey(), TokenExchangeConfig.AUDIENCES); + migrated.put(TokenExchangeConfig.AUDIENCES, entry.getValue()); + break; + case OAuth2Properties.RESOURCE: + warnOnLegacyOption(entry.getKey(), TokenExchangeConfig.RESOURCES); + migrated.put(TokenExchangeConfig.RESOURCES, entry.getValue()); + break; + case OAuth2Properties.ACCESS_TOKEN_TYPE: + case OAuth2Properties.ID_TOKEN_TYPE: + case OAuth2Properties.SAML1_TOKEN_TYPE: + case OAuth2Properties.SAML2_TOKEN_TYPE: + case OAuth2Properties.JWT_TOKEN_TYPE: + case OAuth2Properties.REFRESH_TOKEN_TYPE: + warnOnLegacyOption( + entry.getKey(), + true, + TokenExchangeConfig.SUBJECT_TOKEN, + TokenExchangeConfig.SUBJECT_TOKEN_TYPE, + TokenExchangeConfig.ACTOR_TOKEN); + migrated.put(BasicConfig.GRANT_TYPE, GrantType.TOKEN_EXCHANGE.getValue()); + migrated.put(TokenExchangeConfig.SUBJECT_TOKEN, entry.getValue()); + migrated.put(TokenExchangeConfig.SUBJECT_TOKEN_TYPE, entry.getKey()); + migrated.put(TokenExchangeConfig.ACTOR_TOKEN, ConfigUtil.PARENT_TOKEN); + break; + case OAuth2Properties.TOKEN_EXCHANGE_ENABLED: + warnOnLegacyOption(entry.getKey(), TokenRefreshConfig.TOKEN_EXCHANGE_ENABLED); + migrated.put(TokenRefreshConfig.TOKEN_EXCHANGE_ENABLED, entry.getValue()); + break; + } + } + + // preserve new properties, overriding legacy properties if any + for (Entry entry : properties.entrySet()) { + if (entry.getKey().startsWith(OAuth2Config.PREFIX)) { + migrated.put(entry.getKey(), entry.getValue()); + } + } + + return migrated; + } + + @VisibleForTesting + void handleTokenEndpoint(Map migrated, String catalogUri) { + + Preconditions.checkNotNull(catalogUri, "Catalog URI is required"); + + String tokenEndpoint = migrated.get(BasicConfig.TOKEN_ENDPOINT); + String issuerUrl = migrated.get(BasicConfig.ISSUER_URL); + String token = migrated.get(BasicConfig.TOKEN); + + if (tokenEndpoint == null && issuerUrl == null && token == null) { + + // No token endpoint or issuer URL configured, and no static token: + // default the token endpoint to catalog URI + ResourcePaths.tokens() + tokenEndpoint = RESTUtil.resolveEndpoint(catalogUri, ResourcePaths.tokens()); + migrated.put(BasicConfig.TOKEN_ENDPOINT, tokenEndpoint); + warnOnMissingTokenEndpoint(tokenEndpoint); + + } else if (tokenEndpoint != null && !URI.create(tokenEndpoint).isAbsolute()) { + + // If the token endpoint was provided, but is a relative path: + // assume it's an endpoint internal to the catalog server + // and resolve it against the catalog URI + tokenEndpoint = RESTUtil.resolveEndpoint(catalogUri, tokenEndpoint); + migrated.put(BasicConfig.TOKEN_ENDPOINT, tokenEndpoint); + warnOnRelativeTokenEndpoint(tokenEndpoint); + } + } + + private void warnOnLegacyOption(String icebergOption, String authManagerOption) { + warnOnLegacyOption(icebergOption, false, authManagerOption); + } + + private void warnOnLegacyOption(String legacyOption, boolean and, String... newOptions) { + List options = Lists.newArrayList(newOptions); + String joined = + options.size() == 1 + ? options.get(0) + : options.stream().limit(options.size() - 1).collect(Collectors.joining(", ")) + + (and ? " and " : " or ") + + options.get(options.size() - 1); + logConsumer.accept( + MESSAGE_TEMPLATE_LEGACY_OPTION, + new String[] {legacyOption, options.size() == 1 ? "" : "s", joined}); + } + + private void warnOnMissingTokenEndpoint(String tokenEndpoint) { + logConsumer.accept( + MESSAGE_TEMPLATE_MISSING_TOKEN_ENDPOINT, + new String[] { + tokenEndpoint, BasicConfig.TOKEN_ENDPOINT, BasicConfig.ISSUER_URL, + }); + } + + private void warnOnRelativeTokenEndpoint(String tokenEndpoint) { + logConsumer.accept(MESSAGE_TEMPLATE_RELATIVE_TOKEN_ENDPOINT, new String[] {tokenEndpoint}); + } + + private void warnOnMergedContextualConfig(String mergedOption) { + logConsumer.accept(MESSAGE_TEMPLATE_MERGED_CONTEXTUAL_CONFIG, new String[] {mergedOption}); + } + + private void warnOnVendedToken(String vendedOption) { + logConsumer.accept(MESSAGE_TEMPLATE_VENDED_TOKEN, new String[] {vendedOption}); + } + + private void warnOnForbiddenTableConfig(String tableOption) { + logConsumer.accept(MESSAGE_TEMPLATE_TABLE_CONFIG_NOT_ALLOWED, new String[] {tableOption}); + } +} diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/config/ConfigOption.java b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/config/ConfigOption.java new file mode 100644 index 000000000000..ee3dbd38a7d0 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/config/ConfigOption.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.config; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * An annotation that can be used to mark a method as a configuration option. + * + *

    This annotation can be leveraged by documentation generators to extract the configuration + * options and their descriptions. + */ +@Retention(RetentionPolicy.CLASS) +@Target(ElementType.METHOD) +public @interface ConfigOption { + + /** The fully-qualified name of the configuration option. */ + String value(); + + /** Whether this option is a prefix map option (accepting multiple values). */ + boolean prefixMap() default false; +} diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/config/ConfigUtil.java b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/config/ConfigUtil.java new file mode 100644 index 000000000000..0a9976698c63 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/config/ConfigUtil.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.config; + +import com.nimbusds.oauth2.sdk.GrantType; +import com.nimbusds.oauth2.sdk.auth.ClientAuthenticationMethod; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.annotation.Nullable; +import org.apache.iceberg.relocated.com.google.common.base.Splitter; + +/** + * Helper class for parsing configuration options. It also exposes useful constants for config + * validation. + */ +public final class ConfigUtil { + + public static final List SUPPORTED_GRANT_TYPES = + List.of(GrantType.CLIENT_CREDENTIALS, GrantType.TOKEN_EXCHANGE, GrantType.REFRESH_TOKEN); + + public static final List SUPPORTED_INITIAL_GRANT_TYPES = + List.of(GrantType.CLIENT_CREDENTIALS, GrantType.TOKEN_EXCHANGE); + + public static final List SUPPORTED_CLIENT_AUTH_METHODS = + List.of( + ClientAuthenticationMethod.NONE, + ClientAuthenticationMethod.CLIENT_SECRET_BASIC, + ClientAuthenticationMethod.CLIENT_SECRET_POST); + + /** + * A sentinel value used to indicate that the parent session's token should be used. This is + * useful for the token exchange flow. + */ + public static final String PARENT_TOKEN = "::parent::"; + + public static boolean requiresClientSecret(@Nullable ClientAuthenticationMethod method) { + return Objects.equals(method, ClientAuthenticationMethod.CLIENT_SECRET_BASIC) + || Objects.equals(method, ClientAuthenticationMethod.CLIENT_SECRET_POST); + } + + static Optional parseOptional(Map properties, String option) { + return parseOptional(properties, option, s -> s); + } + + static Optional parseOptional( + Map properties, String option, ConfigParser parser) { + return Optional.ofNullable(properties.get(option)).map(parser::parseUnchecked); + } + + static OptionalInt parseOptionalInt(Map properties, String option) { + ConfigParser parser = Integer::parseInt; + return Optional.ofNullable(properties.get(option)) + .map(parser::parseUnchecked) + .map(OptionalInt::of) + .orElseGet(OptionalInt::empty); + } + + static List parseList(Map properties, String option, String delimiter) { + return parseList(properties, option, delimiter, s -> s); + } + + static List parseList( + Map properties, String option, String delimiter, ConfigParser parser) { + return Optional.ofNullable(properties.get(option)) + .map(s -> Splitter.on(delimiter).trimResults().omitEmptyStrings().splitToStream(s)) + .orElseGet(Stream::empty) + .map(parser::parseUnchecked) + .collect(Collectors.toList()); + } + + @FunctionalInterface + interface ConfigParser { + + T parse(String value) throws Exception; + + default T parseUnchecked(String value) { + try { + return parse(value); + } catch (Exception e) { + throw new IllegalArgumentException( + "Failed to parse configuration value '%s'".formatted(value), e); + } + } + } + + private ConfigUtil() {} +} diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/config/ConfigValidator.java b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/config/ConfigValidator.java new file mode 100644 index 000000000000..e05157b2138c --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/config/ConfigValidator.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.config; + +import com.google.errorprone.annotations.FormatMethod; +import com.google.errorprone.annotations.FormatString; +import java.net.URI; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.immutables.value.Value; + +public final class ConfigValidator { + + @Value.Immutable + interface ConfigViolation { + + @Value.Parameter(order = 1) + List offendingKeys(); + + @Value.Parameter(order = 2) + String message(); + + @Value.Lazy + default String formattedMessage() { + return message() + " (" + String.join(" / ", offendingKeys()) + ")"; + } + } + + private final List violations = Lists.newArrayList(); + + @FormatMethod + public void check( + boolean condition, String offendingKey, @FormatString String msg, Object... args) { + check(condition, List.of(offendingKey), msg, args); + } + + @FormatMethod + public void check( + boolean condition, List offendingKeys, @FormatString String msg, Object... args) { + if (!condition) { + violations.add(ImmutableConfigViolation.of(offendingKeys, String.format(msg, args))); + } + } + + public void checkEndpoint(URI endpoint, String offendingKey, String name) { + check(endpoint.isAbsolute(), offendingKey, "%s %s", name, "must not be relative"); + check( + endpoint.getUserInfo() == null, + offendingKey, + "%s %s", + name, + "must not have a user info part"); + check(endpoint.getQuery() == null, offendingKey, "%s %s", name, "must not have a query part"); + check( + endpoint.getFragment() == null, + offendingKey, + "%s %s", + name, + "must not have a fragment part"); + } + + public void validate() { + if (!violations.isEmpty()) { + throw new IllegalArgumentException( + buildDescription(violations.stream().map(ConfigViolation::formattedMessage))); + } + } + + private static final String DELIMITER = "\n - "; + + @VisibleForTesting + static String buildDescription(Stream violations) { + return "Invalid OAuth2 configuration:" + + violations.collect(Collectors.joining(DELIMITER, DELIMITER, "")); + } +} diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/config/TokenExchangeConfig.java b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/config/TokenExchangeConfig.java new file mode 100644 index 000000000000..94e3f171e08b --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/config/TokenExchangeConfig.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.config; + +import com.nimbusds.oauth2.sdk.id.Audience; +import com.nimbusds.oauth2.sdk.token.TokenTypeURI; +import java.net.URI; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.apache.iceberg.rest.auth.oauth2.OAuth2Config; +import org.immutables.value.Value; + +/** + * Configuration properties for the Token + * Exchange flow. + * + *

    This flow allows a client to exchange one token for another, typically to obtain a token that + * is more suitable for the target resource or service. + */ +@Value.Immutable +@Value.Style(redactedMask = "****") +@SuppressWarnings({"ImmutablesStyle", "SafeLoggingPropagation"}) +public interface TokenExchangeConfig { + + String PREFIX = OAuth2Config.PREFIX + "token-exchange."; + + String SUBJECT_TOKEN = PREFIX + "subject-token"; + String SUBJECT_TOKEN_TYPE = PREFIX + "subject-token-type"; + String ACTOR_TOKEN = PREFIX + "actor-token"; + String ACTOR_TOKEN_TYPE = PREFIX + "actor-token-type"; + String REQUESTED_TOKEN_TYPE = PREFIX + "requested-token-type"; + String RESOURCES = PREFIX + "resources"; + String AUDIENCES = PREFIX + "audiences"; + + /** + * The subject token to exchange. Required. + * + *

    The special value {@code ::parent::} can be used to indicate that the subject token should + * be obtained from the parent OAuth2 session. + */ + @ConfigOption(SUBJECT_TOKEN) + @Value.Redacted + Optional subjectTokenString(); + + /** + * The type of the subject token. Must be a valid URN. Required. If not set, the default is {@code + * urn:ietf:params:oauth:token-type:access_token}. + * + * @see TokenExchangeConfig#SUBJECT_TOKEN_TYPE + */ + @ConfigOption(SUBJECT_TOKEN_TYPE) + Optional subjectTokenType(); + + /** + * The actor token to exchange. Optional. + * + *

    The special value {@code ::parent::} can be used to indicate that the actor token should be + * obtained from the parent OAuth2 session. + */ + @ConfigOption(ACTOR_TOKEN) + @Value.Redacted + Optional actorTokenString(); + + /** + * The type of the actor token. Must be a valid URN. Required if an actor token is used. If not + * set, the default is {@code urn:ietf:params:oauth:token-type:access_token}. + * + * @see TokenExchangeConfig#ACTOR_TOKEN_TYPE + */ + @ConfigOption(ACTOR_TOKEN_TYPE) + Optional actorTokenType(); + + /** The type of the requested token. Must be a valid URN. Optional. */ + @ConfigOption(REQUESTED_TOKEN_TYPE) + Optional requestedTokenType(); + + /** + * One or more URIs that indicate the target service(s) or resource(s) where the client intends to + * use the requested token. + * + *

    Optional. Can be a single value or a comma-separated list of values. + */ + @ConfigOption(RESOURCES) + List resources(); + + /** + * The logical name(s) of the target service where the client intends to use the requested token. + * This serves a purpose similar to the resource parameter but with the client providing a logical + * name for the target service. + * + *

    Optional. Can be a single value or a comma-separated list of values. + */ + @ConfigOption(AUDIENCES) + List audiences(); + + static ImmutableTokenExchangeConfig.Builder parse(Map properties) { + return ImmutableTokenExchangeConfig.builder() + .subjectTokenString(ConfigUtil.parseOptional(properties, SUBJECT_TOKEN)) + .subjectTokenType( + ConfigUtil.parseOptional(properties, SUBJECT_TOKEN_TYPE, TokenTypeURI::parse)) + .actorTokenString(ConfigUtil.parseOptional(properties, ACTOR_TOKEN)) + .actorTokenType(ConfigUtil.parseOptional(properties, ACTOR_TOKEN_TYPE, TokenTypeURI::parse)) + .requestedTokenType( + ConfigUtil.parseOptional(properties, REQUESTED_TOKEN_TYPE, TokenTypeURI::parse)) + .resources(ConfigUtil.parseList(properties, RESOURCES, ",", URI::create)) + .audiences(ConfigUtil.parseList(properties, AUDIENCES, ",", Audience::new)); + } +} diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/config/TokenRefreshConfig.java b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/config/TokenRefreshConfig.java new file mode 100644 index 000000000000..5ec3e6fe17b0 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/config/TokenRefreshConfig.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.config; + +import com.nimbusds.oauth2.sdk.GrantType; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.apache.iceberg.rest.auth.oauth2.OAuth2Config; +import org.immutables.value.Value; + +/** Configuration properties for the token refresh feature. */ +@Value.Immutable +public interface TokenRefreshConfig { + + String PREFIX = OAuth2Config.PREFIX + "token-refresh."; + + String ENABLED = PREFIX + "enabled"; + String TOKEN_EXCHANGE_ENABLED = PREFIX + "token-exchange-enabled"; + String ACCESS_TOKEN_LIFESPAN = PREFIX + "access-token-lifespan"; + String SAFETY_MARGIN = PREFIX + "safety-margin"; + + Duration DEFAULT_ACCESS_TOKEN_LIFESPAN = Duration.parse("PT1H"); + Duration DEFAULT_SAFETY_MARGIN = Duration.parse("PT10S"); + + Duration MIN_ACCESS_TOKEN_LIFESPAN = Duration.parse("PT15S"); + Duration MIN_SAFETY_MARGIN = Duration.parse("PT10S"); + + /** + * Whether to enable token refresh. If enabled, the OAuth2 client will automatically refresh its + * access token when it expires. If disabled, the OAuth2 client will only fetch the initial access + * token, but won't refresh it. Defaults to {@code true}. + */ + @ConfigOption(ENABLED) + @Value.Default + default boolean enabled() { + return true; + } + + /** + * Whether to use the token exchange grant to refresh tokens. + * + *

    When enabled, the token exchange grant will be used to refresh the access token, if no + * refresh token is available. + * + *

    Optional, defaults to {@code true} if the initial grant is {@link + * GrantType#CLIENT_CREDENTIALS}. + */ + @ConfigOption(TOKEN_EXCHANGE_ENABLED) + Optional tokenExchangeEnabled(); + + /** + * Default access token lifespan; if the OAuth2 server returns an access token without specifying + * its expiration time, this value will be used. + * + *

    Optional, defaults to {@link #DEFAULT_ACCESS_TOKEN_LIFESPAN}. Must be a valid ISO-8601 duration. + */ + @ConfigOption(ACCESS_TOKEN_LIFESPAN) + @Value.Default + default Duration accessTokenLifespan() { + return DEFAULT_ACCESS_TOKEN_LIFESPAN; + } + + /** + * Refresh safety margin to use; a new token will be fetched when the current token's remaining + * lifespan is less than this value. Optional, defaults to {@link #DEFAULT_SAFETY_MARGIN}. Must be + * a valid ISO-8601 duration. + */ + @ConfigOption(SAFETY_MARGIN) + @Value.Default + default Duration safetyMargin() { + return DEFAULT_SAFETY_MARGIN; + } + + @Value.Check + default void validate() { + if (enabled()) { + ConfigValidator validator = new ConfigValidator(); + validator.check( + accessTokenLifespan().compareTo(MIN_ACCESS_TOKEN_LIFESPAN) >= 0, + ACCESS_TOKEN_LIFESPAN, + "access token lifespan must be greater than or equal to %s", + MIN_ACCESS_TOKEN_LIFESPAN); + validator.check( + safetyMargin().compareTo(MIN_SAFETY_MARGIN) >= 0, + SAFETY_MARGIN, + "refresh safety margin must be greater than or equal to %s", + MIN_SAFETY_MARGIN); + validator.check( + safetyMargin().compareTo(accessTokenLifespan()) < 0, + List.of(SAFETY_MARGIN, ACCESS_TOKEN_LIFESPAN), + "refresh safety margin must be less than the access token lifespan"); + validator.validate(); + } + } + + static ImmutableTokenRefreshConfig.Builder parse(Map properties) { + return ImmutableTokenRefreshConfig.builder() + .enabled(ConfigUtil.parseOptional(properties, ENABLED, Boolean::parseBoolean).orElse(true)) + .tokenExchangeEnabled( + ConfigUtil.parseOptional(properties, TOKEN_EXCHANGE_ENABLED, Boolean::parseBoolean)) + .accessTokenLifespan( + ConfigUtil.parseOptional(properties, ACCESS_TOKEN_LIFESPAN, Duration::parse) + .orElse(DEFAULT_ACCESS_TOKEN_LIFESPAN)) + .safetyMargin( + ConfigUtil.parseOptional(properties, SAFETY_MARGIN, Duration::parse) + .orElse(DEFAULT_SAFETY_MARGIN)); + } +} diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/flow/BaseFlow.java b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/flow/BaseFlow.java new file mode 100644 index 000000000000..c6e514c08d4d --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/flow/BaseFlow.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.flow; + +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import com.nimbusds.oauth2.sdk.AccessTokenResponse; +import com.nimbusds.oauth2.sdk.AuthorizationGrant; +import com.nimbusds.oauth2.sdk.ErrorObject; +import com.nimbusds.oauth2.sdk.ParseException; +import com.nimbusds.oauth2.sdk.TokenErrorResponse; +import com.nimbusds.oauth2.sdk.TokenRequest; +import com.nimbusds.oauth2.sdk.TokenResponse; +import com.nimbusds.oauth2.sdk.auth.ClientAuthentication; +import com.nimbusds.oauth2.sdk.auth.ClientAuthenticationMethod; +import com.nimbusds.oauth2.sdk.auth.ClientSecretBasic; +import com.nimbusds.oauth2.sdk.auth.ClientSecretPost; +import com.nimbusds.oauth2.sdk.auth.Secret; +import com.nimbusds.oauth2.sdk.http.HTTPRequest; +import com.nimbusds.oauth2.sdk.http.HTTPResponse; +import com.nimbusds.oauth2.sdk.id.ClientID; +import java.net.URI; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.function.Function; +import org.apache.iceberg.rest.auth.oauth2.ImmutableOAuth2Error; +import org.apache.iceberg.rest.auth.oauth2.OAuth2Config; +import org.apache.iceberg.rest.auth.oauth2.OAuth2Exception; +import org.apache.iceberg.rest.auth.oauth2.OAuth2Runtime; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +abstract class BaseFlow implements Flow { + + private static final Logger LOGGER = LoggerFactory.getLogger(BaseFlow.class); + + abstract OAuth2Config config(); + + abstract OAuth2Runtime runtime(); + + abstract EndpointProvider endpointProvider(); + + interface Builder> { + + @CanIgnoreReturnValue + B config(OAuth2Config config); + + @CanIgnoreReturnValue + B runtime(OAuth2Runtime runtime); + + @CanIgnoreReturnValue + B endpointProvider(EndpointProvider endpointProvider); + + F build(); + } + + CompletionStage invokeTokenEndpoint(AuthorizationGrant grant) { + HTTPRequest request; + try { + TokenRequest.Builder builder = newTokenRequestBuilder(grant); + request = builder.build().toHTTPRequest(); + } catch (Exception e) { + return CompletableFuture.failedFuture(e); + } + return CompletableFuture.supplyAsync(() -> sendAndReceive(request), runtime().executor()) + .whenComplete((response, error) -> log(request, response, error)) + .thenApply(this::parseTokenResponse) + .thenCompose(Function.identity()) + .thenApply(this::toTokensResult); + } + + TokenRequest.Builder newTokenRequestBuilder(AuthorizationGrant grant) { + URI tokenEndpoint = endpointProvider().resolvedTokenEndpoint(); + TokenRequest.Builder builder = + publicClient() + ? new TokenRequest.Builder(tokenEndpoint, clientId(), grant) + : new TokenRequest.Builder(tokenEndpoint, createClientAuthentication(), grant); + config().basicConfig().scope().ifPresent(builder::scope); + config().basicConfig().extraRequestParameters().forEach(builder::customParameter); + return builder; + } + + HTTPResponse sendAndReceive(HTTPRequest request) { + try { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Invoking endpoint: {}", request.getURI()); + } + + return request.send(runtime().httpClient()); + } catch (Exception e) { + throw new RuntimeException("Failed to invoke endpoint: " + request.getURI(), e); + } + } + + CompletionStage parseTokenResponse(HTTPResponse httpResponse) { + try { + TokenResponse response = TokenResponse.parse(httpResponse); + if (!response.indicatesSuccess()) { + TokenErrorResponse errorResponse = response.toErrorResponse(); + ErrorObject errorObject = errorResponse.getErrorObject(); + return CompletableFuture.failedFuture(newOAuth2Exception(httpResponse, errorObject)); + } + + return CompletableFuture.completedFuture(response.toSuccessResponse()); + } catch (ParseException e) { + return CompletableFuture.failedFuture(e); + } + } + + OAuth2Exception newOAuth2Exception(HTTPResponse httpResponse, ErrorObject errorObject) { + return new OAuth2Exception( + httpResponse.getStatusCode(), + ImmutableOAuth2Error.builder() + // Code can be null if the error is not a standard OAuth2 error, e.g. a 404 Not Found + .code(Optional.ofNullable(errorObject.getCode()).orElse("unknown_error")) + .description(Optional.ofNullable(errorObject.getDescription())) + .uri(Optional.ofNullable(errorObject.getURI())) + .parameters(errorObject.getCustomParams()) + .build()); + } + + TokensResult toTokensResult(AccessTokenResponse response) { + return TokensResult.of(response, runtime().clock()); + } + + void log(HTTPRequest request, HTTPResponse response, Throwable error) { + if (error == null) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug( + "Received {} response from endpoint: {}", response.getStatusCode(), request.getURI()); + } + } else { + LOGGER.warn("Error invoking endpoint: {}", request.getURI(), error); + } + } + + boolean publicClient() { + return config() + .basicConfig() + .clientAuthenticationMethod() + .equals(ClientAuthenticationMethod.NONE); + } + + ClientID clientId() { + return config() + .basicConfig() + .clientId() + .orElseThrow(() -> new IllegalStateException("Client ID is required")); + } + + Secret clientSecret() { + return config() + .basicConfig() + .clientSecret() + .orElseThrow(() -> new IllegalStateException("Client secret is required")); + } + + ClientAuthentication createClientAuthentication() { + ClientAuthenticationMethod method = config().basicConfig().clientAuthenticationMethod(); + + if (method.equals(ClientAuthenticationMethod.CLIENT_SECRET_BASIC)) { + return new ClientSecretBasic(clientId(), clientSecret()); + + } else if (method.equals(ClientAuthenticationMethod.CLIENT_SECRET_POST)) { + return new ClientSecretPost(clientId(), clientSecret()); + } + + throw new IllegalArgumentException("Unsupported client authentication method: " + method); + } +} diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/flow/ClientCredentialsFlow.java b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/flow/ClientCredentialsFlow.java new file mode 100644 index 000000000000..3166cfd9f0aa --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/flow/ClientCredentialsFlow.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.flow; + +import com.nimbusds.oauth2.sdk.AuthorizationGrant; +import com.nimbusds.oauth2.sdk.ClientCredentialsGrant; +import com.nimbusds.oauth2.sdk.GrantType; +import java.util.concurrent.CompletionStage; +import org.immutables.value.Value; + +/** + * An implementation of the Client Credentials Grant + * flow. + */ +@Value.Immutable +abstract class ClientCredentialsFlow extends BaseFlow { + + private static final AuthorizationGrant GRANT = new ClientCredentialsGrant(); + + interface Builder extends BaseFlow.Builder {} + + @Override + public final GrantType grantType() { + return GrantType.CLIENT_CREDENTIALS; + } + + @Override + public CompletionStage execute() { + return invokeTokenEndpoint(GRANT); + } +} diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/flow/EndpointProvider.java b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/flow/EndpointProvider.java new file mode 100644 index 000000000000..3c99ba920800 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/flow/EndpointProvider.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.flow; + +import com.nimbusds.oauth2.sdk.AbstractConfigurationRequest; +import com.nimbusds.oauth2.sdk.ParseException; +import com.nimbusds.oauth2.sdk.as.AuthorizationServerConfigurationRequest; +import com.nimbusds.oauth2.sdk.as.AuthorizationServerEndpointMetadata; +import com.nimbusds.oauth2.sdk.as.ReadOnlyAuthorizationServerEndpointMetadata; +import com.nimbusds.oauth2.sdk.http.HTTPResponse; +import com.nimbusds.oauth2.sdk.id.Issuer; +import com.nimbusds.openid.connect.sdk.op.OIDCProviderConfigurationRequest; +import com.nimbusds.openid.connect.sdk.op.OIDCProviderEndpointMetadata; +import java.io.IOException; +import java.net.URI; +import java.util.List; +import java.util.Locale; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.rest.auth.oauth2.OAuth2Config; +import org.apache.iceberg.rest.auth.oauth2.OAuth2Runtime; +import org.immutables.value.Value; + +/** + * A provider for OAuth2 endpoints. + * + *

    This component centralizes the logic for fetching endpoint metadata from the issuer ("metadata + * discovery"), and provides a single point of access for all supported OAuth2 endpoints. + */ +@Value.Immutable +public abstract class EndpointProvider { + + public static EndpointProvider of(OAuth2Config config, OAuth2Runtime runtime) { + return ImmutableEndpointProvider.builder().config(config).runtime(runtime).build(); + } + + /** The OAuth2 configuration. */ + protected abstract OAuth2Config config(); + + /** The OAuth2 runtime, required for making HTTP requests. */ + protected abstract OAuth2Runtime runtime(); + + /** + * The resolved token endpoint. If a token endpoint is provided in the configuration, it is used + * as is, otherwise it is resolved through metadata discovery on first access. + */ + @Value.Lazy + public URI resolvedTokenEndpoint() { + return config() + .basicConfig() + .tokenEndpoint() + .orElseGet(() -> serverMetadata().getTokenEndpointURI()); + } + + @Value.Lazy + ReadOnlyAuthorizationServerEndpointMetadata serverMetadata() { + URI issuerUrl = + config() + .basicConfig() + .issuerUrl() + .orElseThrow(() -> new IllegalStateException("No issuer URL configured")); + return fetchServerMetadata(issuerUrl); + } + + private ReadOnlyAuthorizationServerEndpointMetadata fetchServerMetadata(URI issuerUrl) { + Issuer issuer = new Issuer(issuerUrl); + List failures = null; + for (MetadataProvider provider : + List.of(this::oidcProvider, this::oauthProvider)) { + try { + return provider.fetchMetadata(issuer); + } catch (Exception e) { + if (failures == null) { + failures = Lists.newArrayListWithCapacity(2); + } + + failures.add(e); + } + } + + RuntimeException error = + new RuntimeException("Failed to fetch provider metadata", failures.get(0)); + for (int i = 1; i < failures.size(); i++) { + error.addSuppressed(failures.get(i)); + } + + throw error; + } + + private ReadOnlyAuthorizationServerEndpointMetadata oidcProvider(Issuer issuer) + throws IOException, ParseException { + AbstractConfigurationRequest request = new OIDCProviderConfigurationRequest(issuer); + HTTPResponse httpResponse = request.toHTTPRequest().send(runtime().httpClient()); + if (httpResponse.indicatesSuccess()) { + return OIDCProviderEndpointMetadata.parse(httpResponse.getBodyAsJSONObject()); + } + + throw providerFailure("OIDC", httpResponse); + } + + private ReadOnlyAuthorizationServerEndpointMetadata oauthProvider(Issuer issuer) + throws IOException, ParseException { + AbstractConfigurationRequest request = new AuthorizationServerConfigurationRequest(issuer); + HTTPResponse httpResponse = request.toHTTPRequest().send(runtime().httpClient()); + if (httpResponse.indicatesSuccess()) { + return AuthorizationServerEndpointMetadata.parse(httpResponse.getBodyAsJSONObject()); + } + + throw providerFailure("OAuth", httpResponse); + } + + private static RuntimeException providerFailure(String type, HTTPResponse httpResponse) { + return new RuntimeException( + String.format( + Locale.ROOT, + "Failed to fetch %s provider metadata: server returned code %d with message: %s", + type, + httpResponse.getStatusCode(), + httpResponse.getBody())); + } + + @FunctionalInterface + private interface MetadataProvider { + ReadOnlyAuthorizationServerEndpointMetadata fetchMetadata(Issuer issuer) + throws IOException, ParseException; + } +} diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/flow/Flow.java b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/flow/Flow.java new file mode 100644 index 000000000000..5ecd2c75bc3f --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/flow/Flow.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.flow; + +import com.nimbusds.oauth2.sdk.GrantType; +import java.util.concurrent.CompletionStage; + +/** + * An interface representing an OAuth2 flow. + * + *

    A flow is a complete sequence of interactions (generally one, but sometimes more) between the + * client, the authorization server, and (optionally) the resource owner (for human-to-machine + * flows) in order to obtain an access token. + */ +public interface Flow { + + /** Returns the OAuth2 grant type used by this flow. Useful mostly for logging purposes. */ + GrantType grantType(); + + /** + * Executes the flow and returns a {@link CompletionStage} that completes when the flow is done + * and new tokens are available. + * + *

    A flow may be stateful or stateless. Stateful flows should clean up internal resources when + * the returned {@link CompletionStage} completes. + * + * @return A stage that completes when new tokens are fetched. + */ + CompletionStage execute(); +} diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/flow/FlowFactory.java b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/flow/FlowFactory.java new file mode 100644 index 000000000000..13255b7a5517 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/flow/FlowFactory.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.flow; + +import com.nimbusds.oauth2.sdk.GrantType; +import com.nimbusds.oauth2.sdk.token.AccessToken; +import com.nimbusds.oauth2.sdk.token.Tokens; +import com.nimbusds.oauth2.sdk.token.TypelessAccessToken; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import javax.annotation.Nullable; +import org.apache.iceberg.rest.auth.oauth2.OAuth2Config; +import org.apache.iceberg.rest.auth.oauth2.OAuth2Runtime; +import org.apache.iceberg.rest.auth.oauth2.client.OAuth2Client; +import org.apache.iceberg.rest.auth.oauth2.config.ConfigUtil; +import org.immutables.value.Value; + +/** + * A factory for creating {@link Flow} instances. This is one of the main components of the OAuth2 + * client, responsible for creating flows for fetching new tokens and refreshing tokens. + */ +@Value.Immutable +public abstract class FlowFactory { + + public static FlowFactory of(OAuth2Config config, OAuth2Runtime runtime) { + return ImmutableFlowFactory.builder().config(config).runtime(runtime).build(); + } + + /** Creates a flow for fetching new tokens. This is used for the initial token fetch. */ + public Flow newInitialFlow() { + return newInitialFlowBuilder() + .config(config()) + .runtime(runtime()) + .endpointProvider(endpointProvider()) + .build(); + } + + /** + * Creates a flow for refreshing tokens. This is used for refreshing tokens when the access token + * expires. + */ + public Flow newRefreshFlow(Tokens currentTokens) { + return newRefreshFlowBuilder(currentTokens) + .config(config()) + .runtime(runtime()) + .endpointProvider(endpointProvider()) + .build(); + } + + abstract OAuth2Config config(); + + abstract OAuth2Runtime runtime(); + + @Value.Default + EndpointProvider endpointProvider() { + return EndpointProvider.of(config(), runtime()); + } + + private BaseFlow.Builder newInitialFlowBuilder() { + + GrantType grantType = config().basicConfig().grantType(); + + if (grantType.equals(GrantType.CLIENT_CREDENTIALS)) { + return ImmutableClientCredentialsFlow.builder(); + + } else if (grantType.equals(GrantType.TOKEN_EXCHANGE)) { + AccessToken subjectToken = + config() + .tokenExchangeConfig() + .subjectTokenString() + .map(TypelessAccessToken::new) + .orElseThrow(() -> new IllegalStateException("Subject token is required")); + AccessToken actorToken = + config() + .tokenExchangeConfig() + .actorTokenString() + .map(TypelessAccessToken::new) + .orElse(null); + return ImmutableTokenExchangeFlow.builder() + .subjectTokenStage(asTokenStage(subjectToken)) + .actorTokenStage(asTokenStage(actorToken)); + } + + // Should never happen since the grant type is validated by the config. + throw new IllegalArgumentException( + "Unknown or invalid grant type for initial token fetch: " + + config().basicConfig().grantType()); + } + + private CompletionStage asTokenStage(@Nullable AccessToken token) { + if (token != null && token.getValue().equals(ConfigUtil.PARENT_TOKEN)) { + @SuppressWarnings("resource") + OAuth2Client parentClient = + runtime() + .parent() + .orElseThrow(() -> new IllegalStateException("Parent OAuth2 client is required")); + return parentClient.authenticateAsync(); + } else { + return CompletableFuture.completedFuture(token); + } + } + + private BaseFlow.Builder newRefreshFlowBuilder(Tokens currentTokens) { + + if (currentTokens.getRefreshToken() != null) { + return ImmutableRefreshTokenFlow.builder().refreshToken(currentTokens.getRefreshToken()); + } + + boolean refreshWithTokenExchange = + config() + .tokenRefreshConfig() + .tokenExchangeEnabled() + .orElseGet( + () -> config().basicConfig().grantType().equals(GrantType.CLIENT_CREDENTIALS)); + if (refreshWithTokenExchange) { + return ImmutableTokenExchangeFlow.builder() + .subjectTokenStage(CompletableFuture.completedFuture(currentTokens.getAccessToken())) + .actorTokenStage(CompletableFuture.completedFuture(null)); + } + + throw new IllegalStateException( + "Cannot create refresh token flow: no refresh token present and token exchange is disabled"); + } +} diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/flow/RefreshTokenFlow.java b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/flow/RefreshTokenFlow.java new file mode 100644 index 000000000000..7188b33f83ac --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/flow/RefreshTokenFlow.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.flow; + +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import com.nimbusds.oauth2.sdk.AccessTokenResponse; +import com.nimbusds.oauth2.sdk.GrantType; +import com.nimbusds.oauth2.sdk.RefreshTokenGrant; +import com.nimbusds.oauth2.sdk.token.RefreshToken; +import com.nimbusds.oauth2.sdk.token.Tokens; +import java.util.concurrent.CompletionStage; +import org.immutables.value.Value; + +/** + * An implementation of the Token + * Refresh flow. + */ +@Value.Immutable +abstract class RefreshTokenFlow extends BaseFlow { + + interface Builder extends BaseFlow.Builder { + + @CanIgnoreReturnValue + Builder refreshToken(RefreshToken refreshToken); + } + + @Override + public final GrantType grantType() { + return GrantType.REFRESH_TOKEN; + } + + abstract RefreshToken refreshToken(); + + @Override + public CompletionStage execute() { + return invokeTokenEndpoint(new RefreshTokenGrant(refreshToken())); + } + + @Override + TokensResult toTokensResult(AccessTokenResponse response) { + Tokens tokens = response.toSuccessResponse().getTokens(); + // if the server doesn't return a new refresh token, + // this means the current one is still valid, so we can reuse it + if (tokens.getRefreshToken() == null) { + tokens = new Tokens(tokens.getAccessToken(), refreshToken()); + } + + return TokensResult.of(tokens, runtime().clock()); + } +} diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/flow/TokenExchangeFlow.java b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/flow/TokenExchangeFlow.java new file mode 100644 index 000000000000..3c8a2a5c0c65 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/flow/TokenExchangeFlow.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.flow; + +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import com.nimbusds.oauth2.sdk.AuthorizationGrant; +import com.nimbusds.oauth2.sdk.GrantType; +import com.nimbusds.oauth2.sdk.TokenRequest; +import com.nimbusds.oauth2.sdk.token.AccessToken; +import com.nimbusds.oauth2.sdk.token.TokenTypeURI; +import com.nimbusds.oauth2.sdk.tokenexchange.TokenExchangeGrant; +import java.net.URI; +import java.util.Objects; +import java.util.concurrent.CompletionStage; +import javax.annotation.Nullable; +import org.apache.iceberg.rest.auth.oauth2.config.TokenExchangeConfig; +import org.immutables.value.Value; + +/** + * An implementation of the Token + * Exchange flow. + */ +@Value.Immutable +abstract class TokenExchangeFlow extends BaseFlow { + + interface Builder extends BaseFlow.Builder { + @CanIgnoreReturnValue + Builder subjectTokenStage(CompletionStage subjectTokenStage); + + @CanIgnoreReturnValue + Builder actorTokenStage(CompletionStage actorTokenStage); + } + + @Override + public final GrantType grantType() { + return GrantType.TOKEN_EXCHANGE; + } + + abstract CompletionStage subjectTokenStage(); + + abstract CompletionStage actorTokenStage(); + + @Override + public CompletionStage execute() { + return subjectTokenStage() + .thenCombine( + actorTokenStage(), + (subjectToken, actorToken) -> { + Objects.requireNonNull( + subjectToken, "Cannot execute token exchange: missing required subject token"); + return newTokenExchangeGrant( + subjectToken, actorToken, config().tokenExchangeConfig()); + }) + .thenCompose(this::invokeTokenEndpoint); + } + + @Override + TokenRequest.Builder newTokenRequestBuilder(AuthorizationGrant grant) { + return super.newTokenRequestBuilder(grant) + .resources(config().tokenExchangeConfig().resources().toArray(URI[]::new)); + } + + private TokenExchangeGrant newTokenExchangeGrant( + AccessToken subjectToken, + @Nullable AccessToken actorToken, + TokenExchangeConfig tokenExchangeConfig) { + TokenTypeURI subjectTokenType = + tokenType(subjectToken, config().tokenExchangeConfig().subjectTokenType().orElse(null)); + TokenTypeURI actorTokenType = + actorToken == null + ? null + : tokenType(actorToken, config().tokenExchangeConfig().actorTokenType().orElse(null)); + return new TokenExchangeGrant( + subjectToken, + subjectTokenType, + actorToken, + actorTokenType, + tokenExchangeConfig.requestedTokenType().orElse(null), + tokenExchangeConfig.audiences()); + } + + private static TokenTypeURI tokenType(AccessToken token, @Nullable TokenTypeURI tokenType) { + return tokenType != null + ? tokenType + : token.getIssuedTokenType() != null + ? token.getIssuedTokenType() + : TokenTypeURI.ACCESS_TOKEN; + } +} diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/flow/TokensResult.java b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/flow/TokensResult.java new file mode 100644 index 000000000000..dd543a6e0418 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/flow/TokensResult.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.flow; + +import com.nimbusds.jwt.JWTParser; +import com.nimbusds.oauth2.sdk.AccessTokenResponse; +import com.nimbusds.oauth2.sdk.token.AccessToken; +import com.nimbusds.oauth2.sdk.token.Token; +import com.nimbusds.oauth2.sdk.token.Tokens; +import java.time.Clock; +import java.time.Instant; +import java.util.Date; +import java.util.Optional; +import org.immutables.value.Value; + +/** + * The result of a successful token request, capturing the {@linkplain Tokens issued tokens} and the + * time they were received, thus allowing the calculation of their expiration time. + */ +@Value.Immutable +public abstract class TokensResult { + + /** Creates a new {@link TokensResult} from a static access token and a clock. */ + public static TokensResult of(AccessToken token, Clock clock) { + return of(new Tokens(token, null), clock); + } + + /** Creates a new {@link TokensResult} from an {@link AccessTokenResponse} and a clock. */ + public static TokensResult of(AccessTokenResponse response, Clock clock) { + return of(response.toSuccessResponse().getTokens(), clock); + } + + /** + * Creates a new {@link TokensResult} from a {@link Tokens} object, custom parameters, and a + * clock. + */ + public static TokensResult of(Tokens tokens, Clock clock) { + return ImmutableTokensResult.builder().tokens(tokens).clock(clock).build(); + } + + /** The issued tokens. */ + public abstract Tokens tokens(); + + /** + * The time when this result was received (i.e. the local clock time at construction). This is + * used for computing expiration from the OAuth2 {@code expires_in} field, which is relative to + * the response time, not the JWT {@code iat} claim. + */ + @Value.Derived + public Instant receivedAt() { + return clock().instant(); + } + + /** + * The resolved expiration time of the access token, taking into account the response's {@code + * expires_in} field and the JWT claims, if applicable. + */ + @Value.Lazy + public Optional accessTokenExpirationTime() { + return accessTokenResponseExpirationTime().or(this::accessTokenJwtExpirationTime); + } + + /** The clock used to determine the current time. Not exposed publicly. */ + @Value.Auxiliary + abstract Clock clock(); + + /** The access token expiration time as reported in the token response, if any. */ + @Value.Lazy + Optional accessTokenResponseExpirationTime() { + long lifetimeSeconds = tokens().getAccessToken().getLifetime(); + // Note: we don't rely on the JWT's "iat" claim because it was set + // by the server's clock and hence may be off from our own clock. + return lifetimeSeconds > 0 + ? Optional.of(receivedAt().plusSeconds(lifetimeSeconds)) + : Optional.empty(); + } + + /** + * The access token JWT token expiration time, if the token is a JWT token and contains an + * expiration ("exp") claim. + */ + @Value.Lazy + Optional accessTokenJwtExpirationTime() { + Token token = tokens().getAccessToken(); + try { + Date expirationTime = JWTParser.parse(token.getValue()).getJWTClaimsSet().getExpirationTime(); + return Optional.ofNullable(expirationTime).map(Date::toInstant); + } catch (Exception ignored) { + return Optional.empty(); + } + } +} diff --git a/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/http/RESTClientAdapter.java b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/http/RESTClientAdapter.java new file mode 100644 index 000000000000..1be311828d33 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/rest/auth/oauth2/http/RESTClientAdapter.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.http; + +import com.nimbusds.oauth2.sdk.http.HTTPRequestSender; +import com.nimbusds.oauth2.sdk.http.HTTPResponse; +import com.nimbusds.oauth2.sdk.http.ReadOnlyHTTPRequest; +import com.nimbusds.oauth2.sdk.http.ReadOnlyHTTPResponse; +import java.io.IOException; +import java.util.function.Supplier; +import org.apache.hc.client5.http.classic.HttpClient; +import org.apache.hc.core5.http.Header; +import org.apache.hc.core5.http.ParseException; +import org.apache.hc.core5.http.io.entity.EntityUtils; +import org.apache.hc.core5.http.io.entity.StringEntity; +import org.apache.hc.core5.http.message.BasicClassicHttpRequest; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.rest.RESTClient; +import org.apache.iceberg.rest.auth.AuthSession; + +/** + * An adapter that allows using an Iceberg {@link RESTClient} as a Nimbus {@link HTTPRequestSender}. + * + *

    Note: this adapter is only compatible with {@link RESTClient} instances that unwrap to an + * Apache {@link HttpClient}, e.g. {@link org.apache.iceberg.rest.HTTPClient}. + */ +public class RESTClientAdapter implements HTTPRequestSender { + + private final Supplier restClientSupplier; + + public RESTClientAdapter(Supplier restClientSupplier) { + this.restClientSupplier = restClientSupplier; + } + + @Override + public ReadOnlyHTTPResponse send(ReadOnlyHTTPRequest httpRequest) throws IOException { + + RESTClient restClient = restClientSupplier.get().withAuthSession(AuthSession.EMPTY); + + HttpClient httpClient = + restClient + .unwrap(HttpClient.class) + .orElseThrow( + () -> + new IllegalArgumentException( + "RESTClient does not unwrap to Apache HttpClient")); + + BasicClassicHttpRequest request = + new BasicClassicHttpRequest(httpRequest.getMethod().name(), httpRequest.getURI()); + + httpRequest.getHeaderMap().forEach((k, v) -> v.forEach(vv -> request.addHeader(k, vv))); + + String requestBody = httpRequest.getBody(); + if (requestBody != null) { + request.setEntity(new StringEntity(requestBody)); + } + + return httpClient.execute( + request, + response -> { + HTTPResponse httpResponse = new HTTPResponse(response.getCode()); + + if (response.getEntity() != null) { + try { + String body = EntityUtils.toString(response.getEntity()); + if (!body.isEmpty()) { + httpResponse.setBody(body); + } + } catch (ParseException e) { + throw new IOException(e); + } + } + + for (Header header : response.getHeaders()) { + httpResponse + .getHeaderMap() + .computeIfAbsent(header.getName(), k -> Lists.newArrayList()) + .add(header.getValue()); + } + + return httpResponse; + }); + } +} diff --git a/core/src/main/java/org/apache/iceberg/rest/responses/OAuthErrorResponseParser.java b/core/src/main/java/org/apache/iceberg/rest/responses/OAuthErrorResponseParser.java index 9fa6051e2578..180038f99084 100644 --- a/core/src/main/java/org/apache/iceberg/rest/responses/OAuthErrorResponseParser.java +++ b/core/src/main/java/org/apache/iceberg/rest/responses/OAuthErrorResponseParser.java @@ -22,6 +22,11 @@ import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.util.JsonUtil; +/** + * @deprecated will be removed in 1.14.0, use {@link + * org.apache.iceberg.rest.auth.oauth2.OAuth2Manager} instead. + */ +@Deprecated public class OAuthErrorResponseParser { private OAuthErrorResponseParser() {} diff --git a/core/src/test/java/org/apache/iceberg/rest/TestHTTPClient.java b/core/src/test/java/org/apache/iceberg/rest/TestHTTPClient.java index 8cf97bca32ef..a4d7832301f4 100644 --- a/core/src/test/java/org/apache/iceberg/rest/TestHTTPClient.java +++ b/core/src/test/java/org/apache/iceberg/rest/TestHTTPClient.java @@ -42,6 +42,7 @@ import java.util.function.Consumer; import org.apache.hc.client5.http.auth.AuthScope; import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; +import org.apache.hc.client5.http.classic.HttpClient; import org.apache.hc.client5.http.config.ConnectionConfig; import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager; @@ -459,6 +460,22 @@ public void testCloseChild() throws IOException { .doesNotThrowAnyException(); } + @Test + public void testUnwrap() { + assertThat(restClient.unwrap(RESTClient.class)) + .as("HTTPClient should unwrap to RESTClient") + .isPresent(); + assertThat(restClient.unwrap(HTTPClient.class)) + .as("HTTPClient should unwrap to itself") + .isPresent(); + assertThat(restClient.unwrap(HttpClient.class)) + .as("HTTPClient should unwrap to its underlying Apache HttpClient") + .isPresent(); + assertThat(restClient.unwrap(AuthSession.class)) + .as("HTTPClient should not unwrap to unrelated types") + .isNotPresent(); + } + @ParameterizedTest @EnumSource(HttpMethod.class) public void testRetryIdemmpotentMethods(HttpMethod method) throws JsonProcessingException { diff --git a/core/src/test/java/org/apache/iceberg/rest/auth/TestAuthManagers.java b/core/src/test/java/org/apache/iceberg/rest/auth/TestAuthManagers.java index d49f398d7a47..0005c7b3881f 100644 --- a/core/src/test/java/org/apache/iceberg/rest/auth/TestAuthManagers.java +++ b/core/src/test/java/org/apache/iceberg/rest/auth/TestAuthManagers.java @@ -44,18 +44,64 @@ public void after() { } @Test - void oauth2Explicit() { + void oauth2NewExplicitByShortName() { + try (AuthManager manager = + AuthManagers.loadAuthManager( + "test", + Map.of(AuthProperties.AUTH_TYPE, AuthProperties.AUTH_MANAGER_IMPL_OAUTH2_NEW))) { + assertThat(manager).isInstanceOf(org.apache.iceberg.rest.auth.oauth2.OAuth2Manager.class); + } + assertThat(streamCaptor.toString()) + .contains( + "Loading AuthManager implementation: org.apache.iceberg.rest.auth.oauth2.OAuth2Manager"); + } + + @Test + void oauth2NewExplicitByFQCN() { + try (AuthManager manager = + AuthManagers.loadAuthManager( + "test", + Map.of( + AuthProperties.AUTH_TYPE, + org.apache.iceberg.rest.auth.oauth2.OAuth2Manager.class.getName()))) { + assertThat(manager).isInstanceOf(org.apache.iceberg.rest.auth.oauth2.OAuth2Manager.class); + } + assertThat(streamCaptor.toString()) + .contains( + "Loading AuthManager implementation: org.apache.iceberg.rest.auth.oauth2.OAuth2Manager"); + } + + @Test + void oauth2LegacyExplicitByShortName() { try (AuthManager manager = AuthManagers.loadAuthManager( "test", Map.of(AuthProperties.AUTH_TYPE, AuthProperties.AUTH_TYPE_OAUTH2))) { assertThat(manager).isInstanceOf(OAuth2Manager.class); } assertThat(streamCaptor.toString()) - .contains("Loading AuthManager implementation: org.apache.iceberg.rest.auth.OAuth2Manager"); + .contains( + "The AuthManager implementation org.apache.iceberg.rest.auth.OAuth2Manager " + + "is deprecated and will be removed in a future release. " + + "Please migrate to org.apache.iceberg.rest.auth.oauth2.OAuth2Manager."); + } + + @Test + void oauth2LegacyExplicitByFQCN() { + try (AuthManager manager = + AuthManagers.loadAuthManager( + "test", + Map.of(AuthProperties.AUTH_TYPE, AuthProperties.AUTH_MANAGER_IMPL_OAUTH2_LEGACY))) { + assertThat(manager).isInstanceOf(OAuth2Manager.class); + } + assertThat(streamCaptor.toString()) + .contains( + "The AuthManager implementation org.apache.iceberg.rest.auth.OAuth2Manager " + + "is deprecated and will be removed in a future release. " + + "Please migrate to org.apache.iceberg.rest.auth.oauth2.OAuth2Manager."); } @Test - void oauth2InferredFromToken() { + void oauth2LegacyInferredFromToken() { try (AuthManager manager = AuthManagers.loadAuthManager("test", Map.of(OAuth2Properties.TOKEN, "irrelevant"))) { assertThat(manager).isInstanceOf(OAuth2Manager.class); @@ -65,11 +111,14 @@ void oauth2InferredFromToken() { "Inferring rest.auth.type=oauth2 since property token was provided. " + "Please explicitly set rest.auth.type to avoid this warning."); assertThat(streamCaptor.toString()) - .contains("Loading AuthManager implementation: org.apache.iceberg.rest.auth.OAuth2Manager"); + .contains( + "The AuthManager implementation org.apache.iceberg.rest.auth.OAuth2Manager " + + "is deprecated and will be removed in a future release. " + + "Please migrate to org.apache.iceberg.rest.auth.oauth2.OAuth2Manager."); } @Test - void oauth2InferredFromCredential() { + void oauth2LegacyInferredFromCredential() { try (AuthManager manager = AuthManagers.loadAuthManager("test", Map.of(OAuth2Properties.CREDENTIAL, "irrelevant"))) { assertThat(manager).isInstanceOf(OAuth2Manager.class); @@ -79,7 +128,10 @@ void oauth2InferredFromCredential() { "Inferring rest.auth.type=oauth2 since property credential was provided. " + "Please explicitly set rest.auth.type to avoid this warning."); assertThat(streamCaptor.toString()) - .contains("Loading AuthManager implementation: org.apache.iceberg.rest.auth.OAuth2Manager"); + .contains( + "The AuthManager implementation org.apache.iceberg.rest.auth.OAuth2Manager " + + "is deprecated and will be removed in a future release. " + + "Please migrate to org.apache.iceberg.rest.auth.oauth2.OAuth2Manager."); } @Test diff --git a/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/TestOAuth2Config.java b/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/TestOAuth2Config.java new file mode 100644 index 000000000000..a649a5aa9722 --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/TestOAuth2Config.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.nimbusds.oauth2.sdk.GrantType; +import com.nimbusds.oauth2.sdk.Scope; +import com.nimbusds.oauth2.sdk.auth.Secret; +import com.nimbusds.oauth2.sdk.id.ClientID; +import java.net.URI; +import java.time.Duration; +import java.util.Map; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.rest.auth.oauth2.config.BasicConfig; +import org.apache.iceberg.rest.auth.oauth2.config.TokenExchangeConfig; +import org.apache.iceberg.rest.auth.oauth2.config.TokenRefreshConfig; +import org.junit.jupiter.api.Test; + +class TestOAuth2Config { + + @Test + void testFromProperties() { + Map properties = + ImmutableMap.builder() + .put(BasicConfig.TOKEN_ENDPOINT, "https://example.com/token") + .put(BasicConfig.CLIENT_ID, "Client") + .put(BasicConfig.CLIENT_SECRET, "w00t") + .put(BasicConfig.GRANT_TYPE, GrantType.TOKEN_EXCHANGE.getValue()) + .put(BasicConfig.SCOPE, "test") + .put(BasicConfig.EXTRA_PARAMS + ".key1", "value1") + .put(BasicConfig.EXTRA_PARAMS + ".key2", "value2") + .put(TokenRefreshConfig.SAFETY_MARGIN, "PT20S") + .put(TokenExchangeConfig.SUBJECT_TOKEN, "subject-token") + .build(); + OAuth2Config config = OAuth2Config.of(properties); + assertThat(config).isNotNull(); + assertThat(config.basicConfig()).isNotNull(); + assertThat(config.basicConfig().tokenEndpoint()) + .contains(URI.create("https://example.com/token")); + assertThat(config.basicConfig().grantType()).isEqualTo(GrantType.TOKEN_EXCHANGE); + assertThat(config.basicConfig().clientId()).contains(new ClientID("Client")); + assertThat(config.basicConfig().clientSecret()).contains(new Secret("w00t")); + assertThat(config.basicConfig().scope()).contains(new Scope("test")); + assertThat(config.basicConfig().extraRequestParameters()) + .isEqualTo(Map.of("key1", "value1", "key2", "value2")); + assertThat(config.tokenRefreshConfig()).isNotNull(); + assertThat(config.tokenRefreshConfig().safetyMargin()).isEqualTo(Duration.ofSeconds(20)); + assertThat(config.tokenExchangeConfig()).isNotNull(); + assertThat(config.tokenExchangeConfig().subjectTokenString()).contains("subject-token"); + } + + @Test + void testValidate() { + Map properties = + Map.of( + BasicConfig.GRANT_TYPE, + GrantType.TOKEN_EXCHANGE.getValue(), + BasicConfig.TOKEN_ENDPOINT, + "https://example.com/token", + BasicConfig.CLIENT_ID, + "Client1", + BasicConfig.CLIENT_SECRET, + "s3cr3t"); + assertThatThrownBy(() -> OAuth2Config.of(properties)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining( + "subject token must be set if grant type is 'urn:ietf:params:oauth:grant-type:token-exchange' (rest.auth.oauth2.token-exchange.subject-token)"); + } +} diff --git a/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/TestOAuth2Manager.java b/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/TestOAuth2Manager.java new file mode 100644 index 000000000000..0be4b8e96d5d --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/TestOAuth2Manager.java @@ -0,0 +1,640 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.InstanceOfAssertFactories.type; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import com.github.benmanes.caffeine.cache.Cache; +import com.nimbusds.oauth2.sdk.GrantType; +import com.nimbusds.oauth2.sdk.auth.ClientAuthenticationMethod; +import com.nimbusds.oauth2.sdk.token.TokenTypeURI; +import java.io.IOException; +import java.net.URI; +import java.util.Map; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.Table; +import org.apache.iceberg.catalog.SessionCatalog.SessionContext; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.rest.HTTPHeaders.HTTPHeader; +import org.apache.iceberg.rest.HTTPRequest; +import org.apache.iceberg.rest.HTTPRequest.HTTPMethod; +import org.apache.iceberg.rest.ImmutableHTTPRequest; +import org.apache.iceberg.rest.RESTCatalog; +import org.apache.iceberg.rest.auth.AuthSession; +import org.apache.iceberg.rest.auth.OAuth2Properties; +import org.apache.iceberg.rest.auth.oauth2.config.BasicConfig; +import org.apache.iceberg.rest.auth.oauth2.test.TestEnvironment; +import org.apache.iceberg.rest.auth.oauth2.test.junit.EnumLike; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junitpioneer.jupiter.cartesian.CartesianTest; + +/** + * Tests for {@link OAuth2Manager}. + * + *

    The tests in this class are split into two categories: + * + *

      + *
    • Unit tests that instantiate an {@link OAuth2Manager} directly and exercise its public API. + *
    • "Catalog" tests that instantiate a full {@link RESTCatalog} embedding an {@link + * OAuth2Manager}. These tests focus on the interaction between {@link RESTCatalog} and + * {@link OAuth2Manager}. Complete tests for {@link RESTCatalog} functionality can be + * found in {@link TestOAuth2RESTCatalog}. + *
    + * + * @see TestOAuth2RESTCatalog + */ +class TestOAuth2Manager { + + /** Tests that instantiate an {@link OAuth2Manager} directly. */ + @Nested + class UnitTests { + + private final TableIdentifier table = TableIdentifier.of("t1"); + + private final HTTPRequest request = + ImmutableHTTPRequest.builder() + .baseUri(URI.create("http://localhost:8181")) + .method(HTTPMethod.GET) + .path("v1/config") + .build(); + + @Test + void catalogSessionWithoutInit() { + try (TestEnvironment env = TestEnvironment.builder().build(); + OAuth2Manager manager = new OAuth2Manager("test"); + AuthSession session = manager.catalogSession(env.httpClient(), env.catalogProperties())) { + HTTPRequest actual = session.authenticate(request); + assertThat(actual.headers().entries("Authorization")) + .containsOnly(HTTPHeader.of("Authorization", "Bearer access_initial")); + } + } + + @Test + void catalogSessionWithInit() { + try (TestEnvironment env = TestEnvironment.builder().build(); + OAuth2Manager manager = new OAuth2Manager("test")) { + try (AuthSession session = manager.initSession(env.httpClient(), env.catalogProperties())) { + HTTPRequest actual = session.authenticate(request); + assertThat(actual.headers().entries("Authorization")) + .containsOnly(HTTPHeader.of("Authorization", "Bearer access_initial")); + } + + try (AuthSession session = + manager.catalogSession(env.httpClient(), env.catalogProperties())) { + HTTPRequest actual = session.authenticate(request); + assertThat(actual.headers().entries("Authorization")) + .containsOnly(HTTPHeader.of("Authorization", "Bearer access_initial")); + } + } + } + + @Test + void contextualSessionEmpty() { + SessionContext context = SessionContext.createEmpty(); + try (TestEnvironment env = TestEnvironment.builder().build(); + OAuth2Manager manager = new OAuth2Manager("test"); + AuthSession catalogSession = + manager.catalogSession(env.httpClient(), env.catalogProperties()); + AuthSession contextualSession = manager.contextualSession(context, catalogSession)) { + catalogSession.authenticate(request); + assertThat(contextualSession).isSameAs(catalogSession); + HTTPRequest actual = contextualSession.authenticate(request); + assertThat(actual.headers().entries("Authorization")) + .containsOnly(HTTPHeader.of("Authorization", "Bearer access_initial")); + } + } + + @Test + void contextualSessionNotCached() { + try (TestEnvironment env = TestEnvironment.builder().build(); + OAuth2Manager manager = new OAuth2Manager("test")) { + // Identical to catalog properties, so should not be cached + SessionContext context = + new SessionContext("test", "test", env.catalogProperties(), Map.of()); + try (AuthSession catalogSession = + manager.catalogSession(env.httpClient(), env.catalogProperties()); + AuthSession contextualSession = manager.contextualSession(context, catalogSession)) { + catalogSession.authenticate(request); + assertThat(contextualSession).isSameAs(catalogSession); + HTTPRequest actual = contextualSession.authenticate(request); + assertThat(actual.headers().entries("Authorization")) + .containsOnly(HTTPHeader.of("Authorization", "Bearer access_initial")); + } + } + } + + @Test + void contextualSessionCacheHit() { + try (TestEnvironment env = TestEnvironment.builder().build(); + OAuth2Manager manager = new OAuth2Manager("test")) { + SessionContext context = + new SessionContext( + "test", + "test", + Map.of( + BasicConfig.TOKEN_ENDPOINT, + env.tokenEndpoint().toString(), + BasicConfig.CLIENT_ID, + TestEnvironment.CLIENT_ID1.getValue(), + BasicConfig.CLIENT_SECRET, + TestEnvironment.CLIENT_SECRET1.getValue()), + Map.of(BasicConfig.EXTRA_PARAMS + ".extra2", "value2")); + try (AuthSession catalogSession = + manager.catalogSession(env.httpClient(), env.catalogProperties()); + AuthSession contextualSession1 = manager.contextualSession(context, catalogSession); + AuthSession contextualSession2 = manager.contextualSession(context, catalogSession)) { + catalogSession.authenticate(request); + assertThat(contextualSession1).isNotSameAs(catalogSession); + assertThat(contextualSession2).isNotSameAs(catalogSession); + assertThat(contextualSession1).isSameAs(contextualSession2); + HTTPRequest actual = contextualSession1.authenticate(request); + assertThat(actual.headers().entries("Authorization")) + .containsOnly(HTTPHeader.of("Authorization", "Bearer access_initial")); + } + } + } + + @Test + void contextualSessionCacheMiss() { + try (TestEnvironment env = TestEnvironment.builder().build(); + OAuth2Manager manager = new OAuth2Manager("test")) { + SessionContext context1 = + new SessionContext( + "test1", + "test", + Map.of( + BasicConfig.CLIENT_ID, + TestEnvironment.CLIENT_ID1.getValue(), + BasicConfig.CLIENT_SECRET, + TestEnvironment.CLIENT_SECRET1.getValue()), + Map.of( + BasicConfig.TOKEN_ENDPOINT, + env.tokenEndpoint().toString(), + BasicConfig.SCOPE, + TestEnvironment.SCOPE1.toString(), + BasicConfig.EXTRA_PARAMS + ".extra2", + "value2")); + SessionContext context2 = + new SessionContext( + "test2", + "test", + Map.of( + BasicConfig.CLIENT_ID, + TestEnvironment.CLIENT_ID2.getValue(), + BasicConfig.CLIENT_SECRET, + TestEnvironment.CLIENT_SECRET2.getValue()), + Map.of( + BasicConfig.TOKEN_ENDPOINT, + env.tokenEndpoint().toString(), + BasicConfig.SCOPE, + TestEnvironment.SCOPE2.toString(), + BasicConfig.EXTRA_PARAMS + ".extra3", + "value3")); + try (AuthSession catalogSession = + manager.catalogSession(env.httpClient(), env.catalogProperties()); + AuthSession contextualSession1 = manager.contextualSession(context1, catalogSession); + AuthSession contextualSession2 = manager.contextualSession(context2, catalogSession)) { + catalogSession.authenticate(request); + assertThat(contextualSession1).isNotSameAs(catalogSession); + assertThat(contextualSession2).isNotSameAs(catalogSession); + assertThat(contextualSession1).isNotSameAs(contextualSession2); + HTTPRequest actual = contextualSession1.authenticate(request); + assertThat(actual.headers().entries("Authorization")) + .containsOnly(HTTPHeader.of("Authorization", "Bearer access_initial")); + actual = contextualSession2.authenticate(request); + assertThat(actual.headers().entries("Authorization")) + .containsOnly(HTTPHeader.of("Authorization", "Bearer access_initial")); + } + } + } + + @Test + @SuppressWarnings("deprecation") + void contextualSessionLegacyProperties() { + try (TestEnvironment env = TestEnvironment.builder().build(); + OAuth2Manager manager = new OAuth2Manager("test")) { + SessionContext context = + new SessionContext( + "test", + "test", + Map.of( + OAuth2Properties.OAUTH2_SERVER_URI, + env.tokenEndpoint().toString(), + OAuth2Properties.CREDENTIAL, + TestEnvironment.CLIENT_ID2.getValue() + + ":" + + TestEnvironment.CLIENT_SECRET2.getValue(), + BasicConfig.EXTRA_PARAMS + ".extra2", + "value2"), + Map.of(OAuth2Properties.SCOPE, TestEnvironment.SCOPE2.toString())); + try (AuthSession catalogSession = + manager.catalogSession(env.httpClient(), env.catalogProperties()); + AuthSession contextualSession = manager.contextualSession(context, catalogSession)) { + catalogSession.authenticate(request); + assertThat(contextualSession).isNotSameAs(catalogSession); + HTTPRequest actual = contextualSession.authenticate(request); + assertThat(actual.headers().entries("Authorization")) + .containsOnly(HTTPHeader.of("Authorization", "Bearer access_initial")); + } + } + } + + @Test + @SuppressWarnings("deprecation") + void contextualSessionLegacyPropertiesToken() { + try (TestEnvironment env = TestEnvironment.builder().build(); + OAuth2Manager manager = new OAuth2Manager("test")) { + SessionContext context = + new SessionContext( + "test", + "test", + Map.of( + OAuth2Properties.OAUTH2_SERVER_URI, + env.tokenEndpoint().toString(), + OAuth2Properties.TOKEN, + "access_context", + BasicConfig.EXTRA_PARAMS + ".extra2", + "value2"), + Map.of(OAuth2Properties.SCOPE, TestEnvironment.SCOPE2.toString())); + try (AuthSession catalogSession = + manager.catalogSession(env.httpClient(), env.catalogProperties()); + AuthSession contextualSession = manager.contextualSession(context, catalogSession)) { + catalogSession.authenticate(request); + assertThat(contextualSession).isNotSameAs(catalogSession); + HTTPRequest actual = contextualSession.authenticate(request); + assertThat(actual.headers().entries("Authorization")) + .containsOnly(HTTPHeader.of("Authorization", "Bearer access_context")); + } + } + } + + @Test + @SuppressWarnings("deprecation") + void contextualSessionLegacyPropertiesTokenExchange() { + try (TestEnvironment env = + TestEnvironment.builder() + .grantType(GrantType.TOKEN_EXCHANGE) + .subjectTokenType(TokenTypeURI.ACCESS_TOKEN) + .build(); + OAuth2Manager manager = new OAuth2Manager("test")) { + SessionContext context = + new SessionContext( + "test", + "test", + Map.of( + OAuth2Properties.OAUTH2_SERVER_URI, + env.tokenEndpoint().toString(), + OAuth2Properties.ACCESS_TOKEN_TYPE, + "access_context", + BasicConfig.EXTRA_PARAMS + ".extra2", + "value2"), + Map.of(OAuth2Properties.SCOPE, TestEnvironment.SCOPE2.toString())); + try (AuthSession catalogSession = + manager.catalogSession(env.httpClient(), env.catalogProperties()); + AuthSession contextualSession = manager.contextualSession(context, catalogSession)) { + catalogSession.authenticate(request); + assertThat(contextualSession).isNotSameAs(catalogSession); + HTTPRequest actual = contextualSession.authenticate(request); + assertThat(actual.headers().entries("Authorization")) + // access_initial is the exchanged token + .containsOnly(HTTPHeader.of("Authorization", "Bearer access_initial")); + } + } + } + + @Test + void tableSessionEmpty() { + try (TestEnvironment env = TestEnvironment.builder().build(); + OAuth2Manager manager = new OAuth2Manager("test")) { + Map tableProperties = Map.of(); + try (AuthSession catalogSession = + manager.catalogSession(env.httpClient(), env.catalogProperties()); + AuthSession tableSession = + manager.tableSession(table, tableProperties, catalogSession)) { + catalogSession.authenticate(request); + assertThat(tableSession).isSameAs(catalogSession); + HTTPRequest actual = tableSession.authenticate(request); + assertThat(actual.headers().entries("Authorization")) + .containsOnly(HTTPHeader.of("Authorization", "Bearer access_initial")); + } + } + } + + @Test + void tableSessionNotCached() { + try (TestEnvironment env = TestEnvironment.builder().build(); + OAuth2Manager manager = new OAuth2Manager("test")) { + Map catalogProperties = + Map.of( + CatalogProperties.URI, + env.catalogServerUrl().toString(), + BasicConfig.TOKEN, + "token"); + // Virtually identical to catalog properties, so should not be cached + Map tableProperties = Map.of(BasicConfig.TOKEN, "token"); + try (AuthSession catalogSession = + manager.catalogSession(env.httpClient(), catalogProperties); + AuthSession tableSession = + manager.tableSession(table, tableProperties, catalogSession)) { + catalogSession.authenticate(request); + assertThat(tableSession).isSameAs(catalogSession); + HTTPRequest actual = tableSession.authenticate(request); + assertThat(actual.headers().entries("Authorization")) + .containsOnly(HTTPHeader.of("Authorization", "Bearer token")); + } + } + } + + @Test + void tableSessionCacheHit() { + try (TestEnvironment env = TestEnvironment.builder().build(); + OAuth2Manager manager = new OAuth2Manager("test")) { + Map tableProperties = Map.of(BasicConfig.TOKEN, "token"); + try (AuthSession catalogSession = + manager.catalogSession(env.httpClient(), env.catalogProperties()); + AuthSession tableSession1 = + manager.tableSession(table, tableProperties, catalogSession); + AuthSession tableSession2 = + manager.tableSession(table, tableProperties, catalogSession)) { + catalogSession.authenticate(request); + assertThat(tableSession1).isNotSameAs(catalogSession); + assertThat(tableSession2).isNotSameAs(catalogSession); + assertThat(tableSession1).isSameAs(tableSession2); + HTTPRequest actual = tableSession1.authenticate(request); + assertThat(actual.headers().entries("Authorization")) + .containsOnly(HTTPHeader.of("Authorization", "Bearer token")); + } + } + } + + @Test + void tableSessionCacheMiss() { + try (TestEnvironment env = TestEnvironment.builder().build(); + OAuth2Manager manager = new OAuth2Manager("test")) { + Map tableProperties1 = Map.of(BasicConfig.TOKEN, "token1"); + Map tableProperties2 = Map.of(BasicConfig.TOKEN, "token2"); + try (AuthSession catalogSession = + manager.catalogSession(env.httpClient(), env.catalogProperties()); + AuthSession tableSession1 = + manager.tableSession(table, tableProperties1, catalogSession); + AuthSession tableSession2 = + manager.tableSession(table, tableProperties2, catalogSession)) { + catalogSession.authenticate(request); + assertThat(tableSession1).isNotSameAs(catalogSession); + assertThat(tableSession2).isNotSameAs(catalogSession); + assertThat(tableSession1).isNotSameAs(tableSession2); + HTTPRequest actual = tableSession1.authenticate(request); + assertThat(actual.headers().entries("Authorization")) + .containsOnly(HTTPHeader.of("Authorization", "Bearer token1")); + actual = tableSession2.authenticate(request); + assertThat(actual.headers().entries("Authorization")) + .containsOnly(HTTPHeader.of("Authorization", "Bearer token2")); + } + } + } + + @Test + @SuppressWarnings("deprecation") + void tableSessionLegacyPropertiesVendedToken() { + try (TestEnvironment env = TestEnvironment.builder().build(); + OAuth2Manager manager = new OAuth2Manager("test")) { + Map tableProperties = Map.of(OAuth2Properties.TOKEN, "access_vended"); + try (AuthSession catalogSession = + manager.catalogSession(env.httpClient(), env.catalogProperties()); + AuthSession tableSession = + manager.tableSession(table, tableProperties, catalogSession)) { + catalogSession.authenticate(request); + assertThat(tableSession).isNotSameAs(catalogSession); + HTTPRequest actual = tableSession.authenticate(request); + assertThat(actual.headers().entries("Authorization")) + .containsOnly(HTTPHeader.of("Authorization", "Bearer access_vended")); + } + } + } + + @Test + @SuppressWarnings("deprecation") + void tableSessionLegacyPropertiesVendedTokenExchange() { + try (TestEnvironment env = + TestEnvironment.builder().grantType(GrantType.TOKEN_EXCHANGE).build(); + OAuth2Manager manager = new OAuth2Manager("test")) { + Map tableProperties = + Map.of(OAuth2Properties.ACCESS_TOKEN_TYPE, "access_vended"); + try (AuthSession catalogSession = + manager.catalogSession(env.httpClient(), env.catalogProperties()); + AuthSession tableSession = + manager.tableSession(table, tableProperties, catalogSession)) { + catalogSession.authenticate(request); + assertThat(tableSession).isNotSameAs(catalogSession); + HTTPRequest actual = tableSession.authenticate(request); + assertThat(actual.headers().entries("Authorization")) + // access_initial is the exchanged token + .containsOnly(HTTPHeader.of("Authorization", "Bearer access_initial")); + } + } + } + + @Test + void standaloneTableSessionCacheMiss() { + try (TestEnvironment env = TestEnvironment.builder().build(); + OAuth2Manager manager = new OAuth2Manager("test")) { + Map standaloneProperties1 = + Map.of( + CatalogProperties.URI, + env.catalogServerUrl().toString(), + BasicConfig.TOKEN_ENDPOINT, + env.tokenEndpoint().toString(), + BasicConfig.CLIENT_ID, + TestEnvironment.CLIENT_ID1.getValue(), + BasicConfig.CLIENT_SECRET, + TestEnvironment.CLIENT_SECRET1.getValue(), + BasicConfig.SCOPE, + TestEnvironment.SCOPE1.toString(), + BasicConfig.EXTRA_PARAMS + ".extra1", + "value1"); + Map standaloneProperties2 = + Map.of( + CatalogProperties.URI, + env.catalogServerUrl().toString(), + BasicConfig.TOKEN_ENDPOINT, + env.tokenEndpoint().toString(), + BasicConfig.CLIENT_ID, + TestEnvironment.CLIENT_ID2.getValue(), + BasicConfig.CLIENT_SECRET, + TestEnvironment.CLIENT_SECRET2.getValue(), + BasicConfig.SCOPE, + TestEnvironment.SCOPE2.toString(), + BasicConfig.EXTRA_PARAMS + ".extra2", + "value2"); + try (AuthSession standaloneSession1 = + manager.tableSession(env.httpClient(), standaloneProperties1); + AuthSession standaloneSession2 = + manager.tableSession(env.httpClient(), standaloneProperties2)) { + assertThat(standaloneSession1).isNotSameAs(standaloneSession2); + HTTPRequest actual = standaloneSession1.authenticate(request); + assertThat(actual.headers().entries("Authorization")) + .containsOnly(HTTPHeader.of("Authorization", "Bearer access_initial")); + actual = standaloneSession2.authenticate(request); + assertThat(actual.headers().entries("Authorization")) + .containsOnly(HTTPHeader.of("Authorization", "Bearer access_initial")); + } + } + } + + @Test + void standaloneTableSessionCacheHit() { + try (TestEnvironment env = TestEnvironment.builder().build(); + OAuth2Manager manager = new OAuth2Manager("test")) { + Map standaloneProperties1 = + Map.of( + CatalogProperties.URI, + env.catalogServerUrl().toString(), + BasicConfig.TOKEN_ENDPOINT, + env.tokenEndpoint().toString(), + BasicConfig.CLIENT_ID, + TestEnvironment.CLIENT_ID1.getValue(), + BasicConfig.CLIENT_SECRET, + TestEnvironment.CLIENT_SECRET1.getValue(), + BasicConfig.SCOPE, + TestEnvironment.SCOPE1.toString(), + BasicConfig.EXTRA_PARAMS + ".extra1", + "value1"); + // Same OAuth2 config shared by 2 catalog servers => same OAuth2 session + Map standaloneProperties2 = + ImmutableMap.builder() + .putAll(standaloneProperties1) + .put(CatalogProperties.URI, "https://other.com") + .buildKeepingLast(); + try (AuthSession standaloneSession1 = + manager.tableSession(env.httpClient(), standaloneProperties1); + AuthSession standaloneSession2 = + manager.tableSession(env.httpClient(), standaloneProperties2)) { + assertThat(standaloneSession1).isSameAs(standaloneSession2); + HTTPRequest actual = standaloneSession1.authenticate(request); + assertThat(actual.headers().entries("Authorization")) + .containsOnly(HTTPHeader.of("Authorization", "Bearer access_initial")); + } + } + } + } + + /** + * Tests that instantiate a full {@link RESTCatalog} embedding an {@link OAuth2Manager}. + * + *

    These tests focus on the interaction between {@link RESTCatalog} and {@link OAuth2Manager}. + * Complete tests for {@link RESTCatalog} functionality can be found in {@link + * TestOAuth2RESTCatalog}. + * + * @see TestOAuth2RESTCatalog + */ + @Nested + class CatalogTests { + + private static final String BY_SESSION_ID_CACHE = "sessionCatalog.authManager.bySessionId"; + private static final String BY_CONFIG_CACHE = "sessionCatalog.authManager.byConfig"; + + @CartesianTest + void testCatalogProperties( + @EnumLike(excludes = "refresh_token") GrantType grantType, + @EnumLike ClientAuthenticationMethod authenticationMethod) + throws IOException { + assumeTrue( + !grantType.equals(GrantType.CLIENT_CREDENTIALS) + || !authenticationMethod.equals(ClientAuthenticationMethod.NONE)); + try (TestEnvironment env = + TestEnvironment.builder() + .grantType(grantType) + .clientAuthenticationMethod(authenticationMethod) + .sessionContext(SessionContext.createEmpty()) + .tableProperties(Map.of()) + .build(); + RESTCatalog catalog = env.newCatalog(Map.of())) { + Table table = catalog.loadTable(TestEnvironment.TABLE_IDENTIFIER); + assertThat(table).isNotNull(); + assertThat(table.name()).isEqualTo(catalog.name() + "." + TestEnvironment.TABLE_IDENTIFIER); + assertThat(cacheById(catalog)).isNull(); + assertThat(cacheByConfig(catalog)).isNull(); + } + } + + @Test + void testCatalogAndContextProperties() throws IOException { + try (TestEnvironment env = TestEnvironment.builder().tableProperties(Map.of()).build(); + RESTCatalog catalog = env.newCatalog(Map.of())) { + Table table = catalog.loadTable(TestEnvironment.TABLE_IDENTIFIER); + assertThat(table).isNotNull(); + assertThat(table.name()).isEqualTo(catalog.name() + "." + TestEnvironment.TABLE_IDENTIFIER); + assertThat(cacheById(catalog).asMap()) + .hasSize(1) + .containsKey(env.sessionContext().sessionId()); + assertThat(cacheByConfig(catalog)).isNull(); + } + } + + @Test + void testCatalogAndTableProperties() throws IOException { + try (TestEnvironment env = + TestEnvironment.builder() + .sessionContext(SessionContext.createEmpty()) + .tableProperties(Map.of(BasicConfig.TOKEN, "token")) + .build(); + RESTCatalog catalog = env.newCatalog(Map.of())) { + Table table = catalog.loadTable(TestEnvironment.TABLE_IDENTIFIER); + assertThat(table).isNotNull(); + assertThat(table.name()).isEqualTo(catalog.name() + "." + TestEnvironment.TABLE_IDENTIFIER); + assertThat(cacheById(catalog)).isNull(); + assertThat(cacheByConfig(catalog).asMap()).hasSize(1); + } + } + + @Test + void testCatalogAndContextAndTableProperties() throws IOException { + try (TestEnvironment env = TestEnvironment.builder().build(); + RESTCatalog catalog = env.newCatalog(Map.of())) { + Table table = catalog.loadTable(TestEnvironment.TABLE_IDENTIFIER); + assertThat(table).isNotNull(); + assertThat(table.name()).isEqualTo(catalog.name() + "." + TestEnvironment.TABLE_IDENTIFIER); + assertThat(cacheById(catalog).asMap()) + .hasSize(1) + .containsKey(env.sessionContext().sessionId()); + assertThat(cacheByConfig(catalog).asMap()).hasSize(1); + } + } + + private static Cache cacheById(RESTCatalog catalog) { + return assertThat(catalog) + .extracting("sessionCatalog.authManager") + .asInstanceOf(type(OAuth2Manager.class)) + .extracting(OAuth2Manager::cacheById) + .actual(); + } + + private static Cache cacheByConfig(RESTCatalog catalog) { + return assertThat(catalog) + .extracting("sessionCatalog.authManager") + .asInstanceOf(type(OAuth2Manager.class)) + .extracting(OAuth2Manager::cacheByConfig) + .actual(); + } + } +} diff --git a/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/TestOAuth2RESTCatalog.java b/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/TestOAuth2RESTCatalog.java new file mode 100644 index 000000000000..c2e74364e158 --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/TestOAuth2RESTCatalog.java @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.nimbusds.oauth2.sdk.GrantType; +import com.nimbusds.oauth2.sdk.token.AccessToken; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.nio.file.Path; +import java.util.Map; +import java.util.Optional; +import java.util.function.Consumer; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.Table; +import org.apache.iceberg.catalog.CatalogTests; +import org.apache.iceberg.exceptions.NotFoundException; +import org.apache.iceberg.inmemory.InMemoryCatalog; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.rest.HTTPHeaders; +import org.apache.iceberg.rest.HTTPRequest; +import org.apache.iceberg.rest.RESTCatalog; +import org.apache.iceberg.rest.RESTCatalogAdapter; +import org.apache.iceberg.rest.RESTCatalogServlet; +import org.apache.iceberg.rest.RESTResponse; +import org.apache.iceberg.rest.auth.oauth2.client.OAuth2Client; +import org.apache.iceberg.rest.auth.oauth2.config.ConfigUtil; +import org.apache.iceberg.rest.auth.oauth2.test.ImmutableTestEnvironment; +import org.apache.iceberg.rest.auth.oauth2.test.TestCertificates; +import org.apache.iceberg.rest.auth.oauth2.test.TestEnvironment; +import org.apache.iceberg.rest.responses.ErrorResponse; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * {@link CatalogTests} for {@link RESTCatalog} with {@link OAuth2Manager}. + * + *

    In these tests, OAuth2 is configured as follows: + * + *

      + *
    • Catalog session: uses client_credentials as an initial grant, and token exchange for token + * refreshes (legacy behavior). + *
    • Session context: uses token exchange as the initial grant. The subject token is obtained + * off-band with the client_credentials grant. The actor token comes from the parent catalog + * session. Token refreshes use the standard refresh_token grant. + *
    • The authorization server is configured to use HTTPS. + *
    + * + * The equivalent catalog configuration would be: + * + * 
    {@code
+ * rest.auth.type=oauth2
+ * rest.auth.oauth2.issuer-url=https://
+ * rest.auth.oauth2.grant-type=client_credentials
+ * rest.auth.oauth2.client-id=
+ * rest.auth.oauth2.client-secret=
+ * rest.auth.oauth2.scope=catalog
+ * }
    + * + * The equivalent session context configuration would be: + * + * 
    {@code
+ * rest.auth.oauth2.issuer-url=https://
+ * rest.auth.oauth2.grant-type=urn:ietf:params:oauth:grant-type:token-exchange
+ * rest.auth.oauth2.client-id=
+ * rest.auth.oauth2.client-secret=
+ * rest.auth.oauth2.scope=session
+ * rest.auth.oauth2.token-exchange.subject-token=
+ * rest.auth.oauth2.token-exchange.actor-token=::parent::
+ * }
    + */ +public class TestOAuth2RESTCatalog extends CatalogTests { + + @TempDir public static Path tempDir; + + private static InMemoryCatalog backendCatalog; + private static Server httpServer; + private static TestEnvironment testEnvironment; + + private RESTCatalog restCatalog; + + @BeforeAll + static void beforeClass() throws Exception { + backendCatalog = new InMemoryCatalog(); + backendCatalog.initialize( + "in-memory", + ImmutableMap.of( + CatalogProperties.WAREHOUSE_LOCATION, + tempDir.resolve("warehouse").toAbsolutePath().toString())); + + ServletContextHandler servletContext = + new ServletContextHandler(ServletContextHandler.NO_SESSIONS); + servletContext.addServlet( + new ServletHolder( + new RESTCatalogServlet( + new RESTCatalogAdapter(backendCatalog) { + @Override + public T execute( + HTTPRequest request, + Class responseType, + Consumer errorHandler, + Consumer> responseHeaders) { + Optional authHeader = + request.headers().firstEntry("Authorization"); + assertThat(authHeader.map(HTTPHeaders.HTTPHeader::value)) + .contains("Bearer access_initial"); + return super.execute(request, responseType, errorHandler, responseHeaders); + } + })), + "/*"); + + httpServer = new Server(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0)); + httpServer.setHandler(servletContext); + httpServer.start(); + + AccessToken subjectToken; + try (TestEnvironment env = + ImmutableTestEnvironment.builder() + .grantType(GrantType.CLIENT_CREDENTIALS) + .clientId(TestEnvironment.CLIENT_ID2) + .clientSecret(TestEnvironment.CLIENT_SECRET2) + .build(); + OAuth2Client subjectClient = env.newOAuth2Client()) { + subjectToken = subjectClient.authenticate(); + } + + TestCertificates certs = TestCertificates.instance(); + + testEnvironment = + ImmutableTestEnvironment.builder() + .catalogServerUrl(httpServer.getURI()) + .ssl(true) + .sslTrustStorePath(certs.keyStorePath()) + .sslTrustStorePassword(certs.keyStorePassword()) + // Catalog session + .grantType(GrantType.CLIENT_CREDENTIALS) + // Contextual session + .sessionContextGrantType(GrantType.TOKEN_EXCHANGE) + .sessionContextSubjectTokenString(subjectToken.getValue()) + .sessionContextActorTokenString(ConfigUtil.PARENT_TOKEN) + .build(); + } + + @AfterAll + static void afterClass() throws Exception { + if (testEnvironment != null) { + testEnvironment.close(); + } + + if (httpServer != null) { + httpServer.stop(); + httpServer.join(); + } + + if (backendCatalog != null) { + backendCatalog.close(); + } + } + + @BeforeEach + public void before() { + restCatalog = initCatalog("oauth2-test-catalog", Map.of()); + } + + @AfterEach + public void after() throws Exception { + if (restCatalog != null) { + restCatalog.close(); + } + + if (backendCatalog != null) { + backendCatalog.close(); // clears the in-memory data structures + } + } + + @Override + @SuppressWarnings("MustBeClosedChecker") + protected RESTCatalog initCatalog(String catalogName, Map additionalProperties) { + return testEnvironment.newCatalog(additionalProperties); + } + + @Override + protected RESTCatalog catalog() { + return restCatalog; + } + + @Override + protected boolean supportsServerSideRetry() { + return true; + } + + @Override + protected boolean supportsNestedNamespaces() { + return true; + } + + @Override + protected boolean requiresNamespaceCreate() { + return true; + } + + @Test + @Override + @SuppressWarnings("resource") + public void testLoadTableWithMissingMetadataFile(@TempDir Path ignored) { + + if (requiresNamespaceCreate()) { + restCatalog.createNamespace(TBL.namespace()); + } + + restCatalog.buildTable(TBL, SCHEMA).create(); + assertThat(restCatalog.tableExists(TBL)).as("Table should exist").isTrue(); + + Table table = restCatalog.loadTable(TBL); + String metadataFileLocation = + ((HasTableOperations) table).operations().current().metadataFileLocation(); + table.io().deleteFile(metadataFileLocation); + + assertThatThrownBy(() -> restCatalog.loadTable(TBL)) + .isInstanceOf(NotFoundException.class) + .hasMessageContaining("No in-memory file found for location: " + metadataFileLocation); + } +} diff --git a/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/client/TestOAuth2Client.java b/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/client/TestOAuth2Client.java new file mode 100644 index 000000000000..3ec0309e125c --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/client/TestOAuth2Client.java @@ -0,0 +1,606 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.client; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.InstanceOfAssertFactories.throwable; + +import com.nimbusds.oauth2.sdk.GrantType; +import com.nimbusds.oauth2.sdk.auth.ClientAuthenticationMethod; +import com.nimbusds.oauth2.sdk.id.ClientID; +import com.nimbusds.oauth2.sdk.token.AccessToken; +import com.nimbusds.oauth2.sdk.token.BearerAccessToken; +import com.nimbusds.oauth2.sdk.token.Tokens; +import com.nimbusds.oauth2.sdk.token.TypelessAccessToken; +import java.time.Duration; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import org.apache.iceberg.rest.auth.oauth2.OAuth2Exception; +import org.apache.iceberg.rest.auth.oauth2.config.ConfigUtil; +import org.apache.iceberg.rest.auth.oauth2.flow.TokensResult; +import org.apache.iceberg.rest.auth.oauth2.test.TestCertificates; +import org.apache.iceberg.rest.auth.oauth2.test.TestEnvironment; +import org.apache.iceberg.rest.auth.oauth2.test.expectation.ErrorExpectation; +import org.apache.iceberg.rest.auth.oauth2.test.expectation.ImmutableRefreshTokenExpectation; +import org.apache.iceberg.rest.auth.oauth2.test.junit.EnumLike; +import org.junit.jupiter.api.RepeatedTest; +import org.junit.jupiter.api.Test; +import org.junitpioneer.jupiter.cartesian.CartesianTest; +import org.junitpioneer.jupiter.cartesian.CartesianTest.Values; + +class TestOAuth2Client { + + @CartesianTest + void testClientCredentials( + @EnumLike(excludes = "none") ClientAuthenticationMethod authenticationMethod, + @Values(booleans = {true, false}) boolean returnRefreshTokens) { + try (TestEnvironment env = + TestEnvironment.builder() + .clientAuthenticationMethod(authenticationMethod) + .returnRefreshTokens(returnRefreshTokens) + .build(); + OAuth2Client client = env.newOAuth2Client()) { + TokensResult currentTokens = client.authenticateInternal(); + env.assertTokensResult( + currentTokens, "access_initial", returnRefreshTokens ? "refresh_initial" : null); + } + } + + @Test + void testClientCredentialsUnauthorized() { + try (TestEnvironment env = + TestEnvironment.builder().clientId(new ClientID("WrongClient")).build(); + OAuth2Client client = env.newOAuth2Client()) { + assertThatThrownBy(client::authenticate) + .asInstanceOf(throwable(OAuth2Exception.class)) + .extracting(OAuth2Exception::error) + .isEqualTo(ErrorExpectation.OAUTH2_ERROR); + } + } + + @CartesianTest + void testRefreshToken( + @EnumLike(excludes = "none") ClientAuthenticationMethod authenticationMethod) + throws InterruptedException, ExecutionException { + try (TestEnvironment env = + TestEnvironment.builder() + .clientAuthenticationMethod(authenticationMethod) + .returnRefreshTokens(true) + .build(); + OAuth2Client client = env.newOAuth2Client()) { + TokensResult firstTokens = client.authenticateInternal(); + TokensResult refreshedTokens = + client.refreshCurrentTokens(firstTokens).toCompletableFuture().get(); + env.assertTokensResult(refreshedTokens, "access_refreshed", "refresh_refreshed"); + } + } + + @CartesianTest + void testRefreshTokenWithTokenExchange( + @EnumLike(excludes = "none") ClientAuthenticationMethod authenticationMethod) + throws InterruptedException, ExecutionException { + try (TestEnvironment env = + TestEnvironment.builder() + .clientAuthenticationMethod(authenticationMethod) + .returnRefreshTokens(false) + .build(); + OAuth2Client client = env.newOAuth2Client()) { + TokensResult firstTokens = client.authenticateInternal(); + TokensResult refreshedTokens = + client.refreshCurrentTokens(firstTokens).toCompletableFuture().get(); + env.assertTokensResult(refreshedTokens, "access_refreshed", null); + } + } + + @Test + void testRefreshTokenMustFetchNewTokens() { + try (TestEnvironment env = + TestEnvironment.builder().tokenRefreshWithTokenExchangeEnabled(false).build(); + OAuth2Client client = env.newOAuth2Client()) { + TokensResult currentTokens = + TokensResult.of(new Tokens(new BearerAccessToken("access_initial"), null), env.clock()); + assertThat(client.refreshCurrentTokens(currentTokens)) + .completesExceptionallyWithin(Duration.ofSeconds(10)) + .withThrowableOfType(ExecutionException.class) + .withCauseInstanceOf(OAuth2Client.MustFetchNewTokensException.class); + } + } + + @CartesianTest + void testTokenExchangeStaticSubjectAndActorTokens( + @EnumLike ClientAuthenticationMethod authenticationMethod, + @Values(booleans = {true, false}) boolean returnRefreshTokens) + throws ExecutionException, InterruptedException { + try (TestEnvironment env = + TestEnvironment.builder() + .grantType(GrantType.TOKEN_EXCHANGE) + .clientAuthenticationMethod(authenticationMethod) + .returnRefreshTokens(returnRefreshTokens) + .build(); + OAuth2Client client = env.newOAuth2Client()) { + TokensResult currentTokens = client.authenticateInternal(); + env.assertTokensResult( + currentTokens, "access_initial", returnRefreshTokens ? "refresh_initial" : null); + if (returnRefreshTokens) { + currentTokens = client.refreshCurrentTokens(currentTokens).toCompletableFuture().get(); + env.assertTokensResult(currentTokens, "access_refreshed", "refresh_refreshed"); + } + } + } + + @CartesianTest + void testTokenExchangeParentSubjectAndActorTokens( + @EnumLike ClientAuthenticationMethod authenticationMethod, + @Values(booleans = {true, false}) boolean returnRefreshTokens) + throws InterruptedException, ExecutionException { + try (TestEnvironment parent = TestEnvironment.builder().build(); + OAuth2Client parentClient = parent.newOAuth2Client(); + TestEnvironment env = + TestEnvironment.builder() + .grantType(GrantType.TOKEN_EXCHANGE) + .clientAuthenticationMethod(authenticationMethod) + .subjectTokenString(ConfigUtil.PARENT_TOKEN) + .actorTokenString(ConfigUtil.PARENT_TOKEN) + .parentClient(parentClient) + .returnRefreshTokens(returnRefreshTokens) + .build(); + OAuth2Client client = env.newOAuth2Client()) { + TokensResult tokens = client.authenticateInternal(); + env.assertTokensResult( + tokens, "access_initial", returnRefreshTokens ? "refresh_initial" : null); + if (returnRefreshTokens) { + tokens = client.refreshCurrentTokens(tokens).toCompletableFuture().get(); + env.assertTokensResult(tokens, "access_refreshed", "refresh_refreshed"); + } + } + } + + @Test + void testTokenExchangeSubjectUnauthorized() { + try (TestEnvironment env = + TestEnvironment.builder() + .grantType(GrantType.TOKEN_EXCHANGE) + .subjectTokenString("WrongSubjectToken") + .build(); + OAuth2Client client = env.newOAuth2Client()) { + assertThatThrownBy(client::authenticate) + .hasMessageContaining("OAuth2 request failed: Invalid request") + .asInstanceOf(throwable(OAuth2Exception.class)) + .extracting(OAuth2Exception::error) + .isEqualTo(ErrorExpectation.OAUTH2_ERROR); + } + } + + @Test + void testTokenExchangeActorUnauthorized() { + try (TestEnvironment env = + TestEnvironment.builder() + .grantType(GrantType.TOKEN_EXCHANGE) + .actorTokenString("WrongActorToken") + .build(); + OAuth2Client client = env.newOAuth2Client()) { + assertThatThrownBy(client::authenticate) + .asInstanceOf(throwable(OAuth2Exception.class)) + .hasMessageContaining("OAuth2 request failed: Invalid request") + .extracting(OAuth2Exception::error) + .isEqualTo(ErrorExpectation.OAUTH2_ERROR); + } + } + + @Test + void testStaticTokenNoRefreshToken() { + try (TestEnvironment env = + TestEnvironment.builder() + .token(new TypelessAccessToken("access_initial")) + .returnRefreshTokens(false) + .tokenRefreshWithTokenExchangeEnabled(false) + .build(); + OAuth2Client client = env.newOAuth2Client()) { + TokensResult tokens = client.authenticateInternal(); + env.assertAccessToken(tokens.tokens().getAccessToken(), "access_initial", Duration.ZERO); + // Cannot refresh a static token using the refresh_token grant, + // as there is no initial refresh token + assertThat(client.refreshCurrentTokens(tokens)) + .completesExceptionallyWithin(Duration.ofSeconds(10)) + .withThrowableOfType(ExecutionException.class) + .withCauseInstanceOf(OAuth2Client.MustFetchNewTokensException.class); + } + } + + @Test + void testStaticTokenWithTokenExchangeRefreshNoClientId() { + try (TestEnvironment env = + TestEnvironment.builder() + .token(new TypelessAccessToken("access_initial")) + .returnRefreshTokens(false) + .tokenRefreshWithTokenExchangeEnabled(true) + .clientId(Optional.empty()) + .clientSecret(Optional.empty()) + .build(); + OAuth2Client client = env.newOAuth2Client()) { + TokensResult tokens = client.authenticateInternal(); + env.assertAccessToken(tokens.tokens().getAccessToken(), "access_initial", Duration.ZERO); + // Cannot refresh a static token using the token exchange grant, + // when there is no configured client id / secret + assertThat(client.refreshCurrentTokens(tokens)) + .completesExceptionallyWithin(Duration.ofSeconds(10)) + .withThrowableOfType(ExecutionException.class) + .havingCause() + .isInstanceOf(IllegalStateException.class) + .withMessage("Client ID is required"); + } + } + + @CartesianTest + void testStaticTokenWithTokenExchangeRefreshWithClientId() + throws ExecutionException, InterruptedException { + try (TestEnvironment env = + TestEnvironment.builder() + .token(new TypelessAccessToken("access_initial")) + .returnRefreshTokens(false) + .tokenRefreshWithTokenExchangeEnabled(true) + .build(); + OAuth2Client client = env.newOAuth2Client()) { + TokensResult tokens = client.authenticateInternal(); + env.assertAccessToken(tokens.tokens().getAccessToken(), "access_initial", Duration.ZERO); + // Can refresh a static token using the token exchange grant, + // when there is a configured client id / secret + tokens = client.refreshCurrentTokens(tokens).toCompletableFuture().get(); + env.assertTokensResult(tokens, "access_refreshed", null); + } + } + + @Test + void testSsl() { + TestCertificates certs = TestCertificates.instance(); + + try (TestEnvironment env = + TestEnvironment.builder() + .ssl(true) + .sslTrustStorePath(certs.keyStorePath()) + .sslTrustStorePassword(certs.keyStorePassword()) + .build(); + OAuth2Client client = env.newOAuth2Client()) { + assertThatCode(client::authenticate).doesNotThrowAnyException(); + } + } + + // Token renewal tests + + /** + * Tests the whole token renewal cycle, by manually triggering the token renewal task (which is + * normally scheduled). + */ + @CartesianTest + void testTokenRenewal(@Values(booleans = {true, false}) boolean returnRefreshTokens) { + try (TestEnvironment env = + TestEnvironment.builder() + .createDefaultExpectations(false) + .returnRefreshTokens(returnRefreshTokens) + .tokenRefreshEnabled(false) // will be triggered manually below + .build()) { + env.createMetadataDiscoveryExpectations(); + env.createInitialGrantExpectations(); + ImmutableRefreshTokenExpectation.of(env).create(); + try (OAuth2Client client = env.newOAuth2Client()) { + TokensResult initialTokens = client.authenticateInternal(); + env.assertTokensResult( + initialTokens, "access_initial", returnRefreshTokens ? "refresh_initial" : null); + client.renewTokens(); + TokensResult refreshedTokens = client.authenticateInternal(); + env.assertTokensResult( + refreshedTokens, "access_refreshed", returnRefreshTokens ? "refresh_refreshed" : null); + } + } + } + + /** + * Tests the whole token renewal cycle, but in the edge case where the refreshed access token is + * too short and is discarded. + */ + @Test + void testTokenRenewalRefreshedAccessTokenTooShort() { + try (TestEnvironment env = + TestEnvironment.builder() + .returnRefreshTokens(true) + .accessTokenLifespan(Duration.ofSeconds(1)) + .tokenRefreshEnabled(false) // will be triggered manually below + .build()) { + try (OAuth2Client client = env.newOAuth2Client()) { + TokensResult initialTokens = client.authenticateInternal(); + env.assertTokensResult(initialTokens, "access_initial", "refresh_initial"); + client.renewTokens(); + TokensResult nextTokens = client.authenticateInternal(); + // Refreshed access token should have been discarded and a new one fetched + assertThat(nextTokens).isNotSameAs(initialTokens); + env.assertTokensResult(nextTokens, "access_initial", "refresh_initial"); + } + } + } + + // Client copy tests + + /** + * Tests copying a client before and after closing the original client. The typical use case for + * copying a client is when reusing an init session as a catalog session, so the init session is + * already closed when it's copied. But we also want to test the case where the init session is + * not yet closed when it's copied, since nothing in the API prevents that. + */ + @Test + void testCopyAfterSuccessfulAuth() throws InterruptedException, ExecutionException { + try (TestEnvironment env = + TestEnvironment.builder().grantType(GrantType.TOKEN_EXCHANGE).build(); + OAuth2Client client1 = env.newOAuth2Client()) { + TokensResult tokens = client1.authenticateInternal(); + env.assertTokensResult(tokens, "access_initial", "refresh_initial"); + // 1) Test copy before close + try (OAuth2Client client2 = client1.copy()) { + // Should have the same tokens instance + assertThat(client2.authenticateInternal()).isSameAs(tokens); + // Now close client1 + client1.close(); + // Should still have the same tokens instance, and not throw + assertThat(client2.authenticateInternal()).isSameAs(tokens); + // Should have a token refresh future + assertThat(client2).extracting("tokenRefreshFuture").isNotNull(); + // Should be able to refresh tokens + TokensResult refreshedTokens = + client2.refreshCurrentTokens(tokens).toCompletableFuture().get(); + env.assertTokensResult(refreshedTokens, "access_refreshed", "refresh_refreshed"); + // Should be able to fetch new tokens + TokensResult newTokens = client2.fetchNewTokens().toCompletableFuture().get(); + env.assertTokensResult(newTokens, "access_initial", "refresh_initial"); + } + + // 2) Test copy after close + try (OAuth2Client client3 = client1.copy()) { + // Should have the same tokens instance + assertThat(client3.authenticateInternal()).isSameAs(tokens); + // Should have a token refresh future + assertThat(client3).extracting("tokenRefreshFuture").isNotNull(); + // Should be able to refresh tokens + TokensResult refreshedTokens = + client3.refreshCurrentTokens(tokens).toCompletableFuture().get(); + env.assertTokensResult(refreshedTokens, "access_refreshed", "refresh_refreshed"); + // Should be able to fetch new tokens + TokensResult newTokens = client3.fetchNewTokens().toCompletableFuture().get(); + env.assertTokensResult(newTokens, "access_initial", "refresh_initial"); + } + } + } + + /** + * Tests copying a client before and after closing the original client, when the original client + * failed to authenticate. This is a rather contrived scenario since in practice, a failed init + * session would cause the catalog initialization to fail; but it's possible in theory, so we + * should add tests for it. + */ + @Test + @SuppressWarnings("NestedTryDepth") + void testCopyAfterFailedAuth() throws InterruptedException, ExecutionException { + try (TestEnvironment env = + TestEnvironment.builder() + .grantType(GrantType.TOKEN_EXCHANGE) + .createDefaultExpectations(false) + .build()) { + // Emulate success fetching metadata, but failure on initial token fetch + env.createMetadataDiscoveryExpectations(); + env.createErrorExpectations(); + try (OAuth2Client client1 = env.newOAuth2Client()) { + assertThatThrownBy(client1::authenticateInternal) + .isInstanceOf(OAuth2Exception.class) + .hasMessageContaining("Invalid request"); + // Restore expectations so that copied clients can fetch tokens + env.reset(); + env.createExpectations(); + // 1) Test copy before close + try (OAuth2Client client2 = client1.copy()) { + // Should be able to fetch tokens even if the original client failed + assertThat(client2.authenticateInternal()).isNotNull(); + // Now close client1 + client1.close(); + // Should still have tokens + TokensResult tokens = client2.authenticateInternal(); + env.assertTokensResult(tokens, "access_initial", "refresh_initial"); + assertThat(tokens).isNotNull(); + // Should have a token refresh future + assertThat(client2).extracting("tokenRefreshFuture").isNotNull(); + // Should be able to refresh tokens + TokensResult refreshedTokens = + client2.refreshCurrentTokens(tokens).toCompletableFuture().get(); + env.assertTokensResult(refreshedTokens, "access_refreshed", "refresh_refreshed"); + // Should be able to fetch new tokens + TokensResult newTokens = client2.fetchNewTokens().toCompletableFuture().get(); + env.assertTokensResult(newTokens, "access_initial", "refresh_initial"); + } + + // 2) Test copy after close + try (OAuth2Client client3 = client1.copy()) { + // Should be able to fetch tokens even if the original client failed + TokensResult tokens = client3.authenticateInternal(); + env.assertTokensResult(tokens, "access_initial", "refresh_initial"); + assertThat(tokens).isNotNull(); + // Should be able to refresh tokens + TokensResult refreshedTokens = + client3.refreshCurrentTokens(tokens).toCompletableFuture().get(); + env.assertTokensResult(refreshedTokens, "access_refreshed", "refresh_refreshed"); + // Should be able to fetch new tokens + TokensResult newTokens = client3.fetchNewTokens().toCompletableFuture().get(); + env.assertTokensResult(newTokens, "access_initial", "refresh_initial"); + } + } + } + } + + // Concurrency tests + + /** + * Multiple threads calling authenticate() concurrently should all get a valid, non-null access + * token. This exercises the volatile currentTokensFuture field under contention. + */ + @RepeatedTest(100) + void testParallelAuthenticate() throws Exception { + try (TestEnvironment env = TestEnvironment.builder().build(); + OAuth2Client client = env.newOAuth2Client()) { + + CyclicBarrier barrier = new CyclicBarrier(10); + List tokens = new CopyOnWriteArrayList<>(); + List errors = new CopyOnWriteArrayList<>(); + + Thread[] threads = new Thread[10]; + for (int i = 0; i < 10; i++) { + threads[i] = + new Thread( + () -> { + try { + barrier.await(5, TimeUnit.SECONDS); + AccessToken token = client.authenticate(); + tokens.add(token); + } catch (Throwable t) { + errors.add(t); + } + }); + threads[i].start(); + } + + for (Thread t : threads) { + t.join(10_000); + } + + assertThat(errors).as("No thread should have thrown an exception").isEmpty(); + assertThat(tokens) + .hasSize(10) + .allSatisfy(token -> assertThat(token.getValue()).isEqualTo("access_initial")); + } + } + + /** + * Calling close() while multiple threads are authenticating should not cause unhandled + * exceptions. Threads may get a CancellationException or succeed — either is acceptable. + */ + @RepeatedTest(100) + void testCloseWhileAuthenticating() throws Exception { + try (TestEnvironment env = TestEnvironment.builder().build(); + OAuth2Client client = env.newOAuth2Client()) { + + CyclicBarrier barrier = new CyclicBarrier(11); + List errors = new CopyOnWriteArrayList<>(); + + // Repeatedly authenticate while the client may be closing + Runnable action = + () -> { + for (int i = 0; i < 100; i++) { + try { + client.authenticate(); + } catch (CancellationException ignored) { + // Cancellation is expected when client closing + } + } + }; + + Thread[] threads = new Thread[10]; + + for (int i = 0; i < 10; i++) { + threads[i] = + new Thread( + () -> { + try { + barrier.await(5, TimeUnit.SECONDS); + action.run(); + } catch (Throwable t) { + errors.add(t); + } + }); + threads[i].start(); + } + + // Release all threads, then close the client mid-flight + barrier.await(5, TimeUnit.SECONDS); + client.close(); + + for (Thread t : threads) { + t.join(10_000); + } + + assertThat(errors) + .as("No thread should have thrown an exception other than CancellationException") + .isEmpty(); + + // close() is idempotent + assertThatCode(client::close).doesNotThrowAnyException(); + } + } + + /** + * Multiple threads calling authenticate() and renewTokens() concurrently should not corrupt the + * client's internal state. + */ + @RepeatedTest(100) + void testConcurrentAuthenticateAndRenew() throws Exception { + try (TestEnvironment env = + TestEnvironment.builder() + .tokenRefreshEnabled(false) // renewals will be triggered manually + .build(); + OAuth2Client client = env.newOAuth2Client()) { + + // Ensure initial auth completes + client.authenticate(); + + CyclicBarrier barrier = new CyclicBarrier(10); + List errors = new CopyOnWriteArrayList<>(); + + Thread[] threads = new Thread[10]; + for (int i = 0; i < 10; i++) { + boolean renew = i == 0; + threads[i] = + new Thread( + () -> { + try { + barrier.await(5, TimeUnit.SECONDS); + for (int j = 0; j < 50; j++) { + if (renew) { + client.renewTokens(); + } else { + AccessToken token = client.authenticate(); + assertThat(token).isNotNull(); + assertThat(token.getValue()).isIn("access_initial", "access_refreshed"); + } + } + } catch (Throwable t) { + errors.add(t); + } + }); + threads[i].start(); + } + + for (Thread t : threads) { + t.join(10_000); + } + + assertThat(errors).as("No thread should have thrown an exception").isEmpty(); + } + } +} diff --git a/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/config/TestBasicConfig.java b/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/config/TestBasicConfig.java new file mode 100644 index 000000000000..fdcd8677881c --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/config/TestBasicConfig.java @@ -0,0 +1,271 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.config; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; + +import com.nimbusds.oauth2.sdk.GrantType; +import com.nimbusds.oauth2.sdk.Scope; +import com.nimbusds.oauth2.sdk.auth.ClientAuthenticationMethod; +import com.nimbusds.oauth2.sdk.auth.Secret; +import com.nimbusds.oauth2.sdk.id.ClientID; +import com.nimbusds.oauth2.sdk.token.BearerAccessToken; +import java.net.URI; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +class TestBasicConfig { + + @ParameterizedTest + @MethodSource + @SuppressWarnings("ResultOfMethodCallIgnored") + void testValidate(Map properties, List expected) { + assertThatIllegalArgumentException() + .isThrownBy(() -> BasicConfig.parse(properties).build()) + .withMessage(ConfigValidator.buildDescription(expected.stream())); + } + + @SuppressWarnings("MethodLength") + static Stream testValidate() { + return Stream.of( + Arguments.of( + Map.of(BasicConfig.CLIENT_ID, "Client1", BasicConfig.CLIENT_SECRET, "s3cr3t"), + List.of( + "either issuer URL or token endpoint must be set (rest.auth.oauth2.issuer-url / rest.auth.oauth2.token-endpoint)")), + Arguments.of( + Map.of( + BasicConfig.CLIENT_ID, + "Client1", + BasicConfig.CLIENT_SECRET, + "s3cr3t", + BasicConfig.ISSUER_URL, + "realms/master"), + List.of("Issuer URL must not be relative (rest.auth.oauth2.issuer-url)")), + Arguments.of( + Map.of( + BasicConfig.CLIENT_ID, + "Client1", + BasicConfig.CLIENT_SECRET, + "s3cr3t", + BasicConfig.ISSUER_URL, + "https://example.com?query"), + List.of("Issuer URL must not have a query part (rest.auth.oauth2.issuer-url)")), + Arguments.of( + Map.of( + BasicConfig.CLIENT_ID, + "Client1", + BasicConfig.CLIENT_SECRET, + "s3cr3t", + BasicConfig.ISSUER_URL, + "https://example.com#fragment"), + List.of("Issuer URL must not have a fragment part (rest.auth.oauth2.issuer-url)")), + Arguments.of( + Map.of( + BasicConfig.CLIENT_ID, + "Client1", + BasicConfig.CLIENT_SECRET, + "s3cr3t", + BasicConfig.TOKEN_ENDPOINT, + "https://user:pass@example.com"), + List.of( + "Token endpoint must not have a user info part (rest.auth.oauth2.token-endpoint)")), + Arguments.of( + Map.of( + BasicConfig.CLIENT_ID, + "Client1", + BasicConfig.CLIENT_SECRET, + "s3cr3t", + BasicConfig.TOKEN_ENDPOINT, + "https://example.com?query"), + List.of("Token endpoint must not have a query part (rest.auth.oauth2.token-endpoint)")), + Arguments.of( + Map.of( + BasicConfig.CLIENT_ID, + "Client1", + BasicConfig.CLIENT_SECRET, + "s3cr3t", + BasicConfig.TOKEN_ENDPOINT, + "https://example.com#fragment"), + List.of( + "Token endpoint must not have a fragment part (rest.auth.oauth2.token-endpoint)")), + Arguments.of( + Map.of( + BasicConfig.CLIENT_ID, + "Client1", + BasicConfig.CLIENT_SECRET, + "s3cr3t", + BasicConfig.TOKEN_ENDPOINT, + "/token"), + List.of("Token endpoint must not be relative (rest.auth.oauth2.token-endpoint)")), + Arguments.of( + Map.of( + BasicConfig.CLIENT_ID, + "Client1", + BasicConfig.CLIENT_SECRET, + "s3cr3t", + BasicConfig.TOKEN_ENDPOINT, + "token"), + List.of("Token endpoint must not be relative (rest.auth.oauth2.token-endpoint)")), + Arguments.of( + Map.of( + BasicConfig.CLIENT_SECRET, + "s3cr3t", + BasicConfig.TOKEN_ENDPOINT, + "https://example.com/token"), + List.of("client ID must not be empty (rest.auth.oauth2.client-id)")), + Arguments.of( + Map.of( + BasicConfig.CLIENT_ID, + "Client1", + BasicConfig.CLIENT_AUTH, + ClientAuthenticationMethod.CLIENT_SECRET_BASIC.getValue(), + BasicConfig.TOKEN_ENDPOINT, + "https://example.com/token"), + List.of( + "client secret must not be empty when client authentication is 'client_secret_basic' (rest.auth.oauth2.client-auth / rest.auth.oauth2.client-secret)")), + Arguments.of( + Map.of( + BasicConfig.CLIENT_ID, + "Client1", + BasicConfig.CLIENT_AUTH, + ClientAuthenticationMethod.CLIENT_SECRET_POST.getValue(), + BasicConfig.TOKEN_ENDPOINT, + "https://example.com/token"), + List.of( + "client secret must not be empty when client authentication is 'client_secret_post' (rest.auth.oauth2.client-auth / rest.auth.oauth2.client-secret)")), + Arguments.of( + Map.of( + BasicConfig.CLIENT_ID, + "Client1", + BasicConfig.CLIENT_SECRET, + "s3cr3t", + BasicConfig.GRANT_TYPE, + GrantType.TOKEN_EXCHANGE.getValue(), + BasicConfig.CLIENT_AUTH, + ClientAuthenticationMethod.NONE.getValue(), + BasicConfig.TOKEN_ENDPOINT, + "https://example.com/token"), + List.of( + "client secret must not be set when client authentication is 'none' (rest.auth.oauth2.client-auth / rest.auth.oauth2.client-secret)")), + Arguments.of( + Map.of( + BasicConfig.CLIENT_ID, + "Client1", + BasicConfig.GRANT_TYPE, + GrantType.CLIENT_CREDENTIALS.getValue(), + BasicConfig.CLIENT_AUTH, + ClientAuthenticationMethod.NONE.getValue(), + BasicConfig.TOKEN_ENDPOINT, + "https://example.com/token"), + List.of( + "grant type must not be 'client_credentials' when client authentication is 'none' (rest.auth.oauth2.client-auth / rest.auth.oauth2.grant-type)")), + Arguments.of( + Map.of( + BasicConfig.CLIENT_ID, + "Client1", + BasicConfig.CLIENT_SECRET, + "s3cr3t", + BasicConfig.GRANT_TYPE, + GrantType.REFRESH_TOKEN.getValue(), + BasicConfig.TOKEN_ENDPOINT, + "https://example.com/token"), + List.of( + "grant type must be one of: 'client_credentials', 'urn:ietf:params:oauth:grant-type:token-exchange' (rest.auth.oauth2.grant-type)")), + Arguments.of( + Map.of( + BasicConfig.CLIENT_AUTH, + "unknown", + BasicConfig.CLIENT_ID, + "Client1", + BasicConfig.CLIENT_SECRET, + "s3cr3t", + BasicConfig.TOKEN_ENDPOINT, + "https://example.com/token"), + List.of( + "client authentication method must be one of: 'none', 'client_secret_basic', 'client_secret_post' (rest.auth.oauth2.client-auth)")), + Arguments.of( + Map.of( + BasicConfig.TIMEOUT, + "PT1S", + BasicConfig.CLIENT_ID, + "Client1", + BasicConfig.CLIENT_SECRET, + "s3cr3t", + BasicConfig.ISSUER_URL, + "https://example.com"), + List.of("timeout must be greater than or equal to PT30S (rest.auth.oauth2.timeout)"))); + } + + @ParameterizedTest + @MethodSource + void testParse(Map properties, BasicConfig expected) { + BasicConfig actual = BasicConfig.parse(properties).build(); + assertThat(actual).isEqualTo(expected); + } + + static Stream testParse() { + return Stream.of( + Arguments.of( + Map.of(BasicConfig.ISSUER_URL, "https://example.com", BasicConfig.TOKEN, "my-token"), + ImmutableBasicConfig.builder() + .issuerUrl(URI.create("https://example.com")) + .token(new BearerAccessToken("my-token")) + .build()), + Arguments.of( + Map.of( + BasicConfig.TOKEN_ENDPOINT, + "https://example.com/token", + BasicConfig.GRANT_TYPE, + GrantType.TOKEN_EXCHANGE.getValue(), + BasicConfig.CLIENT_AUTH, + "client_secret_post", + BasicConfig.CLIENT_ID, + "my-client", + BasicConfig.CLIENT_SECRET, + "my-secret", + BasicConfig.SCOPE, + "read write", + BasicConfig.TIMEOUT, + "PT10M", + BasicConfig.SESSION_CACHE_TIMEOUT, + "PT30M", + BasicConfig.EXTRA_PARAMS + ".param1", + "value1", + BasicConfig.EXTRA_PARAMS + ".param2", + "value2"), + ImmutableBasicConfig.builder() + .tokenEndpoint(URI.create("https://example.com/token")) + .grantType(GrantType.TOKEN_EXCHANGE) + .clientAuthenticationMethod(ClientAuthenticationMethod.CLIENT_SECRET_POST) + .clientId(new ClientID("my-client")) + .clientSecret(new Secret("my-secret")) + .scope(new Scope("read", "write")) + .tokenAcquisitionTimeout(Duration.ofMinutes(10)) + .sessionCacheTimeout(Duration.ofMinutes(30)) + .putExtraRequestParameters("param1", "value1") + .putExtraRequestParameters("param2", "value2") + .build())); + } +} diff --git a/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/config/TestConfigMigrator.java b/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/config/TestConfigMigrator.java new file mode 100644 index 000000000000..3bb9167a1d2e --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/config/TestConfigMigrator.java @@ -0,0 +1,848 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.config; + +import static org.apache.iceberg.rest.auth.oauth2.config.ConfigMigrator.DEFAULT_CLIENT_ID; +import static org.apache.iceberg.rest.auth.oauth2.config.ConfigMigrator.MESSAGE_TEMPLATE_LEGACY_OPTION; +import static org.apache.iceberg.rest.auth.oauth2.config.ConfigMigrator.MESSAGE_TEMPLATE_MERGED_CONTEXTUAL_CONFIG; +import static org.apache.iceberg.rest.auth.oauth2.config.ConfigMigrator.MESSAGE_TEMPLATE_MISSING_TOKEN_ENDPOINT; +import static org.apache.iceberg.rest.auth.oauth2.config.ConfigMigrator.MESSAGE_TEMPLATE_NO_CLIENT_ID; +import static org.apache.iceberg.rest.auth.oauth2.config.ConfigMigrator.MESSAGE_TEMPLATE_RELATIVE_TOKEN_ENDPOINT; +import static org.apache.iceberg.rest.auth.oauth2.config.ConfigMigrator.MESSAGE_TEMPLATE_TABLE_CONFIG_NOT_ALLOWED; +import static org.apache.iceberg.rest.auth.oauth2.config.ConfigMigrator.MESSAGE_TEMPLATE_VENDED_TOKEN; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.InstanceOfAssertFactories.list; + +import com.nimbusds.oauth2.sdk.GrantType; +import com.nimbusds.oauth2.sdk.ParseException; +import com.nimbusds.oauth2.sdk.Scope; +import com.nimbusds.oauth2.sdk.auth.Secret; +import com.nimbusds.oauth2.sdk.id.Audience; +import com.nimbusds.oauth2.sdk.id.ClientID; +import com.nimbusds.oauth2.sdk.token.TokenTypeURI; +import com.nimbusds.oauth2.sdk.token.TypelessAccessToken; +import java.net.URI; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.function.BiConsumer; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.rest.auth.OAuth2Properties; +import org.apache.iceberg.rest.auth.oauth2.ImmutableOAuth2Config; +import org.apache.iceberg.rest.auth.oauth2.OAuth2Config; +import org.apache.iceberg.util.Pair; +import org.assertj.core.api.ListAssert; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +@SuppressWarnings("deprecation") +class TestConfigMigrator { + + private List>> messages; + private BiConsumer consumer; + + @BeforeEach + void before() { + messages = Lists.newArrayList(); + consumer = (msg, args) -> messages.add(Pair.of(msg, List.of(args))); + } + + @AfterEach + void after() { + messages.clear(); + } + + // Legacy properties migration tests + + @Test + void noLegacyProperties() { + Map input = + Map.of( + BasicConfig.TOKEN_ENDPOINT, + "https://example.com/token", + BasicConfig.CLIENT_ID, + "client1", + BasicConfig.CLIENT_SECRET, + "secret", + "non.oauth2.property", + "value"); + Map actual = new ConfigMigrator(consumer).migrateProperties(input); + // Only OAuth2 properties should be included + assertThat(actual) + .isEqualTo( + Map.of( + BasicConfig.TOKEN_ENDPOINT, + "https://example.com/token", + BasicConfig.CLIENT_ID, + "client1", + BasicConfig.CLIENT_SECRET, + "secret")); + assertThat(messages).isEmpty(); + } + + @Test + void credentialValid() { + Map input = Map.of(OAuth2Properties.CREDENTIAL, "client1:secret1"); + Map actual = new ConfigMigrator(consumer).migrateProperties(input); + assertThat(actual) + .isEqualTo(Map.of(BasicConfig.CLIENT_ID, "client1", BasicConfig.CLIENT_SECRET, "secret1")); + assertThat(messages).hasSize(1); + assertThatMessage(messages.get(0), MESSAGE_TEMPLATE_LEGACY_OPTION) + .containsExactly( + OAuth2Properties.CREDENTIAL, + "s", + BasicConfig.CLIENT_ID + " and " + BasicConfig.CLIENT_SECRET); + } + + @Test + void credentialNoClientId() { + Map input = Map.of(OAuth2Properties.CREDENTIAL, "secret1"); + Map actual = new ConfigMigrator(consumer).migrateProperties(input); + assertThat(actual) + .isEqualTo( + Map.of( + BasicConfig.CLIENT_ID, + DEFAULT_CLIENT_ID.getValue(), + BasicConfig.CLIENT_SECRET, + "secret1")); + assertThat(messages).hasSize(2); + assertThatMessage(messages.get(0), MESSAGE_TEMPLATE_LEGACY_OPTION) + .containsExactly( + OAuth2Properties.CREDENTIAL, + "s", + BasicConfig.CLIENT_ID + " and " + BasicConfig.CLIENT_SECRET); + assertThatMessage(messages.get(1), MESSAGE_TEMPLATE_NO_CLIENT_ID) + .containsExactly(DEFAULT_CLIENT_ID.getValue()); + } + + @Test + void token() { + Map input = Map.of(OAuth2Properties.TOKEN, "access-token-123"); + Map actual = new ConfigMigrator(consumer).migrateProperties(input); + assertThat(actual).isEqualTo(Map.of(BasicConfig.TOKEN, "access-token-123")); + assertThat(messages).hasSize(1); + assertThatMessage(messages.get(0), MESSAGE_TEMPLATE_LEGACY_OPTION) + .containsExactly(OAuth2Properties.TOKEN, "", BasicConfig.TOKEN); + } + + @Test + void tokenExpiresInMs() { + Map input = Map.of(OAuth2Properties.TOKEN_EXPIRES_IN_MS, "300000"); + Map actual = new ConfigMigrator(consumer).migrateProperties(input); + assertThat(actual) + .isEqualTo( + Map.of(TokenRefreshConfig.ACCESS_TOKEN_LIFESPAN, Duration.ofMillis(300000).toString())); + assertThat(messages).hasSize(1); + assertThatMessage(messages.get(0), MESSAGE_TEMPLATE_LEGACY_OPTION) + .containsExactly( + OAuth2Properties.TOKEN_EXPIRES_IN_MS, "", TokenRefreshConfig.ACCESS_TOKEN_LIFESPAN); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void tokenRefreshEnabled(boolean enabled) { + Map input = + Map.of(OAuth2Properties.TOKEN_REFRESH_ENABLED, String.valueOf(enabled)); + Map actual = new ConfigMigrator(consumer).migrateProperties(input); + assertThat(actual).isEqualTo(Map.of(TokenRefreshConfig.ENABLED, String.valueOf(enabled))); + assertThat(messages).hasSize(1); + assertThatMessage(messages.get(0), MESSAGE_TEMPLATE_LEGACY_OPTION) + .containsExactly(OAuth2Properties.TOKEN_REFRESH_ENABLED, "", TokenRefreshConfig.ENABLED); + } + + @Test + void oauth2ServerUri() { + Map input = + Map.of(OAuth2Properties.OAUTH2_SERVER_URI, "https://example.com/token"); + Map actual = new ConfigMigrator(consumer).migrateProperties(input); + assertThat(actual).isEqualTo(Map.of(BasicConfig.TOKEN_ENDPOINT, "https://example.com/token")); + assertThat(messages).hasSize(1); + assertThatMessage(messages.get(0), MESSAGE_TEMPLATE_LEGACY_OPTION) + .containsExactly( + OAuth2Properties.OAUTH2_SERVER_URI, + "s", + BasicConfig.ISSUER_URL + " or " + BasicConfig.TOKEN_ENDPOINT); + } + + @Test + void scope() { + Map input = Map.of(OAuth2Properties.SCOPE, "read write admin"); + Map actual = new ConfigMigrator(consumer).migrateProperties(input); + assertThat(actual).isEqualTo(Map.of(BasicConfig.SCOPE, "read write admin")); + assertThat(messages).hasSize(1); + assertThatMessage(messages.get(0), MESSAGE_TEMPLATE_LEGACY_OPTION) + .containsExactly(OAuth2Properties.SCOPE, "", BasicConfig.SCOPE); + } + + @Test + void audience() { + Map input = Map.of(OAuth2Properties.AUDIENCE, "https://api.example.com"); + Map actual = new ConfigMigrator(consumer).migrateProperties(input); + assertThat(actual).isEqualTo(Map.of(TokenExchangeConfig.AUDIENCES, "https://api.example.com")); + assertThat(messages).hasSize(1); + assertThatMessage(messages.get(0), MESSAGE_TEMPLATE_LEGACY_OPTION) + .containsExactly(OAuth2Properties.AUDIENCE, "", TokenExchangeConfig.AUDIENCES); + } + + @Test + void resource() { + Map input = Map.of(OAuth2Properties.RESOURCE, "urn:example:resource"); + Map actual = new ConfigMigrator(consumer).migrateProperties(input); + assertThat(actual).isEqualTo(Map.of(TokenExchangeConfig.RESOURCES, "urn:example:resource")); + assertThat(messages).hasSize(1); + assertThatMessage(messages.get(0), MESSAGE_TEMPLATE_LEGACY_OPTION) + .containsExactly(OAuth2Properties.RESOURCE, "", TokenExchangeConfig.RESOURCES); + } + + @ParameterizedTest + @MethodSource + void vendedTokenExchange(String tokenTypeProperty) { + Map input = Map.of(tokenTypeProperty, "some-value"); + Map actual = new ConfigMigrator(consumer).migrateProperties(input); + assertThat(actual) + .isEqualTo( + Map.of( + BasicConfig.GRANT_TYPE, + GrantType.TOKEN_EXCHANGE.getValue(), + TokenExchangeConfig.SUBJECT_TOKEN, + "some-value", + TokenExchangeConfig.SUBJECT_TOKEN_TYPE, + tokenTypeProperty, + TokenExchangeConfig.ACTOR_TOKEN, + ConfigUtil.PARENT_TOKEN)); + assertThat(messages).hasSize(1); + assertThatMessage(messages.get(0), MESSAGE_TEMPLATE_LEGACY_OPTION) + .containsExactly( + tokenTypeProperty, + "s", + TokenExchangeConfig.SUBJECT_TOKEN + + ", " + + TokenExchangeConfig.SUBJECT_TOKEN_TYPE + + " and " + + TokenExchangeConfig.ACTOR_TOKEN); + } + + static Stream vendedTokenExchange() { + return Stream.of( + OAuth2Properties.ACCESS_TOKEN_TYPE, + OAuth2Properties.ID_TOKEN_TYPE, + OAuth2Properties.SAML1_TOKEN_TYPE, + OAuth2Properties.SAML2_TOKEN_TYPE, + OAuth2Properties.JWT_TOKEN_TYPE, + OAuth2Properties.REFRESH_TOKEN_TYPE); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void tokenExchangeEnabled(boolean enabled) { + Map input = + Map.of(OAuth2Properties.TOKEN_EXCHANGE_ENABLED, String.valueOf(enabled)); + Map actual = new ConfigMigrator(consumer).migrateProperties(input); + assertThat(actual) + .isEqualTo(Map.of(TokenRefreshConfig.TOKEN_EXCHANGE_ENABLED, String.valueOf(enabled))); + assertThat(messages).hasSize(1); + assertThatMessage(messages.get(0), MESSAGE_TEMPLATE_LEGACY_OPTION) + .containsExactly( + OAuth2Properties.TOKEN_EXCHANGE_ENABLED, "", TokenRefreshConfig.TOKEN_EXCHANGE_ENABLED); + } + + @Test + void fullLegacyMigrationScenario() { + Map input = + ImmutableMap.builder() + .put(OAuth2Properties.CREDENTIAL, "client1:secret1") + .put(OAuth2Properties.TOKEN_EXPIRES_IN_MS, "300000") + .put(OAuth2Properties.TOKEN_REFRESH_ENABLED, "true") + .put(OAuth2Properties.OAUTH2_SERVER_URI, "custom/path/to/token") + .put(OAuth2Properties.SCOPE, "read write") + .put(OAuth2Properties.AUDIENCE, "https://api.example.com") + .put(OAuth2Properties.RESOURCE, "urn:example:resource") + .put(OAuth2Properties.TOKEN_EXCHANGE_ENABLED, "false") + .put( + BasicConfig.ISSUER_URL, + "https://idp.example.com") // New property should be preserved + .put("non.oauth2.property", "ignored") // Non-OAuth2 property should be filtered out + .build(); + + Map expected = + ImmutableMap.builder() + .put(BasicConfig.CLIENT_ID, "client1") + .put(BasicConfig.CLIENT_SECRET, "secret1") + .put(BasicConfig.TOKEN_ENDPOINT, "custom/path/to/token") + .put(BasicConfig.ISSUER_URL, "https://idp.example.com") + .put(BasicConfig.SCOPE, "read write") + .put(TokenRefreshConfig.ENABLED, "true") + .put(TokenRefreshConfig.TOKEN_EXCHANGE_ENABLED, "false") + .put(TokenRefreshConfig.ACCESS_TOKEN_LIFESPAN, Duration.ofMillis(300000).toString()) + .put(TokenExchangeConfig.RESOURCES, "urn:example:resource") + .put(TokenExchangeConfig.AUDIENCES, "https://api.example.com") + .build(); + + Map actual = new ConfigMigrator(consumer).migrateProperties(input); + assertThat(actual).isEqualTo(expected); + + assertThat(messages).hasSize(8); + List legacyProperties = + messages.stream().map(Pair::second).map(args -> args.get(0)).collect(Collectors.toList()); + + assertThat(legacyProperties) + .containsExactlyInAnyOrder( + OAuth2Properties.CREDENTIAL, + OAuth2Properties.TOKEN_EXPIRES_IN_MS, + OAuth2Properties.TOKEN_REFRESH_ENABLED, + OAuth2Properties.OAUTH2_SERVER_URI, + OAuth2Properties.SCOPE, + OAuth2Properties.AUDIENCE, + OAuth2Properties.RESOURCE, + OAuth2Properties.TOKEN_EXCHANGE_ENABLED); + } + + @ParameterizedTest + @MethodSource + void newPropertyOverridesLegacy( + Map input, Map expectedOutput, String[] expectedWarningArgs) { + Map actual = new ConfigMigrator(consumer).migrateProperties(input); + assertThat(actual).isEqualTo(expectedOutput); + assertThat(messages).hasSize(1); + assertThatMessage(messages.get(0), MESSAGE_TEMPLATE_LEGACY_OPTION) + .containsExactly(expectedWarningArgs); + } + + static Stream newPropertyOverridesLegacy() { + return Stream.of( + Arguments.of( + ImmutableMap.of( + OAuth2Properties.CREDENTIAL, + "legacy-client:legacy-secret", + BasicConfig.CLIENT_ID, + "new-client", + BasicConfig.CLIENT_SECRET, + "new-secret"), + Map.of(BasicConfig.CLIENT_ID, "new-client", BasicConfig.CLIENT_SECRET, "new-secret"), + new String[] { + OAuth2Properties.CREDENTIAL, + "s", + BasicConfig.CLIENT_ID + " and " + BasicConfig.CLIENT_SECRET + }), + Arguments.of( + ImmutableMap.of(OAuth2Properties.TOKEN, "legacy-token", BasicConfig.TOKEN, "new-token"), + Map.of(BasicConfig.TOKEN, "new-token"), + new String[] {OAuth2Properties.TOKEN, "", BasicConfig.TOKEN}), + Arguments.of( + ImmutableMap.of( + OAuth2Properties.OAUTH2_SERVER_URI, + "https://legacy.example.com/token", + BasicConfig.TOKEN_ENDPOINT, + "https://new.example.com/token"), + Map.of(BasicConfig.TOKEN_ENDPOINT, "https://new.example.com/token"), + new String[] { + OAuth2Properties.OAUTH2_SERVER_URI, + "s", + BasicConfig.ISSUER_URL + " or " + BasicConfig.TOKEN_ENDPOINT + }), + Arguments.of( + ImmutableMap.of(OAuth2Properties.SCOPE, "legacy-scope", BasicConfig.SCOPE, "new-scope"), + Map.of(BasicConfig.SCOPE, "new-scope"), + new String[] {OAuth2Properties.SCOPE, "", BasicConfig.SCOPE}), + Arguments.of( + ImmutableMap.of( + OAuth2Properties.AUDIENCE, + "https://legacy.example.com", + TokenExchangeConfig.AUDIENCES, + "https://new.example.com"), + Map.of(TokenExchangeConfig.AUDIENCES, "https://new.example.com"), + new String[] {OAuth2Properties.AUDIENCE, "", TokenExchangeConfig.AUDIENCES}), + Arguments.of( + ImmutableMap.of( + OAuth2Properties.RESOURCE, + "urn:legacy:resource", + TokenExchangeConfig.RESOURCES, + "urn:new:resource"), + Map.of(TokenExchangeConfig.RESOURCES, "urn:new:resource"), + new String[] {OAuth2Properties.RESOURCE, "", TokenExchangeConfig.RESOURCES}), + Arguments.of( + ImmutableMap.of( + OAuth2Properties.TOKEN_REFRESH_ENABLED, + "false", + TokenRefreshConfig.ENABLED, + "true"), + Map.of(TokenRefreshConfig.ENABLED, "true"), + new String[] {OAuth2Properties.TOKEN_REFRESH_ENABLED, "", TokenRefreshConfig.ENABLED}), + Arguments.of( + ImmutableMap.of( + OAuth2Properties.TOKEN_EXCHANGE_ENABLED, + "false", + TokenRefreshConfig.TOKEN_EXCHANGE_ENABLED, + "true"), + Map.of(TokenRefreshConfig.TOKEN_EXCHANGE_ENABLED, "true"), + new String[] { + OAuth2Properties.TOKEN_EXCHANGE_ENABLED, "", TokenRefreshConfig.TOKEN_EXCHANGE_ENABLED + }), + Arguments.of( + ImmutableMap.of( + OAuth2Properties.TOKEN_EXPIRES_IN_MS, + "300000", + TokenRefreshConfig.ACCESS_TOKEN_LIFESPAN, + "PT10M"), + Map.of(TokenRefreshConfig.ACCESS_TOKEN_LIFESPAN, "PT10M"), + new String[] { + OAuth2Properties.TOKEN_EXPIRES_IN_MS, "", TokenRefreshConfig.ACCESS_TOKEN_LIFESPAN + })); + } + + // Token endpoint URL handling tests + + @ParameterizedTest + @CsvSource({ + "https://example.com , /oauth2/token , https://example.com/oauth2/token", + "https://example.com , oauth2/token , https://example.com/oauth2/token", + "https://example.com/ , /oauth2/token , https://example.com/oauth2/token", + "https://example.com/ , oauth2/token , https://example.com/oauth2/token", + "https://example.com/api , /oauth2/token , https://example.com/api/oauth2/token", + "https://example.com/api/ , oauth2/token , https://example.com/api/oauth2/token" + }) + void legacyTokenEndpoint(String catalogUri, String oauth2ServerUri, String expected) { + Map input = Map.of(OAuth2Properties.OAUTH2_SERVER_URI, oauth2ServerUri); + ConfigMigrator migrator = new ConfigMigrator(consumer); + Map actual = migrator.migrateProperties(input); + migrator.handleTokenEndpoint(actual, catalogUri); + assertThat(actual).isEqualTo(Map.of(BasicConfig.TOKEN_ENDPOINT, expected)); + assertThat(messages).hasSize(2); + assertThatMessage(messages.get(0), MESSAGE_TEMPLATE_LEGACY_OPTION) + .containsExactly( + OAuth2Properties.OAUTH2_SERVER_URI, + "s", + BasicConfig.ISSUER_URL + " or " + BasicConfig.TOKEN_ENDPOINT); + assertThatMessage(messages.get(1), MESSAGE_TEMPLATE_RELATIVE_TOKEN_ENDPOINT) + .containsExactly(expected); + } + + @ParameterizedTest + @CsvSource({ + "https://example.com , https://example.com/v1/oauth/tokens", + "https://example.com/ , https://example.com/v1/oauth/tokens", + "https://example.com/api , https://example.com/api/v1/oauth/tokens", + "https://example.com/api/ , https://example.com/api/v1/oauth/tokens" + }) + void tokenEndpointMissing(String catalogUri, String expected) { + Map input = + Map.of(BasicConfig.CLIENT_ID, "client-id", BasicConfig.CLIENT_SECRET, "client-secret"); + ConfigMigrator migrator = new ConfigMigrator(consumer); + Map actual = migrator.migrateProperties(input); + migrator.handleTokenEndpoint(actual, catalogUri); + assertThat(actual) + .isEqualTo( + Map.of( + BasicConfig.CLIENT_ID, + "client-id", + BasicConfig.CLIENT_SECRET, + "client-secret", + BasicConfig.TOKEN_ENDPOINT, + expected)); + assertThat(messages).hasSize(1); + assertThatMessage(messages.get(0), MESSAGE_TEMPLATE_MISSING_TOKEN_ENDPOINT) + .containsExactly(expected, BasicConfig.TOKEN_ENDPOINT, BasicConfig.ISSUER_URL); + } + + @Test + void tokenEndpointMissingWithIssuerUrl() { + Map input = + Map.of( + BasicConfig.ISSUER_URL, + "https://issuer.com", + BasicConfig.CLIENT_ID, + "client-id", + BasicConfig.CLIENT_SECRET, + "client-secret"); + ConfigMigrator migrator = new ConfigMigrator(consumer); + Map actual = migrator.migrateProperties(input); + migrator.handleTokenEndpoint(actual, "https://catalog.com"); + assertThat(actual).isEqualTo(input); + assertThat(messages).hasSize(0); + } + + @Test + void tokenEndpointMissingWithStaticToken() { + Map input = Map.of(BasicConfig.TOKEN, "static-token"); + ConfigMigrator migrator = new ConfigMigrator(consumer); + Map actual = migrator.migrateProperties(input); + migrator.handleTokenEndpoint(actual, "https://catalog.com"); + assertThat(actual).isEqualTo(input); + assertThat(messages).hasSize(0); + } + + @Test + void tokenEndpointRelative() { + Map input = + Map.of( + BasicConfig.TOKEN_ENDPOINT, + "/relative/token/endpoint", + BasicConfig.CLIENT_ID, + "client-id", + BasicConfig.CLIENT_SECRET, + "client-secret"); + ConfigMigrator migrator = new ConfigMigrator(consumer); + Map actual = migrator.migrateProperties(input); + migrator.handleTokenEndpoint(actual, "https://catalog.com"); + assertThat(actual) + .isEqualTo( + Map.of( + BasicConfig.TOKEN_ENDPOINT, + "https://catalog.com/relative/token/endpoint", + BasicConfig.CLIENT_ID, + "client-id", + BasicConfig.CLIENT_SECRET, + "client-secret")); + assertThat(messages).hasSize(1); + assertThatMessage(messages.get(0), MESSAGE_TEMPLATE_RELATIVE_TOKEN_ENDPOINT) + .containsExactly("https://catalog.com/relative/token/endpoint"); + } + + @Test + void tokenEndpointAbsolute() { + Map input = + Map.of( + BasicConfig.TOKEN_ENDPOINT, + "https://token-endpoint.com/token", + BasicConfig.CLIENT_ID, + "client-id", + BasicConfig.CLIENT_SECRET, + "client-secret"); + ConfigMigrator migrator = new ConfigMigrator(consumer); + Map actual = migrator.migrateProperties(input); + migrator.handleTokenEndpoint(actual, "https://catalog.com"); + assertThat(actual).isEqualTo(input); + assertThat(messages).hasSize(0); + } + + // Full config migration tests + + @Test + void migrateCatalogConfig() { + Map input = + ImmutableMap.builder() + .put(BasicConfig.TOKEN_ENDPOINT, "https://example.com/token") + .put(BasicConfig.CLIENT_ID, "client-id") + .put(BasicConfig.CLIENT_SECRET, "client-secret") + .build(); + ConfigMigrator migrator = new ConfigMigrator(consumer); + OAuth2Config actual = migrator.migrateCatalogConfig(input, "https://example.com"); + assertThat(actual) + .isEqualTo( + ImmutableOAuth2Config.builder() + .basicConfig( + ImmutableBasicConfig.builder() + .tokenEndpoint(URI.create("https://example.com/token")) + .clientId(new ClientID("client-id")) + .clientSecret(new Secret("client-secret")) + .build()) + .build()); + assertThat(messages).hasSize(0); + } + + @Test + void migrateContextualConfigFromEmptyInputEmptyParent() { + // minimal parent config with just a token + OAuth2Config parent = + ImmutableOAuth2Config.builder() + .basicConfig( + ImmutableBasicConfig.builder() + .token(new TypelessAccessToken("access-token-123")) + .build()) + .build(); + Map input = Map.of(); + ConfigMigrator migrator = new ConfigMigrator(consumer); + OAuth2Config actual = migrator.migrateContextualConfig(parent, input, "https://example.com"); + assertThat(actual).isSameAs(parent); + assertThat(messages).hasSize(0); + } + + @Test + void migrateContextualConfigFromEmptyInputNonEmptyParent() { + OAuth2Config parent = + ImmutableOAuth2Config.builder() + .basicConfig( + ImmutableBasicConfig.builder() + .tokenEndpoint(URI.create("https://example.com/token")) + .clientId(new ClientID("parent-client-id")) + .clientSecret(new Secret("parent-client-secret")) + .scope(Scope.parse("parent-scope")) + .build()) + .tokenExchangeConfig( + ImmutableTokenExchangeConfig.builder() + .addAudiences(new Audience("parent-audience")) + .addResources(URI.create("parent-resource")) + .build()) + .build(); + Map input = Map.of(); + ConfigMigrator migrator = new ConfigMigrator(consumer); + OAuth2Config actual = migrator.migrateContextualConfig(parent, input, "https://example.com"); + assertThat(actual).isEqualTo(parent); + assertThat(messages).hasSize(6); + assertThatMessage(messages.get(0), MESSAGE_TEMPLATE_MERGED_CONTEXTUAL_CONFIG) + .containsExactly(BasicConfig.CLIENT_ID); + assertThatMessage(messages.get(1), MESSAGE_TEMPLATE_MERGED_CONTEXTUAL_CONFIG) + .containsExactly(BasicConfig.CLIENT_SECRET); + assertThatMessage(messages.get(2), MESSAGE_TEMPLATE_MERGED_CONTEXTUAL_CONFIG) + .containsExactly(BasicConfig.TOKEN_ENDPOINT); + assertThatMessage(messages.get(3), MESSAGE_TEMPLATE_MERGED_CONTEXTUAL_CONFIG) + .containsExactly(BasicConfig.SCOPE); + assertThatMessage(messages.get(4), MESSAGE_TEMPLATE_MERGED_CONTEXTUAL_CONFIG) + .containsExactly(TokenExchangeConfig.RESOURCES); + assertThatMessage(messages.get(5), MESSAGE_TEMPLATE_MERGED_CONTEXTUAL_CONFIG) + .containsExactly(TokenExchangeConfig.AUDIENCES); + } + + @Test + void migrateContextualConfigFromNonEmptyInputEmptyParent() { + // minimal parent config with just a token + OAuth2Config parent = + ImmutableOAuth2Config.builder() + .basicConfig( + ImmutableBasicConfig.builder() + .token(new TypelessAccessToken("access-token-123")) + .build()) + .build(); + Map input = + Map.of( + BasicConfig.CLIENT_ID, "child-client-id", + BasicConfig.CLIENT_SECRET, "child-client-secret", + BasicConfig.TOKEN_ENDPOINT, "https://example.com/token/child", + BasicConfig.SCOPE, "child-scope", + TokenExchangeConfig.RESOURCES, "child-resource", + TokenExchangeConfig.AUDIENCES, "child-audience"); + ConfigMigrator migrator = new ConfigMigrator(consumer); + OAuth2Config actual = migrator.migrateContextualConfig(parent, input, "https://example.com"); + assertThat(actual) + .isEqualTo( + ImmutableOAuth2Config.builder() + .basicConfig( + ImmutableBasicConfig.builder() + .tokenEndpoint(URI.create("https://example.com/token/child")) + .clientId(new ClientID("child-client-id")) + .clientSecret(new Secret("child-client-secret")) + .scope(Scope.parse("child-scope")) + .build()) + .tokenExchangeConfig( + ImmutableTokenExchangeConfig.builder() + .addAudiences(new Audience("child-audience")) + .addResources(URI.create("child-resource")) + .build()) + .build()); + assertThat(messages).hasSize(0); + } + + @Test + void migrateContextualConfigFromNonEmptyInputNonEmptyParent() { + OAuth2Config parent = + ImmutableOAuth2Config.builder() + .basicConfig( + ImmutableBasicConfig.builder() + .tokenEndpoint(URI.create("https://example.com/token/parent")) + .clientId(new ClientID("parent-client-id")) + .clientSecret(new Secret("parent-client-secret")) + .scope(Scope.parse("parent-scope")) + .build()) + .tokenExchangeConfig( + ImmutableTokenExchangeConfig.builder() + .addAudiences(new Audience("parent-audience")) + .addResources(URI.create("parent-resource")) + .build()) + .build(); + Map input = + Map.of( + BasicConfig.CLIENT_ID, "child-client-id", + BasicConfig.CLIENT_SECRET, "child-client-secret", + BasicConfig.TOKEN_ENDPOINT, "https://example.com/token/child", + BasicConfig.SCOPE, "child-scope", + TokenExchangeConfig.RESOURCES, "child-resource", + TokenExchangeConfig.AUDIENCES, "child-audience"); + ConfigMigrator migrator = new ConfigMigrator(consumer); + OAuth2Config actual = migrator.migrateContextualConfig(parent, input, "https://example.com"); + assertThat(actual) + .isEqualTo( + ImmutableOAuth2Config.builder() + .basicConfig( + ImmutableBasicConfig.builder() + .tokenEndpoint(URI.create("https://example.com/token/child")) + .clientId(new ClientID("child-client-id")) + .clientSecret(new Secret("child-client-secret")) + .scope(Scope.parse("child-scope")) + .build()) + .tokenExchangeConfig( + ImmutableTokenExchangeConfig.builder() + .addAudiences(new Audience("child-audience")) + .addResources(URI.create("child-resource")) + .build()) + .build()); + assertThat(messages).hasSize(0); + } + + @Test + void migrateTableConfigFromEmptyInput() { + OAuth2Config parent = + ImmutableOAuth2Config.builder() + .basicConfig( + ImmutableBasicConfig.builder() + .tokenEndpoint(URI.create("https://example.com/token/parent")) + .clientId(new ClientID("parent-client-id")) + .clientSecret(new Secret("parent-client-secret")) + .scope(Scope.parse("parent-scope")) + .build()) + .build(); + Map input = Map.of(); + ConfigMigrator migrator = new ConfigMigrator(consumer); + OAuth2Config actual = migrator.migrateTableConfig(parent, input); + assertThat(actual).isEqualTo(parent); + assertThat(messages).hasSize(0); + } + + @Test + void migrateTableConfigFromDisallowedInput() { + OAuth2Config parent = + ImmutableOAuth2Config.builder() + .basicConfig( + ImmutableBasicConfig.builder() + .tokenEndpoint(URI.create("https://example.com/token/parent")) + .clientId(new ClientID("parent-client-id")) + .clientSecret(new Secret("parent-client-secret")) + .scope(Scope.parse("parent-scope")) + .build()) + .build(); + // should be filtered out + Map input = + ImmutableMap.of( + BasicConfig.CLIENT_ID, "child-client-id", + BasicConfig.CLIENT_SECRET, "child-client-secret", + BasicConfig.SCOPE, "table-scope"); + ConfigMigrator migrator = new ConfigMigrator(consumer); + OAuth2Config actual = migrator.migrateTableConfig(parent, input); + assertThat(actual).isEqualTo(parent); + assertThat(messages).hasSize(3); + assertThatMessage(messages.get(0), MESSAGE_TEMPLATE_TABLE_CONFIG_NOT_ALLOWED) + .containsExactly(BasicConfig.CLIENT_ID); + assertThatMessage(messages.get(1), MESSAGE_TEMPLATE_TABLE_CONFIG_NOT_ALLOWED) + .containsExactly(BasicConfig.CLIENT_SECRET); + assertThatMessage(messages.get(2), MESSAGE_TEMPLATE_TABLE_CONFIG_NOT_ALLOWED) + .containsExactly(BasicConfig.SCOPE); + } + + @Test + void migrateTableConfigFromNonEmptyInputWithVendedStaticToken() { + OAuth2Config parent = + ImmutableOAuth2Config.builder() + .basicConfig( + ImmutableBasicConfig.builder() + .tokenEndpoint(URI.create("https://example.com/token/parent")) + .clientId(new ClientID("parent-client-id")) + .clientSecret(new Secret("parent-client-secret")) + .scope(Scope.parse("parent-scope")) + .build()) + .build(); + Map input = + ImmutableMap.builder().put(BasicConfig.TOKEN, "access-token-123").build(); + ConfigMigrator migrator = new ConfigMigrator(consumer); + OAuth2Config actual = migrator.migrateTableConfig(parent, input); + assertThat(actual) + .isEqualTo( + ImmutableOAuth2Config.builder() + .basicConfig( + ImmutableBasicConfig.builder() + // from table config + .token(new TypelessAccessToken("access-token-123")) + // from parent config + .tokenEndpoint(URI.create("https://example.com/token/parent")) + .clientId(new ClientID("parent-client-id")) + .clientSecret(new Secret("parent-client-secret")) + .scope(Scope.parse("parent-scope")) + .build()) + .build()); + assertThat(messages).hasSize(1); + assertThatMessage(messages.get(0), MESSAGE_TEMPLATE_VENDED_TOKEN) + .containsExactly(BasicConfig.TOKEN); + } + + @Test + void migrateTableConfigFromNonEmptyInputWithVendedTokenExchange() throws ParseException { + OAuth2Config parent = + ImmutableOAuth2Config.builder() + .basicConfig( + ImmutableBasicConfig.builder() + .tokenEndpoint(URI.create("https://example.com/token/parent")) + .clientId(new ClientID("parent-client-id")) + .clientSecret(new Secret("parent-client-secret")) + .scope(Scope.parse("parent-scope")) + .build()) + .build(); + Map input = + ImmutableMap.builder() + .put(TokenExchangeConfig.SUBJECT_TOKEN, "id-token-123") + .put(TokenExchangeConfig.SUBJECT_TOKEN_TYPE, OAuth2Properties.ID_TOKEN_TYPE) + .put(TokenExchangeConfig.ACTOR_TOKEN, ConfigUtil.PARENT_TOKEN) + .put(TokenExchangeConfig.ACTOR_TOKEN_TYPE, OAuth2Properties.ACCESS_TOKEN_TYPE) + .build(); + ConfigMigrator migrator = new ConfigMigrator(consumer); + OAuth2Config actual = migrator.migrateTableConfig(parent, input); + assertThat(actual) + .isEqualTo( + ImmutableOAuth2Config.builder() + // from parent config + .basicConfig( + ImmutableBasicConfig.builder() + .grantType(GrantType.TOKEN_EXCHANGE) + .tokenEndpoint(URI.create("https://example.com/token/parent")) + .clientId(new ClientID("parent-client-id")) + .clientSecret(new Secret("parent-client-secret")) + .scope(Scope.parse("parent-scope")) + .build()) + // from table config + .tokenExchangeConfig( + ImmutableTokenExchangeConfig.builder() + .subjectTokenString("id-token-123") + .subjectTokenType(TokenTypeURI.parse(OAuth2Properties.ID_TOKEN_TYPE)) + .actorTokenString(ConfigUtil.PARENT_TOKEN) + .actorTokenType(TokenTypeURI.parse(OAuth2Properties.ACCESS_TOKEN_TYPE)) + .build()) + .build()); + assertThat(messages).hasSize(4); + assertThatMessage(messages.get(0), MESSAGE_TEMPLATE_VENDED_TOKEN) + .containsExactly(TokenExchangeConfig.SUBJECT_TOKEN); + assertThatMessage(messages.get(1), MESSAGE_TEMPLATE_VENDED_TOKEN) + .containsExactly(TokenExchangeConfig.SUBJECT_TOKEN_TYPE); + assertThatMessage(messages.get(2), MESSAGE_TEMPLATE_VENDED_TOKEN) + .containsExactly(TokenExchangeConfig.ACTOR_TOKEN); + assertThatMessage(messages.get(3), MESSAGE_TEMPLATE_VENDED_TOKEN) + .containsExactly(TokenExchangeConfig.ACTOR_TOKEN_TYPE); + } + + private static ListAssert assertThatMessage( + Pair> message, String template) { + assertThat(message).extracting(Pair::first).isEqualTo(template); + return assertThat(message).extracting(Pair::second).asInstanceOf(list(String.class)); + } +} diff --git a/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/config/TestConfigUtil.java b/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/config/TestConfigUtil.java new file mode 100644 index 000000000000..66b53a57b91b --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/config/TestConfigUtil.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.config; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.nimbusds.oauth2.sdk.auth.ClientAuthenticationMethod; +import java.util.Map; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +class TestConfigUtil { + + @ParameterizedTest + @MethodSource + void requiresClientSecret(ClientAuthenticationMethod method, boolean expectedResult) { + assertThat(ConfigUtil.requiresClientSecret(method)).isEqualTo(expectedResult); + } + + static Stream requiresClientSecret() { + return Stream.of( + Arguments.of(null, false), + Arguments.of(ClientAuthenticationMethod.CLIENT_SECRET_BASIC, true), + Arguments.of(ClientAuthenticationMethod.CLIENT_SECRET_POST, true), + Arguments.of(ClientAuthenticationMethod.NONE, false)); + } + + @Test + void parseOptional() { + assertThat(ConfigUtil.parseOptional(Map.of("a", "1"), "a")).hasValue("1"); + assertThat(ConfigUtil.parseOptional(Map.of("a", "1"), "b")).isEmpty(); + } + + @Test + void parseOptionalWithParser() { + assertThat(ConfigUtil.parseOptional(Map.of("a", "1"), "a", Integer::parseInt)).hasValue(1); + assertThat(ConfigUtil.parseOptional(Map.of("a", "1"), "b", Integer::parseInt)).isEmpty(); + } + + @Test + void parseOptionalWithParserFailure() { + assertThatThrownBy( + () -> ConfigUtil.parseOptional(Map.of("a", "invalid"), "a", Integer::parseInt)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Failed to parse configuration value 'invalid'") + .hasRootCauseInstanceOf(NumberFormatException.class); + } + + @Test + void parseOptionalInt() { + assertThat(ConfigUtil.parseOptionalInt(Map.of("a", "1"), "a")).hasValue(1); + assertThat(ConfigUtil.parseOptionalInt(Map.of("a", "1"), "b")).isEmpty(); + } + + @Test + void parseOptionalIntFailure() { + assertThatThrownBy(() -> ConfigUtil.parseOptionalInt(Map.of("a", "invalid"), "a")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Failed to parse configuration value 'invalid'") + .hasRootCauseInstanceOf(NumberFormatException.class); + } + + @Test + void parseList() { + assertThat(ConfigUtil.parseList(Map.of("a", ""), "a", ",")).isEmpty(); + assertThat(ConfigUtil.parseList(Map.of("a", "1,2,3"), "a", ",")).containsExactly("1", "2", "3"); + assertThat(ConfigUtil.parseList(Map.of("a", "1 , 2 , 3"), "a", ",")) + .containsExactly("1", "2", "3"); + assertThat(ConfigUtil.parseList(Map.of("a", "1 2 3"), "a", " ")) + .containsExactly("1", "2", "3"); + assertThat(ConfigUtil.parseList(Map.of("a", "1,2,3"), "b", ",")).isEmpty(); + } + + @Test + void parseListWithParser() { + assertThat(ConfigUtil.parseList(Map.of("a", ""), "a", ",", Integer::parseInt)).isEmpty(); + assertThat(ConfigUtil.parseList(Map.of("a", "1,2,3"), "a", ",", Integer::parseInt)) + .containsExactly(1, 2, 3); + assertThat(ConfigUtil.parseList(Map.of("a", "1 , 2 , 3"), "a", ",", Integer::parseInt)) + .containsExactly(1, 2, 3); + assertThat(ConfigUtil.parseList(Map.of("a", "1 2 3"), "a", " ", Integer::parseInt)) + .containsExactly(1, 2, 3); + assertThat(ConfigUtil.parseList(Map.of("a", "1,2,3"), "b", ",", Integer::parseInt)).isEmpty(); + } + + @Test + void parseListWithParserFailure() { + assertThatThrownBy( + () -> ConfigUtil.parseList(Map.of("a", "1,invalid,3"), "a", ",", Integer::parseInt)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Failed to parse configuration value 'invalid'") + .hasRootCauseInstanceOf(NumberFormatException.class); + } +} diff --git a/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/config/TestTokenExchangeConfig.java b/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/config/TestTokenExchangeConfig.java new file mode 100644 index 000000000000..b971681b42c5 --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/config/TestTokenExchangeConfig.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.config; + +import static org.apache.iceberg.rest.auth.oauth2.config.TokenExchangeConfig.ACTOR_TOKEN; +import static org.apache.iceberg.rest.auth.oauth2.config.TokenExchangeConfig.ACTOR_TOKEN_TYPE; +import static org.apache.iceberg.rest.auth.oauth2.config.TokenExchangeConfig.AUDIENCES; +import static org.apache.iceberg.rest.auth.oauth2.config.TokenExchangeConfig.REQUESTED_TOKEN_TYPE; +import static org.apache.iceberg.rest.auth.oauth2.config.TokenExchangeConfig.RESOURCES; +import static org.apache.iceberg.rest.auth.oauth2.config.TokenExchangeConfig.SUBJECT_TOKEN; +import static org.apache.iceberg.rest.auth.oauth2.config.TokenExchangeConfig.SUBJECT_TOKEN_TYPE; +import static org.assertj.core.api.Assertions.assertThat; + +import com.nimbusds.oauth2.sdk.ParseException; +import com.nimbusds.oauth2.sdk.id.Audience; +import com.nimbusds.oauth2.sdk.token.TokenTypeURI; +import java.net.URI; +import java.util.Map; +import java.util.stream.Stream; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +class TestTokenExchangeConfig { + + @ParameterizedTest + @MethodSource + void testParse(Map properties, TokenExchangeConfig expected) { + TokenExchangeConfig actual = TokenExchangeConfig.parse(properties).build(); + assertThat(actual).isEqualTo(expected); + } + + static Stream testParse() throws ParseException { + return Stream.of( + Arguments.of(Map.of(), ImmutableTokenExchangeConfig.builder().build()), + Arguments.of( + Map.of( + SUBJECT_TOKEN, + "my-subject-token", + SUBJECT_TOKEN_TYPE, + "urn:ietf:params:oauth:token-type:jwt"), + ImmutableTokenExchangeConfig.builder() + .subjectTokenString("my-subject-token") + .subjectTokenType(TokenTypeURI.parse("urn:ietf:params:oauth:token-type:jwt")) + .build()), + Arguments.of( + Map.of( + ACTOR_TOKEN, + "my-actor-token", + ACTOR_TOKEN_TYPE, + "urn:ietf:params:oauth:token-type:jwt"), + ImmutableTokenExchangeConfig.builder() + .actorTokenString("my-actor-token") + .actorTokenType(TokenTypeURI.parse("urn:ietf:params:oauth:token-type:jwt")) + .build()), + Arguments.of( + Map.of(REQUESTED_TOKEN_TYPE, "urn:ietf:params:oauth:token-type:jwt"), + ImmutableTokenExchangeConfig.builder() + .requestedTokenType(TokenTypeURI.parse("urn:ietf:params:oauth:token-type:jwt")) + .build()), + Arguments.of( + Map.of(RESOURCES, "https://example.com/api"), + ImmutableTokenExchangeConfig.builder() + .addResources(URI.create("https://example.com/api")) + .build()), + Arguments.of( + Map.of(AUDIENCES, "https://example.com/resource"), + ImmutableTokenExchangeConfig.builder() + .addAudiences(new Audience("https://example.com/resource")) + .build()), + Arguments.of( + Map.of(AUDIENCES, "https://example.com/resource1,https://example.com/resource2"), + ImmutableTokenExchangeConfig.builder() + .addAudiences( + new Audience("https://example.com/resource1"), + new Audience("https://example.com/resource2")) + .build())); + } +} diff --git a/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/config/TestTokenRefreshConfig.java b/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/config/TestTokenRefreshConfig.java new file mode 100644 index 000000000000..6c1847121847 --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/config/TestTokenRefreshConfig.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.config; + +import static org.apache.iceberg.rest.auth.oauth2.config.TokenRefreshConfig.ACCESS_TOKEN_LIFESPAN; +import static org.apache.iceberg.rest.auth.oauth2.config.TokenRefreshConfig.ENABLED; +import static org.apache.iceberg.rest.auth.oauth2.config.TokenRefreshConfig.SAFETY_MARGIN; +import static org.apache.iceberg.rest.auth.oauth2.config.TokenRefreshConfig.TOKEN_EXCHANGE_ENABLED; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; + +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +class TestTokenRefreshConfig { + + @ParameterizedTest + @MethodSource + @SuppressWarnings("ResultOfMethodCallIgnored") + void testValidate(Map properties, List expected) { + assertThatIllegalArgumentException() + .isThrownBy(() -> TokenRefreshConfig.parse(properties).build()) + .withMessage(ConfigValidator.buildDescription(expected.stream())); + } + + static Stream testValidate() { + return Stream.of( + Arguments.of( + Map.of(ACCESS_TOKEN_LIFESPAN, "PT2S"), + List.of( + "access token lifespan must be greater than or equal to PT15S (rest.auth.oauth2.token-refresh.access-token-lifespan)", + "refresh safety margin must be less than the access token lifespan (rest.auth.oauth2.token-refresh.safety-margin / rest.auth.oauth2.token-refresh.access-token-lifespan)")), + Arguments.of( + Map.of(SAFETY_MARGIN, "PT0.1S"), + List.of( + "refresh safety margin must be greater than or equal to PT10S (rest.auth.oauth2.token-refresh.safety-margin)")), + Arguments.of( + Map.of(SAFETY_MARGIN, "PT10M", ACCESS_TOKEN_LIFESPAN, "PT5M"), + List.of( + "refresh safety margin must be less than the access token lifespan (rest.auth.oauth2.token-refresh.safety-margin / rest.auth.oauth2.token-refresh.access-token-lifespan)"))); + } + + @ParameterizedTest + @MethodSource + void testParse(Map properties, TokenRefreshConfig expected) { + TokenRefreshConfig actual = TokenRefreshConfig.parse(properties).build(); + assertThat(actual).isEqualTo(expected); + } + + static Stream testParse() { + return Stream.of( + Arguments.of(Map.of(), ImmutableTokenRefreshConfig.builder().build()), + Arguments.of( + Map.of(ENABLED, "false"), ImmutableTokenRefreshConfig.builder().enabled(false).build()), + Arguments.of( + Map.of(TOKEN_EXCHANGE_ENABLED, "false"), + ImmutableTokenRefreshConfig.builder().tokenExchangeEnabled(false).build()), + Arguments.of( + Map.of(ACCESS_TOKEN_LIFESPAN, "PT10M"), + ImmutableTokenRefreshConfig.builder() + .accessTokenLifespan(Duration.ofMinutes(10)) + .build()), + Arguments.of( + Map.of(SAFETY_MARGIN, "PT30S"), + ImmutableTokenRefreshConfig.builder().safetyMargin(Duration.ofSeconds(30)).build())); + } +} diff --git a/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/flow/TestClientCredentialsFlow.java b/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/flow/TestClientCredentialsFlow.java new file mode 100644 index 000000000000..f9b2c3416e68 --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/flow/TestClientCredentialsFlow.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.flow; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.nimbusds.oauth2.sdk.auth.ClientAuthenticationMethod; +import java.util.concurrent.ExecutionException; +import org.apache.iceberg.rest.auth.oauth2.test.TestEnvironment; +import org.apache.iceberg.rest.auth.oauth2.test.junit.EnumLike; +import org.junitpioneer.jupiter.cartesian.CartesianTest; + +class TestClientCredentialsFlow { + + @CartesianTest + void fetchNewTokens(@EnumLike(excludes = "none") ClientAuthenticationMethod authenticationMethod) + throws InterruptedException, ExecutionException { + try (TestEnvironment env = + TestEnvironment.builder().clientAuthenticationMethod(authenticationMethod).build()) { + FlowFactory flowFactory = env.newFlowFactory(); + Flow flow = flowFactory.newInitialFlow(); + assertThat(flow).isInstanceOf(ClientCredentialsFlow.class); + TokensResult tokens = flow.execute().toCompletableFuture().get(); + env.assertTokensResult(tokens, "access_initial", null); + } + } +} diff --git a/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/flow/TestEndpointProvider.java b/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/flow/TestEndpointProvider.java new file mode 100644 index 000000000000..aed4be32e15f --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/flow/TestEndpointProvider.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.flow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; +import static org.assertj.core.api.InstanceOfAssertFactories.throwable; + +import com.nimbusds.oauth2.sdk.ParseException; +import org.apache.iceberg.rest.auth.oauth2.test.TestEnvironment; +import org.apache.iceberg.rest.auth.oauth2.test.TestServer; +import org.junit.jupiter.api.Test; +import org.junitpioneer.jupiter.cartesian.CartesianTest; +import org.junitpioneer.jupiter.cartesian.CartesianTest.Values; +import org.mockserver.model.HttpRequest; +import org.mockserver.model.HttpResponse; +import org.mockserver.model.JsonBody; +import org.mockserver.model.MediaType; + +class TestEndpointProvider { + + private static final String INVALID_METADATA = "{\"token_endpoint\":\"not a valid URL\"}"; + + @Test + void withoutDiscovery() { + try (TestEnvironment env = TestEnvironment.builder().discoveryEnabled(false).build()) { + EndpointProvider endpointProvider = EndpointProvider.of(env.config(), env.runtime()); + assertThat(endpointProvider.resolvedTokenEndpoint()).isEqualTo(env.tokenEndpoint()); + } + } + + @CartesianTest + void withDiscovery( + @Values( + strings = { + ".well-known/openid-configuration", + ".well-known/oauth-authorization-server" + }) + String wellKnownPath) { + try (TestEnvironment env = + TestEnvironment.builder().discoveryEnabled(true).wellKnownPath(wellKnownPath).build()) { + EndpointProvider endpointProvider = EndpointProvider.of(env.config(), env.runtime()); + assertThat(endpointProvider.resolvedTokenEndpoint()).isEqualTo(env.tokenEndpoint()); + } + } + + @Test + void fetchServerMetadataWrongEndpoint() { + try (TestEnvironment env = TestEnvironment.builder().createDefaultExpectations(false).build()) { + env.createErrorExpectations(); + EndpointProvider endpointProvider = EndpointProvider.of(env.config(), env.runtime()); + Throwable throwable = catchThrowable(endpointProvider::serverMetadata); + assertThat(throwable) + .isInstanceOf(RuntimeException.class) + .hasMessageContaining("Failed to fetch provider metadata"); + // OIDC well-known path + assertThat(throwable.getCause()) + .asInstanceOf(throwable(RuntimeException.class)) + .hasMessageContaining("Failed to fetch OIDC provider metadata") + .hasMessageContaining("Invalid request"); + // OAuth well-known path + assertThat(throwable.getSuppressed()) + .singleElement() + .asInstanceOf(throwable(RuntimeException.class)) + .hasMessageContaining("Failed to fetch OAuth provider metadata") + .hasMessageContaining("Invalid request"); + } + } + + @Test + @SuppressWarnings("resource") + void fetchServerMetadataWrongData() { + try (TestEnvironment env = TestEnvironment.builder().createDefaultExpectations(false).build()) { + TestServer.instance() + .when( + HttpRequest.request() + .withPath(env.authorizationServerUrl().getPath() + ".well-known/.*")) + .respond( + HttpResponse.response() + .withStatusCode(200) + .withContentType(MediaType.APPLICATION_JSON) + .withBody(JsonBody.json(INVALID_METADATA))); + EndpointProvider endpointProvider = EndpointProvider.of(env.config(), env.runtime()); + Throwable throwable = catchThrowable(endpointProvider::serverMetadata); + // OIDC well-known path + assertThat(throwable) + .isInstanceOf(RuntimeException.class) + .hasMessageContaining("Failed to fetch provider metadata"); + assertThat(throwable.getCause()) + .asInstanceOf(throwable(ParseException.class)) + .hasMessageContaining("Illegal character in path at index 3: not a valid URL"); + // OAuth well-known path + assertThat(throwable.getSuppressed()) + .singleElement() + .asInstanceOf(throwable(ParseException.class)) + .hasMessageContaining("Illegal character in path at index 3: not a valid URL"); + } + } +} diff --git a/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/flow/TestFlowFactory.java b/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/flow/TestFlowFactory.java new file mode 100644 index 000000000000..7894bddf0d3a --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/flow/TestFlowFactory.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.flow; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.nimbusds.oauth2.sdk.GrantType; +import com.nimbusds.oauth2.sdk.token.BearerAccessToken; +import com.nimbusds.oauth2.sdk.token.RefreshToken; +import com.nimbusds.oauth2.sdk.token.Tokens; +import java.util.stream.Stream; +import org.apache.iceberg.rest.auth.oauth2.test.TestEnvironment; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +class TestFlowFactory { + + @ParameterizedTest + @MethodSource + void testNewInitialFlow(Class flowClass, GrantType grantType) { + try (TestEnvironment env = TestEnvironment.builder().grantType(grantType).build()) { + FlowFactory flowFactory = env.newFlowFactory(); + Flow flow = flowFactory.newInitialFlow(); + assertThat(flow).isNotNull(); + assertThat(flow).isInstanceOf(flowClass); + assertThat(flow.grantType()).isEqualTo(grantType); + } + } + + static Stream testNewInitialFlow() { + return Stream.of( + Arguments.of(ClientCredentialsFlow.class, GrantType.CLIENT_CREDENTIALS), + Arguments.of(TokenExchangeFlow.class, GrantType.TOKEN_EXCHANGE)); + } + + @Test + void testNewRefreshFlow() { + Tokens currentTokens = + new Tokens(new BearerAccessToken("access_token"), new RefreshToken("refresh_token")); + try (TestEnvironment env = + TestEnvironment.builder().grantType(GrantType.TOKEN_EXCHANGE).build()) { + FlowFactory flowFactory = env.newFlowFactory(); + Flow flow = flowFactory.newRefreshFlow(currentTokens); + assertThat(flow).isNotNull(); + assertThat(flow).isInstanceOf(RefreshTokenFlow.class); + assertThat(flow.grantType()).isEqualTo(GrantType.REFRESH_TOKEN); + } + } + + @Test + void testNewRefreshFlowWithTokenExchange() { + Tokens currentTokens = new Tokens(new BearerAccessToken("access_token"), null); + try (TestEnvironment env = + TestEnvironment.builder().tokenRefreshWithTokenExchangeEnabled(true).build()) { + FlowFactory flowFactory = env.newFlowFactory(); + Flow flow = flowFactory.newRefreshFlow(currentTokens); + assertThat(flow).isNotNull(); + assertThat(flow).isInstanceOf(TokenExchangeFlow.class); + assertThat(flow.grantType()).isEqualTo(GrantType.TOKEN_EXCHANGE); + } + } + + @Test + void testNewRefreshFlowFailure() { + Tokens currentTokens = new Tokens(new BearerAccessToken("access_token"), null); + try (TestEnvironment env = + TestEnvironment.builder().tokenRefreshWithTokenExchangeEnabled(false).build()) { + FlowFactory flowFactory = env.newFlowFactory(); + assertThatThrownBy(() -> flowFactory.newRefreshFlow(currentTokens)) + .isInstanceOf(IllegalStateException.class) + .hasMessage( + "Cannot create refresh token flow: no refresh token present and token exchange is disabled"); + } + } +} diff --git a/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/flow/TestRefreshTokenFlow.java b/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/flow/TestRefreshTokenFlow.java new file mode 100644 index 000000000000..6d8207866a22 --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/flow/TestRefreshTokenFlow.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.flow; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.nimbusds.oauth2.sdk.GrantType; +import com.nimbusds.oauth2.sdk.auth.ClientAuthenticationMethod; +import com.nimbusds.oauth2.sdk.token.BearerAccessToken; +import com.nimbusds.oauth2.sdk.token.RefreshToken; +import com.nimbusds.oauth2.sdk.token.Tokens; +import java.util.concurrent.ExecutionException; +import org.apache.iceberg.rest.auth.oauth2.test.TestEnvironment; +import org.apache.iceberg.rest.auth.oauth2.test.junit.EnumLike; +import org.junitpioneer.jupiter.cartesian.CartesianTest; +import org.junitpioneer.jupiter.cartesian.CartesianTest.Values; + +class TestRefreshTokenFlow { + + /** + * Emulates token refresh with standard refresh_token grant. The initial grant is token exchange, + * in order to exercise public clients as well. + */ + @CartesianTest + void fetchNewTokens( + @EnumLike ClientAuthenticationMethod authenticationMethod, + @Values(booleans = {true, false}) boolean returnRefreshTokens) + throws ExecutionException, InterruptedException { + Tokens currentTokens = + new Tokens(new BearerAccessToken("access_initial"), new RefreshToken("refresh_initial")); + try (TestEnvironment env = + TestEnvironment.builder() + .grantType(GrantType.TOKEN_EXCHANGE) + .clientAuthenticationMethod(authenticationMethod) + .tokenRefreshWithTokenExchangeEnabled(false) + .returnRefreshTokens(returnRefreshTokens) + .build()) { + FlowFactory flowFactory = env.newFlowFactory(); + Flow flow = flowFactory.newRefreshFlow(currentTokens); + TokensResult tokens = flow.execute().toCompletableFuture().get(); + assertThat(flow).isInstanceOf(RefreshTokenFlow.class); + env.assertTokensResult( + tokens, + "access_refreshed", + returnRefreshTokens ? "refresh_refreshed" : "refresh_initial"); + } + } + + /** Emulates token refresh with legacy catalog servers using token exchange. */ + @CartesianTest + void fetchNewTokensWithTokenExchange( + @EnumLike(excludes = "none") ClientAuthenticationMethod authenticationMethod) + throws ExecutionException, InterruptedException { + Tokens currentTokens = new Tokens(new BearerAccessToken("access_initial"), null); + try (TestEnvironment env = + TestEnvironment.builder() + .grantType(GrantType.CLIENT_CREDENTIALS) + .clientAuthenticationMethod(authenticationMethod) + .tokenRefreshWithTokenExchangeEnabled(true) + .returnRefreshTokens(false) + .build()) { + FlowFactory flowFactory = env.newFlowFactory(); + Flow flow = flowFactory.newRefreshFlow(currentTokens); + TokensResult tokens = flow.execute().toCompletableFuture().get(); + assertThat(flow).isInstanceOf(TokenExchangeFlow.class); + env.assertTokensResult(tokens, "access_refreshed", null); + } + } +} diff --git a/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/flow/TestTokenExchangeFlow.java b/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/flow/TestTokenExchangeFlow.java new file mode 100644 index 000000000000..f39a201510a8 --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/flow/TestTokenExchangeFlow.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.flow; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.nimbusds.oauth2.sdk.GrantType; +import com.nimbusds.oauth2.sdk.auth.ClientAuthenticationMethod; +import com.nimbusds.oauth2.sdk.id.Audience; +import com.nimbusds.oauth2.sdk.token.TokenTypeURI; +import java.net.URI; +import java.util.List; +import java.util.concurrent.ExecutionException; +import org.apache.iceberg.rest.auth.oauth2.client.OAuth2Client; +import org.apache.iceberg.rest.auth.oauth2.config.ConfigUtil; +import org.apache.iceberg.rest.auth.oauth2.test.TestEnvironment; +import org.apache.iceberg.rest.auth.oauth2.test.junit.EnumLike; +import org.junitpioneer.jupiter.cartesian.CartesianTest; +import org.junitpioneer.jupiter.cartesian.CartesianTest.Values; + +class TestTokenExchangeFlow { + + @CartesianTest + void fetchNewTokensStatic( + @EnumLike ClientAuthenticationMethod authenticationMethod, + @Values(booleans = {true, false}) boolean returnRefreshTokens) + throws InterruptedException, ExecutionException { + try (TestEnvironment env = + TestEnvironment.builder() + .grantType(GrantType.TOKEN_EXCHANGE) + .clientAuthenticationMethod(authenticationMethod) + .returnRefreshTokens(returnRefreshTokens) + // test without audiences and resources + .audiences(List.of()) + .resources(List.of()) + .build()) { + FlowFactory flowFactory = env.newFlowFactory(); + Flow flow = flowFactory.newInitialFlow(); + assertThat(flow).isInstanceOf(TokenExchangeFlow.class); + TokensResult tokens = flow.execute().toCompletableFuture().get(); + env.assertTokensResult( + tokens, "access_initial", returnRefreshTokens ? "refresh_initial" : null); + } + } + + @CartesianTest + void fetchNewTokensStaticWithTokenTypes( + @EnumLike ClientAuthenticationMethod authenticationMethod, + @EnumLike( + includes = { + "urn:ietf:params:oauth:token-type:access_token", + "urn:ietf:params:oauth:token-type:jwt" + }) + TokenTypeURI tokenType, + @Values(booleans = {true, false}) boolean returnRefreshTokens) + throws InterruptedException, ExecutionException { + try (TestEnvironment env = + TestEnvironment.builder() + .grantType(GrantType.TOKEN_EXCHANGE) + .clientAuthenticationMethod(authenticationMethod) + .returnRefreshTokens(returnRefreshTokens) + .subjectTokenType(tokenType) + .actorTokenType(tokenType) + .requestedTokenType(tokenType) + // test multiple audiences and resources + .addAudiences(new Audience("audience1")) + .addAudiences(new Audience("audience2")) + .addResources(URI.create("https://example.com/api1")) + .addResources(URI.create("https://example.com/api2")) + .build()) { + FlowFactory flowFactory = env.newFlowFactory(); + Flow flow = flowFactory.newInitialFlow(); + assertThat(flow).isInstanceOf(TokenExchangeFlow.class); + TokensResult tokens = flow.execute().toCompletableFuture().get(); + env.assertTokensResult( + tokens, "access_initial", returnRefreshTokens ? "refresh_initial" : null); + } + } + + @CartesianTest + void fetchNewTokensParent( + @EnumLike ClientAuthenticationMethod authenticationMethod, + @Values(booleans = {true, false}) boolean returnRefreshTokens) + throws InterruptedException, ExecutionException { + try (TestEnvironment parent = TestEnvironment.builder().build(); + OAuth2Client parentClient = parent.newOAuth2Client(); + TestEnvironment env = + TestEnvironment.builder() + .grantType(GrantType.TOKEN_EXCHANGE) + .clientAuthenticationMethod(authenticationMethod) + .returnRefreshTokens(returnRefreshTokens) + .subjectTokenString(ConfigUtil.PARENT_TOKEN) + .actorTokenString(ConfigUtil.PARENT_TOKEN) + .parentClient(parentClient) + .build()) { + FlowFactory flowFactory = env.newFlowFactory(); + Flow flow = flowFactory.newInitialFlow(); + assertThat(flow).isInstanceOf(TokenExchangeFlow.class); + TokensResult tokens = flow.execute().toCompletableFuture().get(); + env.assertTokensResult( + tokens, "access_initial", returnRefreshTokens ? "refresh_initial" : null); + } + } +} diff --git a/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/flow/TestTokensResult.java b/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/flow/TestTokensResult.java new file mode 100644 index 000000000000..ed0b0cb5f0df --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/flow/TestTokensResult.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.flow; + +import static org.apache.iceberg.rest.auth.oauth2.test.TestEnvironment.NOW; +import static org.assertj.core.api.Assertions.assertThat; + +import com.nimbusds.jose.JOSEException; +import com.nimbusds.jose.JWSAlgorithm; +import com.nimbusds.jose.JWSHeader; +import com.nimbusds.jose.crypto.MACSigner; +import com.nimbusds.jwt.JWTClaimsSet; +import com.nimbusds.jwt.SignedJWT; +import com.nimbusds.oauth2.sdk.AccessTokenResponse; +import com.nimbusds.oauth2.sdk.token.BearerAccessToken; +import com.nimbusds.oauth2.sdk.token.RefreshToken; +import com.nimbusds.oauth2.sdk.token.Tokens; +import java.time.Instant; +import java.util.Date; +import org.apache.iceberg.rest.auth.oauth2.test.TestClock; +import org.junit.jupiter.api.Test; + +class TestTokensResult { + + private final TestClock clock = new TestClock(NOW); + + @Test + void testCreateFromOpaqueAccessToken() { + BearerAccessToken accessToken = new BearerAccessToken("test-access-token"); + TokensResult result = TokensResult.of(accessToken, clock); + assertThat(result.tokens().getAccessToken()).isEqualTo(accessToken); + assertThat(result.tokens().getRefreshToken()).isNull(); + assertThat(result.receivedAt()).isEqualTo(NOW); + assertThat(result.accessTokenExpirationTime()).isEmpty(); + } + + @Test + void testCreateFromJwtAccessToken() throws JOSEException { + JWTClaimsSet claimsSet = + new JWTClaimsSet.Builder() + .subject("test-subject") + .expirationTime(Date.from(NOW.plusSeconds(100))) + .build(); + BearerAccessToken accessToken = new BearerAccessToken(createJwtToken(claimsSet)); + TokensResult result = TokensResult.of(accessToken, clock); + assertThat(result.tokens().getAccessToken()).isEqualTo(accessToken); + assertThat(result.tokens().getRefreshToken()).isNull(); + assertThat(result.receivedAt()).isEqualTo(NOW); + assertThat(result.accessTokenExpirationTime()).hasValue(NOW.plusSeconds(100)); + } + + @Test + void testCreateFromAccessTokenResponse() { + BearerAccessToken accessToken = new BearerAccessToken("test-access-token"); + RefreshToken refreshToken = new RefreshToken("test-refresh-token"); + Tokens tokens = new Tokens(accessToken, refreshToken); + AccessTokenResponse accessTokenResponse = new AccessTokenResponse(tokens); + TokensResult result = TokensResult.of(accessTokenResponse, clock); + assertThat(result.tokens()).isEqualTo(tokens); + assertThat(result.receivedAt()).isEqualTo(NOW); + } + + // Access token expiration tests + + @Test + void testAccessTokenNoExpirationTime() { + BearerAccessToken accessToken = new BearerAccessToken("test-access-token"); + TokensResult result = TokensResult.of(accessToken, clock); + assertThat(result.accessTokenExpirationTime()).isEmpty(); + } + + @Test + void testAccessTokenExpirationTimeFromResponse() { + BearerAccessToken accessToken = new BearerAccessToken("test-access-token", 7200, null); + TokensResult result = TokensResult.of(accessToken, clock); + assertThat(result.accessTokenExpirationTime()).hasValue(NOW.plusSeconds(7200)); + } + + @Test + void testAccessTokenExpirationTimeFromJwt() throws JOSEException { + Instant jwtExpiration = NOW.plusSeconds(7200); + String jwtToken = createJwtToken(jwtExpiration); + BearerAccessToken accessToken = new BearerAccessToken(jwtToken); + TokensResult result = TokensResult.of(accessToken, clock); + assertThat(result.accessTokenExpirationTime()).hasValue(jwtExpiration); + } + + @Test + void testAccessTokenExpirationTimeResponseTakesPrecedenceOverJwt() throws JOSEException { + Instant jwtExpiration = NOW.plusSeconds(7200); + String jwtToken = createJwtToken(jwtExpiration); + BearerAccessToken accessToken = new BearerAccessToken(jwtToken, 3600, null); + TokensResult result = TokensResult.of(accessToken, clock); + Instant expectedExpiration = NOW.plusSeconds(3600); + assertThat(result.accessTokenExpirationTime()).hasValue(expectedExpiration); + } + + @Test + void testAccessTokenExpirationTimeFromResponseUsesReceivedAtNotJwtIat() throws JOSEException { + // JWT iat is 10 seconds ahead of local clock; expires_in should be relative to local clock + JWTClaimsSet claimsSet = + new JWTClaimsSet.Builder() + .subject("test-subject") + .issueTime(Date.from(NOW.plusSeconds(10))) + .expirationTime(Date.from(NOW.plusSeconds(7200))) + .build(); + BearerAccessToken accessToken = new BearerAccessToken(createJwtToken(claimsSet), 3600, null); + TokensResult result = TokensResult.of(accessToken, clock); + // expires_in uses receivedAt() (NOW), not the JWT iat (NOW+10) + assertThat(result.receivedAt()).isEqualTo(NOW); + assertThat(result.accessTokenExpirationTime()).hasValue(NOW.plusSeconds(3600)); + } + + @Test + void testAccessTokenExpirationTimeNullWhenZeroLifetime() { + BearerAccessToken accessToken = new BearerAccessToken("test-access-token", 0, null); + Tokens tokens = new Tokens(accessToken, null); + TokensResult result = TokensResult.of(tokens, clock); + assertThat(result.accessTokenExpirationTime()).isEmpty(); + } + + @Test + void testAccessTokenExpirationTimeNullWhenNegativeLifetime() { + BearerAccessToken accessToken = new BearerAccessToken("test-access-token", -1, null); + Tokens tokens = new Tokens(accessToken, null); + TokensResult result = TokensResult.of(tokens, clock); + assertThat(result.accessTokenExpirationTime()).isEmpty(); + } + + private static String createJwtToken(Instant expiration) throws JOSEException { + return createJwtToken( + new JWTClaimsSet.Builder() + .subject("test-subject") + .expirationTime(Date.from(expiration)) + .build()); + } + + private static String createJwtToken(JWTClaimsSet claimsSet) throws JOSEException { + SignedJWT signedJWT = new SignedJWT(new JWSHeader(JWSAlgorithm.HS256), claimsSet); + signedJWT.sign(new MACSigner("a-secret-key-with-at-least-256-bits")); + return signedJWT.serialize(); + } +} diff --git a/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/http/TestRESTClientAdapter.java b/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/http/TestRESTClientAdapter.java new file mode 100644 index 000000000000..7a5a02e10249 --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/rest/auth/oauth2/http/TestRESTClientAdapter.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.http; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.nimbusds.oauth2.sdk.auth.ClientAuthenticationMethod; +import com.nimbusds.oauth2.sdk.http.HTTPRequest; +import com.nimbusds.oauth2.sdk.http.HTTPResponse; +import com.nimbusds.oauth2.sdk.http.ReadOnlyHTTPResponse; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.rest.RESTUtil; +import org.apache.iceberg.rest.auth.oauth2.test.TestEnvironment; +import org.junit.jupiter.api.Test; + +public class TestRESTClientAdapter { + + @Test + void testGETSuccess() throws IOException { + try (TestEnvironment env = TestEnvironment.builder().build()) { + + HTTPRequest httpRequest = new HTTPRequest(HTTPRequest.Method.GET, env.discoveryEndpoint()); + httpRequest.setHeader("Accept", "application/json"); + + RESTClientAdapter adapter = env.restClientAdapter(); + ReadOnlyHTTPResponse response = adapter.send(httpRequest); + + assertThat(response.getStatusCode()).isEqualTo(HTTPResponse.SC_OK); + assertThat(response.getBody()).contains("\"issuer\" : \"" + env.authorizationServerUrl()); + assertThat(response.getHeaderMap()) + .containsEntry("content-type", List.of("application/json; charset=utf-8")); + } + } + + @Test + void testGETFailure() throws IOException { + try (TestEnvironment env = TestEnvironment.builder().createDefaultExpectations(false).build()) { + + HTTPRequest httpRequest = new HTTPRequest(HTTPRequest.Method.GET, env.discoveryEndpoint()); + httpRequest.setHeader("Accept", "application/json"); + + RESTClientAdapter adapter = env.restClientAdapter(); + ReadOnlyHTTPResponse response = adapter.send(httpRequest); + + assertThat(response.getStatusCode()).isEqualTo(404); + assertThat(response.getBody()).isNull(); + assertThat(response.getHeaderMap()).containsEntry("content-length", List.of("0")); + } + } + + @Test + void testPOSTSuccess() throws IOException { + try (TestEnvironment env = + TestEnvironment.builder() + .clientAuthenticationMethod(ClientAuthenticationMethod.CLIENT_SECRET_POST) + .build()) { + + HTTPRequest httpRequest = new HTTPRequest(HTTPRequest.Method.POST, env.tokenEndpoint()); + Map formData = + ImmutableMap.builder() + .put("grant_type", "client_credentials") + .put("client_id", TestEnvironment.CLIENT_ID1.getValue()) + .put("client_secret", TestEnvironment.CLIENT_SECRET1.getValue()) + .put("scope", TestEnvironment.SCOPE1.toString()) + .put("extra1", "value1") + .build(); + httpRequest.setBody(RESTUtil.encodeFormData(formData)); + httpRequest.setHeader("Content-Type", "application/x-www-form-urlencoded"); + httpRequest.setHeader("Accept", "application/json"); + + RESTClientAdapter adapter = env.restClientAdapter(); + ReadOnlyHTTPResponse response = adapter.send(httpRequest); + + assertThat(response.getStatusCode()).isEqualTo(HTTPResponse.SC_OK); + assertThat(response.getBody()).contains("\"access_token\" : \"access_initial\""); + assertThat(response.getHeaderMap()) + .containsEntry("content-type", List.of("application/json; charset=utf-8")); + } + } + + @Test + void testPOSTFailure() throws IOException { + try (TestEnvironment env = TestEnvironment.builder().build()) { + + HTTPRequest httpRequest = new HTTPRequest(HTTPRequest.Method.POST, env.tokenEndpoint()); + httpRequest.setBody("grant_type=invalid"); + httpRequest.setHeader("Content-Type", "application/x-www-form-urlencoded"); + httpRequest.setHeader("Accept", "application/json"); + + RESTClientAdapter adapter = env.restClientAdapter(); + ReadOnlyHTTPResponse response = adapter.send(httpRequest); + + assertThat(response.getStatusCode()).isEqualTo(400); + assertThat(response.getBody()).contains("\"error\"").contains("\"error_description\""); + assertThat(response.getHeaderMap()) + .containsEntry("content-type", List.of("application/json")); + } + } +} diff --git a/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/TestCertificates.java b/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/TestCertificates.java new file mode 100644 index 000000000000..828a7d8aff78 --- /dev/null +++ b/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/TestCertificates.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.test; + +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.Reader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.KeyStore; +import java.security.PrivateKey; +import java.security.cert.Certificate; +import java.security.cert.CertificateFactory; +import java.security.cert.X509Certificate; +import org.apache.commons.io.FileUtils; +import org.bouncycastle.asn1.pkcs.PrivateKeyInfo; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.openssl.PEMKeyPair; +import org.bouncycastle.openssl.PEMParser; +import org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter; + +/** + * Generates test certificates and keystores at runtime. All materials are generated lazily (once + * per JVM) and stored in a temp directory. + */ +public final class TestCertificates { + + private static final class Holder { + private static final TestCertificates INSTANCE = new TestCertificates(); + } + + public static TestCertificates instance() { + return Holder.INSTANCE; + } + + private static final String KEYSTORE_PASSWORD = "s3cr3t"; + + private final Path mockServerP12; + + private TestCertificates() { + try { + Path baseDir = Files.createTempDirectory("iceberg-test-certs"); + Runtime.getRuntime() + .addShutdownHook(new Thread(() -> FileUtils.deleteQuietly(baseDir.toFile()))); + + // Generate mock server PKCS#12 keystore from MockServer's CA materials + mockServerP12 = baseDir.resolve("mockserver.p12"); + writeMockServerKeyStore(mockServerP12); + + } catch (Exception e) { + throw new RuntimeException("Failed to generate test certificates", e); + } + } + + /** PKCS#12 keystore containing MockServer's CA certificate and private key (password: s3cr3t). */ + public Path keyStorePath() { + return mockServerP12; + } + + /** The keystore password: {@code s3cr3t}. */ + public String keyStorePassword() { + return KEYSTORE_PASSWORD; + } + + private static void writeKeyStore(PrivateKey privateKey, X509Certificate certificate, Path path) + throws Exception { + KeyStore ks = KeyStore.getInstance("PKCS12"); + ks.load(null, null); + ks.setKeyEntry( + "1", privateKey, KEYSTORE_PASSWORD.toCharArray(), new Certificate[] {certificate}); + try (OutputStream os = Files.newOutputStream(path)) { + ks.store(os, KEYSTORE_PASSWORD.toCharArray()); + } + } + + private static void writeMockServerKeyStore(Path path) throws Exception { + X509Certificate cert; + PrivateKey key; + try (InputStream certStream = + TestCertificates.class.getResourceAsStream( + "/org/mockserver/socket/CertificateAuthorityCertificate.pem")) { + if (certStream == null) { + throw new IllegalStateException( + "MockServer CA certificate not found on classpath. " + + "Ensure org.mock-server:mockserver-netty is a dependency."); + } + cert = readCertificateFromPem(certStream); + } + try (InputStream keyStream = + TestCertificates.class.getResourceAsStream( + "/org/mockserver/socket/CertificateAuthorityPrivateKey.pem")) { + if (keyStream == null) { + throw new IllegalStateException( + "MockServer CA private key not found on classpath. " + + "Ensure org.mock-server:mockserver-netty is a dependency."); + } + key = readPrivateKeyFromPem(keyStream); + } + writeKeyStore(key, cert, path); + } + + private static X509Certificate readCertificateFromPem(InputStream is) throws Exception { + return (X509Certificate) CertificateFactory.getInstance("X.509").generateCertificate(is); + } + + private static PrivateKey readPrivateKeyFromPem(InputStream is) throws Exception { + try (Reader reader = new InputStreamReader(is, StandardCharsets.UTF_8); + PEMParser parser = new PEMParser(reader)) { + Object pemObject = parser.readObject(); + JcaPEMKeyConverter converter = + new JcaPEMKeyConverter().setProvider(new BouncyCastleProvider()); + if (pemObject instanceof PEMKeyPair) { + return converter.getPrivateKey(((PEMKeyPair) pemObject).getPrivateKeyInfo()); + } else if (pemObject instanceof PrivateKeyInfo) { + return converter.getPrivateKey((PrivateKeyInfo) pemObject); + } + throw new IllegalStateException("Unexpected PEM object: " + pemObject.getClass()); + } + } +} diff --git a/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/TestClock.java b/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/TestClock.java new file mode 100644 index 000000000000..3ffb84f1e090 --- /dev/null +++ b/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/TestClock.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.test; + +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.temporal.TemporalAmount; + +public class TestClock extends Clock { + + private Instant now; + + public TestClock(Instant now) { + this.now = now; + } + + @Override + public Instant instant() { + return now; + } + + @Override + public ZoneId getZone() { + return ZoneOffset.UTC; + } + + @Override + public Clock withZone(ZoneId zone) { + throw new UnsupportedOperationException(); + } + + public void plus(TemporalAmount amount) { + now = now.plus(amount); + } +} diff --git a/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/TestEnvironment.java b/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/TestEnvironment.java new file mode 100644 index 000000000000..fdf59d842683 --- /dev/null +++ b/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/TestEnvironment.java @@ -0,0 +1,737 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.test; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.google.errorprone.annotations.MustBeClosed; +import com.nimbusds.jose.JWSAlgorithm; +import com.nimbusds.oauth2.sdk.GrantType; +import com.nimbusds.oauth2.sdk.Scope; +import com.nimbusds.oauth2.sdk.auth.ClientAuthenticationMethod; +import com.nimbusds.oauth2.sdk.auth.Secret; +import com.nimbusds.oauth2.sdk.id.Audience; +import com.nimbusds.oauth2.sdk.id.ClientID; +import com.nimbusds.oauth2.sdk.token.AccessToken; +import com.nimbusds.oauth2.sdk.token.RefreshToken; +import com.nimbusds.oauth2.sdk.token.TokenTypeURI; +import com.nimbusds.oauth2.sdk.token.TypelessAccessToken; +import java.io.IOException; +import java.net.URI; +import java.nio.file.Path; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import javax.annotation.Nullable; +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.SSLContext; +import org.apache.hc.client5.http.ssl.HttpsSupport; +import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; +import org.apache.hc.core5.ssl.SSLContextBuilder; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.catalog.SessionCatalog; +import org.apache.iceberg.catalog.SessionCatalog.SessionContext; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; +import org.apache.iceberg.relocated.com.google.common.util.concurrent.MoreExecutors; +import org.apache.iceberg.rest.HTTPClient; +import org.apache.iceberg.rest.RESTCatalog; +import org.apache.iceberg.rest.RESTUtil; +import org.apache.iceberg.rest.ResourcePaths; +import org.apache.iceberg.rest.auth.AuthProperties; +import org.apache.iceberg.rest.auth.AuthSession; +import org.apache.iceberg.rest.auth.TLSConfigurer; +import org.apache.iceberg.rest.auth.oauth2.ImmutableOAuth2Config; +import org.apache.iceberg.rest.auth.oauth2.ImmutableOAuth2Runtime; +import org.apache.iceberg.rest.auth.oauth2.OAuth2Config; +import org.apache.iceberg.rest.auth.oauth2.OAuth2Manager; +import org.apache.iceberg.rest.auth.oauth2.OAuth2Runtime; +import org.apache.iceberg.rest.auth.oauth2.client.OAuth2Client; +import org.apache.iceberg.rest.auth.oauth2.config.BasicConfig; +import org.apache.iceberg.rest.auth.oauth2.config.ConfigUtil; +import org.apache.iceberg.rest.auth.oauth2.config.ImmutableBasicConfig; +import org.apache.iceberg.rest.auth.oauth2.config.ImmutableTokenExchangeConfig; +import org.apache.iceberg.rest.auth.oauth2.config.ImmutableTokenRefreshConfig; +import org.apache.iceberg.rest.auth.oauth2.config.TokenExchangeConfig; +import org.apache.iceberg.rest.auth.oauth2.config.TokenRefreshConfig; +import org.apache.iceberg.rest.auth.oauth2.flow.FlowFactory; +import org.apache.iceberg.rest.auth.oauth2.flow.TokensResult; +import org.apache.iceberg.rest.auth.oauth2.http.RESTClientAdapter; +import org.apache.iceberg.rest.auth.oauth2.test.expectation.ImmutableClientCredentialsExpectation; +import org.apache.iceberg.rest.auth.oauth2.test.expectation.ImmutableConfigEndpointExpectation; +import org.apache.iceberg.rest.auth.oauth2.test.expectation.ImmutableErrorExpectation; +import org.apache.iceberg.rest.auth.oauth2.test.expectation.ImmutableLoadTableEndpointExpectation; +import org.apache.iceberg.rest.auth.oauth2.test.expectation.ImmutableMetadataDiscoveryExpectation; +import org.apache.iceberg.rest.auth.oauth2.test.expectation.ImmutableRefreshTokenExpectation; +import org.apache.iceberg.rest.auth.oauth2.test.expectation.ImmutableTokenExchangeExpectation; +import org.apache.iceberg.util.ThreadPools; +import org.immutables.value.Value; + +/** + * A test environment for OAuth2. + * + *

    The environment provides easy configuration of {@link OAuth2Config} and {@link OAuth2Runtime} + * objects, as well as a MockServer instance for unit testing, with a set of pre-configured + * expectations for the OAuth2 endpoints that match the desired client behavior. + * + *

    It also provides a set of utility methods for creating and managing OAuth2 clients and REST + * catalogs. + */ +@Value.Immutable(copy = false) +@SuppressWarnings("resource") +public abstract class TestEnvironment implements AutoCloseable { + + public static final Instant NOW = Instant.parse("2025-01-01T00:00:00Z"); + + public static final int ACCESS_TOKEN_EXPIRES_IN_SECONDS = 3600; + + public static final ClientID CLIENT_ID1 = new ClientID("Client1"); + public static final ClientID CLIENT_ID2 = new ClientID("Client2"); + + public static final Secret CLIENT_SECRET1 = new Secret("s3cr3t"); + public static final Secret CLIENT_SECRET2 = new Secret("sEcrEt"); + + public static final String USERNAME = "Alice"; + public static final Secret PASSWORD = new Secret("s3cr3t"); + + public static final Scope SCOPE1 = new Scope("catalog"); + public static final Scope SCOPE2 = new Scope("session"); + + public static final String SUBJECT_TOKEN = "subject"; + public static final String ACTOR_TOKEN = "actor"; + + public static final Audience AUDIENCE = new Audience("audience"); + public static final URI RESOURCE = URI.create("urn:iceberg-oauth2-client:test:resource"); + + public static final TableIdentifier TABLE_IDENTIFIER = TableIdentifier.of("namespace1", "table1"); + + private static final AtomicInteger ID_COUNTER = new AtomicInteger(0); + + public static ImmutableTestEnvironment.Builder builder() { + return ImmutableTestEnvironment.builder(); + } + + // OAuth2 properties + + @Value.Default + public GrantType grantType() { + return GrantType.CLIENT_CREDENTIALS; + } + + @Value.Default + public Optional clientId() { + return Optional.of(CLIENT_ID1); + } + + @Value.Default + public Optional clientSecret() { + return Optional.of(CLIENT_SECRET1); + } + + @Value.Default + public ClientAuthenticationMethod clientAuthenticationMethod() { + return ClientAuthenticationMethod.CLIENT_SECRET_BASIC; + } + + @Value.Default + public Map extraRequestParameters() { + return Map.of("extra1", "value1"); + } + + public abstract Optional token(); + + @Value.Default + public Scope scope() { + return SCOPE1; + } + + @Value.Default + public Duration tokenAcquisitionTimeout() { + return unitTest() ? Duration.ofSeconds(5) : BasicConfig.DEFAULT_TIMEOUT; + } + + @Value.Default + public boolean tokenRefreshEnabled() { + return true; + } + + @Value.Default + public boolean tokenRefreshWithTokenExchangeEnabled() { + return grantType().equals(GrantType.CLIENT_CREDENTIALS); + } + + @Value.Default + public Duration accessTokenLifespan() { + return Duration.ofSeconds(ACCESS_TOKEN_EXPIRES_IN_SECONDS); + } + + @Value.Default + public Optional subjectTokenString() { + return Optional.of(SUBJECT_TOKEN); + } + + public abstract Optional subjectTokenType(); + + @Value.Default + public Optional actorTokenString() { + return Optional.of(ACTOR_TOKEN); + } + + public abstract Optional actorTokenType(); + + public abstract Optional requestedTokenType(); + + @Value.Default + public List audiences() { + return List.of(AUDIENCE); + } + + @Value.Default + public List resources() { + return List.of(RESOURCE); + } + + public abstract Optional jwsAlgorithm(); + + public abstract Optional privateKey(); + + // General configuration + + @Value.Default + public boolean unitTest() { + return true; + } + + @Value.Default + public String environmentId() { + return "env" + ID_COUNTER.incrementAndGet(); + } + + @Value.Default + public boolean discoveryEnabled() { + return true; + } + + @Value.Default + public boolean returnRefreshTokens() { + return !grantType().equals(GrantType.CLIENT_CREDENTIALS); + } + + @Value.Default + public boolean createDefaultExpectations() { + return unitTest(); + } + + @Value.Default + public boolean ssl() { + return false; + } + + // URLs, endpoints and paths + + @Value.Default + public URI serverRootUrl() { + if (!unitTest()) { + throw new IllegalStateException( + "serverRootUrl() must be provided explicitly for integration tests"); + } + return URI.create( + "http://localhost:%d/%s/".formatted(TestServer.instance().getLocalPort(), environmentId())); + } + + @Value.Default + public String authorizationServerContextPath() { + return "auth-server/"; + } + + @Value.Default + public String catalogServerContextPath() { + return "catalog-server/"; + } + + @Value.Default + public URI authorizationServerUrl() { + return serverRootUrl().resolve(authorizationServerContextPath()); + } + + @Value.Default + public URI catalogServerUrl() { + return serverRootUrl().resolve(catalogServerContextPath()); + } + + @Value.Default + public URI tokenEndpoint() { + return authorizationServerUrl().resolve("token"); + } + + @Value.Default + public String wellKnownPath() { + return ".well-known/openid-configuration"; + } + + @Value.Default + public URI discoveryEndpoint() { + return authorizationServerUrl().resolve(wellKnownPath()); + } + + @Value.Default + public URI configEndpoint() { + return catalogServerUrl().resolve(ResourcePaths.config()); + } + + @Value.Default + public URI loadTableEndpoint() { + return catalogServerUrl() + .resolve(ResourcePaths.forCatalogProperties(catalogProperties()).table(TABLE_IDENTIFIER)); + } + + // REST Catalog configuration + + @Value.Default + public Map catalogProperties() { + + ImmutableMap.Builder builder = + ImmutableMap.builder() + .put(CatalogProperties.URI, catalogServerUrl().toString()) + .put(CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO") + .put(CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key1", "catalog-default-key1") + .put(CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key2", "catalog-default-key2") + .put(CatalogProperties.TABLE_DEFAULT_PREFIX + "override-key3", "catalog-default-key3") + .put(CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key3", "catalog-override-key3") + .put(CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key4", "catalog-override-key4") + .put(AuthProperties.AUTH_TYPE, OAuth2Manager.class.getName()); + + // Note: we don't include all possible OAuth2 properties here, only a few ones that are + // relevant for catalog tests. + + builder + .put(BasicConfig.GRANT_TYPE, grantType().toString()) + .put(BasicConfig.SCOPE, scope().toString()) + .put(BasicConfig.CLIENT_AUTH, clientAuthenticationMethod().toString()); + + token().ifPresent(t -> builder.put(BasicConfig.TOKEN, t.getValue())); + clientId().ifPresent(id -> builder.put(BasicConfig.CLIENT_ID, id.getValue())); + + if (ConfigUtil.requiresClientSecret(clientAuthenticationMethod())) { + clientSecret().ifPresent(secret -> builder.put(BasicConfig.CLIENT_SECRET, secret.getValue())); + } + + extraRequestParameters().forEach((k, v) -> builder.put(BasicConfig.EXTRA_PARAMS + '.' + k, v)); + + if (discoveryEnabled()) { + builder.put(BasicConfig.ISSUER_URL, authorizationServerUrl().toString()); + } else { + builder.put(BasicConfig.TOKEN_ENDPOINT, tokenEndpoint().toString()); + } + + if (grantType().equals(GrantType.TOKEN_EXCHANGE)) { + subjectTokenString().ifPresent(t -> builder.put(TokenExchangeConfig.SUBJECT_TOKEN, t)); + actorTokenString().ifPresent(t -> builder.put(TokenExchangeConfig.ACTOR_TOKEN, t)); + + subjectTokenType() + .ifPresent(t -> builder.put(TokenExchangeConfig.SUBJECT_TOKEN_TYPE, t.toString())); + actorTokenType() + .ifPresent(t -> builder.put(TokenExchangeConfig.ACTOR_TOKEN_TYPE, t.toString())); + requestedTokenType() + .ifPresent(t -> builder.put(TokenExchangeConfig.REQUESTED_TOKEN_TYPE, t.toString())); + + if (!resources().isEmpty()) { + builder.put( + TokenExchangeConfig.RESOURCES, + resources().stream().map(URI::toString).collect(Collectors.joining(","))); + } + + if (!audiences().isEmpty()) { + builder.put( + TokenExchangeConfig.AUDIENCES, + audiences().stream().map(Audience::getValue).collect(Collectors.joining(","))); + } + } + + return builder.build(); + } + + @Value.Default + public GrantType sessionContextGrantType() { + return GrantType.CLIENT_CREDENTIALS; + } + + public abstract Optional sessionContextSubjectTokenString(); + + public abstract Optional sessionContextActorTokenString(); + + @Value.Default + public SessionContext sessionContext() { + + // Note: we don't include all possible OAuth2 properties here, only a few ones that are + // relevant for catalog tests. + + Map credentials = + Map.of( + BasicConfig.CLIENT_ID, + TestEnvironment.CLIENT_ID1.getValue(), + BasicConfig.CLIENT_SECRET, + TestEnvironment.CLIENT_SECRET1.getValue()); + ImmutableMap.Builder propertiesBuilder = + ImmutableMap.builder() + .put(BasicConfig.ISSUER_URL, authorizationServerUrl().toString()) + .put(BasicConfig.GRANT_TYPE, sessionContextGrantType().getValue()) + .put(BasicConfig.SCOPE, TestEnvironment.SCOPE2.toString()) + .put(BasicConfig.EXTRA_PARAMS + ".extra2", "value2"); + if (sessionContextGrantType().equals(GrantType.TOKEN_EXCHANGE)) { + sessionContextSubjectTokenString() + .ifPresent(t -> propertiesBuilder.put(TokenExchangeConfig.SUBJECT_TOKEN, t)); + sessionContextActorTokenString() + .ifPresent(t -> propertiesBuilder.put(TokenExchangeConfig.ACTOR_TOKEN, t)); + + subjectTokenType() + .ifPresent( + t -> propertiesBuilder.put(TokenExchangeConfig.SUBJECT_TOKEN_TYPE, t.toString())); + actorTokenType() + .ifPresent( + t -> propertiesBuilder.put(TokenExchangeConfig.ACTOR_TOKEN_TYPE, t.toString())); + requestedTokenType() + .ifPresent( + t -> propertiesBuilder.put(TokenExchangeConfig.REQUESTED_TOKEN_TYPE, t.toString())); + + if (!resources().isEmpty()) { + propertiesBuilder.put( + TokenExchangeConfig.RESOURCES, + resources().stream().map(URI::toString).collect(Collectors.joining(","))); + } + + if (!audiences().isEmpty()) { + propertiesBuilder.put( + TokenExchangeConfig.AUDIENCES, + audiences().stream().map(Audience::getValue).collect(Collectors.joining(","))); + } + } + return new SessionCatalog.SessionContext( + UUID.randomUUID().toString(), "user", credentials, propertiesBuilder.build()); + } + + @Value.Default + public Map tableProperties() { + return Map.of(BasicConfig.TOKEN, "token"); // vended token + } + + // OAuth2 Configuration objects + + @Value.Derived + public BasicConfig basicConfig() { + ImmutableBasicConfig.Builder builder = + ImmutableBasicConfig.builder() + .grantType(grantType()) + .token(token()) + .clientId(clientId()) + .clientAuthenticationMethod(clientAuthenticationMethod()) + .clientSecret( + ConfigUtil.requiresClientSecret(clientAuthenticationMethod()) + ? clientSecret() + : Optional.empty()) + .scope(scope()) + .extraRequestParameters(extraRequestParameters()) + .tokenAcquisitionTimeout(tokenAcquisitionTimeout()) + .minTokenAcquisitionTimeout(tokenAcquisitionTimeout()); + if (discoveryEnabled()) { + builder.issuerUrl(authorizationServerUrl()); + } else { + builder.tokenEndpoint(tokenEndpoint()); + } + + return builder.build(); + } + + @Value.Derived + public TokenRefreshConfig tokenRefreshConfig() { + return ImmutableTokenRefreshConfig.builder() + .enabled(tokenRefreshEnabled()) + .tokenExchangeEnabled(tokenRefreshWithTokenExchangeEnabled()) + .accessTokenLifespan(accessTokenLifespan()) + .build(); + } + + @Value.Derived + public TokenExchangeConfig tokenExchangeConfig() { + return ImmutableTokenExchangeConfig.builder() + .subjectTokenString(subjectTokenString()) + .subjectTokenType(subjectTokenType()) + .actorTokenString(actorTokenString()) + .actorTokenType(actorTokenType()) + .requestedTokenType(requestedTokenType()) + .audiences(audiences()) + .resources(resources()) + .build(); + } + + @Value.Derived + public OAuth2Config config() { + return ImmutableOAuth2Config.builder() + .basicConfig(basicConfig()) + .tokenRefreshConfig(tokenRefreshConfig()) + .tokenExchangeConfig(tokenExchangeConfig()) + .build(); + } + + // User Emulation + + @Value.Default + public String username() { + return USERNAME; + } + + @Value.Default + public Secret password() { + return PASSWORD; + } + + @Value.Default + public boolean forceInactiveUser() { + return false; + } + + @Value.Default + public boolean emulateFailure() { + return false; + } + + // OAuth2 Runtime + + public abstract Optional parentClient(); + + @Value.Default + public ScheduledExecutorService executor() { + return ThreadPools.authRefreshPool(); + } + + @Value.Default + public boolean sslHostnameVerificationEnabled() { + return true; + } + + public abstract Optional sslTrustStorePath(); + + public abstract Optional sslTrustStorePassword(); + + @Value.Default + public HostnameVerifier sslHostnameVerifier() { + return sslHostnameVerificationEnabled() + ? HttpsSupport.getDefaultHostnameVerifier() + : NoopHostnameVerifier.INSTANCE; + } + + @Value.Derived + public SSLContext sslContext() { + try { + if (sslTrustStorePath().isEmpty()) { + return SSLContext.getDefault(); + } + + return SSLContextBuilder.create() + .loadTrustMaterial( + sslTrustStorePath().get(), + sslTrustStorePassword().map(String::toCharArray).orElse(null)) + .build(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Value.Derived + public HTTPClient.Builder httpClientBuilder() { + return HTTPClient.builder( + Map.of("rest.client.tls.configurer-impl", TestTLSConfigurer.class.getName())) + .uri(catalogServerUrl()) + .withAuthSession(AuthSession.EMPTY); + } + + @Value.Derived + public HTTPClient httpClient() { + HTTPClient.Builder builder = httpClientBuilder(); + TestTLSConfigurer.SSL_CONTEXT.set(sslContext()); + TestTLSConfigurer.HOSTNAME_VERIFIER.set(sslHostnameVerifier()); + return builder.build(); + } + + @Value.Derived + public RESTClientAdapter restClientAdapter() { + return new RESTClientAdapter(this::httpClient); + } + + @Value.Default + public Clock clock() { + return unitTest() ? new TestClock(NOW) : Clock.systemUTC(); + } + + @Value.Derived + public OAuth2Runtime runtime() { + return ImmutableOAuth2Runtime.builder() + .parent(parentClient()) + .httpClient(restClientAdapter()) + .executor(executor()) + .clock(clock()) + .build(); + } + + // Lifecycle methods + + @Value.Check + public void initialize() { + if (createDefaultExpectations()) { + createExpectations(); + } + } + + public void reset() { + if (unitTest()) { + TestServer.clearExpectations(serverRootUrl().getPath() + ".*"); + } + } + + @Override + public void close() { + if (executor() != ThreadPools.authRefreshPool()) { + MoreExecutors.shutdownAndAwaitTermination(executor(), Duration.ofSeconds(10)); + } + try { + httpClient().close(); + } catch (IOException e) { + throw new RuntimeException(e); + } finally { + reset(); + } + } + + // Factory methods + + @MustBeClosed + public FlowFactory newFlowFactory() { + return FlowFactory.of(config(), runtime()); + } + + @MustBeClosed + public OAuth2Client newOAuth2Client() { + return new OAuth2Client(config(), runtime()); + } + + @MustBeClosed + public RESTCatalog newCatalog(Map additionalProperties) { + RESTCatalog catalog = + new RESTCatalog( + sessionContext(), + config -> httpClientBuilder().withHeaders(RESTUtil.configHeaders(config)).build()); + catalog.initialize( + "catalog-" + System.nanoTime(), + ImmutableMap.builder() + .putAll(catalogProperties()) + .putAll(additionalProperties) + .buildKeepingLast()); + return catalog; + } + + // MockServer Expectations + + public void createExpectations() { + createInitialGrantExpectations(); + createRefreshTokenExpectations(); + createCatalogExpectations(); + createMetadataDiscoveryExpectations(); + createErrorExpectations(); + } + + public void createInitialGrantExpectations() { + Set grantTypes = ImmutableSet.of(grantType(), sessionContextGrantType()); + for (GrantType grantType : grantTypes) { + if (grantType.equals(GrantType.CLIENT_CREDENTIALS)) { + ImmutableClientCredentialsExpectation.of(this).create(); + } else if (grantType.equals(GrantType.TOKEN_EXCHANGE)) { + ImmutableTokenExchangeExpectation.of(this).create(); + } + } + } + + public void createRefreshTokenExpectations() { + if (tokenRefreshEnabled()) { + ImmutableRefreshTokenExpectation.of(this).create(); + } + } + + public void createCatalogExpectations() { + ImmutableConfigEndpointExpectation.of(this).create(); + ImmutableLoadTableEndpointExpectation.of(this).create(); + } + + public void createMetadataDiscoveryExpectations() { + if (discoveryEnabled()) { + ImmutableMetadataDiscoveryExpectation.of(this).create(); + } + } + + public void createErrorExpectations() { + ImmutableErrorExpectation.of(this).create(); + } + + // Useful token assertions + + public void assertTokensResult( + TokensResult result, String accessToken, @Nullable String refreshToken) { + assertThat(result.accessTokenExpirationTime()).hasValue(NOW.plus(accessTokenLifespan())); + assertAccessToken(result.tokens().getAccessToken(), accessToken, accessTokenLifespan()); + assertRefreshToken(result.tokens().getRefreshToken(), refreshToken); + } + + public void assertAccessToken(AccessToken actual, String expected, Duration lifespan) { + assertThat(actual.getValue()).as("Access token value").isEqualTo(expected); + assertThat(actual.getLifetime()).as("Access token lifetime").isEqualTo(lifespan.getSeconds()); + } + + public void assertRefreshToken(RefreshToken actual, String expected) { + if (expected == null) { + assertThat(actual).as("Refresh token").isNull(); + } else { + assertThat(actual).as("Refresh token").isNotNull(); + assertThat(actual.getValue()).as("Refresh token value").isEqualTo(expected); + } + } + + public static class TestTLSConfigurer implements TLSConfigurer { + + private static final ThreadLocal SSL_CONTEXT = new ThreadLocal<>(); + private static final ThreadLocal HOSTNAME_VERIFIER = new ThreadLocal<>(); + + @Override + public SSLContext sslContext() { + return SSL_CONTEXT.get(); + } + + @Override + public HostnameVerifier hostnameVerifier() { + return HOSTNAME_VERIFIER.get(); + } + } +} diff --git a/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/TestServer.java b/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/TestServer.java new file mode 100644 index 000000000000..a2bd52099652 --- /dev/null +++ b/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/TestServer.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.test; + +import org.mockserver.integration.ClientAndServer; +import org.mockserver.model.HttpRequest; + +/** + * A singleton test server for OAuth2 testing. It can be shared by many tests, even running in + * parallel, provided that each test uses a different root path. + */ +public final class TestServer { + + private static final class Holder { + + private static final ClientAndServer INSTANCE; + + static { + INSTANCE = ClientAndServer.startClientAndServer(); + Runtime.getRuntime().addShutdownHook(new Thread(INSTANCE::close)); + } + } + + public static ClientAndServer instance() { + return Holder.INSTANCE; + } + + private TestServer() {} + + /** Clears all expectations and logs for the given path. */ + @SuppressWarnings("resource") + public static void clearExpectations(String path) { + instance().clear(HttpRequest.request().withPath(path)); + } +} diff --git a/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/container/KeycloakContainer.java b/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/container/KeycloakContainer.java new file mode 100644 index 000000000000..9f7813a30c69 --- /dev/null +++ b/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/container/KeycloakContainer.java @@ -0,0 +1,358 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.test.container; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; + +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import com.nimbusds.jwt.JWTClaimsSet; +import com.nimbusds.jwt.JWTParser; +import dasniko.testcontainers.keycloak.ExtendableKeycloakContainer; +import jakarta.ws.rs.core.Response; +import java.net.URI; +import java.text.ParseException; +import java.time.Duration; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Collectors; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.keycloak.admin.client.Keycloak; +import org.keycloak.admin.client.resource.RealmResource; +import org.keycloak.representations.idm.ClientRepresentation; +import org.keycloak.representations.idm.ClientScopeRepresentation; +import org.keycloak.representations.idm.CredentialRepresentation; +import org.keycloak.representations.idm.ProtocolMapperRepresentation; +import org.keycloak.representations.idm.RealmRepresentation; +import org.keycloak.representations.idm.UserRepresentation; +import org.keycloak.representations.idm.authorization.PolicyEnforcementMode; +import org.keycloak.representations.idm.authorization.ResourceServerRepresentation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.output.Slf4jLogConsumer; + +public class KeycloakContainer extends ExtendableKeycloakContainer { + + private static final Logger LOGGER = LoggerFactory.getLogger(KeycloakContainer.class); + + public static final Duration ACCESS_TOKEN_LIFESPAN = Duration.ofMinutes(10); + + private static final String CONTEXT_PATH = "/realms/master"; + + private final List scopes = Lists.newArrayList(); + private final List clients = Lists.newArrayList(); + private final List users = Lists.newArrayList(); + private final Map> clientAudiences = Maps.newHashMap(); + + private URI rootUrl; + private URI issuerUrl; + private URI tokenEndpoint; + private URI authEndpoint; + + @SuppressWarnings("resource") + public KeycloakContainer() { + super("keycloak/keycloak:26.4"); + withNetworkAliases("keycloak"); + withLogConsumer(new Slf4jLogConsumer(LOGGER)); + withEnv("KC_LOG_LEVEL", rootLoggerLevel() + ",org.keycloak:" + keycloakLoggerLevel()); + // Useful when debugging Keycloak REST endpoints: + addExposedPorts(5005); + withEnv( + "JAVA_TOOL_OPTIONS", + "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005"); + } + + @CanIgnoreReturnValue + public KeycloakContainer withScope(String scope) { + scopes.add(newScope(scope)); + return this; + } + + @CanIgnoreReturnValue + public KeycloakContainer withClient( + String clientId, String clientSecret, String authenticationMethod) { + clients.add(newClient(clientId, clientSecret, authenticationMethod)); + return this; + } + + @CanIgnoreReturnValue + public KeycloakContainer withClientAudience(String clientId, String audience) { + clientAudiences.computeIfAbsent(clientId, k -> Lists.newArrayList()).add(audience); + return this; + } + + @CanIgnoreReturnValue + public KeycloakContainer withUser(String username, String password) { + users.add(newUser(username, password)); + return this; + } + + @Override + public void start() { + if (getContainerId() != null) { + return; + } + + super.start(); + rootUrl = URI.create(getAuthServerUrl()); + issuerUrl = rootUrl.resolve(CONTEXT_PATH); + tokenEndpoint = rootUrl.resolve(CONTEXT_PATH + "/protocol/openid-connect/token"); + authEndpoint = rootUrl.resolve(CONTEXT_PATH + "/protocol/openid-connect/auth"); + try (Keycloak client = getKeycloakAdminClient()) { + RealmResource master = client.realms().realm("master"); + updateMasterRealm(master); + scopes.forEach(scope -> createScope(master, scope)); + users.forEach(user -> createUser(master, user)); + clients.forEach(cl -> createClient(master, cl)); + } + } + + public URI rootUrl() { + return rootUrl; + } + + public URI issuerUrl() { + return issuerUrl; + } + + public URI tokenEndpoint() { + return tokenEndpoint; + } + + public URI authEndpoint() { + return authEndpoint; + } + + /** + * Verifies a JWT token by parsing it and doing a basic validation of its claims. This method does + * NOT verify the JWT signature. + */ + public JWTClaimsSet verifyToken(String token) { + assertThat(token).isNotNull(); + try { + JWTClaimsSet claims = JWTParser.parse(token).getJWTClaimsSet(); + assertThat(claims.getIssuer()).isEqualTo(issuerUrl.toString()); + assertThat(claims.getExpirationTime()).isInTheFuture(); + return claims; + } catch (ParseException e) { + return fail("Failed to parse JWT token", e); + } + } + + protected void updateMasterRealm(RealmResource master) { + RealmRepresentation masterRep = master.toRepresentation(); + masterRep.setAccessTokenLifespan((int) ACCESS_TOKEN_LIFESPAN.toSeconds()); + // Minimum polling interval for device auth flow + masterRep.setOAuth2DevicePollingInterval(1); + master.update(masterRep); + } + + protected void createScope(RealmResource master, ClientScopeRepresentation scope) { + try (Response response = master.clientScopes().create(scope)) { + if (response.getStatus() != 201) { + throw new IllegalStateException( + "Failed to create scope: " + response.readEntity(String.class)); + } + } + } + + protected void createClient(RealmResource master, ClientRepresentation client) { + client.setOptionalClientScopes( + scopes.stream().map(ClientScopeRepresentation::getName).collect(Collectors.toList())); + try (Response response = master.clients().create(client)) { + if (response.getStatus() != 201) { + throw new IllegalStateException( + "Failed to create client: " + response.readEntity(String.class)); + } + } + + // Required for Polaris + addPrincipalIdClaimMapper(master, client.getId()); + addPrincipalRoleClaimMapper(master, client.getId()); + + // Add audience mappers if configured + List audiences = clientAudiences.get(client.getClientId()); + if (audiences != null) { + for (String audience : audiences) { + addAudienceMapper(master, client.getId(), audience); + } + } + } + + protected void createUser(RealmResource master, UserRepresentation user) { + try (Response response = master.users().create(user)) { + if (response.getStatus() != 201) { + throw new IllegalStateException( + "Failed to create user: " + response.readEntity(String.class)); + } + } + } + + private static ClientScopeRepresentation newScope(String scopeName) { + ClientScopeRepresentation scope = new ClientScopeRepresentation(); + scope.setId(UUID.randomUUID().toString()); + scope.setName(scopeName); + scope.setProtocol("openid-connect"); + scope.setAttributes( + Map.of( + "include.in.token.scope", + "true", + "consent.screen.text", + "REST Catalog", + "display.on.consent.screen", + "true")); + return scope; + } + + private static ClientRepresentation newClient( + String clientId, String clientSecret, String authenticationMethod) { + ClientRepresentation client = new ClientRepresentation(); + String clientUuid = UUID.randomUUID().toString(); + client.setId(clientUuid); + client.setClientId(clientId); + boolean publicClient = authenticationMethod.equals("none"); + client.setPublicClient(publicClient); + client.setServiceAccountsEnabled(!publicClient); // required for client credentials grant + client.setStandardFlowEnabled(true); // required for authorization code grant + client.setRedirectUris(List.of("http://localhost:*", "https://localhost:*")); + ImmutableMap.Builder attributes = + ImmutableMap.builder() + .put("use.refresh.tokens", "true") + .put("client_credentials.use_refresh_token", "false") + .put("oauth2.device.authorization.grant.enabled", "true") + .put("standard.token.exchange.enabled", "true") + .put("standard.token.exchange.enableRefreshRequestedTokenType", "SAME_SESSION"); + switch (authenticationMethod) { + case "client_secret_basic": + case "client_secret_post": + client.setSecret(clientSecret); + break; + } + + if (!publicClient) { + ResourceServerRepresentation settings = new ResourceServerRepresentation(); + settings.setPolicyEnforcementMode(PolicyEnforcementMode.DISABLED); + client.setAuthorizationSettings(settings); + } + + client.setAttributes(attributes.build()); + return client; + } + + private static UserRepresentation newUser(String username, String password) { + UserRepresentation user = new UserRepresentation(); + user.setId(UUID.randomUUID().toString()); + user.setUsername(username); + user.setFirstName(username); + user.setLastName(username); + CredentialRepresentation credential = new CredentialRepresentation(); + credential.setType(CredentialRepresentation.PASSWORD); + credential.setValue(password); + credential.setTemporary(false); + user.setCredentials(ImmutableList.of(credential)); + user.setEnabled(true); + user.setEmail(username.toLowerCase(Locale.ROOT) + "@example.com"); + user.setEmailVerified(true); + user.setRequiredActions(Collections.emptyList()); + return user; + } + + private void addPrincipalIdClaimMapper(RealmResource master, String clientUuid) { + ProtocolMapperRepresentation mapper = new ProtocolMapperRepresentation(); + mapper.setId(UUID.randomUUID().toString()); + mapper.setName("principal-id-claim-mapper"); + mapper.setProtocol("openid-connect"); + mapper.setProtocolMapper("oidc-hardcoded-claim-mapper"); + mapper.setConfig( + ImmutableMap.builder() + .put("claim.name", "principal_id") + .put("claim.value", "1") + .put("jsonType.label", "long") + .put("id.token.claim", "true") + .put("access.token.claim", "true") + .put("userinfo.token.claim", "true") + .build()); + try (Response response = + master.clients().get(clientUuid).getProtocolMappers().createMapper(mapper)) { + if (response.getStatus() != 201) { + throw new IllegalStateException( + "Failed to create mapper: " + response.readEntity(String.class)); + } + } + } + + private void addPrincipalRoleClaimMapper(RealmResource master, String clientUuid) { + ProtocolMapperRepresentation mapper = new ProtocolMapperRepresentation(); + mapper.setId(UUID.randomUUID().toString()); + mapper.setName("principal-role-claim-mapper"); + mapper.setProtocol("openid-connect"); + mapper.setProtocolMapper("oidc-hardcoded-claim-mapper"); + mapper.setConfig( + ImmutableMap.builder() + .put("claim.name", "groups") + .put("claim.value", "[\"PRINCIPAL_ROLE:ALL\"]") + .put("jsonType.label", "JSON") + .put("id.token.claim", "true") + .put("access.token.claim", "true") + .put("userinfo.token.claim", "true") + .build()); + try (Response response = + master.clients().get(clientUuid).getProtocolMappers().createMapper(mapper)) { + if (response.getStatus() != 201) { + throw new IllegalStateException( + "Failed to create role claim mapper: " + response.readEntity(String.class)); + } + } + } + + private void addAudienceMapper(RealmResource master, String clientUuid, String audience) { + ProtocolMapperRepresentation mapper = new ProtocolMapperRepresentation(); + mapper.setId(UUID.randomUUID().toString()); + mapper.setName("audience-mapper-" + audience); + mapper.setProtocol("openid-connect"); + mapper.setProtocolMapper("oidc-audience-mapper"); + mapper.setConfig( + ImmutableMap.builder() + .put("included.client.audience", audience) + .put("id.token.claim", "false") + .put("access.token.claim", "true") + .build()); + try (Response response = + master.clients().get(clientUuid).getProtocolMappers().createMapper(mapper)) { + if (response.getStatus() != 201) { + throw new IllegalStateException( + "Failed to create audience mapper: " + response.readEntity(String.class)); + } + } + } + + private static String rootLoggerLevel() { + return LOGGER.isInfoEnabled() ? "INFO" : LOGGER.isWarnEnabled() ? "WARN" : "ERROR"; + } + + private static String keycloakLoggerLevel() { + return LOGGER.isDebugEnabled() ? "DEBUG" : rootLoggerLevel(); + } +} diff --git a/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/expectation/BaseExpectation.java b/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/expectation/BaseExpectation.java new file mode 100644 index 000000000000..031bec4a326d --- /dev/null +++ b/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/expectation/BaseExpectation.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.test.expectation; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.nimbusds.oauth2.sdk.util.URLUtils; +import java.util.List; +import java.util.Map; +import org.apache.iceberg.rest.RESTResponse; +import org.apache.iceberg.rest.RESTSerializers; +import org.apache.iceberg.rest.auth.oauth2.test.TestEnvironment; +import org.immutables.value.Value; +import org.mockserver.model.HttpMessage; +import org.mockserver.model.JsonBody; +import org.mockserver.model.Parameter; +import org.mockserver.model.ParameterBody; + +/** A base class for all test expectations. */ +public abstract class BaseExpectation { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + static { + RESTSerializers.registerAll(OBJECT_MAPPER); + } + + protected static JsonBody jsonBody(RESTResponse body) { + try { + return JsonBody.json(OBJECT_MAPPER.writeValueAsString(body)); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + protected static JsonBody jsonBody(Map body) { + try { + return JsonBody.json(OBJECT_MAPPER.writeValueAsString(body)); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + protected static ParameterBody parameterBody(List parameters) { + return ParameterBody.params(parameters); + } + + protected static Map> decodeBodyParameters(HttpMessage httpMessage) { + // See https://github.com/mock-server/mockserver/issues/1468 + String body = httpMessage.getBodyAsString(); + return URLUtils.parseParameters(body); + } + + @Value.Parameter(order = 1) + protected abstract TestEnvironment testEnvironment(); + + public abstract void create(); +} diff --git a/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/expectation/ClientCredentialsExpectation.java b/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/expectation/ClientCredentialsExpectation.java new file mode 100644 index 000000000000..4bc5f79cddb9 --- /dev/null +++ b/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/expectation/ClientCredentialsExpectation.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.test.expectation; + +import com.nimbusds.oauth2.sdk.GrantType; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.rest.auth.oauth2.test.TestServer; +import org.immutables.value.Value; +import org.mockserver.model.Parameter; + +@Value.Immutable +@SuppressWarnings("resource") +public abstract class ClientCredentialsExpectation extends TokenEndpointExpectation { + + @Override + public void create() { + TestServer.instance().when(request()).respond(response("access_initial", "refresh_initial")); + } + + @Override + protected ImmutableList.Builder requestBody() { + return super.requestBody() + .add(param("grant_type", GrantType.CLIENT_CREDENTIALS)) + .add(param("scope", ACCEPTED_SCOPES)); + } +} diff --git a/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/expectation/ConfigEndpointExpectation.java b/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/expectation/ConfigEndpointExpectation.java new file mode 100644 index 000000000000..bd985134729a --- /dev/null +++ b/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/expectation/ConfigEndpointExpectation.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.test.expectation; + +import java.util.List; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.rest.Endpoint; +import org.apache.iceberg.rest.auth.oauth2.test.TestServer; +import org.apache.iceberg.rest.responses.ConfigResponse; +import org.immutables.value.Value; +import org.mockserver.model.HttpRequest; +import org.mockserver.model.HttpResponse; + +@Value.Immutable +public abstract class ConfigEndpointExpectation extends BaseExpectation { + + @Override + @SuppressWarnings("resource") + public void create() { + List endpoints = + ImmutableList.builder() + .add(Endpoint.V1_LIST_NAMESPACES) + .add(Endpoint.V1_LOAD_NAMESPACE) + .add(Endpoint.V1_CREATE_NAMESPACE) + .add(Endpoint.V1_UPDATE_NAMESPACE) + .add(Endpoint.V1_DELETE_NAMESPACE) + .add(Endpoint.V1_LIST_TABLES) + .add(Endpoint.V1_LOAD_TABLE) + .add(Endpoint.V1_CREATE_TABLE) + .add(Endpoint.V1_UPDATE_TABLE) + .add(Endpoint.V1_DELETE_TABLE) + .add(Endpoint.V1_RENAME_TABLE) + .add(Endpoint.V1_REGISTER_TABLE) + .add(Endpoint.V1_REPORT_METRICS) + .add(Endpoint.V1_LIST_VIEWS) + .add(Endpoint.V1_LOAD_VIEW) + .add(Endpoint.V1_CREATE_VIEW) + .add(Endpoint.V1_UPDATE_VIEW) + .add(Endpoint.V1_DELETE_VIEW) + .add(Endpoint.V1_RENAME_VIEW) + .add(Endpoint.V1_COMMIT_TRANSACTION) + .build(); + ConfigResponse response = + ConfigResponse.builder() + .withDefaults(Maps.newHashMap()) + .withOverrides(Maps.newHashMap()) + .withEndpoints(endpoints) + .build(); + TestServer.instance() + .when( + HttpRequest.request() + .withMethod("GET") + .withPath(testEnvironment().configEndpoint().getPath()) + .withHeader("Accept", "application/json") + .withHeader("Authorization", "Bearer access_initial")) + .respond(HttpResponse.response().withBody(jsonBody(response))); + } +} diff --git a/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/expectation/ErrorExpectation.java b/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/expectation/ErrorExpectation.java new file mode 100644 index 000000000000..aabe0f04c282 --- /dev/null +++ b/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/expectation/ErrorExpectation.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.test.expectation; + +import org.apache.iceberg.rest.auth.oauth2.ImmutableOAuth2Error; +import org.apache.iceberg.rest.auth.oauth2.test.TestServer; +import org.immutables.value.Value; +import org.mockserver.model.HttpRequest; +import org.mockserver.model.HttpResponse; +import org.mockserver.model.JsonBody; +import org.mockserver.model.MediaType; + +@Value.Immutable +public abstract class ErrorExpectation extends BaseExpectation { + + public static final ImmutableOAuth2Error OAUTH2_ERROR = + ImmutableOAuth2Error.builder() + .code("invalid_request") + .description("Invalid request (MockServer Authorization Server)") + .build(); + + public static final HttpResponse AUTHORIZATION_SERVER_ERROR_RESPONSE = + HttpResponse.response() + .withStatusCode(400) // recommended by RFC 6749, section 5.2 + .withContentType(MediaType.APPLICATION_JSON) + .withBody( + JsonBody.json( + "{\"error\":\"invalid_request\", " + + "\"error_description\":\"Invalid request (MockServer Authorization Server)\"}")); + + public static final HttpResponse CATALOG_SERVER_ERROR_RESPONSE = + HttpResponse.response() + .withStatusCode(401) + .withContentType(MediaType.APPLICATION_JSON) + .withBody( + JsonBody.json( + "{\"error\":{" + + "\"code\":401," + + "\"type\":\"invalid_request\"," + + "\"message\":\"Invalid request (MockServer Catalog Server)\"}}")); + + @Override + @SuppressWarnings("resource") + public void create() { + TestServer.instance() + .when( + HttpRequest.request() + .withPath(testEnvironment().authorizationServerUrl().getPath() + ".*")) + .respond(AUTHORIZATION_SERVER_ERROR_RESPONSE); + TestServer.instance() + .when(HttpRequest.request().withPath(testEnvironment().catalogServerUrl().getPath() + ".*")) + .respond(CATALOG_SERVER_ERROR_RESPONSE); + } +} diff --git a/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/expectation/LoadTableEndpointExpectation.java b/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/expectation/LoadTableEndpointExpectation.java new file mode 100644 index 000000000000..1eba2e2ea72d --- /dev/null +++ b/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/expectation/LoadTableEndpointExpectation.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.test.expectation; + +import java.util.UUID; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SortOrder; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.rest.auth.oauth2.test.TestServer; +import org.apache.iceberg.rest.responses.LoadTableResponse; +import org.apache.iceberg.types.Types; +import org.immutables.value.Value; +import org.mockserver.model.HttpRequest; +import org.mockserver.model.HttpResponse; + +@Value.Immutable +public abstract class LoadTableEndpointExpectation extends BaseExpectation { + + @Override + @SuppressWarnings("resource") + public void create() { + TableMetadata metadata = + TableMetadata.buildFromEmpty(1) + .assignUUID(UUID.randomUUID().toString()) + .setLocation("s3://bucket") + .setCurrentSchema( + new Schema(Types.NestedField.required(1, "x", Types.LongType.get())), 1) + .addPartitionSpec(PartitionSpec.unpartitioned()) + .addSortOrder(SortOrder.unsorted()) + .discardChanges() + .withMetadataLocation("s3://bucket/metadata") + .build(); + LoadTableResponse response = + LoadTableResponse.builder() + .withTableMetadata(metadata) + .addAllConfig(testEnvironment().tableProperties()) + .build(); + TestServer.instance() + .when( + HttpRequest.request() + .withMethod("GET") + .withPath(testEnvironment().loadTableEndpoint().getPath()) + .withHeader("Content-Type", "application/json") + .withHeader("Accept", "application/json") + .withHeader("Authorization", "Bearer access_initial2?")) + .respond(HttpResponse.response().withBody(jsonBody(response))); + } +} diff --git a/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/expectation/MetadataDiscoveryExpectation.java b/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/expectation/MetadataDiscoveryExpectation.java new file mode 100644 index 000000000000..4584f3acfe6b --- /dev/null +++ b/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/expectation/MetadataDiscoveryExpectation.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.test.expectation; + +import java.net.URI; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.rest.auth.oauth2.test.TestServer; +import org.immutables.value.Value; +import org.mockserver.model.HttpRequest; +import org.mockserver.model.HttpResponse; + +@Value.Immutable +public abstract class MetadataDiscoveryExpectation extends BaseExpectation { + + @Override + @SuppressWarnings("resource") + public void create() { + if (testEnvironment().discoveryEnabled()) { + URI issuerUrl = testEnvironment().authorizationServerUrl(); + URI discoveryEndpoint = testEnvironment().discoveryEndpoint(); + ImmutableMap.Builder builder = + ImmutableMap.builder() + .put("issuer", issuerUrl.toString()) + .put("token_endpoint", testEnvironment().tokenEndpoint().toString()); + + TestServer.instance() + .when(HttpRequest.request().withMethod("GET").withPath(discoveryEndpoint.getPath())) + .respond(HttpResponse.response().withBody(jsonBody(builder.build()))); + } + } +} diff --git a/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/expectation/RefreshTokenExpectation.java b/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/expectation/RefreshTokenExpectation.java new file mode 100644 index 000000000000..5b752f0115b7 --- /dev/null +++ b/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/expectation/RefreshTokenExpectation.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.test.expectation; + +import com.nimbusds.oauth2.sdk.GrantType; +import com.nimbusds.oauth2.sdk.id.Audience; +import com.nimbusds.oauth2.sdk.token.TokenTypeURI; +import java.net.URI; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.rest.auth.oauth2.test.TestServer; +import org.immutables.value.Value; +import org.mockserver.model.Parameter; + +@Value.Immutable +@SuppressWarnings("resource") +public abstract class RefreshTokenExpectation extends TokenEndpointExpectation { + + @Override + public void create() { + TestServer.instance() + .when(request()) + .respond(response("access_refreshed", "refresh_refreshed")); + if (testEnvironment().tokenRefreshWithTokenExchangeEnabled()) { + TestServer.instance() + .when(requestStub().withBody(parameterBody(requestBodyTokenExchange().build()))) + .respond(response("access_refreshed", "refresh_refreshed")); + } + } + + @Override + protected ImmutableList.Builder requestBody() { + return super.requestBody() + .add(param("grant_type", GrantType.REFRESH_TOKEN)) + .add(param("refresh_token", "refresh_.*")) + .add(param("scope", ACCEPTED_SCOPES)); + } + + /** + * Note: this expectation is very similar to the {@link TokenExchangeExpectation}, but token + * refreshes never use actor tokens, so we need a separate expectation. + */ + protected ImmutableList.Builder requestBodyTokenExchange() { + ImmutableList.Builder builder = + super.requestBody() + .add(param("grant_type", GrantType.TOKEN_EXCHANGE)) + .add(param("subject_token", "access_.*")) + .add( + param( + "subject_token_type", + testEnvironment().subjectTokenType().orElse(TokenTypeURI.ACCESS_TOKEN))) + .add(param("scope", ACCEPTED_SCOPES)); + + testEnvironment() + .requestedTokenType() + .ifPresent(tokenType -> builder.add(param("requested_token_type", tokenType))); + + for (URI resource : testEnvironment().resources()) { + builder.add(param("resource", resource)); + } + + for (Audience audience : testEnvironment().audiences()) { + builder.add(param("audience", audience.getValue())); + } + + return builder; + } +} diff --git a/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/expectation/TokenEndpointExpectation.java b/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/expectation/TokenEndpointExpectation.java new file mode 100644 index 000000000000..4e02ce88d571 --- /dev/null +++ b/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/expectation/TokenEndpointExpectation.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.test.expectation; + +import com.nimbusds.oauth2.sdk.auth.ClientAuthenticationMethod; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.rest.auth.oauth2.test.TestEnvironment; +import org.mockserver.model.Header; +import org.mockserver.model.HttpRequest; +import org.mockserver.model.HttpResponse; +import org.mockserver.model.Parameter; + +@SuppressWarnings("resource") +public abstract class TokenEndpointExpectation extends BaseExpectation { + + protected static final String ACCEPTED_SCOPES = + String.format("(%s|%s)", TestEnvironment.SCOPE1, TestEnvironment.SCOPE2); + + protected static final String ACCEPTED_CLIENT_IDS = + String.format("(%s|%s)", TestEnvironment.CLIENT_ID1, TestEnvironment.CLIENT_ID2); + + protected static final String ACCEPTED_CLIENT_SECRETS = + String.format( + "(%s|%s)", + TestEnvironment.CLIENT_SECRET1.getValue(), TestEnvironment.CLIENT_SECRET2.getValue()); + + protected static final String ACCEPTED_EXTRA_PARAM_NAMES = "(extra1|extra2|extra3)"; + protected static final String ACCEPTED_EXTRA_PARAM_VALUES = "(value1|value2|value3)"; + + private static final String CLIENT1_AUTH_HEADER = + "Basic " + + Base64.getEncoder() + .encodeToString( + (TestEnvironment.CLIENT_ID1.getValue() + + ":" + + TestEnvironment.CLIENT_SECRET1.getValue()) + .getBytes(StandardCharsets.UTF_8)); + private static final String CLIENT2_AUTH_HEADER = + "Basic " + + Base64.getEncoder() + .encodeToString( + (TestEnvironment.CLIENT_ID2.getValue() + + ":" + + TestEnvironment.CLIENT_SECRET2.getValue()) + .getBytes(StandardCharsets.UTF_8)); + + private static final String ACCEPTED_AUTH_HEADERS = + String.format("(%s|%s)", CLIENT1_AUTH_HEADER, CLIENT2_AUTH_HEADER); + + protected HttpRequest request() { + return requestStub().withBody(parameterBody(requestBody().build())); + } + + protected static Parameter param(String name, Object value) { + return Parameter.param(name, value.toString()); + } + + protected HttpRequest requestStub() { + URI tokenEndpoint = testEnvironment().tokenEndpoint(); + String path = + tokenEndpoint.isAbsolute() + ? tokenEndpoint.getPath() + : testEnvironment().catalogServerContextPath() + tokenEndpoint.getPath(); + return HttpRequest.request() + .withMethod("POST") + .withPath(path) + .withHeader("Content-Type", "application/x-www-form-urlencoded(; charset=UTF-8)?") + .withHeader("Accept", "application/json") + .withHeaders(requestHeaders().build()); + } + + protected ImmutableList.Builder

    requestHeaders() { + ImmutableList.Builder
    builder = ImmutableList.builder(); + if (testEnvironment() + .clientAuthenticationMethod() + .equals(ClientAuthenticationMethod.CLIENT_SECRET_BASIC)) { + builder.add(new Header("Authorization", ACCEPTED_AUTH_HEADERS)); + } + + return builder; + } + + protected ImmutableList.Builder requestBody() { + ImmutableList.Builder builder = + ImmutableList.builder() + .add(param(ACCEPTED_EXTRA_PARAM_NAMES, ACCEPTED_EXTRA_PARAM_VALUES)); + if (testEnvironment().clientAuthenticationMethod().equals(ClientAuthenticationMethod.NONE)) { + builder.add(param("client_id", ACCEPTED_CLIENT_IDS)); + } else if (testEnvironment() + .clientAuthenticationMethod() + .equals(ClientAuthenticationMethod.CLIENT_SECRET_POST)) { + builder.add(param("client_id", ACCEPTED_CLIENT_IDS)); + builder.add(param("client_secret", ACCEPTED_CLIENT_SECRETS)); + } + + return builder; + } + + protected HttpResponse response(String accessToken, String refreshToken) { + return HttpResponse.response() + .withBody(jsonBody(responseBody(accessToken, refreshToken).build())); + } + + protected ImmutableMap.Builder responseBody( + String accessToken, String refreshToken) { + ImmutableMap.Builder builder = + ImmutableMap.builder() + .put("access_token", accessToken) + .put("token_type", "bearer") + .put("expires_in", testEnvironment().accessTokenLifespan().toSeconds()); + if (testEnvironment().returnRefreshTokens()) { + builder.put("refresh_token", refreshToken); + } + + return builder; + } +} diff --git a/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/expectation/TokenExchangeExpectation.java b/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/expectation/TokenExchangeExpectation.java new file mode 100644 index 000000000000..8dd4f3972f75 --- /dev/null +++ b/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/expectation/TokenExchangeExpectation.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.test.expectation; + +import com.nimbusds.oauth2.sdk.GrantType; +import com.nimbusds.oauth2.sdk.id.Audience; +import com.nimbusds.oauth2.sdk.token.TokenTypeURI; +import java.net.URI; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.rest.auth.oauth2.test.TestEnvironment; +import org.apache.iceberg.rest.auth.oauth2.test.TestServer; +import org.immutables.value.Value; +import org.mockserver.model.Parameter; + +@Value.Immutable +@SuppressWarnings("resource") +public abstract class TokenExchangeExpectation extends TokenEndpointExpectation { + + // Accept both constant tokens defined in TestEnvironment + // and tokens starting with "access_" + // (since other expectations return tokens starting with "access_") + private static final String ACCEPTED_SUBJECT_TOKENS = + String.format("(%s|%s)", TestEnvironment.SUBJECT_TOKEN, "access_.*"); + private static final String ACCEPTED_ACTOR_TOKENS = + String.format("(%s|%s)", TestEnvironment.ACTOR_TOKEN, "access_.*"); + + @Override + public void create() { + TestServer.instance().when(request()).respond(response("access_initial", "refresh_initial")); + } + + @Override + protected ImmutableList.Builder requestBody() { + ImmutableList.Builder builder = + super.requestBody() + .add(param("grant_type", GrantType.TOKEN_EXCHANGE)) + .add(param("subject_token", ACCEPTED_SUBJECT_TOKENS)) + .add(param("scope", ACCEPTED_SCOPES)) + .add( + param( + "subject_token_type", + testEnvironment().subjectTokenType().orElse(TokenTypeURI.ACCESS_TOKEN))); + + testEnvironment() + .requestedTokenType() + .ifPresent(tokenType -> builder.add(param("requested_token_type", tokenType))); + + if (testEnvironment().actorTokenString().isPresent()) { + builder + .add(param("actor_token", ACCEPTED_ACTOR_TOKENS)) + .add( + param( + "actor_token_type", + testEnvironment().actorTokenType().orElse(TokenTypeURI.ACCESS_TOKEN))); + } + + for (URI resource : testEnvironment().resources()) { + builder.add(param("resource", resource)); + } + + for (Audience audience : testEnvironment().audiences()) { + builder.add(param("audience", audience.getValue())); + } + + return builder; + } + + @Override + protected ImmutableMap.Builder responseBody( + String accessToken, String refreshToken) { + return super.responseBody(accessToken, refreshToken) + .put( + "issued_token_type", + testEnvironment().requestedTokenType().orElse(TokenTypeURI.ACCESS_TOKEN).toString()); + } +} diff --git a/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/junit/EnumLike.java b/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/junit/EnumLike.java new file mode 100644 index 000000000000..f7c481e1d3cd --- /dev/null +++ b/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/junit/EnumLike.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.test.junit; + +import com.nimbusds.oauth2.sdk.GrantType; +import com.nimbusds.oauth2.sdk.auth.ClientAuthenticationMethod; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.lang.reflect.Parameter; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ConcurrentMap; +import java.util.stream.Stream; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.rest.auth.oauth2.config.ConfigUtil; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junitpioneer.jupiter.cartesian.CartesianArgumentsSource; +import org.junitpioneer.jupiter.cartesian.CartesianParameterArgumentsProvider; + +/** + * A JUnit 5 cartesian test parameter annotation that provides a stream of values from an + * "Enum-like" type. + * + *

    This annotation can be used with any type that exposes a set of public static final constants, + * as long as their {@code toString()} method returns the constant's canonical string + * representation. + * + *

    This is the case for many Nimbus SDK types, such as {@link GrantType} and {@link + * ClientAuthenticationMethod}. + */ +@Target(ElementType.PARAMETER) +@Retention(RetentionPolicy.RUNTIME) +@CartesianArgumentsSource(EnumLike.EnumLikeMethodArgumentsProvider.class) +public @interface EnumLike { + + String[] includes() default {}; + + String[] excludes() default {}; + + class EnumLikeMethodArgumentsProvider implements CartesianParameterArgumentsProvider { + + private static final ConcurrentMap, List> CONSTANTS_CACHE = + Maps.newConcurrentMap(); + + @Override + public Stream provideArguments(ExtensionContext context, Parameter parameter) { + EnumLike ann = parameter.getAnnotation(EnumLike.class); + return enumLikeConstants(parameter.getType()).stream() + .filter(constant -> applyIncludes(ann, constant)) + .filter(constant -> applyExcludes(ann, constant)); + } + + private List enumLikeConstants(Class type) { + return CONSTANTS_CACHE.computeIfAbsent(type, this::computeEnumLikeConstants); + } + + private boolean applyIncludes(EnumLike ann, Object constant) { + return ann.includes().length == 0 + || Stream.of(ann.includes()).anyMatch(name -> matchByName(name, constant)); + } + + private boolean applyExcludes(EnumLike ann, Object constant) { + return ann.excludes().length == 0 + || Stream.of(ann.excludes()).noneMatch(name -> matchByName(name, constant)); + } + + private boolean matchByName(String name, Object constant) { + // This relies on the toString() method of the constant to return its canonical string + // representation. + return name.equals(constant.toString()); + } + + private List computeEnumLikeConstants(Class type) { + if (type.equals(GrantType.class)) { + return List.copyOf(ConfigUtil.SUPPORTED_GRANT_TYPES); + } else if (type.equals(ClientAuthenticationMethod.class)) { + return ConfigUtil.SUPPORTED_CLIENT_AUTH_METHODS.stream().map(Object.class::cast).toList(); + } else { + return Arrays.stream(type.getFields()) + .filter(f -> constantField(type, f)) + .map(this::constantValue) + .toList(); + } + } + + private boolean constantField(Class type, Field field) { + return field.getModifiers() == (Modifier.PUBLIC | Modifier.STATIC | Modifier.FINAL) + && field.getType().equals(type); + } + + private Object constantValue(Field field) { + try { + return field.get(null); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + } + } +} diff --git a/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/junit/KeycloakExtension.java b/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/junit/KeycloakExtension.java new file mode 100644 index 000000000000..66d693d9a613 --- /dev/null +++ b/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/junit/KeycloakExtension.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.test.junit; + +import com.nimbusds.oauth2.sdk.auth.ClientAuthenticationMethod; +import com.nimbusds.oauth2.sdk.id.Audience; +import java.util.List; +import java.util.Optional; +import org.apache.iceberg.rest.auth.oauth2.test.ImmutableTestEnvironment; +import org.apache.iceberg.rest.auth.oauth2.test.TestCertificates; +import org.apache.iceberg.rest.auth.oauth2.test.TestEnvironment; +import org.apache.iceberg.rest.auth.oauth2.test.container.KeycloakContainer; +import org.junit.jupiter.api.extension.AfterAllCallback; +import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.ParameterContext; +import org.junit.jupiter.api.extension.ParameterResolutionException; +import org.junit.jupiter.api.extension.ParameterResolver; + +public class KeycloakExtension extends TestEnvironmentExtension + implements BeforeAllCallback, AfterAllCallback, ParameterResolver { + + // Client1 is used for client_secret_basic and client_secret_post authentication + public static final String CLIENT_ID1 = TestEnvironment.CLIENT_ID1.getValue(); + public static final String CLIENT_SECRET1 = TestEnvironment.CLIENT_SECRET1.getValue(); + public static final String CLIENT_AUTH1 = + ClientAuthenticationMethod.CLIENT_SECRET_BASIC.getValue(); + + // Client2 is used for dynamic subject tokens + public static final String CLIENT_ID2 = TestEnvironment.CLIENT_ID2.getValue(); + public static final String CLIENT_SECRET2 = TestEnvironment.CLIENT_SECRET2.getValue(); + public static final String CLIENT_AUTH2 = + ClientAuthenticationMethod.CLIENT_SECRET_BASIC.getValue(); + + // Client3 is used for dynamic actor tokens + public static final String CLIENT_ID3 = "Client3"; + public static final String CLIENT_SECRET3 = "s€cr€t"; + public static final String CLIENT_AUTH3 = + ClientAuthenticationMethod.CLIENT_SECRET_BASIC.getValue(); + + // Client4 is used for "none" authentication (public client) + public static final String CLIENT_ID4 = "Client4"; + public static final String CLIENT_AUTH4 = ClientAuthenticationMethod.NONE.getValue(); + + public static final String USERNAME = TestEnvironment.USERNAME; + public static final String PASSWORD = TestEnvironment.PASSWORD.getValue(); + + public static final String SCOPE1 = TestEnvironment.SCOPE1.toString(); + public static final String SCOPE2 = TestEnvironment.SCOPE2.toString(); + + // The intended audience for tokens issued by the clients is CLIENT_ID1 itself, + // which will be used in token exchange scenarios. + public static final String AUDIENCE = CLIENT_ID1; + + @Override + public void beforeAll(ExtensionContext context) { + context + .getStore(ExtensionContext.Namespace.GLOBAL) + .getOrComputeIfAbsent(KeycloakContainer.class.getName(), key -> createKeycloak()); + } + + @Override + public void afterAll(ExtensionContext context) { + KeycloakContainer keycloak = + context + .getStore(ExtensionContext.Namespace.GLOBAL) + .remove(KeycloakContainer.class.getName(), KeycloakContainer.class); + if (keycloak != null) { + keycloak.close(); + } + } + + @Override + public boolean supportsParameter( + ParameterContext parameterContext, ExtensionContext extensionContext) + throws ParameterResolutionException { + return parameterContext.getParameter().getType().equals(KeycloakContainer.class) + || super.supportsParameter(parameterContext, extensionContext); + } + + @Override + public Object resolveParameter( + ParameterContext parameterContext, ExtensionContext extensionContext) + throws ParameterResolutionException { + if (parameterContext.getParameter().getType().equals(KeycloakContainer.class)) { + return extensionContext + .getStore(ExtensionContext.Namespace.GLOBAL) + .getOrComputeIfAbsent(KeycloakContainer.class.getName(), key -> createKeycloak()); + } + return super.resolveParameter(parameterContext, extensionContext); + } + + @Override + protected ImmutableTestEnvironment.Builder newTestEnvironmentBuilder(ExtensionContext context) { + KeycloakContainer keycloak = + context + .getStore(ExtensionContext.Namespace.GLOBAL) + .get(KeycloakContainer.class.getName(), KeycloakContainer.class); + return TestEnvironment.builder() + .unitTest(false) + .accessTokenLifespan(KeycloakContainer.ACCESS_TOKEN_LIFESPAN) + .serverRootUrl(keycloak.rootUrl()) + .authorizationServerUrl(keycloak.issuerUrl()) + .tokenEndpoint(keycloak.tokenEndpoint()) + // Do not use the default test tokens for Keycloak + .subjectTokenString(Optional.empty()) + .actorTokenString(Optional.empty()) + .addAudiences(new Audience(AUDIENCE)) + // Keycloak does not yet support the "resource" parameter in token exchange + .resources(List.of()); + } + + private KeycloakContainer createKeycloak() { + TestCertificates certs = TestCertificates.instance(); + KeycloakContainer keycloak = + new KeycloakContainer() + .withScope(SCOPE1) + .withScope(SCOPE2) + .withUser(USERNAME, PASSWORD) + .withClient(CLIENT_ID1, CLIENT_SECRET1, CLIENT_AUTH1) + .withClient(CLIENT_ID2, CLIENT_SECRET2, CLIENT_AUTH2) + .withClient(CLIENT_ID3, CLIENT_SECRET3, CLIENT_AUTH3) + .withClient(CLIENT_ID4, null, CLIENT_AUTH4) + // Include CLIENT_ID1 in the aud claim of tokens issued by these clients. + // This will allow CLIENT_ID1 to exchange tokens obtained by all clients + // (except CLIENT_ID4), since it's an authorized audience for them. + // In Keycloak V2 standard token exchange, the audience parameter is restrictive: + // it filters the resulting token's aud claim, so audiences must be declared in advance. + .withClientAudience(CLIENT_ID1, AUDIENCE) + .withClientAudience(CLIENT_ID2, AUDIENCE) + .withClientAudience(CLIENT_ID3, AUDIENCE); + keycloak.start(); + return keycloak; + } +} diff --git a/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/junit/TestEnvironmentExtension.java b/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/junit/TestEnvironmentExtension.java new file mode 100644 index 000000000000..c7b12e6f1a8f --- /dev/null +++ b/core/src/testFixtures/java/org/apache/iceberg/rest/auth/oauth2/test/junit/TestEnvironmentExtension.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.auth.oauth2.test.junit; + +import org.apache.iceberg.rest.auth.oauth2.test.ImmutableTestEnvironment; +import org.apache.iceberg.rest.auth.oauth2.test.TestEnvironment; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.ParameterContext; +import org.junit.jupiter.api.extension.ParameterResolutionException; +import org.junit.jupiter.api.extension.ParameterResolver; + +/** + * JUnit 5 extension that provides a {@link TestEnvironment} for integration tests. + * + *

    Unit tests should create their own {@link TestEnvironment} using the {@link + * TestEnvironment#builder()} method. + */ +public abstract class TestEnvironmentExtension implements ParameterResolver { + + @Override + public boolean supportsParameter( + ParameterContext parameterContext, ExtensionContext extensionContext) + throws ParameterResolutionException { + return parameterContext.getParameter().getType().equals(ImmutableTestEnvironment.Builder.class); + } + + @Override + public Object resolveParameter( + ParameterContext parameterContext, ExtensionContext extensionContext) + throws ParameterResolutionException { + if (parameterContext.getParameter().getType().equals(ImmutableTestEnvironment.Builder.class)) { + return newTestEnvironmentBuilder(extensionContext); + } + + throw new ParameterResolutionException("Unsupported parameter type"); + } + + protected abstract ImmutableTestEnvironment.Builder newTestEnvironmentBuilder( + ExtensionContext extensionContext); +} diff --git a/core/src/testFixtures/resources/mockserver.properties b/core/src/testFixtures/resources/mockserver.properties new file mode 100644 index 000000000000..877cad0dab61 --- /dev/null +++ b/core/src/testFixtures/resources/mockserver.properties @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# See https://www.mock-server.com/mock_server/configuration_properties.html + +# Memory optimization +mockserver.maxLogEntries=1000 +mockserver.maxExpectations=1000 + +# Disable logging by default, set this to INFO or DEBUG to investigate test failures +mockserver.logLevel=OFF diff --git a/docs/docs/oauth2-configuration.md b/docs/docs/oauth2-configuration.md new file mode 100644 index 000000000000..bdb77a5c239d --- /dev/null +++ b/docs/docs/oauth2-configuration.md @@ -0,0 +1,177 @@ + + + + +# REST OAuth2 Configuration + +## Basic Settings + +Basic OAuth2 properties. These properties are used to configure the basic OAuth2 options such as the issuer URL, token endpoint, client ID, and client secret. + +### `rest.auth.oauth2.token` + +The initial access token to use. Optional. If this is set, the OAuth2 client will not attempt to fetch an initial token from the Authorization server, but will use this token instead. + +This option should be avoided as in most cases, the token cannot be refreshed. + +### `rest.auth.oauth2.issuer-url` + +The root URL of the Authorization server, which will be used for discovering supported endpoints and their locations. For Keycloak, this is typically the realm URL: `https:///realms/`. + +Two "well-known" paths are supported for endpoint discovery: `.well-known/openid-configuration` and `.well-known/oauth-authorization-server`. The full metadata discovery URL will be constructed by appending these paths to the issuer URL. + +Unless a static token (`rest.auth.oauth2.token`) is provided, either this property or `rest.auth.oauth2.token-endpoint` must be set. + +### `rest.auth.oauth2.token-endpoint` + +URL of the OAuth2 token endpoint. For Keycloak, this is typically `https:///realms//protocol/openid-connect/token`. + +Unless a static token (`rest.auth.oauth2.token`) is provided, either this property or `rest.auth.oauth2.issuer-url` must be set. In case it is not set, the token endpoint will be discovered from the issuer URL (`rest.auth.oauth2.issuer-url`), using the OpenID Connect Discovery metadata published by the issuer. + +### `rest.auth.oauth2.grant-type` + +The grant type to use when authenticating against the OAuth2 server. Valid values are: + +- `client_credentials` +- `urn:ietf:params:oauth:grant-type:token-exchange` + +Optional, defaults to `client_credentials`. + +### `rest.auth.oauth2.client-id` + +Client ID to use when authenticating against the OAuth2 server. Required, unless a static token (`rest.auth.oauth2.token`) is provided. + +### `rest.auth.oauth2.client-auth` + +The OAuth2 client authentication method to use. Valid values are: + +- `none`: the client does not authenticate itself at the token endpoint, because it is a public client with no client secret or other authentication mechanism. +- `client_secret_basic`: client secret is sent in the HTTP Basic Authorization header. +- `client_secret_post`: client secret is sent in the request body as a form parameter. + +The default is `client_secret_basic`. + +### `rest.auth.oauth2.client-secret` + +Client secret to use when authenticating against the OAuth2 server. Required if the client is private and is authenticated using the standard "client-secret" methods. + +### `rest.auth.oauth2.scope` + +Space-separated list of scopes to include in each request to the OAuth2 server. Optional, defaults to empty (no scopes). + +The scope names will not be validated by the OAuth2 client; make sure they are valid according to [RFC 6749 Section 3.3](https://datatracker.ietf.org/doc/html/rfc6749#section-3.3). + +### `rest.auth.oauth2.extra-params.*` + +Extra parameters to include in each request to the token endpoint. This is useful for custom parameters that are not covered by the standard OAuth2 specification. Optional, defaults to empty. + +This is a prefix property, and multiple values can be set, each with a different key and value. The values must NOT be URL-encoded. Example: + +``` +rest.auth.oauth2.extra-params.custom_param1=custom_value1 +rest.auth.oauth2.extra-params.custom_param2=custom_value2 +``` + +For example, Auth0 requires the `audience` parameter to be set to the API identifier. This can be done by setting the following configuration: + +``` +rest.auth.oauth2.extra-params.audience=https://iceberg-rest-catalog/api +``` + +### `rest.auth.oauth2.timeout` + +The token acquisition timeout. Optional, defaults to `PT5M`. The default timeout is intentionally large, in order to accommodate for long-running flows that require human intervention (e.g. Authorization Code flow). + +Must be a valid [ISO-8601 duration](https://en.wikipedia.org/wiki/ISO_8601#Durations). + +### `rest.auth.oauth2.session-cache.timeout` + +The session cache timeout. Cached sessions will become eligible for eviction after this duration of inactivity. Defaults to 1 hour. Must be a valid [ISO-8601 duration](https://en.wikipedia.org/wiki/ISO_8601#Durations). + +This value is used for housekeeping; it does not mean that cached sessions will stop working after this time, but that the session cache will evict the session after this time of inactivity. If the context is used again, a new session will be created and cached. + +This property can only be specified at catalog session level. It is ignored if present in other levels. +## Token Refresh Settings + +Configuration properties for the token refresh feature. + +### `rest.auth.oauth2.token-refresh.enabled` + +Whether to enable token refresh. If enabled, the OAuth2 client will automatically refresh its access token when it expires. If disabled, the OAuth2 client will only fetch the initial access token, but won't refresh it. Defaults to `true`. + +### `rest.auth.oauth2.token-refresh.token-exchange-enabled` + +Whether to use the token exchange grant to refresh tokens. + +When enabled, the token exchange grant will be used to refresh the access token, if no refresh token is available. + +Optional, defaults to `true` if the initial grant is `client_credentials`. + +### `rest.auth.oauth2.token-refresh.access-token-lifespan` + +Default access token lifespan; if the OAuth2 server returns an access token without specifying its expiration time, this value will be used. + +Optional, defaults to `PT1H`. Must be a valid [ISO-8601 duration](https://en.wikipedia.org/wiki/ISO_8601#Durations). + +### `rest.auth.oauth2.token-refresh.safety-margin` + +Refresh safety margin to use; a new token will be fetched when the current token's remaining lifespan is less than this value. Optional, defaults to `PT10S`. Must be a valid [ISO-8601 duration](https://en.wikipedia.org/wiki/ISO_8601#Durations). +## Token Exchange Settings + +Configuration properties for the [Token Exchange](https://datatracker.ietf.org/doc/html/rfc8693) flow. + +This flow allows a client to exchange one token for another, typically to obtain a token that is more suitable for the target resource or service. + +### `rest.auth.oauth2.token-exchange.subject-token` + +The subject token to exchange. Required. + +The special value `::parent::` can be used to indicate that the subject token should be obtained from the parent OAuth2 session. + +### `rest.auth.oauth2.token-exchange.subject-token-type` + +The type of the subject token. Must be a valid URN. Required. If not set, the default is `urn:ietf:params:oauth:token-type:access_token`. + +### `rest.auth.oauth2.token-exchange.actor-token` + +The actor token to exchange. Optional. + +The special value `::parent::` can be used to indicate that the actor token should be obtained from the parent OAuth2 session. + +### `rest.auth.oauth2.token-exchange.actor-token-type` + +The type of the actor token. Must be a valid URN. Required if an actor token is used. If not set, the default is `urn:ietf:params:oauth:token-type:access_token`. + +### `rest.auth.oauth2.token-exchange.requested-token-type` + +The type of the requested token. Must be a valid URN. Optional. + +### `rest.auth.oauth2.token-exchange.resources` + +One or more URIs that indicate the target service(s) or resource(s) where the client intends to use the requested token. + +Optional. Can be a single value or a comma-separated list of values. + +### `rest.auth.oauth2.token-exchange.audiences` + +The logical name(s) of the target service where the client intends to use the requested token. This serves a purpose similar to the resource parameter but with the client providing a logical name for the target service. + +Optional. Can be a single value or a comma-separated list of values. diff --git a/docs/docs/oauth2-migration.md b/docs/docs/oauth2-migration.md new file mode 100644 index 000000000000..bfb53ad4bd02 --- /dev/null +++ b/docs/docs/oauth2-migration.md @@ -0,0 +1,186 @@ + + +# REST OAuth2 Migration Guide + +This guide explains how to migrate from the legacy REST OAuth2 configuration to the new +OAuth2 Auth Manager (v2). + +## Overview + +The new OAuth2 Auth Manager (`org.apache.iceberg.rest.auth.oauth2.OAuth2Manager`) replaces the +legacy implementation (`org.apache.iceberg.rest.auth.OAuth2Manager`). It provides: + +- Standards-compliant OAuth2/OIDC support via the [Nimbus OAuth2 SDK](https://connect2id.com/products/nimbus-oauth-openid-connect-sdk) +- OpenID Connect Discovery for automatic endpoint resolution +- Proper client authentication methods (`client_secret_basic`, `client_secret_post`, `none`) +- Token Exchange support ([RFC 8693](https://datatracker.ietf.org/doc/html/rfc8693)) +- Custom token endpoint parameters (e.g. Auth0 `audience`) +- Automatic background token refresh + +## Enabling the New Auth Manager + +The legacy implementation remains the default. To opt in to the new Auth Manager, set: + +```properties +rest.auth.type=org.apache.iceberg.rest.auth.oauth2.OAuth2Manager +``` + +## Automatic Migration + +If you enable the new Auth Manager but keep using legacy property names, they will be +**automatically migrated** at runtime with deprecation warnings. No immediate changes to your +configuration are required. + +However, the automatic migration will be removed in a future Iceberg release. You should update +your configuration to use the new property names as described below. + +## Property Mapping + +### Basic Properties + +| Legacy Property | New Property | Notes | +|---------------------|--------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------| +| `credential` | `rest.auth.oauth2.client-id` + `rest.auth.oauth2.client-secret` | Legacy format was `client_id:client_secret`. If no colon is present (client secret alone), the client ID defaults to `iceberg`. | +| `token` | `rest.auth.oauth2.token` | Static Bearer token. | +| `oauth2-server-uri` | `rest.auth.oauth2.token-endpoint` or `rest.auth.oauth2.issuer-url` | Prefer `issuer-url` for automatic endpoint discovery. See [Endpoint Configuration](#endpoint-configuration). | +| `scope` | `rest.auth.oauth2.scope` | No change in semantics. | + +### Token Refresh Properties + +| Legacy Property | New Property | Notes | +|--------------------------|---------------------------------------------------------|------------------------------------------------------------------------| +| `token-refresh-enabled` | `rest.auth.oauth2.token-refresh.enabled` | Same semantics, defaults to `true`. | +| `token-expires-in-ms` | `rest.auth.oauth2.token-refresh.access-token-lifespan` | New format is an ISO-8601 duration (e.g. `PT1H` instead of `3600000`). | +| `token-exchange-enabled` | `rest.auth.oauth2.token-refresh.token-exchange-enabled` | Same semantics. | + +### Token Exchange Properties + +| Legacy Property | New Property | Notes | +|---------------------------|--------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `audience` | `rest.auth.oauth2.token-exchange.audiences` | Same semantics. Supports comma-separated list. | +| `resource` | `rest.auth.oauth2.token-exchange.resources` | Same semantics. Supports comma-separated list. | +| *(token type URN as key)* | `rest.auth.oauth2.token-exchange.subject-token` + `rest.auth.oauth2.token-exchange.subject-token-type` | Legacy usage of token type URNs (e.g. `urn:ietf:params:oauth:token-type:access_token`) as property keys is replaced by explicit subject/actor token configuration. | + +### New Properties (No Legacy Equivalent) + +Many properties are new in v2 and have no legacy counterpart, here are some of them: + +| Property | Description | +|--------------------------------------------------------|--------------------------------------------------------------------------------------------------| +| `rest.auth.oauth2.issuer-url` | Authorization server root URL for OIDC Discovery. | +| `rest.auth.oauth2.grant-type` | Explicit grant type (`client_credentials` or `urn:ietf:params:oauth:grant-type:token-exchange`). | +| `rest.auth.oauth2.client-auth` | Client authentication method (`client_secret_basic`, `client_secret_post`, `none`). | +| `rest.auth.oauth2.extra-params.*` | Additional token endpoint parameters (e.g. `rest.auth.oauth2.extra-params.audience` for Auth0). | +| `rest.auth.oauth2.timeout` | Token acquisition timeout (ISO-8601 duration, default `PT5M`). | +| `rest.auth.oauth2.session-cache.timeout` | Session cache eviction timeout (ISO-8601 duration, default `PT1H`). | +| `rest.auth.oauth2.token-exchange.actor-token` | Actor token for token exchange. | +| `rest.auth.oauth2.token-exchange.requested-token-type` | Requested token type for token exchange. | +| `rest.auth.oauth2.token-refresh.safety-margin` | How early to refresh before expiration (ISO-8601 duration, default `PT10S`). | + +For the full reference of all new configuration properties, see [REST OAuth2 Configuration](oauth2-configuration.md). + +## Endpoint Configuration + +The legacy `oauth2-server-uri` pointed to a single token endpoint. The new Auth Manager supports +two approaches: + +**Option 1: Issuer URL with automatic discovery (recommended)** + +```properties +rest.auth.oauth2.issuer-url=https://keycloak.example.com/realms/my-realm +``` + +The token endpoint will be automatically discovered via the +`.well-known/openid-configuration` or `.well-known/oauth-authorization-server` metadata endpoints. + +**Option 2: Explicit token endpoint** + +If the authorization server does not support discovery, you can also specify the token endpoint explicitly: + +```properties +rest.auth.oauth2.token-endpoint=https://keycloak.example.com/realms/my-realm/protocol/openid-connect/token +``` + +### Legacy Fallback Behavior + +If neither `issuer-url` nor `token-endpoint` is set, and no static token is provided, the new Auth +Manager will fall back to the catalog server's built-in token endpoint (i.e. `{catalog-uri}/v1/oauth/tokens`). +A warning will be logged. This fallback will be removed in a future Iceberg release. + +Similarly, relative token endpoint URLs (e.g. `v1/oauth/tokens`) will be resolved against the +catalog URI. This behavior will also be removed in a future release. + +## Migration Examples + +### Client Credentials Flow + +Before: + +```properties +rest.auth.type=oauth2 +credential=my-client-id:my-client-secret +oauth2-server-uri=https://auth.example.com/token +scope=catalog +``` + +After: + +```properties +rest.auth.type=org.apache.iceberg.rest.auth.oauth2.OAuth2Manager +rest.auth.oauth2.issuer-url=https://auth.example.com +rest.auth.oauth2.client-id=my-client-id +rest.auth.oauth2.client-secret=my-client-secret +rest.auth.oauth2.scope=catalog +``` + +### Static Token + +Before: + +```properties +rest.auth.type=oauth2 +token=eyJhbGciOiJSUzI1NiIs... +``` + +After: + +```properties +rest.auth.type=org.apache.iceberg.rest.auth.oauth2.OAuth2Manager +rest.auth.oauth2.token=eyJhbGciOiJSUzI1NiIs... +``` + +### Auth0 with Audience Parameter + +This was not possible with the legacy Auth Manager. With v2: + +```properties +rest.auth.type=org.apache.iceberg.rest.auth.oauth2.OAuth2Manager +rest.auth.oauth2.issuer-url=https://my-tenant.auth0.com +rest.auth.oauth2.client-id=my-client-id +rest.auth.oauth2.client-secret=my-client-secret +rest.auth.oauth2.extra-params.audience=https://iceberg-rest-catalog/api +``` + +## Deprecation Timeline + +- In Iceberg **1.12.0**, the legacy `org.apache.iceberg.rest.auth.OAuth2Manager` and all properties in + `org.apache.iceberg.rest.auth.OAuth2Properties` are deprecated. The default Auth Manager remains v1 + (`org.apache.iceberg.rest.auth.OAuth2Manager`) for compatibility. +- In Iceberg **1.13.0**, the default Auth Manager will be changed to v2 + (`org.apache.iceberg.rest.auth.oauth2.OAuth2Manager`). +- In Iceberg **1.14.0**, the legacy Auth Manager and properties will be removed. diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index abe2c3f543eb..36d9f0877fb8 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -36,6 +36,7 @@ awaitility = "4.3.0" awssdk-bom = "2.42.13" azuresdk-bom = "1.3.5" awssdk-s3accessgrants = "2.4.1" +bouncycastle = "1.83" bson-ver = "4.11.5" caffeine = "2.9.3" calcite = "1.41.0" @@ -71,6 +72,7 @@ junit = "5.14.3" junit-platform = "1.14.3" junit-pioneer = "2.3.0" kafka = "3.9.2" +keycloak-admin-client = "26.0.6" kryo-shaded = "4.0.3" lz4Java = "1.10.4" microprofile-openapi-api = "3.1.2" @@ -78,6 +80,8 @@ mockito = "4.11.0" mockserver = "5.15.0" nessie = "0.107.4" netty-buffer = "4.2.10.Final" +nimbus-oauth2 = "11.34" +nimbus-jose-jwt = "10.8" object-client-bundle = "3.3.2" orc = "1.9.8" parquet = "1.17.0" @@ -91,6 +95,7 @@ spark40 = "4.0.2" spark41 = "4.1.1" sqlite-jdbc = "3.51.2.0" testcontainers = "2.0.3" +testcontainers-keycloak = "3.6.0" tez08 = { strictly = "0.8.4"} # see rich version usage explanation above [libraries] @@ -111,6 +116,7 @@ awssdk-bom = { module = "software.amazon.awssdk:bom", version.ref = "awssdk-bom" awssdk-s3accessgrants = { module = "software.amazon.s3.accessgrants:aws-s3-accessgrants-java-plugin", version.ref = "awssdk-s3accessgrants" } azuresdk-bom = { module = "com.azure:azure-sdk-bom", version.ref = "azuresdk-bom" } bson = { module = "org.mongodb:bson", version.ref = "bson-ver"} +bouncycastle-bcpkix = { module = "org.bouncycastle:bcpkix-jdk18on", version.ref = "bouncycastle" } caffeine = { module = "com.github.ben-manes.caffeine:caffeine", version.ref = "caffeine" } calcite-core = { module = "org.apache.calcite:calcite-core", version.ref = "calcite" } calcite-druid = { module = "org.apache.calcite:calcite-druid", version.ref = "calcite" } @@ -166,6 +172,8 @@ lz4Java = { module = "at.yawk.lz4:lz4-java", version.ref = "lz4Java" } microprofile-openapi-api = { module = "org.eclipse.microprofile.openapi:microprofile-openapi-api", version.ref = "microprofile-openapi-api" } nessie-client = { module = "org.projectnessie.nessie:nessie-client", version.ref = "nessie" } netty-buffer = { module = "io.netty:netty-buffer", version.ref = "netty-buffer" } +nimbus-oauth2-oidc-sdk = { module = "com.nimbusds:oauth2-oidc-sdk", version.ref = "nimbus-oauth2" } +nimbus-jose-jwt = { module = "com.nimbusds:nimbus-jose-jwt", version.ref = "nimbus-jose-jwt" } object-client-bundle = { module = "com.emc.ecs:object-client-bundle", version.ref = "object-client-bundle" } orc-core = { module = "org.apache.orc:orc-core", version.ref = "orc" } parquet-avro = { module = "org.apache.parquet:parquet-avro", version.ref = "parquet" } @@ -211,6 +219,7 @@ junit-pioneer = { module = "org.junit-pioneer:junit-pioneer", version.ref = "jun junit-platform-launcher = { module = "org.junit.platform:junit-platform-launcher", version.ref = "junit-platform" } junit-suite-api = { module = "org.junit.platform:junit-platform-suite-api", version.ref = "junit-platform" } junit-suite-engine = { module = "org.junit.platform:junit-platform-suite-engine", version.ref = "junit-platform" } +keycloak-admin-client = { module = "org.keycloak:keycloak-admin-client", version.ref = "keycloak-admin-client" } kryo-shaded = { module = "com.esotericsoftware:kryo-shaded", version.ref = "kryo-shaded" } mockito-core = { module = "org.mockito:mockito-core", version.ref = "mockito" } mockito-inline = { module = "org.mockito:mockito-inline", version.ref = "mockito" } @@ -224,6 +233,7 @@ orc-tools = { module = "org.apache.orc:orc-tools", version.ref = "orc" } sqlite-jdbc = { module = "org.xerial:sqlite-jdbc", version.ref = "sqlite-jdbc" } testcontainers = { module = "org.testcontainers:testcontainers", version.ref = "testcontainers" } testcontainers-junit-jupiter = { module = "org.testcontainers:testcontainers-junit-jupiter", version.ref = "testcontainers" } +testcontainers-keycloak = { module = "com.github.dasniko:testcontainers-keycloak", version.ref = "testcontainers-keycloak" } testcontainers-minio = { module = "org.testcontainers:testcontainers-minio", version.ref = "testcontainers" } tez08-dag = { module = "org.apache.tez:tez-dag", version.ref = "tez08" } tez08-mapreduce = { module = "org.apache.tez:tez-mapreduce", version.ref = "tez08" }