From 97fa03e0f2ee42fe28340d9600891a0af8012866 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandeep=20More=CC=81?= Date: Wed, 19 Nov 2025 13:52:14 -0500 Subject: [PATCH 1/3] initial commit --- .github/workflows/build/Dockerfile | 55 ++++++ .github/workflows/build/gateway-site.xml | 197 +++++++++++++++++++ .github/workflows/build/gateway.sh | 7 + .github/workflows/build/ldap.sh | 3 + .github/workflows/build/master | 3 + .github/workflows/build/settings.xml | 30 +++ .github/workflows/compose/docker-compose.yml | 26 +++ .github/workflows/tests.yml | 58 ++++++ 8 files changed, 379 insertions(+) create mode 100644 .github/workflows/build/Dockerfile create mode 100644 .github/workflows/build/gateway-site.xml create mode 100755 .github/workflows/build/gateway.sh create mode 100755 .github/workflows/build/ldap.sh create mode 100644 .github/workflows/build/master create mode 100644 .github/workflows/build/settings.xml create mode 100644 .github/workflows/compose/docker-compose.yml create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/build/Dockerfile b/.github/workflows/build/Dockerfile new file mode 100644 index 0000000000..a5ed1d738a --- /dev/null +++ b/.github/workflows/build/Dockerfile @@ -0,0 +1,55 @@ +FROM maven:3.8.4-openjdk-8 + +MAINTAINER moresandeep + +# Install dependencies +RUN apt-get update + +RUN apt-get install -y git + +RUN useradd -ms /bin/bash gateway + +# Clone our dev branch +ARG knoxurl +ARG branch + +RUN git clone -b $branch $knoxurl knox + +RUN mkdir /knox/knox-temp-artifacts +RUN mkdir /knox/knoxshell-temp-artifacts + +# Update maven settings to ignore jcenter repo +#ADD settings.xml /home/gateway/.m2/settings.xml +#RUN mv /home/gateway/.m2/settings.xml ~/.m2/settings.xml +ADD settings.xml /usr/share/maven/ref/ + +# Skipping tests here for faster turnaround. +RUN cd knox && mvn -settings /usr/share/maven/ref/settings.xml clean -Ppackage,release install -DskipTests && tar -xvzf target/*/knox-*.tar.gz -C knox-temp-artifacts && tar -xvzf /knox/target/*/knoxshell-*.tar.gz -C knoxshell-temp-artifacts + +# move runtime to new location +# See https://github.com/docker/compose/issues/4581#issuecomment-321386605 +# for KnoxShell dance +RUN mkdir /knox-runtime +RUN mkdir /knoxshell +RUN mkdir /knox-runtime/knoxshell +RUN mv /knox/knox-temp-artifacts/*/* /knox-runtime +RUN mv /knox/knoxshell-temp-artifacts/*/* /knox-runtime/knoxshell + +# delete build artifacts +RUN rm -rf /knox + +ADD master /knox-runtime/data/security/master +# Enable websockets +ADD gateway-site.xml /knox-runtime/conf/gateway-site.xml + +RUN chown -R gateway /knox-runtime/ + +# Cleanup +RUN rm -rf /home/gateway/.m2/repository + +ADD ldap.sh /ldap.sh +ADD gateway.sh /gateway.sh + +RUN chmod +x /ldap.sh +RUN chmod +x /gateway.sh + diff --git a/.github/workflows/build/gateway-site.xml b/.github/workflows/build/gateway-site.xml new file mode 100644 index 0000000000..694b00c52e --- /dev/null +++ b/.github/workflows/build/gateway-site.xml @@ -0,0 +1,197 @@ + + + + + + gateway.service.alias.impl + org.apache.knox.gateway.services.security.impl.RemoteAliasService + + + gateway.port + 8443 + The HTTP port for the Gateway. + + + + gateway.path + gateway + The default context path for the gateway. + + + + gateway.gateway.conf.dir + deployments + The directory within GATEWAY_HOME that contains gateway topology files and deployments. + + + + gateway.hadoop.kerberos.secured + false + Boolean flag indicating whether the Hadoop cluster protected by Gateway is secured with Kerberos + + + + java.security.krb5.conf + /etc/knox/conf/krb5.conf + Absolute path to krb5.conf file + + + + java.security.auth.login.config + /etc/knox/conf/krb5JAASLogin.conf + Absolute path to JAAS login config file + + + + sun.security.krb5.debug + false + Boolean flag indicating whether to enable debug messages for krb5 authentication + + + + + gateway.websocket.feature.enabled + true + Enable/Disable websocket feature. + + + + gateway.scope.cookies.feature.enabled + false + Enable/Disable cookie scoping feature. + + + + gateway.cluster.config.monitor.ambari.enabled + false + Enable/disable Ambari cluster configuration monitoring. + + + + gateway.cluster.config.monitor.ambari.interval + 60 + The interval (in seconds) for polling Ambari for cluster configuration changes. + + + + + gateway.webshell.feature.enabled + true + Enable/Disable webshell feature. + + + gateway.webshell.max.concurrent.sessions + 20 + Maximum number of total concurrent webshell sessions + + + gateway.webshell.audit.logging.enabled + false + [Experimental Feature] Enable/Disable webshell command audit logging. + NOTE: Turning this on might log secrets that might be part of + command line arguments, please consider this before turning this on. + + + gateway.webshell.read.buffer.size + 1024 + Web Shell buffer size for reading + + + + + gateway.websocket.JWT.validation.feature.enabled + true + Enable/Disable websocket JWT validation at websocket layer. + + + + + knox.homepage.logout.enabled + true + Enable/disable logout from the Knox Homepage. + + + + + gateway.knox.token.eviction.grace.period + 0 + A duration (in seconds) beyond a token’s expiration to wait before evicting its state. This configuration only applies when server-managed token state is enabled either in gateway-site or at the topology level. + + + + + gateway.knox.admin.groups + admin + + + + + gateway.group.config.hadoop.security.group.mapping + org.apache.hadoop.security.LdapGroupsMapping + + + gateway.group.config.hadoop.security.group.mapping.ldap.bind.user + uid=guest,ou=people,dc=hadoop,dc=apache,dc=org + + + gateway.group.config.hadoop.security.group.mapping.ldap.bind.password + guest-password + + + gateway.group.config.hadoop.security.group.mapping.ldap.url + ldap://localhost:33389 + + + gateway.group.config.hadoop.security.group.mapping.ldap.base + + + + gateway.group.config.hadoop.security.group.mapping.ldap.search.filter.user + (&(|(objectclass=person)(objectclass=applicationProcess))(cn={0})) + + + gateway.group.config.hadoop.security.group.mapping.ldap.search.filter.group + (objectclass=groupOfNames) + + + gateway.group.config.hadoop.security.group.mapping.ldap.search.attr.member + member + + + gateway.group.config.hadoop.security.group.mapping.ldap.search.attr.group.name + cn + + + gateway.dispatch.whitelist.services + DATANODE,HBASEUI,HDFSUI,JOBHISTORYUI,NODEUI,YARNUI,knoxauth + The comma-delimited list of service roles for which the gateway.dispatch.whitelist should be applied. + + + gateway.dispatch.whitelist + ^https?:\/\/(www\.local\.com|localhost|127\.0\.0\.1|0:0:0:0:0:0:0:1|::1):[0-9].*$ + The whitelist to be applied for dispatches associated with the service roles specified by gateway.dispatch.whitelist.services. + If the value is DEFAULT, a domain-based whitelist will be derived from the Knox host. + + + gateway.xforwarded.header.context.append.servicename + LIVYSERVER + Add service name to x-forward-context header for the list of services defined above. + + + diff --git a/.github/workflows/build/gateway.sh b/.github/workflows/build/gateway.sh new file mode 100755 index 0000000000..f73d014b48 --- /dev/null +++ b/.github/workflows/build/gateway.sh @@ -0,0 +1,7 @@ +#!/bin/sh +# Move the KnoxShell directory to proper place +# This is vecause of https://github.com/docker/compose/issues/4581#issuecomment-321386605 +mv /knox-runtime/knoxshell/* /knoxshell + +# Start Knox +java -jar /knox-runtime/bin/gateway.jar \ No newline at end of file diff --git a/.github/workflows/build/ldap.sh b/.github/workflows/build/ldap.sh new file mode 100755 index 0000000000..f1b3ea42d4 --- /dev/null +++ b/.github/workflows/build/ldap.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +java -jar /knox-runtime/bin/ldap.jar /knox-runtime/conf \ No newline at end of file diff --git a/.github/workflows/build/master b/.github/workflows/build/master new file mode 100644 index 0000000000..3a9f1b4ea5 --- /dev/null +++ b/.github/workflows/build/master @@ -0,0 +1,3 @@ +#1.0# Tue, Apr 30 2024 14:19:35.337 +RHdXbkpJTXUzcGc9Ojp5WWFSV0JYeG1GaFdjSHhIUEJlYWZ3PT06OmdOWC9CRDJ5dFpQeVExVkM2V2lYRkE9PQ== + diff --git a/.github/workflows/build/settings.xml b/.github/workflows/build/settings.xml new file mode 100644 index 0000000000..ea1f0dea48 --- /dev/null +++ b/.github/workflows/build/settings.xml @@ -0,0 +1,30 @@ + + + + + + replace-jcenter-with-central + jcenter + https://repo.maven.apache.org/maven2 + + + diff --git a/.github/workflows/compose/docker-compose.yml b/.github/workflows/compose/docker-compose.yml new file mode 100644 index 0000000000..6c633b87d4 --- /dev/null +++ b/.github/workflows/compose/docker-compose.yml @@ -0,0 +1,26 @@ +version: '3' + +services: + knox-dev: + build: + context: ../build + args: + knoxurl: ${knoxurl:-https://github.com/apache/knox.git} + branch: ${branch:-master} + image: moresandeep/knox-dev:master + + ldap: + image: moresandeep/knox-dev:master + command: /ldap.sh + + knox: + image: moresandeep/knox-dev:master + ports: + - "8443:8443" + command: /gateway.sh + volumes: + - ./topologies:/knox-runtime/conf/topologies + - ./logs:/knox-runtime/logs + - ./knoxshell:/knoxshell + depends_on: + - ldap diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000000..06b2f8a5ba --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Apache Knox Docker Compose Tests + +on: + pull_request: + branches: + - '**' # triggers for all PRs + workflow_dispatch: + +jobs: + build-and-test: + runs-on: ubuntu-latest + + env: + KNOX_URL: "https://github.com/apache/knox.git" + BRANCH: "${{ github.head_ref }}" # PR branch being tested + + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Set up Docker Compose + run: docker compose version + + - name: Build Docker Images + run: | + export knoxurl=${KNOX_URL} + export branch=${BRANCH} + docker compose -f ./.github/workflows/compose/docker-compose.yml build + + - name: Start Knox and LDAP Services + run: docker compose -f ./.github/workflows/compose/docker-compose.yml up -d + + - name: Wait for services to stabilize + run: sleep 30 # Adjust as needed for services startup time + + - name: Run Knox Tests + run: | + # Example: Run integration or custom tests here + # docker compose -f ./.github/workflows/compose/docker-compose.yml exec -T knox bash -c "cd /knoxshell && ./run-tests.sh" + + - name: Tear Down Docker Compose + if: always() + run: docker compose -f ./.github/workflows/compose/docker-compose.yml down --volumes From 1725a5519cb8485743d5006594387c132e2ece73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandeep=20More=CC=81?= Date: Wed, 19 Nov 2025 16:15:06 -0500 Subject: [PATCH 2/3] Add license --- .github/workflows/build/Dockerfile | 15 +++++++++++++++ .github/workflows/build/gateway.sh | 15 +++++++++++++++ .github/workflows/build/ldap.sh | 15 ++++++++++++++- .github/workflows/build/master | 14 ++++++++++++++ .github/workflows/compose/docker-compose.yml | 15 +++++++++++++++ 5 files changed, 73 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build/Dockerfile b/.github/workflows/build/Dockerfile index a5ed1d738a..58423f5320 100644 --- a/.github/workflows/build/Dockerfile +++ b/.github/workflows/build/Dockerfile @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + FROM maven:3.8.4-openjdk-8 MAINTAINER moresandeep diff --git a/.github/workflows/build/gateway.sh b/.github/workflows/build/gateway.sh index f73d014b48..e5e1794621 100755 --- a/.github/workflows/build/gateway.sh +++ b/.github/workflows/build/gateway.sh @@ -1,4 +1,19 @@ #!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Move the KnoxShell directory to proper place # This is vecause of https://github.com/docker/compose/issues/4581#issuecomment-321386605 mv /knox-runtime/knoxshell/* /knoxshell diff --git a/.github/workflows/build/ldap.sh b/.github/workflows/build/ldap.sh index f1b3ea42d4..ad3d9c8e3f 100755 --- a/.github/workflows/build/ldap.sh +++ b/.github/workflows/build/ldap.sh @@ -1,3 +1,16 @@ #!/bin/sh - +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. java -jar /knox-runtime/bin/ldap.jar /knox-runtime/conf \ No newline at end of file diff --git a/.github/workflows/build/master b/.github/workflows/build/master index 3a9f1b4ea5..acd09cbb91 100644 --- a/.github/workflows/build/master +++ b/.github/workflows/build/master @@ -1,3 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. #1.0# Tue, Apr 30 2024 14:19:35.337 RHdXbkpJTXUzcGc9Ojp5WWFSV0JYeG1GaFdjSHhIUEJlYWZ3PT06OmdOWC9CRDJ5dFpQeVExVkM2V2lYRkE9PQ== diff --git a/.github/workflows/compose/docker-compose.yml b/.github/workflows/compose/docker-compose.yml index 6c633b87d4..14ae736404 100644 --- a/.github/workflows/compose/docker-compose.yml +++ b/.github/workflows/compose/docker-compose.yml @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +#

+# http://www.apache.org/licenses/LICENSE-2.0 +#

+# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + version: '3' services: From cb822b1ecfcb18f31ed1be7c03b006c21d1f455d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandeep=20More=CC=81?= Date: Thu, 20 Nov 2025 07:00:58 -0500 Subject: [PATCH 3/3] Update node version to fix instability --- .github/workflows/build/Dockerfile | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build/Dockerfile b/.github/workflows/build/Dockerfile index 58423f5320..c757079e9a 100644 --- a/.github/workflows/build/Dockerfile +++ b/.github/workflows/build/Dockerfile @@ -39,7 +39,7 @@ RUN mkdir /knox/knoxshell-temp-artifacts ADD settings.xml /usr/share/maven/ref/ # Skipping tests here for faster turnaround. -RUN cd knox && mvn -settings /usr/share/maven/ref/settings.xml clean -Ppackage,release install -DskipTests && tar -xvzf target/*/knox-*.tar.gz -C knox-temp-artifacts && tar -xvzf /knox/target/*/knoxshell-*.tar.gz -C knoxshell-temp-artifacts +RUN cd knox && mvn -settings /usr/share/maven/ref/settings.xml clean -Ppackage,release install -Dforbiddenapis.skip=true -Denforcer.skip -Dpmd.failOnViolation=false -DskipTests=true -Dcheckstyle.skip=true -Dspotbugs.skip=true -Dpmd.skip=true -Drat.skip -DskipTests && tar -xvzf target/*/knox-*.tar.gz -C knox-temp-artifacts && tar -xvzf /knox/target/*/knoxshell-*.tar.gz -C knoxshell-temp-artifacts # move runtime to new location # See https://github.com/docker/compose/issues/4581#issuecomment-321386605 diff --git a/pom.xml b/pom.xml index ef66768f8d..6a6deb4e5e 100644 --- a/pom.xml +++ b/pom.xml @@ -256,7 +256,7 @@ 2.2.4 4.1.127.Final 10.0.2 - v16.10.0 + v16.20.2 4.12.0 3.4.5 4.5.6