From 2d71fdbec9841cc696121c843796c4ec70fdf7a6 Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Thu, 19 Mar 2026 09:15:00 +0100 Subject: [PATCH 1/2] e2e: Remove cert-manager from all test environments Signed-off-by: Andrea Mazzotti --- .github/workflows/test_chart.yaml | 36 +++-- Makefile | 10 +- charts/rancher-turtles/questions.yml | 20 --- docs/release-v2.md | 1 - scripts/create-rancher-certs.sh | 68 ++++++++ scripts/kind-cluster-with-extramounts.yaml | 9 +- scripts/turtles-dev.sh | 29 +++- test/e2e/config/operator.yaml | 5 - test/e2e/data/rancher/test-nodeport.yaml | 8 +- test/e2e/suites/capiprovider/suite_test.go | 4 - .../chart-upgrade/chart_upgrade_test.go | 1 + test/e2e/suites/chart-upgrade/suite_test.go | 4 - test/e2e/suites/import-gitops/suite_test.go | 4 - test/e2e/suites/v2prov/suite_test.go | 4 - test/e2e/suites/v2prov/v2prov_test.go | 5 +- test/framework/command_helper.go | 6 +- test/framework/kube_helper.go | 5 +- test/framework/rancher_helpers.go | 5 +- test/framework/wrangler.go | 39 ++--- test/go.mod | 2 +- test/testenv/aws.go | 5 +- test/testenv/certmanager.go | 89 ----------- test/testenv/eks.go | 5 +- test/testenv/eksctl_provider.go | 10 +- test/testenv/providers.go | 7 +- test/testenv/rancher.go | 147 +++++++++++++++--- test/testenv/rancher_system_chart.go | 6 + 27 files changed, 312 insertions(+), 222 deletions(-) delete mode 100644 charts/rancher-turtles/questions.yml create mode 100755 scripts/create-rancher-certs.sh delete mode 100644 test/testenv/certmanager.go diff --git a/.github/workflows/test_chart.yaml b/.github/workflows/test_chart.yaml index 6e5ee25b9..f4240dded 100644 --- a/.github/workflows/test_chart.yaml +++ b/.github/workflows/test_chart.yaml @@ -15,7 +15,6 @@ env: MANIFEST_IMG: controller CONTROLLER_IMG: controller PULL_POLICY: Never - CERT_MANAGER_VERSION: v1.16.3 RANCHER_VERSION: v2.14.0-alpha9 jobs: @@ -66,17 +65,24 @@ jobs: - name: Add local docker image run: kind load docker-image ${{ env.MANIFEST_IMG }}:${{ env.TAG }} - - name: Add cert-manager chart repo - run: helm repo add jetstack https://charts.jetstack.io - - name: Add rancher chart repo run: helm repo add rancher-alpha https://releases.rancher.com/server-charts/alpha - - name: Install cert-manager - run: helm install cert-manager jetstack/cert-manager --namespace cert-manager --create-namespace --version ${{ env.CERT_MANAGER_VERSION }} --set crds.enabled=true --set crds.keep=true --wait + - name: Setup Rancher Private CA + run: | + echo "Configuring Private CA Certificate..." + ./scripts/create-rancher-certs.sh + + echo "Create secrets" + kubectl create namespace cattle-system + kubectl -n cattle-system create secret tls tls-rancher-ingress \ + --cert /tmp/rancher-private-ca/tls.crt \ + --key /tmp/rancher-private-ca/tls.key + kubectl -n cattle-system create secret generic tls-ca \ + --from-file /tmp/rancher-private-ca/cacerts.pem - name: Install Rancher - run: helm install rancher rancher-alpha/rancher --namespace cattle-system --create-namespace --set bootstrapPassword=rancheradmin --set replicas=1 --set hostname="e2e.dev.rancher" --set 'extraEnv[0].name=CATTLE_FEATURES' --set 'extraEnv[0].value=turtles=false' --version ${{ env.RANCHER_VERSION }} --wait + run: helm install rancher rancher-alpha/rancher --namespace cattle-system --create-namespace --set bootstrapPassword=rancheradmin --set replicas=1 --set hostname="e2e.dev.rancher" --set 'extraEnv[0].name=CATTLE_FEATURES' --set 'extraEnv[0].value=turtles=false' --version ${{ env.RANCHER_VERSION }} --set ingress.tls.source=secret --set privateCA=true --wait - name: Wait for rancher-webhook run: | @@ -164,11 +170,21 @@ jobs: - name: Add local docker image run: kind load docker-image ${{ env.MANIFEST_IMG }}:${{ env.TAG }} - - name: Install cert-manager - run: helm repo add jetstack https://charts.jetstack.io && helm install cert-manager jetstack/cert-manager --namespace cert-manager --create-namespace --version ${{ env.CERT_MANAGER_VERSION }} --set crds.enabled=true --set crds.keep=true --wait + - name: Setup Rancher Private CA + run: | + echo "Configuring Private CA Certificate..." + ./scripts/create-rancher-certs.sh + + echo "Create secrets" + kubectl create namespace cattle-system + kubectl -n cattle-system create secret tls tls-rancher-ingress \ + --cert /tmp/rancher-private-ca/tls.crt \ + --key /tmp/rancher-private-ca/tls.key + kubectl -n cattle-system create secret generic tls-ca \ + --from-file /tmp/rancher-private-ca/cacerts.pem - name: Install Rancher - run: helm repo add rancher-alpha https://releases.rancher.com/server-charts/alpha && helm install rancher rancher-alpha/rancher --namespace cattle-system --create-namespace --set bootstrapPassword=rancheradmin --set replicas=1 --set hostname="e2e.dev.rancher" --set 'extraEnv[0].name=CATTLE_FEATURES' --set 'extraEnv[0].value=turtles=false' --version ${{ env.RANCHER_VERSION }} --wait + run: helm repo add rancher-alpha https://releases.rancher.com/server-charts/alpha && helm install rancher rancher-alpha/rancher --namespace cattle-system --create-namespace --set bootstrapPassword=rancheradmin --set replicas=1 --set hostname="e2e.dev.rancher" --set 'extraEnv[0].name=CATTLE_FEATURES' --set 'extraEnv[0].value=turtles=false' --version ${{ env.RANCHER_VERSION }} --set ingress.tls.source=secret --set privateCA=true --wait - name: Wait for rancher-webhook run: | diff --git a/Makefile b/Makefile index ce3c9c7a6..5629d2da5 100644 --- a/Makefile +++ b/Makefile @@ -203,6 +203,13 @@ export RANCHER_CHARTS_REPO_DIR ?= $(abspath $(RELEASE_DIR)/rancher-charts) export RANCHER_CHART_DEV_VERSION ?= 108.0.0+up99.99.99 export RANCHER_CHARTS_BASE_BRANCH ?= dev-v2.14 +# Rancher Private CA setup +CREATE_RANCHER_CERTS_SCRIPT_PATH ?= $(ROOT_DIR)/scripts/create-rancher-certs.sh +export RANCHER_CERT_DIR ?= /tmp/rancher-private-ca +export RANCHER_CERT_PATH ?= $(RANCHER_CERT_DIR)/tls.crt +export RANCHER_CERT_KEY_PATH ?= $(RANCHER_CERT_DIR)/tls.key +export RANCHER_CACERT_PATH ?= $(RANCHER_CERT_DIR)/cacerts.pem + # Allow overriding the imagePullPolicy PULL_POLICY ?= IfNotPresent @@ -637,7 +644,8 @@ CLUSTERCTL_BINARY_PATH=$(CLUSTERCTL) \ SKIP_RESOURCE_CLEANUP=$(SKIP_RESOURCE_CLEANUP) \ USE_EXISTING_CLUSTER=$(USE_EXISTING_CLUSTER) \ TURTLES_PROVIDERS=$(TURTLES_PROVIDERS) \ -TURTLES_PROVIDERS_PATH=$(ROOT_DIR)/$(CHART_PACKAGE_DIR)/rancher-turtles-providers-$(RANCHER_CHART_DEV_VERSION).tgz +TURTLES_PROVIDERS_PATH=$(ROOT_DIR)/$(CHART_PACKAGE_DIR)/rancher-turtles-providers-$(RANCHER_CHART_DEV_VERSION).tgz \ +CREATE_RANCHER_CERTS_SCRIPT_PATH=$(CREATE_RANCHER_CERTS_SCRIPT_PATH) E2E_RUN_COMMAND=$(E2ECONFIG_VARS) $(GINKGO) -v --trace -p -procs=10 -poll-progress-after=$(GINKGO_POLL_PROGRESS_AFTER) \ -poll-progress-interval=$(GINKGO_POLL_PROGRESS_INTERVAL) --tags=e2e --focus="$(GINKGO_FOCUS)" --label-filter="$(GINKGO_LABEL_FILTER)" \ diff --git a/charts/rancher-turtles/questions.yml b/charts/rancher-turtles/questions.yml deleted file mode 100644 index a2bc376ac..000000000 --- a/charts/rancher-turtles/questions.yml +++ /dev/null @@ -1,20 +0,0 @@ -namespace: cattle-turtles-system -questions: - - variable: cluster-api-operator.cleanup - default: true - description: "Specify that the CAPI Operator post-delete cleanup job will be performed." - type: boolean - label: Cleanup CAPI Operator installation - group: "CAPI Operator cleanup settings" - - variable: features.agent-tls-mode.enabled - default: true - description: "[BETA] If enabled Turtles will use the agent-tls-mode setting to determine CA cert trust mode for importing clusters." - type: boolean - label: Enable Agent TLS Mode - group: "Rancher Turtles Features Settings" - - variable: features.no-cert-manager.enabled - default: true - description: "[ALPHA] If enabled Turtles will remove cert-manager." - type: boolean - label: Remove cert-manager - group: "Rancher Turtles Features Settings" diff --git a/docs/release-v2.md b/docs/release-v2.md index 6bb5c6c40..875cd53a1 100644 --- a/docs/release-v2.md +++ b/docs/release-v2.md @@ -214,7 +214,6 @@ It is generally not accepted to submit pull requests directly against release br - Critical bugs fixes, security issue fixes, or fixes for bugs without easy workarounds. - Dependency bumps for CVE (usually limited to CVE resolution; backports of non-CVE related version bumps are considered exceptions to be evaluated case by case) -- Cert-manager version bumps (to avoid having releases with cert-manager versions that are out of support, when possible) - Changes required to support new Kubernetes versions, when possible. - Changes to use the latest Go patch version to build controller images. - Improvements to existing docs (the latest supported branch hosts the current version of the book) diff --git a/scripts/create-rancher-certs.sh b/scripts/create-rancher-certs.sh new file mode 100755 index 000000000..9347b9830 --- /dev/null +++ b/scripts/create-rancher-certs.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +# Copyright © 2026 SUSE LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Creates self signed certificates to configure Rancher's ingress. +# Files can be loaded before installing Rancher with: +# kubectl create namespace cattle-system +# kubectl -n cattle-system create secret tls tls-rancher-ingress \ +# --cert=$RANCHER_CERT_PATH \ +# --key=$RANCHER_KEY_PATH +# kubectl -n cattle-system create secret generic tls-ca \ +# --from-file=$RANCHER_CACERT_PATH + +set -xe + +RANCHER_HOSTNAME=${RANCHER_HOSTNAME:-localhost} +RANCHER_CERT_DIR=${RANCHER_CERT_DIR:-/tmp/rancher-private-ca} +RANCHER_CERT_PATH=${RANCHER_CERT_PATH:-$RANCHER_CERT_DIR/tls.crt} +RANCHER_CERT_KEY_PATH=${RANCHER_CERT_KEY_PATH:-$RANCHER_CERT_DIR/tls.key} +RANCHER_CACERT_PATH=${RANCHER_CACERT_PATH:-$RANCHER_CERT_DIR/cacerts.pem} + +mkdir -p $RANCHER_CERT_DIR + +# Generate CA cert +openssl genrsa -out "$RANCHER_CERT_DIR/cacerts.key" 4096 +openssl req -x509 -new -nodes \ + -key "$RANCHER_CERT_DIR/cacerts.key" \ + -sha256 -days 3650 \ + -out "$RANCHER_CACERT_PATH" \ + -subj "/CN=Rancher Test" + +# Generate tls cert +openssl genrsa -out "$RANCHER_CERT_KEY_PATH" 2048 +openssl req -new \ + -key "$RANCHER_CERT_KEY_PATH" \ + -out "$RANCHER_CERT_DIR/tls.csr" \ + -subj "/CN=localhost" +cat > $RANCHER_CERT_DIR/tls.v3.ext << EOF +authorityKeyIdentifier=keyid,issuer +basicConstraints=CA:FALSE +keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment +subjectAltName = @alt_names +[alt_names] +DNS.1 = $RANCHER_HOSTNAME +DNS.2 = localhost +IP.1 = 127.0.0.1 +EOF +openssl x509 -req \ + -in "$RANCHER_CERT_DIR/tls.csr" \ + -CA "$RANCHER_CACERT_PATH" \ + -CAkey "$RANCHER_CERT_DIR/cacerts.key" \ + -CAcreateserial \ + -out "$RANCHER_CERT_PATH" \ + -days 3650 \ + -sha256 \ + -extfile "$RANCHER_CERT_DIR/tls.v3.ext" diff --git a/scripts/kind-cluster-with-extramounts.yaml b/scripts/kind-cluster-with-extramounts.yaml index 6edc22dec..ba25eebce 100644 --- a/scripts/kind-cluster-with-extramounts.yaml +++ b/scripts/kind-cluster-with-extramounts.yaml @@ -12,7 +12,10 @@ nodes: - containerPort: 30001 hostPort: 30001 protocol: TCP - # Rancher test Nodeport - - containerPort: 30002 - hostPort: 30002 + # Rancher test Nodeports (HTTP and HTTPs) + - containerPort: 30080 + hostPort: 30080 + protocol: TCP + - containerPort: 30443 + hostPort: 30443 protocol: TCP diff --git a/scripts/turtles-dev.sh b/scripts/turtles-dev.sh index 7e36cd19f..e4b379260 100755 --- a/scripts/turtles-dev.sh +++ b/scripts/turtles-dev.sh @@ -38,6 +38,11 @@ RANCHER_CHARTS_REPO_DIR=${RANCHER_CHARTS_REPO_DIR} RANCHER_CHART_DEV_VERSION=${RANCHER_CHART_DEV_VERSION} RANCHER_CHARTS_BASE_BRANCH=${RANCHER_CHARTS_BASE_BRANCH} +RANCHER_CERT_DIR=${RANCHER_CERT_DIR:-/tmp/rancher-private-ca} +RANCHER_CERT_PATH=${RANCHER_CERT_PATH:-$RANCHER_CERT_DIR/tls.crt} +RANCHER_KEY_PATH=${RANCHER_KEY_PATH:-$RANCHER_CERT_DIR/tls.key} +RANCHER_CACERT_PATH=${RANCHER_CACERT_PATH:-$RANCHER_CERT_DIR/cacerts.pem} + BASEDIR=$(dirname "$0") if pgrep -x ngrok > /dev/null; then @@ -53,15 +58,21 @@ kind load docker-image $RANCHER_IMAGE --name $CLUSTER_NAME kubectl rollout status deployment coredns -n kube-system --timeout=90s helm repo add rancher-$RANCHER_CHANNEL https://releases.rancher.com/server-charts/$RANCHER_CHANNEL --force-update -helm repo add jetstack https://charts.jetstack.io --force-update helm repo add gitea-charts https://dl.gitea.com/charts/ --force-update helm repo update -helm install cert-manager jetstack/cert-manager \ - --namespace cert-manager \ - --create-namespace \ - --set crds.enabled=true +echo "Configuring Private CA Certificate..." +./scripts/create-rancher-certs.sh + +# Create secrets +kubectl create namespace cattle-system +kubectl -n cattle-system create secret tls tls-rancher-ingress \ + --cert $RANCHER_CERT_PATH \ + --key $RANCHER_KEY_PATH +kubectl -n cattle-system create secret generic tls-ca \ + --from-file $RANCHER_CACERT_PATH +echo "Installing Gitea..." helm install gitea gitea-charts/gitea \ -f test/e2e/data/gitea/values.yaml \ --set gitea.admin.password=$GITEA_PASSWORD \ @@ -112,6 +123,8 @@ helm install rancher rancher-$RANCHER_CHANNEL/rancher \ --set image.tag=$RANCHER_IMAGE_TAG \ --set debug=true \ --version="$RANCHER_VERSION" \ + --set ingress.tls.source=secret \ + --set privateCA=true \ --wait # Deploy Rancher test Nodeport @@ -119,8 +132,8 @@ echo "Deploying Rancher test Nodeport..." kubectl apply -f test/e2e/data/rancher/test-nodeport.yaml # Wait for Rancher to be accessible locally -echo "Waiting for Rancher to be accessible on localhost:30002..." -until curl -s -o /dev/null -w "%{http_code}" http://localhost:30002 | grep -q "200\|302\|301"; do +echo "Waiting for Rancher to be accessible on localhost:30080..." +until curl -s -o /dev/null -w "%{http_code}" http://localhost:30080 | grep -q "200\|302\|301"; do echo "Waiting for test Rancher Nodeport..." sleep 2 done @@ -134,7 +147,7 @@ authtoken: $NGROK_AUTHTOKEN tunnels: rancher: proto: http - addr: http://localhost:30002 + addr: http://localhost:30080 hostname: $RANCHER_HOSTNAME gitea: proto: http diff --git a/test/e2e/config/operator.yaml b/test/e2e/config/operator.yaml index 76b2d19c1..6358a3a4b 100644 --- a/test/e2e/config/operator.yaml +++ b/test/e2e/config/operator.yaml @@ -132,11 +132,6 @@ variables: TURTLES_PROVIDERS_URL: "https://rancher.github.io/turtles" TURTLES_PROVIDERS_PATH: "turtles/rancher-turtles-providers" - # External Charts and Dependencies - CERT_MANAGER_REPO_NAME: "jetstack" - CERT_MANAGER_URL: "https://charts.jetstack.io" - CERT_MANAGER_PATH: "jetstack/cert-manager" - # Ingress Configuration (Ngrok) NGROK_REPO_NAME: "ngrok" NGROK_URL: "https://charts.ngrok.com" diff --git a/test/e2e/data/rancher/test-nodeport.yaml b/test/e2e/data/rancher/test-nodeport.yaml index a42d7899e..8443060fd 100644 --- a/test/e2e/data/rancher/test-nodeport.yaml +++ b/test/e2e/data/rancher/test-nodeport.yaml @@ -8,7 +8,13 @@ spec: selector: app: rancher ports: - - nodePort: 30002 + - nodePort: 30080 + name: http port: 80 protocol: TCP targetPort: 80 + - nodePort: 30443 + name: https + port: 443 + protocol: TCP + targetPort: 443 diff --git a/test/e2e/suites/capiprovider/suite_test.go b/test/e2e/suites/capiprovider/suite_test.go index be02f0cd2..51456b973 100644 --- a/test/e2e/suites/capiprovider/suite_test.go +++ b/test/e2e/suites/capiprovider/suite_test.go @@ -69,10 +69,6 @@ var _ = SynchronizedBeforeSuite( Scheme: e2e.InitScheme(), }) - testenv.DeployCertManager(ctx, testenv.DeployCertManagerInput{ - BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, - }) - testenv.RancherDeployIngress(ctx, testenv.RancherDeployIngressInput{ BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, CustomIngress: e2e.TraefikIngress, diff --git a/test/e2e/suites/chart-upgrade/chart_upgrade_test.go b/test/e2e/suites/chart-upgrade/chart_upgrade_test.go index b5fa7481f..f8041fd51 100644 --- a/test/e2e/suites/chart-upgrade/chart_upgrade_test.go +++ b/test/e2e/suites/chart-upgrade/chart_upgrade_test.go @@ -198,6 +198,7 @@ var _ = Describe("Chart upgrade functionality should work", Ordered, Label(e2e.S TurtlesImageTag: "v0.0.1", RancherHostname: hostName, RancherWaitInterval: e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-rancher"), + SkipPrivateCASetup: true, }) By("Waiting for Rancher to be ready after upgrade") diff --git a/test/e2e/suites/chart-upgrade/suite_test.go b/test/e2e/suites/chart-upgrade/suite_test.go index 046d35f0d..83a549a69 100644 --- a/test/e2e/suites/chart-upgrade/suite_test.go +++ b/test/e2e/suites/chart-upgrade/suite_test.go @@ -86,10 +86,6 @@ var _ = SynchronizedBeforeSuite( KubernetesVersion: e2eConfig.GetVariableOrEmpty(e2e.KubernetesVersionChartUpgradeVar), }) - testenv.DeployCertManager(ctx, testenv.DeployCertManagerInput{ - BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, - }) - testenv.RancherDeployIngress(ctx, testenv.RancherDeployIngressInput{ BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, CustomIngress: e2e.TraefikIngress, diff --git a/test/e2e/suites/import-gitops/suite_test.go b/test/e2e/suites/import-gitops/suite_test.go index c0af97d14..1cb43b61d 100644 --- a/test/e2e/suites/import-gitops/suite_test.go +++ b/test/e2e/suites/import-gitops/suite_test.go @@ -66,10 +66,6 @@ var _ = SynchronizedBeforeSuite( Scheme: e2e.InitScheme(), }) - testenv.DeployCertManager(ctx, testenv.DeployCertManagerInput{ - BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, - }) - testenv.RancherDeployIngress(ctx, testenv.RancherDeployIngressInput{ BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, CustomIngress: e2e.TraefikIngress, diff --git a/test/e2e/suites/v2prov/suite_test.go b/test/e2e/suites/v2prov/suite_test.go index 72cdc716e..411743b66 100644 --- a/test/e2e/suites/v2prov/suite_test.go +++ b/test/e2e/suites/v2prov/suite_test.go @@ -71,10 +71,6 @@ var _ = SynchronizedBeforeSuite( Scheme: e2e.InitScheme(), }) - testenv.DeployCertManager(ctx, testenv.DeployCertManagerInput{ - BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, - }) - testenv.RancherDeployIngress(ctx, testenv.RancherDeployIngressInput{ BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy, CustomIngress: e2e.TraefikIngress, diff --git a/test/e2e/suites/v2prov/v2prov_test.go b/test/e2e/suites/v2prov/v2prov_test.go index 152e18d9c..3704d87b8 100644 --- a/test/e2e/suites/v2prov/v2prov_test.go +++ b/test/e2e/suites/v2prov/v2prov_test.go @@ -150,8 +150,7 @@ var _ = Describe("[v2prov] [Azure] Creating a cluster with v2prov should still w WaitInterval: e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-rancher"), }) - rancherConnectRes := &turtlesframework.RunCommandResult{} - turtlesframework.RunCommand(ctx, turtlesframework.RunCommandInput{ + rancherConnectRes := turtlesframework.RunCommand(ctx, turtlesframework.RunCommandInput{ Command: "kubectl", Args: []string{ "--kubeconfig", @@ -160,7 +159,7 @@ var _ = Describe("[v2prov] [Azure] Creating a cluster with v2prov should still w "nodes", "--insecure-skip-tls-verify", }, - }, rancherConnectRes) + }) Expect(rancherConnectRes.Error).NotTo(HaveOccurred(), "Failed getting nodes with Rancher Kubeconfig") Expect(rancherConnectRes.ExitCode).To(Equal(0), "Getting nodes return non-zero exit code") }) diff --git a/test/framework/command_helper.go b/test/framework/command_helper.go index ccd38f9e0..f1cdff15d 100644 --- a/test/framework/command_helper.go +++ b/test/framework/command_helper.go @@ -53,10 +53,12 @@ type RunCommandResult struct { } // RunCommand will run a command with the given args and environment variables. -func RunCommand(ctx context.Context, input RunCommandInput, result *RunCommandResult) { +func RunCommand(ctx context.Context, input RunCommandInput) RunCommandResult { Expect(ctx).NotTo(BeNil(), "ctx is required for RunCommand") Expect(input.Command).ToNot(BeEmpty(), "Invalid argument. input.Command can't be empty when calling RunCommand") + result := RunCommandResult{} + cmd := exec.Command(input.Command, input.Args...) for name, val := range input.EnvironmentVariables { @@ -77,4 +79,6 @@ func RunCommand(ctx context.Context, input RunCommandInput, result *RunCommandRe if exitError, ok := err.(*exec.ExitError); ok { result.ExitCode = exitError.ExitCode() } + + return result } diff --git a/test/framework/kube_helper.go b/test/framework/kube_helper.go index f9551ef58..9eafa7589 100644 --- a/test/framework/kube_helper.go +++ b/test/framework/kube_helper.go @@ -258,8 +258,7 @@ func CreateDockerRegistrySecret(ctx context.Context, input CreateDockerRegistryS Byf("Creating docker registry k8s secret (%s\\%s)", input.Namespace, input.Name) - cmdCreateSecret := &RunCommandResult{} - RunCommand(ctx, RunCommandInput{ + cmdCreateSecret := RunCommand(ctx, RunCommandInput{ Command: "kubectl", Args: []string{ "--kubeconfig", @@ -277,7 +276,7 @@ func CreateDockerRegistrySecret(ctx context.Context, input CreateDockerRegistryS "--docker-password", input.DockerPassword, }, - }, cmdCreateSecret) + }) Expect(cmdCreateSecret.Error).NotTo(HaveOccurred(), "Failed creating docker registry k8s secret") Expect(cmdCreateSecret.ExitCode).To(Equal(0), "Creating secret return non-zero exit code") diff --git a/test/framework/rancher_helpers.go b/test/framework/rancher_helpers.go index f3d45b783..301aeac5e 100644 --- a/test/framework/rancher_helpers.go +++ b/test/framework/rancher_helpers.go @@ -393,8 +393,7 @@ func ValidateRancherCluster(ctx context.Context, input ValidateRancherClusterInp }) Eventually(func() bool { - rancherConnectRes := &RunCommandResult{} - RunCommand(ctx, RunCommandInput{ + rancherConnectRes := RunCommand(ctx, RunCommandInput{ Command: "kubectl", Args: []string{ "--kubeconfig", @@ -403,7 +402,7 @@ func ValidateRancherCluster(ctx context.Context, input ValidateRancherClusterInp "nodes", "--insecure-skip-tls-verify", }, - }, rancherConnectRes) + }) log.FromContext(ctx).Info("kubectl stdout", "output", string(rancherConnectRes.Stdout)) diff --git a/test/framework/wrangler.go b/test/framework/wrangler.go index 3e467e820..6abb264c6 100644 --- a/test/framework/wrangler.go +++ b/test/framework/wrangler.go @@ -21,6 +21,9 @@ import ( . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" @@ -29,33 +32,25 @@ import ( ) func VerifyCertificatesInNamespace(ctx context.Context, cl client.Client, namespace string) { - Byf("Verifying no Certificates are used in namespace: %s", namespace) - certs := schema.GroupVersionKind{ - Group: "cert-manager.io", - Version: "v1", - Kind: "Certificate", + Byf("Verifying no certificates.cert-manager.io CRD is installed") + crd := &apiextensionsv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: "certificates.cert-manager.io", + }, } - - certList := &unstructured.UnstructuredList{} - certList.SetGroupVersionKind(certs) - - Expect(cl.List(ctx, certList, &client.ListOptions{Namespace: namespace})).Should(Succeed()) - Expect(certList.Items).Should(BeEmpty(), "cert-manager Certificates should not have been deployed") + err := cl.Get(ctx, client.ObjectKeyFromObject(crd), crd) + Expect(apierrors.IsNotFound(err)).Should(BeTrue(), "certificates.cert-manager.io CRD should not be installed") } func VerifyIssuersInNamespace(ctx context.Context, cl client.Client, namespace string) { - Byf("Should verify no Issuers are used in namespace: %s", namespace) - issuers := schema.GroupVersionKind{ - Group: "cert-manager.io", - Version: "v1", - Kind: "Issuer", + Byf("Verifying no issuers.cert-manager.io CRD is installed") + crd := &apiextensionsv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: "issuers.cert-manager.io", + }, } - - issuerList := &unstructured.UnstructuredList{} - issuerList.SetGroupVersionKind(issuers) - - Expect(cl.List(ctx, issuerList, &client.ListOptions{Namespace: namespace})).Should(Succeed()) - Expect(issuerList.Items).Should(BeEmpty(), "cert-manager Issuers should not have been deployed") + err := cl.Get(ctx, client.ObjectKeyFromObject(crd), crd) + Expect(apierrors.IsNotFound(err)).Should(BeTrue(), "issuers.cert-manager.io CRD should not be installed") } func VerifyCertManagerAnnotationsForProvider(ctx context.Context, cl client.Client, providerName string) { diff --git a/test/go.mod b/test/go.mod index 85acf1590..232f152fc 100644 --- a/test/go.mod +++ b/test/go.mod @@ -149,7 +149,7 @@ require ( gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect - k8s.io/apiextensions-apiserver v0.34.5 // indirect + k8s.io/apiextensions-apiserver v0.34.5 k8s.io/apiserver v0.34.5 // indirect k8s.io/cluster-bootstrap v0.34.2 // indirect k8s.io/component-base v0.34.5 // indirect diff --git a/test/testenv/aws.go b/test/testenv/aws.go index b77253316..4fe30fb43 100644 --- a/test/testenv/aws.go +++ b/test/testenv/aws.go @@ -56,14 +56,13 @@ func CreateECRCreds(ctx context.Context, input CreateECRCredsInput) { Expect(input.Region).ToNot(BeEmpty(), "Region is required for CreateECRCreds") By("Getting password for ECR") - cmdPwdRes := &turtlesframework.RunCommandResult{} - turtlesframework.RunCommand(ctx, turtlesframework.RunCommandInput{ + cmdPwdRes := turtlesframework.RunCommand(ctx, turtlesframework.RunCommandInput{ Command: "aws", Args: []string{ "ecr", "get-login-password", }, - }, cmdPwdRes) + }) Expect(cmdPwdRes.Error).NotTo(HaveOccurred(), "Failed getting ecr password") Expect(cmdPwdRes.ExitCode).To(Equal(0), "Getting password return non-zero exit code") ecrPassword := string(cmdPwdRes.Stdout) diff --git a/test/testenv/certmanager.go b/test/testenv/certmanager.go deleted file mode 100644 index 2c9b4df9b..000000000 --- a/test/testenv/certmanager.go +++ /dev/null @@ -1,89 +0,0 @@ -/* -Copyright © 2023 - 2024 SUSE LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package testenv - -import ( - "context" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - turtlesframework "github.com/rancher/turtles/test/framework" - - opframework "sigs.k8s.io/cluster-api-operator/test/framework" - "sigs.k8s.io/cluster-api/test/framework" -) - -// DeployCertManagerInput represents the input parameters for deploying Cert Manager. -type DeployCertManagerInput struct { - // BootstrapClusterProxy is the cluster proxy for bootstrapping. - BootstrapClusterProxy framework.ClusterProxy - - // HelmBinaryPath is the path to the Helm binary. - HelmBinaryPath string `env:"HELM_BINARY_PATH"` - - // CertManagerChartPath is the path to the Cert Manager chart. - CertManagerChartPath string `env:"CERT_MANAGER_PATH"` - - // CertManagerUrl is the URL for Cert Manager. - CertManagerUrl string `env:"CERT_MANAGER_URL"` - - // CertManagerRepoName is the repository name for Cert Manager. - CertManagerRepoName string `env:"CERT_MANAGER_REPO_NAME"` -} - -// DeployCertManager deploys Cert Manager using the provided input parameters. -func DeployCertManager(ctx context.Context, input DeployCertManagerInput) { - Expect(turtlesframework.Parse(&input)).To(Succeed(), "Failed to parse environment variables") - - Expect(ctx).NotTo(BeNil(), "ctx is required for DeployRancher") - Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "BootstrapClusterProxy is required for DeployCertManager") - - Expect(input.CertManagerRepoName).ToNot(BeEmpty(), "CertManagerRepoName is required for DeployRancher") - Expect(input.CertManagerUrl).ToNot(BeEmpty(), "CertManagerUrl is required for DeployRancher") - Expect(input.CertManagerChartPath).ToNot(BeEmpty(), "CertManagerChartPath is required for DeployRancher") - - By("Adding cert-manager chart repo") - certChart := &opframework.HelmChart{ - BinaryPath: input.HelmBinaryPath, - Name: input.CertManagerRepoName, - Path: input.CertManagerUrl, - Commands: opframework.Commands(opframework.Repo, opframework.Add), - AdditionalFlags: opframework.Flags("--force-update"), - Kubeconfig: input.BootstrapClusterProxy.GetKubeconfigPath(), - } - _, certErr := certChart.Run(nil) - Expect(certErr).ToNot(HaveOccurred()) - - By("Installing cert-manager") - certManagerChart := &opframework.HelmChart{ - BinaryPath: input.HelmBinaryPath, - Path: input.CertManagerChartPath, - Name: "cert-manager", - Kubeconfig: input.BootstrapClusterProxy.GetKubeconfigPath(), - AdditionalFlags: opframework.Flags( - "--namespace", "cert-manager", - "--version", "v1.16.3", - "--create-namespace", - ), - Wait: true, - } - _, err := certManagerChart.Run(map[string]string{ - "crds.enabled": "true", - "crds.keep": "true", - }) - Expect(err).ToNot(HaveOccurred()) -} diff --git a/test/testenv/eks.go b/test/testenv/eks.go index b24fd79c7..f0e373a9b 100644 --- a/test/testenv/eks.go +++ b/test/testenv/eks.go @@ -58,15 +58,14 @@ func CreateEKSBootstrapClusterAndValidateImages(ctx context.Context, input Creat By("Checking images are present in registry") for _, image := range input.Images { turtlesframework.Byf("Checking image: %s", image.Name) - cmdImgRes := &turtlesframework.RunCommandResult{} - turtlesframework.RunCommand(ctx, turtlesframework.RunCommandInput{ + cmdImgRes := turtlesframework.RunCommand(ctx, turtlesframework.RunCommandInput{ Command: "docker", Args: []string{ "manifest", "inspect", image.Name, }, - }, cmdImgRes) + }) Expect(cmdImgRes.Error).NotTo(HaveOccurred(), "Failed checking if image is available %s error", image.Name) Expect(cmdImgRes.ExitCode).To(Equal(0), "Image not found %s", image.Name) diff --git a/test/testenv/eksctl_provider.go b/test/testenv/eksctl_provider.go index 4e9eaae6d..3e288fabf 100644 --- a/test/testenv/eksctl_provider.go +++ b/test/testenv/eksctl_provider.go @@ -59,9 +59,8 @@ func (k *EKSClusterProvider) Create(ctx context.Context) { turtlesframework.Byf("Creating cluster using eksctl (version %s)", eksVersion) - createClusterRes := &turtlesframework.RunCommandResult{} numWorkerNodes := strconv.Itoa(k.numWorkers) - turtlesframework.RunCommand(ctx, turtlesframework.RunCommandInput{ + createClusterRes := turtlesframework.RunCommand(ctx, turtlesframework.RunCommandInput{ Command: "eksctl", Args: []string{ "create", @@ -88,7 +87,7 @@ func (k *EKSClusterProvider) Create(ctx context.Context) { "--node-type", "m5.xlarge", }, - }, createClusterRes) + }) Expect(createClusterRes.Error).NotTo(HaveOccurred(), "Failed to create cluster using eksctl. Stderr: %s", createClusterRes.Stderr) Expect(createClusterRes.ExitCode).To(Equal(0), "Creating cluster returned non-zero exit code. Stderr: %s", createClusterRes.Stderr) @@ -106,8 +105,7 @@ func (k *EKSClusterProvider) Dispose(ctx context.Context) { By("Deleting cluster using eksctl") - deleteClusterRes := &turtlesframework.RunCommandResult{} - turtlesframework.RunCommand(ctx, turtlesframework.RunCommandInput{ + deleteClusterRes := turtlesframework.RunCommand(ctx, turtlesframework.RunCommandInput{ Command: "eksctl", Args: []string{ "delete", @@ -116,7 +114,7 @@ func (k *EKSClusterProvider) Dispose(ctx context.Context) { k.name, "--wait", }, - }, deleteClusterRes) + }) Expect(deleteClusterRes.Error).NotTo(HaveOccurred(), "Failed to delete cluster using eksctl. Stderr: %s", deleteClusterRes.Stderr) Expect(deleteClusterRes.ExitCode).To(Equal(0), "Deleting cluster returned non-zero exit code. Stderr: %s", deleteClusterRes.Stderr) diff --git a/test/testenv/providers.go b/test/testenv/providers.go index 557ab2e2a..1755e710f 100644 --- a/test/testenv/providers.go +++ b/test/testenv/providers.go @@ -290,8 +290,7 @@ func DeployRancherTurtlesProviders(ctx context.Context, input DeployRancherTurtl // UninstallRancherTurtlesProviders uninstalls the rancher-turtles-providers chart. func UninstallRancherTurtlesProviders(ctx context.Context, namespace string, clusterProxy framework.ClusterProxy) { - var cmd turtlesframework.RunCommandResult - turtlesframework.RunCommand(ctx, turtlesframework.RunCommandInput{ + cmdResult := turtlesframework.RunCommand(ctx, turtlesframework.RunCommandInput{ Command: "helm", Args: []string{ "uninstall", e2e.ProvidersChartName, @@ -299,7 +298,9 @@ func UninstallRancherTurtlesProviders(ctx context.Context, namespace string, clu "--kubeconfig", clusterProxy.GetKubeconfigPath(), "--wait", }, - }, &cmd) + }) + Expect(cmdResult.Error).NotTo(HaveOccurred(), "Failed to uninstall providers chart. Stderr: %s", cmdResult.Stderr) + Expect(cmdResult.ExitCode).To(Equal(0), "Uninstalling providers chart returned non-zero exit code. Stderr: %s", cmdResult.Stderr) } func enableAllProviders(values map[string]string) { diff --git a/test/testenv/rancher.go b/test/testenv/rancher.go index 2c7472d05..10900ed5e 100644 --- a/test/testenv/rancher.go +++ b/test/testenv/rancher.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "os" + "os/exec" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -112,8 +113,6 @@ type deployRancherIngressValuesFile struct { // DeployRancher deploys Rancher using the provided input parameters. // It expects the required input parameters to be non-nil. -// If InstallCertManager is true, the function will install cert-manager. -// The function adds the cert-manager chart repository and the Rancher chart repository. // It then updates the Rancher chart repository. // The function generates the extra values file for Rancher and writes it to the Helm extra values path.// // If RancherIngressConfig is provided, the function sets up the ingress for Rancher. @@ -185,6 +184,8 @@ func DeployRancher(ctx context.Context, input DeployRancherInput) PreRancherInst "--namespace", input.RancherNamespace, "--create-namespace", "--values", input.HelmExtraValuesPath, + "--set", "ingress.tls.source=secret", + "--set", "privateCA=true", ) if input.RancherDebug { installFlags = append(installFlags, "--set", "debug=true") @@ -492,6 +493,106 @@ func deployTraefikIngressLoadBalancer(ctx context.Context, input RancherDeployIn Eventually(komega.Object(ingressDeployment), input.IngressWaitInterval...).Should(HaveField("Status.AvailableReplicas", Equal(int32(1)))) } +type SetupRancherPrivateCAInput struct { + // BootstrapClusterProxy is the cluster proxy for bootstrapping. + BootstrapClusterProxy framework.ClusterProxy + + // RancherHostname is the digested Rancher hostname. This depends on the environment type. + RancherHostname string + + // CreateRancherCertsScriptPath is the file path to the certs creation script. + CreateRancherCertsScriptPath string `env:"CREATE_RANCHER_CERTS_SCRIPT_PATH"` + + // RancherCertPath Rancher self-signed certificate. + RancherCertPath string `env:"RANCHER_CERT_PATH"` + + // RancherCertKeyPath is the Rancher self-signed certificate key. + RancherCertKeyPath string `env:"RANCHER_CERT_KEY_PATH"` + + // RancherCACertPath is the Rancher Private CA certificate. + RancherCACertPath string `env:"RANCHER_CACERT_PATH"` +} + +// SetupRancherPrivateCA initializes the Rancher Private CA setups and loads the certificates into Secrets. +func SetupRancherPrivateCA(ctx context.Context, input SetupRancherPrivateCAInput) { + Expect(turtlesframework.Parse(&input)).To(Succeed(), "Failed to parse environment variables") + + Expect(ctx).NotTo(BeNil(), "ctx is required for RancherDeployIngress") + Expect(input.BootstrapClusterProxy).NotTo(BeNil(), "BootstrapClusterProxy is required for SetupRancherPrivateCA") + Expect(input.RancherHostname).NotTo(BeEmpty(), "RancherHostname can not be empty") + Expect(input.CreateRancherCertsScriptPath).NotTo(BeEmpty(), "CREATE_RANCHER_CERTS_SCRIPT_PATH can not be empty") + Expect(input.RancherCertPath).NotTo(BeEmpty(), "RANCHER_CERT_PATH can not be empty") + Expect(input.RancherCertKeyPath).NotTo(BeEmpty(), "RANCHER_CERT_KEY_PATH can not be empty") + Expect(input.RancherCACertPath).NotTo(BeEmpty(), "RANCHER_CACERT_PATH can not be empty") + + if _, err := os.Stat(input.CreateRancherCertsScriptPath); err != nil { + Expect(fmt.Errorf("Create Rancher certs script not found in path: %s", input.CreateRancherCertsScriptPath)).Should(Succeed()) + } + + By("Running create Rancher certs script") + runGenerateCertsCmd := exec.Command(input.CreateRancherCertsScriptPath) + runGenerateCertsCmd.Env = append(os.Environ(), + "RANCHER_HOSTNAME="+input.RancherHostname) + out, err := runGenerateCertsCmd.CombinedOutput() + if err != nil { + Expect(fmt.Errorf("create Rancher certs script failed: %w\nOutput: %s", err, out)).Should(Succeed()) + } + + By(fmt.Sprintf("Creating Rancher Namespace %s", e2e.RancherNamespace)) + cmdResult := turtlesframework.RunCommand(ctx, turtlesframework.RunCommandInput{ + Command: "kubectl", + Args: []string{ + "--kubeconfig", + input.BootstrapClusterProxy.GetKubeconfigPath(), + "create", + "namespace", + e2e.RancherNamespace, + }, + }) + Expect(cmdResult.Error).NotTo(HaveOccurred(), "Failed creating Rancher namespace") + Expect(cmdResult.ExitCode).To(Equal(0), "Creating Rancher namespace returned non-zero exit code") + + By("Creating Rancher tls-rancher-ingress Secret") + cmdResult = turtlesframework.RunCommand(ctx, turtlesframework.RunCommandInput{ + Command: "kubectl", + Args: []string{ + "--kubeconfig", + input.BootstrapClusterProxy.GetKubeconfigPath(), + "--namespace", + e2e.RancherNamespace, + "create", + "secret", + "tls", + "tls-rancher-ingress", + "--cert", + input.RancherCertPath, + "--key", + input.RancherCertKeyPath, + }, + }) + Expect(cmdResult.Error).NotTo(HaveOccurred(), "Failed creating Rancher tls-rancher-ingress secret") + Expect(cmdResult.ExitCode).To(Equal(0), "Creating Rancher tls-rancher-ingress secret returned non-zero exit code") + + By("Creating Rancher tls-ca Secret") + cmdResult = turtlesframework.RunCommand(ctx, turtlesframework.RunCommandInput{ + Command: "kubectl", + Args: []string{ + "--kubeconfig", + input.BootstrapClusterProxy.GetKubeconfigPath(), + "--namespace", + e2e.RancherNamespace, + "create", + "secret", + "generic", + "tls-ca", + "--from-file", + input.RancherCACertPath, + }, + }) + Expect(cmdResult.Error).NotTo(HaveOccurred(), "Failed creating Rancher tls-ca secret") + Expect(cmdResult.ExitCode).To(Equal(0), "Creating Rancher tls-ca secret returned non-zero exit code") +} + // PreRancherInstallHookInput represents the input parameters for the pre-Rancher install hook. type PreRancherInstallHookInput struct { // Ctx is the context for the hook execution. @@ -511,6 +612,9 @@ type PreRancherInstallHookInput struct { // RancherHostname is a maunally specified RancherHostname value RancherHostname string `env:"RANCHER_HOSTNAME"` + + // SkipPrivateCASetup is a flag that can be used to not repeat the Private CA setup. + SkipPrivateCASetup bool } // PreRancherInstallHookResult represents the result of a pre-Rancher install hook. @@ -532,6 +636,8 @@ type PreRancherInstallHookResult struct { func PreRancherInstallHook(input PreRancherInstallHookInput) PreRancherInstallHookResult { Expect(turtlesframework.Parse(&input)).To(Succeed(), "Failed to parse environment variables") + result := PreRancherInstallHookResult{} + switch input.EnvironmentType { case e2e.ManagementClusterEnvironmentEKS: By("Getting ingress hostname") @@ -548,34 +654,35 @@ func PreRancherInstallHook(input PreRancherInstallHookInput) PreRancherInstallHo BootstrapClusterProxy: input.BootstrapClusterProxy, }) - return PreRancherInstallHookResult{ - Hostname: svcRes.Hostname, - IngressClassName: "traefik", - } + result.Hostname = svcRes.Hostname + result.IngressClassName = "traefik" case e2e.ManagementClusterEnvironmentIsolatedKind: By("Getting internal cluster hostname") hostname := getInternalClusterHostname(input.Ctx, input.BootstrapClusterProxy) - return PreRancherInstallHookResult{ - Hostname: hostname, - IngressClassName: input.RancherIngressClassName, - } + + result.Hostname = hostname + result.IngressClassName = input.RancherIngressClassName case e2e.ManagementClusterEnvironmentKind: By("Using RANCHER_HOSTNAME") // i.e. we are using ngrok locally - return PreRancherInstallHookResult{ - Hostname: input.RancherHostname, - IngressClassName: input.RancherIngressClassName, - ConfigPatches: [][]byte{e2e.RancherServicePatch, e2e.IngressConfig, e2e.SystemStoreSettingPatch}, - } - + result.Hostname = input.RancherHostname + result.IngressClassName = input.RancherIngressClassName + result.ConfigPatches = [][]byte{e2e.RancherServicePatch, e2e.IngressConfig, e2e.SystemStoreSettingPatch} case e2e.ManagementClusterEnvironmentInternalKind: By("Using RANCHER_HOSTNAME for internal kind") - return PreRancherInstallHookResult{ - Hostname: input.RancherHostname, - } - + result.Hostname = input.RancherHostname default: Fail(fmt.Sprintf("Unknown MANAGEMENT_CLUSTER_ENVIRONMENT: %s", input.EnvironmentType)) return PreRancherInstallHookResult{} } + + if !input.SkipPrivateCASetup { + By("Setting up Rancher Private CA") + SetupRancherPrivateCA(input.Ctx, SetupRancherPrivateCAInput{ + BootstrapClusterProxy: input.BootstrapClusterProxy, + RancherHostname: result.Hostname, + }) + } + + return result } diff --git a/test/testenv/rancher_system_chart.go b/test/testenv/rancher_system_chart.go index 2d5886522..4c8ef426e 100644 --- a/test/testenv/rancher_system_chart.go +++ b/test/testenv/rancher_system_chart.go @@ -162,6 +162,9 @@ type UpgradeInstallRancherWithGiteaInput struct { // RancherWaitInterval is the wait interval for Rancher. RancherWaitInterval []interface{} `envDefault:"15m,30s"` + + // SkipPrivateCASetup is a flag that can be used to not repeat the Private CA setup. + SkipPrivateCASetup bool } // UpgradeInstallRancherWithGitea upgrades Rancher to a new version and configures it with Gitea chart repository @@ -210,6 +213,7 @@ func UpgradeInstallRancherWithGitea(ctx context.Context, input UpgradeInstallRan BootstrapClusterProxy: input.BootstrapClusterProxy, RancherIngressClassName: input.RancherIngressClassName, RancherHostname: input.RancherHostname, + SkipPrivateCASetup: input.SkipPrivateCASetup, }) input.RancherPatches = append(input.RancherPatches, rancherHookResult.ConfigPatches...) @@ -230,6 +234,8 @@ func UpgradeInstallRancherWithGitea(ctx context.Context, input UpgradeInstallRan "--set", "extraEnv[2].name=CATTLE_RANCHER_TURTLES_VERSION", "--set", fmt.Sprintf("extraEnv[2].value=%s", input.ChartVersion), "--set", "networkExposure.type=ingress", + "--set", "ingress.tls.source=secret", + "--set", "privateCA=true", "--wait", } From f779312ce332e29c5df9d3aa9dc92fd2e12cec3b Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Thu, 19 Mar 2026 13:24:22 +0100 Subject: [PATCH 2/2] e2e: wait for GitRepo to be Ready Signed-off-by: Andrea Mazzotti --- test/e2e/specs/import_gitops.go | 4 +- test/framework/fleet_helper.go | 75 ++++++++++++++++++++++++++++++--- 2 files changed, 71 insertions(+), 8 deletions(-) diff --git a/test/e2e/specs/import_gitops.go b/test/e2e/specs/import_gitops.go index d76a868fa..e9422ea89 100644 --- a/test/e2e/specs/import_gitops.go +++ b/test/e2e/specs/import_gitops.go @@ -173,7 +173,7 @@ func CreateUsingGitOpsSpec(ctx context.Context, inputGetter func() CreateUsingGi additionalRepo.TargetNamespace = namespace.Name } - turtlesframework.FleetCreateGitRepo(ctx, additionalRepo) + turtlesframework.FleetCreateAndWaitGitRepo(ctx, additionalRepo) } additionalVars := map[string]string{ @@ -468,7 +468,7 @@ func CreateUsingGitOpsV1Beta1Spec(ctx context.Context, inputGetter func() Create additionalRepo.TargetNamespace = namespace.Name } - turtlesframework.FleetCreateGitRepo(ctx, additionalRepo) + turtlesframework.FleetCreateAndWaitGitRepo(ctx, additionalRepo) } additionalVars := map[string]string{ diff --git a/test/framework/fleet_helper.go b/test/framework/fleet_helper.go index ed01a3c30..d9df5851f 100644 --- a/test/framework/fleet_helper.go +++ b/test/framework/fleet_helper.go @@ -23,6 +23,7 @@ import ( "os" "strings" "text/template" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -37,6 +38,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +var gvkGitRepo = schema.GroupVersionKind{Group: "fleet.cattle.io", Version: "v1alpha1", Kind: "GitRepo"} + // FleetCreateGitRepoInput represents the input parameters for creating a Git repository in Fleet. type FleetCreateGitRepoInput struct { // Name is the name of the Git repository. @@ -84,7 +87,7 @@ type FleetCreateGitRepoInput struct { // FleetCreateGitRepo will create and apply a GitRepo resource to the cluster. See the Fleet docs // for further information: https://fleet.rancher.io/gitrepo-add -func FleetCreateGitRepo(ctx context.Context, input FleetCreateGitRepoInput) { +func FleetCreateAndWaitGitRepo(ctx context.Context, input FleetCreateGitRepoInput) { Expect(Parse(&input)).To(Succeed(), "Failed to parse environment variables") defaultToCurrentGitRepo(&input) @@ -118,10 +121,72 @@ func FleetCreateGitRepo(ctx context.Context, input FleetCreateGitRepoInput) { err = t.Execute(&renderedTemplate, input) Expect(err).NotTo(HaveOccurred(), "Failed to execute template") + pollingTime := 1 * time.Minute Eventually(func() error { - Byf("Applying GitRepo: %s", renderedTemplate.String()) - return Apply(ctx, input.ClusterProxy, renderedTemplate.Bytes()) - }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to apply GitRepo") + Byf("Checking if GitRepo %s/%s exists", input.Namespace, input.Name) + repo := &unstructured.Unstructured{} + repo.SetGroupVersionKind(gvkGitRepo) + + err := input.ClusterProxy.GetClient().Get(ctx, client.ObjectKey{ + Namespace: input.Namespace, + Name: input.Name, + }, repo) + + if err != nil { + if apierrors.IsNotFound(err) { + Byf("Applying GitRepo: %s", renderedTemplate.String()) + if err := Apply(ctx, input.ClusterProxy, renderedTemplate.Bytes()); err != nil { + return fmt.Errorf("applying GitRepo: %w", err) + } + } else { + return fmt.Errorf("fetching unstructured GitRepo: %w", err) + } + } + + // GitRepo does not seem to reconcile on some errors. + // If after pollingTime/2 it's not ready, delete and recreate. + // See: https://github.com/rancher/fleet/issues/4865 + time.Sleep(pollingTime / 2) + + if err := input.ClusterProxy.GetClient().Get(ctx, client.ObjectKey{ + Namespace: input.Namespace, + Name: input.Name, + }, repo); err != nil { + return fmt.Errorf("refreshing unstructured GitRepo: %w", err) + } + + By("Checking if GitRepo is Ready") + if repo.GetDeletionTimestamp() != nil { + return fmt.Errorf("GitRepo is deleting") + } + readyClusters, statusFieldFound, err := unstructured.NestedInt64(repo.Object, "status", "readyClusters") + if err != nil { + FleetDeleteGitRepo(ctx, FleetDeleteGitRepoInput{ + Name: input.Name, + Namespace: input.Namespace, + ClusterProxy: input.ClusterProxy, + }) + return fmt.Errorf("fetching GitRepo.status.readyClusters: %w", err) + } + if !statusFieldFound { + FleetDeleteGitRepo(ctx, FleetDeleteGitRepoInput{ + Name: input.Name, + Namespace: input.Namespace, + ClusterProxy: input.ClusterProxy, + }) + return fmt.Errorf("GitRepo.status.readyClusters field not found. Can not determine if GitRepo is Ready.") + } + if readyClusters != 1 { + FleetDeleteGitRepo(ctx, FleetDeleteGitRepoInput{ + Name: input.Name, + Namespace: input.Namespace, + ClusterProxy: input.ClusterProxy, + }) + return fmt.Errorf("GitRepo is not Ready. Expected to be ready in 1 Cluster. readyclusters = %d", readyClusters) + } + return nil + }, 15*time.Minute, pollingTime).Should(Succeed(), "Failed to apply GitRepo") + } // FleetDeleteGitRepoInput represents the input parameters for deleting a Git repository in the fleet. @@ -150,8 +215,6 @@ func FleetDeleteGitRepo(ctx context.Context, input FleetDeleteGitRepoInput) { By("Getting GitRepo from cluster") - gvkGitRepo := schema.GroupVersionKind{Group: "fleet.cattle.io", Version: "v1alpha1", Kind: "GitRepo"} - repo := &unstructured.Unstructured{} repo.SetGroupVersionKind(gvkGitRepo) err := input.ClusterProxy.GetClient().Get(ctx, client.ObjectKey{Namespace: input.Namespace, Name: input.Name}, repo)