diff --git a/vertical-pod-autoscaler/e2e/v1/admission_controller.go b/vertical-pod-autoscaler/e2e/v1/admission_controller.go index 9936c0fcfc0a..d6d8e03d4121 100644 --- a/vertical-pod-autoscaler/e2e/v1/admission_controller.go +++ b/vertical-pod-autoscaler/e2e/v1/admission_controller.go @@ -25,8 +25,10 @@ import ( apiv1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/autoscaler/vertical-pod-autoscaler/e2e/utils" vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/features" "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/test" "k8s.io/kubernetes/test/e2e/framework" podsecurity "k8s.io/pod-security-admission/api" @@ -35,12 +37,20 @@ import ( "github.com/onsi/gomega" ) +func init() { + // Dynamically register feature gates from the VPA's versioned feature gate configuration + // This ensures consistency with the main VPA feature gate definitions + if err := utilfeature.DefaultMutableFeatureGate.Add(features.MutableFeatureGate.GetAll()); err != nil { + panic(fmt.Sprintf("Failed to add VPA feature gates: %v", err)) + } +} + const ( webhookConfigName = "vpa-webhook-config" webhookName = "vpa.k8s.io" ) -var _ = AdmissionControllerE2eDescribe("Admission-controller", ginkgo.Label("FG:InPlaceOrRecreate"), func() { +var _ = AdmissionControllerE2eDescribe("Admission-controller", func() { f := framework.NewDefaultFramework("vertical-pod-autoscaling") f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline @@ -48,7 +58,7 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", ginkgo.Label("FG: waitForVpaWebhookRegistration(f) }) - ginkgo.It("starts pods with new recommended request with InPlaceOrRecreate mode", func() { + f.It("starts pods with new recommended request with InPlaceOrRecreate mode", framework.WithFeatureGate(features.InPlaceOrRecreate), func() { d := NewHamsterDeploymentWithResources(f, ParseQuantityOrDie("100m") /*cpu*/, ParseQuantityOrDie("100Mi") /*memory*/) ginkgo.By("Setting up a VPA CRD") @@ -878,33 +888,49 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", ginkgo.Label("FG: expectedErr string }{ { - name: "Invalid oomBumpUpRatio (negative value)", + name: "Invalid minAllowed (invalid requests field)", vpaJSON: `{ "apiVersion": "autoscaling.k8s.io/v1", "kind": "VerticalPodAutoscaler", - "metadata": {"name": "oom-test-vpa"}, + "metadata": {"name": "hamster-vpa-invalid"}, "spec": { "targetRef": { "apiVersion": "apps/v1", "kind": "Deployment", - "name": "oom-test" - }, - "updatePolicy": { - "updateMode": "Auto" + "name": "hamster" }, "resourcePolicy": { "containerPolicies": [{ "containerName": "*", - "oomBumpUpRatio": -1, - "oomMinBumpUp": 104857600 + "minAllowed": { + "requests": { + "cpu": "50m" + } + } }] } } }`, - expectedErr: "spec.resourcePolicy.containerPolicies[0].oomBumpUpRatio: Invalid value: -1: spec.resourcePolicy.containerPolicies[0].oomBumpUpRatio in body should be greater than or equal to 1", + expectedErr: "admission webhook .*vpa.* denied the request:", }, + } + for _, tc := range testCases { + ginkgo.By(fmt.Sprintf("Testing %s", tc.name)) + err := InstallRawVPA(f, []byte(tc.vpaJSON)) + gomega.Expect(err).To(gomega.HaveOccurred(), "Invalid VPA object accepted") + gomega.Expect(err.Error()).To(gomega.MatchRegexp(tc.expectedErr)) + } + }) + + f.It("accepts valid and rejects invalid VPA object with features.PerVPAConfig enabled", framework.WithFeatureGate(features.PerVPAConfig), func() { + ginkgo.By("Setting up invalid VPA objects") + testCases := []struct { + name string + vpaJSON string + expectedErr string + }{ { - name: "Invalid oomBumpUpRatio (string value)", + name: "Invalid oomBumpUpRatio (negative value)", vpaJSON: `{ "apiVersion": "autoscaling.k8s.io/v1", "kind": "VerticalPodAutoscaler", @@ -921,16 +947,16 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", ginkgo.Label("FG: "resourcePolicy": { "containerPolicies": [{ "containerName": "*", - "oomBumpUpRatio": "12", + "oomBumpUpRatio": -1, "oomMinBumpUp": 104857600 }] } } }`, - expectedErr: "json: cannot unmarshal string into Go struct field ContainerResourcePolicy.spec.resourcePolicy.containerPolicies.oomBumpUpRatio of type float64", + expectedErr: "admission webhook \"vpa.k8s.io\" denied the request: oomBumpUpRatio must be greater than or equal to 1.0, got -1", }, { - name: "Invalid oomBumpUpRatio (less than 1)", + name: "Invalid oomBumpUpRatio (string value)", vpaJSON: `{ "apiVersion": "autoscaling.k8s.io/v1", "kind": "VerticalPodAutoscaler", @@ -947,16 +973,16 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", ginkgo.Label("FG: "resourcePolicy": { "containerPolicies": [{ "containerName": "*", - "oomBumpUpRatio": 0.5, + "oomBumpUpRatio": "not-a-number", "oomMinBumpUp": 104857600 }] } } }`, - expectedErr: "spec.resourcePolicy.containerPolicies[0].oomBumpUpRatio: Invalid value: 0.5: spec.resourcePolicy.containerPolicies[0].oomBumpUpRatio in body should be greater than or equal to 1", + expectedErr: "admission webhook \"vpa\\.k8s\\.io\" denied the request: quantities must match the regular expression", }, { - name: "Invalid oomMinBumpUp (negative value)", + name: "Invalid oomBumpUpRatio (less than 1)", vpaJSON: `{ "apiVersion": "autoscaling.k8s.io/v1", "kind": "VerticalPodAutoscaler", @@ -973,45 +999,45 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", ginkgo.Label("FG: "resourcePolicy": { "containerPolicies": [{ "containerName": "*", - "oomBumpUpRatio": 2, - "oomMinBumpUp": -1 + "oomBumpUpRatio": 0.5, + "oomMinBumpUp": 104857600 }] } } }`, - expectedErr: "spec.resourcePolicy.containerPolicies[0].oomMinBumpUp: Invalid value: -1: spec.resourcePolicy.containerPolicies[0].oomMinBumpUp in body should be greater than or equal to 0", + expectedErr: "admission webhook \"vpa.k8s.io\" denied the request: oomBumpUpRatio must be greater than or equal to 1.0, got 0.5", }, { - name: "Invalid minAllowed (invalid requests field)", + name: "Invalid oomMinBumpUp (negative value)", vpaJSON: `{ "apiVersion": "autoscaling.k8s.io/v1", "kind": "VerticalPodAutoscaler", - "metadata": {"name": "hamster-vpa-invalid"}, + "metadata": {"name": "oom-test-vpa"}, "spec": { "targetRef": { "apiVersion": "apps/v1", "kind": "Deployment", - "name": "hamster" + "name": "oom-test" + }, + "updatePolicy": { + "updateMode": "Auto" }, "resourcePolicy": { "containerPolicies": [{ "containerName": "*", - "minAllowed": { - "requests": { - "cpu": "50m" - } - } + "oomBumpUpRatio": 2, + "oomMinBumpUp": -1 }] } } }`, - expectedErr: "admission webhook .*vpa.* denied the request:", + expectedErr: "admission webhook \"vpa.k8s.io\" denied the request: oomMinBumpUp must be greater than or equal to 0, got -1 bytes", }, } for _, tc := range testCases { ginkgo.By(fmt.Sprintf("Testing %s", tc.name)) err := InstallRawVPA(f, []byte(tc.vpaJSON)) - gomega.Expect(err).To(gomega.HaveOccurred(), "Invalid VPA object accepted") + gomega.Expect(err).To(gomega.HaveOccurred(), fmt.Sprintf("Invalid VPA object accepted, name: \"%s\"", tc.name)) gomega.Expect(err.Error()).To(gomega.MatchRegexp(tc.expectedErr)) } }) diff --git a/vertical-pod-autoscaler/e2e/v1/common.go b/vertical-pod-autoscaler/e2e/v1/common.go index efa63162660a..17517ff15918 100644 --- a/vertical-pod-autoscaler/e2e/v1/common.go +++ b/vertical-pod-autoscaler/e2e/v1/common.go @@ -20,7 +20,6 @@ import ( "context" "encoding/json" "fmt" - "strings" "time" ginkgo "github.com/onsi/ginkgo/v2" @@ -37,7 +36,6 @@ import ( "k8s.io/autoscaler/vertical-pod-autoscaler/e2e/utils" vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" vpa_clientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned" - "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/features" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" framework_deployment "k8s.io/kubernetes/test/e2e/framework/deployment" @@ -468,26 +466,3 @@ func WaitForPodsUpdatedWithoutEviction(f *framework.Framework, initialPods *apiv framework.Logf("finished waiting for at least one pod to be updated without eviction") return err } - -// checkPerVPAConfigTestsEnabled checks if the PerVPAConfig feature gate is enabled -// in the VPA recommender. -func checkPerVPAConfigTestsEnabled(f *framework.Framework) { - ginkgo.By("Checking PerVPAConfig feature gate is enabled for recommender") - deploy, err := f.ClientSet.AppsV1().Deployments(VpaNamespace).Get(context.TODO(), "vpa-recommender", metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(deploy.Spec.Template.Spec.Containers).To(gomega.HaveLen(1)) - vpaRecommenderPod := deploy.Spec.Template.Spec.Containers[0] - gomega.Expect(vpaRecommenderPod.Name).To(gomega.Equal("recommender")) - if !anyContainsSubstring(vpaRecommenderPod.Args, fmt.Sprintf("%s=true", string(features.PerVPAConfig))) { - ginkgo.Skip("Skipping suite: PerVPAConfig feature gate is not enabled for the VPA recommender") - } -} - -func anyContainsSubstring(arr []string, substr string) bool { - for _, s := range arr { - if strings.Contains(s, substr) { - return true - } - } - return false -} diff --git a/vertical-pod-autoscaler/e2e/v1/full_vpa.go b/vertical-pod-autoscaler/e2e/v1/full_vpa.go index 1d4e26a2e463..3c73fab1c900 100644 --- a/vertical-pod-autoscaler/e2e/v1/full_vpa.go +++ b/vertical-pod-autoscaler/e2e/v1/full_vpa.go @@ -26,8 +26,10 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/autoscaler/vertical-pod-autoscaler/e2e/utils" vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/features" "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/test" "k8s.io/kubernetes/test/e2e/framework" podsecurity "k8s.io/pod-security-admission/api" @@ -47,6 +49,14 @@ const ( oomTestTimeout = 8 * time.Minute ) +func init() { + // Dynamically register feature gates from the VPA's versioned feature gate configuration + // This ensures consistency with the main VPA feature gate definitions + if err := utilfeature.DefaultMutableFeatureGate.Add(features.MutableFeatureGate.GetAll()); err != nil { + panic(fmt.Sprintf("Failed to add VPA feature gates: %v", err)) + } +} + var _ = FullVpaE2eDescribe("Pods under VPA", func() { var ( rc *ResourceConsumer @@ -62,7 +72,7 @@ var _ = FullVpaE2eDescribe("Pods under VPA", func() { f := framework.NewDefaultFramework("vertical-pod-autoscaling") f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline - ginkgo.Describe("with InPlaceOrRecreate update mode", ginkgo.Label("FG:InPlaceOrRecreate"), func() { + f.Describe("with InPlaceOrRecreate update mode", framework.WithFeatureGate(features.InPlaceOrRecreate), func() { ginkgo.BeforeEach(func() { ns := f.Namespace.Name ginkgo.By("Setting up a hamster deployment") diff --git a/vertical-pod-autoscaler/e2e/v1/recommender.go b/vertical-pod-autoscaler/e2e/v1/recommender.go index 55560cee8835..5de6d4deb791 100644 --- a/vertical-pod-autoscaler/e2e/v1/recommender.go +++ b/vertical-pod-autoscaler/e2e/v1/recommender.go @@ -29,9 +29,11 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/autoscaler/vertical-pod-autoscaler/e2e/utils" vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" vpa_clientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/features" "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/model" "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/test" clientset "k8s.io/client-go/kubernetes" @@ -41,6 +43,14 @@ import ( podsecurity "k8s.io/pod-security-admission/api" ) +func init() { + // Dynamically register feature gates from the VPA's versioned feature gate configuration + // This ensures consistency with the main VPA feature gate definitions + if err := utilfeature.DefaultMutableFeatureGate.Add(features.MutableFeatureGate.GetAll()); err != nil { + panic(fmt.Sprintf("Failed to add VPA feature gates: %v", err)) + } +} + type resourceRecommendation struct { target, lower, upper int64 } @@ -51,7 +61,6 @@ func (r *resourceRecommendation) sub(other *resourceRecommendation) resourceReco lower: r.lower - other.lower, upper: r.upper - other.upper, } - } func getResourceRecommendation(containerRecommendation *vpa_types.RecommendedContainerResources, r apiv1.ResourceName) resourceRecommendation { @@ -411,28 +420,21 @@ var _ = utils.RecommenderE2eDescribe("VPA CRD object", func() { gomega.Expect(vpa.Status.Recommendation.ContainerRecommendations).Should(gomega.HaveLen(1), errMsg) gomega.Expect(vpa.Status.Recommendation.ContainerRecommendations[0].ContainerName).To(gomega.Equal(utils.GetHamsterContainerNameByIndex(1)), errMsg) }) -}) + f.It("have memory requests growing with OOMs more than the default", framework.WithFeatureGate(features.PerVPAConfig), func() { + const replicas = 1 + const defaultOOMBumpUpRatio = model.DefaultOOMBumpUpRatio + const oomBumpUpRatio = 3 -var _ = utils.RecommenderE2eDescribe("OOM with custom config", ginkgo.Label("FG:PerVPAConfig"), func() { - const replicas = 1 - const defaultOOMBumpUpRatio = model.DefaultOOMBumpUpRatio - const oomBumpUpRatio = 3 - f := framework.NewDefaultFramework("vertical-pod-autoscaling") - f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline - var ( - vpaCRD *vpa_types.VerticalPodAutoscaler - vpaClientSet vpa_clientset.Interface - ) - ginkgo.BeforeEach(func() { - checkPerVPAConfigTestsEnabled(f) ns := f.Namespace.Name vpaClientSet = utils.GetVpaClientSet(f) + ginkgo.By("Setting up a hamster deployment") runOomingReplicationController( f.ClientSet, ns, "hamster", replicas) + ginkgo.By("Setting up a VPA CRD") targetRef := &autoscaling.CrossVersionObjectReference{ APIVersion: "v1", @@ -440,7 +442,7 @@ var _ = utils.RecommenderE2eDescribe("OOM with custom config", ginkgo.Label("FG: Name: "hamster", } containerName := utils.GetHamsterContainerNameByIndex(0) - vpaCRD = test.VerticalPodAutoscaler(). + vpaCRD := test.VerticalPodAutoscaler(). WithName("hamster-vpa"). WithNamespace(f.Namespace.Name). WithTargetRef(targetRef). @@ -448,8 +450,7 @@ var _ = utils.RecommenderE2eDescribe("OOM with custom config", ginkgo.Label("FG: WithOOMBumpUpRatio(resource.NewQuantity(oomBumpUpRatio, resource.DecimalSI)). Get() utils.InstallVPA(f, vpaCRD) - }) - ginkgo.It("have memory requests growing with OOMs more than the default", func() { + ginkgo.By("Waiting for recommendation to be filled") vpa, err := utils.WaitForRecommendationPresent(vpaClientSet, vpaCRD) gomega.Expect(err).NotTo(gomega.HaveOccurred()) diff --git a/vertical-pod-autoscaler/hack/run-e2e-locally.sh b/vertical-pod-autoscaler/hack/run-e2e-locally.sh index 2449e57ab028..386c7b6b22f3 100755 --- a/vertical-pod-autoscaler/hack/run-e2e-locally.sh +++ b/vertical-pod-autoscaler/hack/run-e2e-locally.sh @@ -87,6 +87,14 @@ echo " loading image into kind" kind load docker-image localhost:5001/write-metrics:dev +export FEATURE_GATES="" +export TEST_WITH_FEATURE_GATES_ENABLED="" + +if [ "${ENABLE_ALL_FEATURE_GATES:-}" == "true" ] ; then + export FEATURE_GATES='AllAlpha=true,AllBeta=true' + export TEST_WITH_FEATURE_GATES_ENABLED="true" +fi + case ${SUITE} in recommender|recommender-externalmetrics|updater|admission-controller|actuation|full-vpa) ${SCRIPT_ROOT}/hack/vpa-down.sh diff --git a/vertical-pod-autoscaler/hack/run-e2e-tests.sh b/vertical-pod-autoscaler/hack/run-e2e-tests.sh index f7f724925cba..20d53e176a95 100755 --- a/vertical-pod-autoscaler/hack/run-e2e-tests.sh +++ b/vertical-pod-autoscaler/hack/run-e2e-tests.sh @@ -46,11 +46,17 @@ export GO111MODULE=on export WORKSPACE=${WORKSPACE:-/workspace/_artifacts} +SKIP="--ginkgo.skip=\[Feature\:OffByDefault\]" + +if [ "${TEST_WITH_FEATURE_GATES_ENABLED:-}" == "true" ]; then + SKIP="" +fi + case ${SUITE} in recommender|updater|admission-controller|actuation|full-vpa) export KUBECONFIG=$HOME/.kube/config pushd ${SCRIPT_ROOT}/e2e - go test ./v1/*go -v --test.timeout=150m --args --ginkgo.v=true --ginkgo.focus="\[VPA\] \[${SUITE}\]" --report-dir=${WORKSPACE} --disable-log-dump --ginkgo.timeout=150m + go test ./v1/*go -v --test.timeout=150m --args --ginkgo.v=true --ginkgo.focus="\[VPA\] \[${SUITE}\]" --report-dir=${WORKSPACE} --disable-log-dump --ginkgo.timeout=150m ${SKIP} V1_RESULT=$? popd echo v1 test result: ${V1_RESULT}