diff --git a/vertical-pod-autoscaler/e2e/v1/admission_controller.go b/vertical-pod-autoscaler/e2e/v1/admission_controller.go index 2fa12fdc4a6..80ab1d9195f 100644 --- a/vertical-pod-autoscaler/e2e/v1/admission_controller.go +++ b/vertical-pod-autoscaler/e2e/v1/admission_controller.go @@ -1098,6 +1098,122 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", func() { }) }) +var _ = AdmissionControllerE2eDescribe("Admission-controller", func() { + f := framework.NewDefaultFramework("vertical-pod-autoscaling") + f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline + + ginkgo.BeforeEach(func() { + waitForVpaWebhookRegistration(f) + }) + + f.It("boosts CPU by factor on pod creation", framework.WithFeatureGate(features.CPUStartupBoost), func() { + initialCPU := ParseQuantityOrDie("100m") + expectedCPU := ParseQuantityOrDie("200m") + d := NewHamsterDeploymentWithResources(f, initialCPU, ParseQuantityOrDie("100Mi")) + + ginkgo.By("Setting up a VPA with a startup boost policy (factor)") + containerName := utils.GetHamsterContainerNameByIndex(0) + factor := int32(2) + vpaCRD := test.VerticalPodAutoscaler(). + WithName("hamster-vpa"). + WithNamespace(f.Namespace.Name). + WithTargetRef(utils.HamsterTargetRef). + WithContainer(containerName). + WithCPUStartupBoost(vpa_types.FactorStartupBoostType, &factor, nil, "15s"). + AppendRecommendation( + test.Recommendation(). + WithContainer(containerName). + WithTarget("100m", "100Mi"). + GetContainerResources(), + ). + Get() + utils.InstallVPA(f, vpaCRD) + + ginkgo.By("Starting the deployment and verifying the pod is boosted") + podList := utils.StartDeploymentPods(f, d) + pod := podList.Items[0] + gomega.Expect(pod.Spec.Containers[0].Resources.Requests.Cpu().Cmp(expectedCPU)).To(gomega.Equal(0)) + gomega.Expect(pod.Spec.Containers[0].Resources.Limits.Cpu().Cmp(expectedCPU)).To(gomega.Equal(0)) + }) + + f.It("boosts CPU by quantity on pod creation", framework.WithFeatureGate(features.CPUStartupBoost), func() { + initialCPU := ParseQuantityOrDie("100m") + boostCPUQuantity := ParseQuantityOrDie("500m") + expectedCPU := ParseQuantityOrDie("600m") + d := NewHamsterDeploymentWithResources(f, initialCPU, ParseQuantityOrDie("100Mi")) + + ginkgo.By("Setting up a VPA with a startup boost policy (quantity)") + containerName := utils.GetHamsterContainerNameByIndex(0) + vpaCRD := test.VerticalPodAutoscaler(). + WithName("hamster-vpa"). + WithNamespace(f.Namespace.Name). + WithTargetRef(utils.HamsterTargetRef). + WithContainer(containerName). + WithCPUStartupBoost(vpa_types.QuantityStartupBoostType, nil, &boostCPUQuantity, "15s"). + AppendRecommendation( + test.Recommendation(). + WithContainer(containerName). + WithTarget("100m", "100Mi"). + GetContainerResources(), + ). + Get() + utils.InstallVPA(f, vpaCRD) + + ginkgo.By("Starting the deployment and verifying the pod is boosted") + podList := utils.StartDeploymentPods(f, d) + pod := podList.Items[0] + gomega.Expect(pod.Spec.Containers[0].Resources.Requests.Cpu().Cmp(expectedCPU)).To(gomega.Equal(0)) + gomega.Expect(pod.Spec.Containers[0].Resources.Limits.Cpu().Cmp(expectedCPU)).To(gomega.Equal(0)) + }) + + f.It("boosts CPU on pod creation when VPA update mode is Off", framework.WithFeatureGate(features.CPUStartupBoost), func() { + initialCPU := ParseQuantityOrDie("100m") + expectedCPU := ParseQuantityOrDie("200m") + d := NewHamsterDeploymentWithResources(f, initialCPU, ParseQuantityOrDie("100Mi")) + + ginkgo.By("Setting up a VPA with updateMode Off and a startup boost policy") + containerName := utils.GetHamsterContainerNameByIndex(0) + factor := int32(2) + vpaCRD := test.VerticalPodAutoscaler(). + WithName("hamster-vpa"). + WithNamespace(f.Namespace.Name). + WithTargetRef(utils.HamsterTargetRef). + WithContainer(containerName). + WithUpdateMode(vpa_types.UpdateModeOff). // VPA is off, but boost should still work + WithCPUStartupBoost(vpa_types.FactorStartupBoostType, &factor, nil, "15s"). + Get() + utils.InstallVPA(f, vpaCRD) + + ginkgo.By("Starting the deployment and verifying the pod is boosted") + podList := utils.StartDeploymentPods(f, d) + pod := podList.Items[0] + gomega.Expect(pod.Spec.Containers[0].Resources.Requests.Cpu().Cmp(expectedCPU)).To(gomega.Equal(0)) + }) + + f.It("doesn't boost CPU on pod creation when scaling mode is Off", framework.WithFeatureGate(features.CPUStartupBoost), func() { + initialCPU := ParseQuantityOrDie("100m") + d := NewHamsterDeploymentWithResources(f, initialCPU, ParseQuantityOrDie("100Mi")) + + ginkgo.By("Setting up a VPA with a startup boost policy and scaling mode Off") + containerName := utils.GetHamsterContainerNameByIndex(0) + factor := int32(2) + vpaCRD := test.VerticalPodAutoscaler(). + WithName("hamster-vpa"). + WithNamespace(f.Namespace.Name). + WithTargetRef(utils.HamsterTargetRef). + WithContainer(containerName). + WithCPUStartupBoost(vpa_types.FactorStartupBoostType, &factor, nil, "15s"). + WithScalingMode(containerName, vpa_types.ContainerScalingModeOff). + Get() + utils.InstallVPA(f, vpaCRD) + + ginkgo.By("Starting the deployment and verifying the pod is NOT boosted") + podList := utils.StartDeploymentPods(f, d) + pod := podList.Items[0] + gomega.Expect(pod.Spec.Containers[0].Resources.Requests.Cpu().Cmp(initialCPU)).To(gomega.Equal(0)) + }) +}) + func waitForVpaWebhookRegistration(f *framework.Framework) { ginkgo.By("Waiting for VPA webhook registration") gomega.Eventually(func() bool { diff --git a/vertical-pod-autoscaler/e2e/v1/common.go b/vertical-pod-autoscaler/e2e/v1/common.go index 17517ff1591..10d432558ee 100644 --- a/vertical-pod-autoscaler/e2e/v1/common.go +++ b/vertical-pod-autoscaler/e2e/v1/common.go @@ -244,14 +244,30 @@ func InstallRawVPA(f *framework.Framework, obj interface{}) error { // AnnotatePod adds annotation for an existing pod. func AnnotatePod(f *framework.Framework, podName, annotationName, annotationValue string) { - bytes, err := json.Marshal([]utils.PatchRecord{{ + pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(context.TODO(), podName, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to get pod.") + + patches := []utils.PatchRecord{} + if pod.Annotations == nil { + patches = append(patches, utils.PatchRecord{ + Op: "add", + Path: "/metadata/annotations", + Value: make(map[string]string), + }) + } + + patches = append(patches, utils.PatchRecord{ Op: "add", - Path: fmt.Sprintf("/metadata/annotations/%v", annotationName), + Path: fmt.Sprintf("/metadata/annotations/%s", annotationName), Value: annotationValue, - }}) - pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Patch(context.TODO(), podName, types.JSONPatchType, bytes, metav1.PatchOptions{}) + }) + + bytes, err := json.Marshal(patches) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + patchedPod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Patch(context.TODO(), podName, types.JSONPatchType, bytes, metav1.PatchOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to patch pod.") - gomega.Expect(pod.Annotations[annotationName]).To(gomega.Equal(annotationValue)) + gomega.Expect(patchedPod.Annotations[annotationName]).To(gomega.Equal(annotationValue)) } // ParseQuantityOrDie parses quantity from string and dies with an error if diff --git a/vertical-pod-autoscaler/e2e/v1/full_vpa.go b/vertical-pod-autoscaler/e2e/v1/full_vpa.go index 480031ae83b..dc0c5285db9 100644 --- a/vertical-pod-autoscaler/e2e/v1/full_vpa.go +++ b/vertical-pod-autoscaler/e2e/v1/full_vpa.go @@ -356,6 +356,121 @@ var _ = FullVpaE2eDescribe("Pods under VPA with non-recognized recommender expli }) }) +var _ = FullVpaE2eDescribe("Pods under VPA with CPUStartupBoost", func() { + var ( + rc *ResourceConsumer + ) + replicas := 3 + + ginkgo.AfterEach(func() { + rc.CleanUp() + }) + + f := framework.NewDefaultFramework("vertical-pod-autoscaling") + f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline + + ginkgo.Describe("have CPU startup boost recommendation applied", func() { + ginkgo.BeforeEach(func() { + waitForVpaWebhookRegistration(f) + }) + + f.It("to all containers of a pod", framework.WithFeatureGate(features.CPUStartupBoost), func() { + ns := f.Namespace.Name + ginkgo.By("Setting up a VPA CRD with CPUStartupBoost") + targetRef := &autoscaling.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "hamster", + } + + containerName := utils.GetHamsterContainerNameByIndex(0) + factor := int32(100) + vpaCRD := test.VerticalPodAutoscaler(). + WithName("hamster-vpa"). + WithNamespace(f.Namespace.Name). + WithTargetRef(targetRef). + WithUpdateMode(vpa_types.UpdateModeInPlaceOrRecreate). + WithContainer(containerName). + WithCPUStartupBoost(vpa_types.FactorStartupBoostType, &factor, nil, "10s"). + Get() + utils.InstallVPA(f, vpaCRD) + + ginkgo.By("Setting up a hamster deployment") + rc = NewDynamicResourceConsumer("hamster", ns, KindDeployment, + replicas, + 1, /*initCPUTotal*/ + 10, /*initMemoryTotal*/ + 1, /*initCustomMetric*/ + initialCPU, /*cpuRequest*/ + initialMemory, /*memRequest*/ + f.ClientSet, + f.ScalesGetter) + + // Pods should be created with boosted CPU (10m * 100 = 1000m) + err := waitForResourceRequestInRangeInPods( + f, utils.PollTimeout, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceCPU, + ParseQuantityOrDie("900m"), ParseQuantityOrDie("1100m")) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // Pods should be scaled back down in-place after they become Ready and + // StartupBoost.CPU.Duration has elapsed + err = waitForResourceRequestInRangeInPods( + f, utils.PollTimeout, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceCPU, + ParseQuantityOrDie(minimalCPULowerBound), ParseQuantityOrDie(minimalCPUUpperBound)) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + f.It("to a subset of containers in a pod", framework.WithFeatureGate(features.CPUStartupBoost), func() { + ns := f.Namespace.Name + + ginkgo.By("Setting up a VPA CRD with CPUStartupBoost") + targetRef := &autoscaling.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "hamster", + } + + containerName := utils.GetHamsterContainerNameByIndex(0) + factor := int32(100) + vpaCRD := test.VerticalPodAutoscaler(). + WithName("hamster-vpa"). + WithNamespace(f.Namespace.Name). + WithTargetRef(targetRef). + WithUpdateMode(vpa_types.UpdateModeInPlaceOrRecreate). + WithContainer(containerName). + WithCPUStartupBoost(vpa_types.FactorStartupBoostType, &factor, nil, "10s"). + Get() + + utils.InstallVPA(f, vpaCRD) + + ginkgo.By("Setting up a hamster deployment") + rc = NewDynamicResourceConsumer("hamster", ns, KindDeployment, + replicas, + 1, /*initCPUTotal*/ + 10, /*initMemoryTotal*/ + 1, /*initCustomMetric*/ + initialCPU, /*cpuRequest*/ + initialMemory, /*memRequest*/ + f.ClientSet, + f.ScalesGetter) + + // Pods should be created with boosted CPU (10m * 100 = 1000m) + err := waitForResourceRequestInRangeInPods( + f, utils.PollTimeout, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceCPU, + ParseQuantityOrDie("900m"), ParseQuantityOrDie("1100m")) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // Pods should be scaled back down in-place after they become Ready and + // StartupBoost.CPU.Duration has elapsed + err = waitForResourceRequestInRangeInPods( + f, utils.PollTimeout, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceCPU, + ParseQuantityOrDie(minimalCPULowerBound), ParseQuantityOrDie(minimalCPUUpperBound)) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + }) + +}) + var _ = FullVpaE2eDescribe("OOMing pods under VPA", func() { const replicas = 3 diff --git a/vertical-pod-autoscaler/e2e/v1/updater.go b/vertical-pod-autoscaler/e2e/v1/updater.go index 19327021c8e..18af0e8d08b 100644 --- a/vertical-pod-autoscaler/e2e/v1/updater.go +++ b/vertical-pod-autoscaler/e2e/v1/updater.go @@ -26,6 +26,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/autoscaler/vertical-pod-autoscaler/e2e/utils" vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/features" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/annotations" "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/status" "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/test" "k8s.io/kubernetes/test/e2e/framework" @@ -207,6 +209,85 @@ var _ = UpdaterE2eDescribe("Updater", func() { }) }) +var _ = UpdaterE2eDescribe("Updater", func() { + f := framework.NewDefaultFramework("vertical-pod-autoscaling") + f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline + + f.It("Unboost pods when they become Ready", framework.WithFeatureGate(features.CPUStartupBoost), func() { + const statusUpdateInterval = 10 * time.Second + + ginkgo.By("Setting up the Admission Controller status") + stopCh := make(chan struct{}) + statusUpdater := status.NewUpdater( + f.ClientSet, + status.AdmissionControllerStatusName, + status.AdmissionControllerStatusNamespace, + statusUpdateInterval, + "e2e test", + ) + defer func() { + // Schedule a cleanup of the Admission Controller status. + // Status is created outside the test namespace. + ginkgo.By("Deleting the Admission Controller status") + close(stopCh) + err := f.ClientSet.CoordinationV1().Leases(status.AdmissionControllerStatusNamespace). + Delete(context.TODO(), status.AdmissionControllerStatusName, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }() + statusUpdater.Run(stopCh) + + podList := setupPodsForCPUBoost(f, "100m", "100Mi") + initialPods := podList.DeepCopy() + + ginkgo.By("Waiting for pods to be in-place updated") + err := WaitForPodsUpdatedWithoutEviction(f, initialPods) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + +}) + +func setupPodsForCPUBoost(f *framework.Framework, hamsterCPU, hamsterMemory string) *apiv1.PodList { + controller := &autoscaling.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "hamster-deployment", + } + ginkgo.By(fmt.Sprintf("Setting up a hamster %v", controller.Kind)) + // Create pods with boosted CPU, which is 2x the target recommendation + boostedCPU := "200m" + setupHamsterController(f, controller.Kind, boostedCPU, hamsterMemory, utils.DefaultHamsterReplicas) + podList, err := GetHamsterPods(f) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Setting up a VPA CRD") + containerName := utils.GetHamsterContainerNameByIndex(0) + factor := int32(2) + vpaCRD := test.VerticalPodAutoscaler(). + WithName("hamster-vpa"). + WithNamespace(f.Namespace.Name). + WithTargetRef(controller). + WithUpdateMode(vpa_types.UpdateModeAuto). + WithContainer(containerName). + WithCPUStartupBoost(vpa_types.FactorStartupBoostType, &factor, nil, "1s"). + AppendRecommendation( + test.Recommendation(). + WithContainer(containerName). + WithTarget(hamsterCPU, hamsterMemory). + GetContainerResources(), + ). + Get() + + utils.InstallVPA(f, vpaCRD) + + ginkgo.By("Annotating pods with boost annotation") + for _, pod := range podList.Items { + original, err := annotations.GetOriginalResourcesAnnotationValue(&pod.Spec.Containers[0]) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + AnnotatePod(f, pod.Name, annotations.StartupCPUBoostAnnotation, original) + } + return podList +} + func setupPodsForUpscalingEviction(f *framework.Framework) *apiv1.PodList { return setupPodsForEviction(f, "100m", "100Mi", nil) }