Skip to content

Add e2e tests for CPU startup boost #8672

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: experimental-cpu-boost
Choose a base branch
from
Open

Add e2e tests for CPU startup boost #8672

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
118 changes: 118 additions & 0 deletions vertical-pod-autoscaler/e2e/v1/admission_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
apiv1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1"
"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/features"
"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/test"
"k8s.io/kubernetes/test/e2e/framework"
framework_deployment "k8s.io/kubernetes/test/e2e/framework/deployment"
Expand Down Expand Up @@ -961,6 +962,123 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", func() {
})
})

var _ = AdmissionControllerE2eDescribe("Admission-controller", ginkgo.Label("FG:CPUStartupBoost"), func() {
f := framework.NewDefaultFramework("vertical-pod-autoscaling")
f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline

ginkgo.BeforeEach(func() {
checkFeatureGateTestsEnabled(f, features.CPUStartupBoost, true, false)
waitForVpaWebhookRegistration(f)
})

ginkgo.It("boosts CPU by factor on pod creation", func() {
initialCPU := ParseQuantityOrDie("100m")
expectedCPU := ParseQuantityOrDie("200m")
d := NewHamsterDeploymentWithResources(f, initialCPU, ParseQuantityOrDie("100Mi"))

ginkgo.By("Setting up a VPA with a startup boost policy (factor)")
containerName := GetHamsterContainerNameByIndex(0)
factor := int32(2)
vpaCRD := test.VerticalPodAutoscaler().
WithName("hamster-vpa").
WithNamespace(f.Namespace.Name).
WithTargetRef(hamsterTargetRef).
WithContainer(containerName).
WithCPUStartupBoost(vpa_types.FactorStartupBoostType, &factor, nil, "15s").
AppendRecommendation(
test.Recommendation().
WithContainer(containerName).
WithTarget("100m", "100Mi").
GetContainerResources(),
).
Get()
InstallVPA(f, vpaCRD)

ginkgo.By("Starting the deployment and verifying the pod is boosted")
podList := startDeploymentPods(f, d)
pod := podList.Items[0]
gomega.Expect(pod.Spec.Containers[0].Resources.Requests.Cpu().Cmp(expectedCPU)).To(gomega.Equal(0))
gomega.Expect(pod.Spec.Containers[0].Resources.Limits.Cpu().Cmp(expectedCPU)).To(gomega.Equal(0))
})

ginkgo.It("boosts CPU by quantity on pod creation", func() {
initialCPU := ParseQuantityOrDie("100m")
boostCPUQuantity := ParseQuantityOrDie("500m")
expectedCPU := ParseQuantityOrDie("600m")
d := NewHamsterDeploymentWithResources(f, initialCPU, ParseQuantityOrDie("100Mi"))

ginkgo.By("Setting up a VPA with a startup boost policy (quantity)")
containerName := GetHamsterContainerNameByIndex(0)
vpaCRD := test.VerticalPodAutoscaler().
WithName("hamster-vpa").
WithNamespace(f.Namespace.Name).
WithTargetRef(hamsterTargetRef).
WithContainer(containerName).
WithCPUStartupBoost(vpa_types.QuantityStartupBoostType, nil, &boostCPUQuantity, "15s").
AppendRecommendation(
test.Recommendation().
WithContainer(containerName).
WithTarget("100m", "100Mi").
GetContainerResources(),
).
Get()
InstallVPA(f, vpaCRD)

ginkgo.By("Starting the deployment and verifying the pod is boosted")
podList := startDeploymentPods(f, d)
pod := podList.Items[0]
gomega.Expect(pod.Spec.Containers[0].Resources.Requests.Cpu().Cmp(expectedCPU)).To(gomega.Equal(0))
gomega.Expect(pod.Spec.Containers[0].Resources.Limits.Cpu().Cmp(expectedCPU)).To(gomega.Equal(0))
})

ginkgo.It("boosts CPU on pod creation when VPA update mode is Off", func() {
initialCPU := ParseQuantityOrDie("100m")
expectedCPU := ParseQuantityOrDie("200m")
d := NewHamsterDeploymentWithResources(f, initialCPU, ParseQuantityOrDie("100Mi"))

ginkgo.By("Setting up a VPA with updateMode Off and a startup boost policy")
containerName := GetHamsterContainerNameByIndex(0)
factor := int32(2)
vpaCRD := test.VerticalPodAutoscaler().
WithName("hamster-vpa").
WithNamespace(f.Namespace.Name).
WithTargetRef(hamsterTargetRef).
WithContainer(containerName).
WithUpdateMode(vpa_types.UpdateModeOff). // VPA is off, but boost should still work
WithCPUStartupBoost(vpa_types.FactorStartupBoostType, &factor, nil, "15s").
Get()
InstallVPA(f, vpaCRD)

ginkgo.By("Starting the deployment and verifying the pod is boosted")
podList := startDeploymentPods(f, d)
pod := podList.Items[0]
gomega.Expect(pod.Spec.Containers[0].Resources.Requests.Cpu().Cmp(expectedCPU)).To(gomega.Equal(0))
})

ginkgo.It("doesn't boost CPU on pod creation when scaling mode is Off", func() {
initialCPU := ParseQuantityOrDie("100m")
d := NewHamsterDeploymentWithResources(f, initialCPU, ParseQuantityOrDie("100Mi"))

ginkgo.By("Setting up a VPA with a startup boost policy and scaling mode Off")
containerName := GetHamsterContainerNameByIndex(0)
factor := int32(2)
vpaCRD := test.VerticalPodAutoscaler().
WithName("hamster-vpa").
WithNamespace(f.Namespace.Name).
WithTargetRef(hamsterTargetRef).
WithContainer(containerName).
WithCPUStartupBoost(vpa_types.FactorStartupBoostType, &factor, nil, "15s").
WithScalingMode(containerName, vpa_types.ContainerScalingModeOff).
Get()
InstallVPA(f, vpaCRD)

ginkgo.By("Starting the deployment and verifying the pod is NOT boosted")
podList := startDeploymentPods(f, d)
pod := podList.Items[0]
gomega.Expect(pod.Spec.Containers[0].Resources.Requests.Cpu().Cmp(initialCPU)).To(gomega.Equal(0))
})
})

func startDeploymentPods(f *framework.Framework, deployment *appsv1.Deployment) *apiv1.PodList {
// Apiserver watch can lag depending on cached object count and apiserver resource usage.
// We assume that watch can lag up to 5 seconds.
Expand Down
60 changes: 55 additions & 5 deletions vertical-pod-autoscaler/e2e/v1/common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ import (
vpa_clientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned"
"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/features"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/component-base/featuregate"
"k8s.io/kubernetes/test/e2e/framework"
framework_deployment "k8s.io/kubernetes/test/e2e/framework/deployment"
)
Expand Down Expand Up @@ -359,14 +360,30 @@ func PatchVpaRecommendation(f *framework.Framework, vpa *vpa_types.VerticalPodAu

// AnnotatePod adds annotation for an existing pod.
func AnnotatePod(f *framework.Framework, podName, annotationName, annotationValue string) {
bytes, err := json.Marshal([]patchRecord{{
pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(context.TODO(), podName, metav1.GetOptions{})
gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to get pod.")

patches := []patchRecord{}
if pod.Annotations == nil {
patches = append(patches, patchRecord{
Op: "add",
Path: "/metadata/annotations",
Value: make(map[string]string),
})
}

patches = append(patches, patchRecord{
Op: "add",
Path: fmt.Sprintf("/metadata/annotations/%v", annotationName),
Path: fmt.Sprintf("/metadata/annotations/%s", annotationName),
Value: annotationValue,
}})
pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Patch(context.TODO(), podName, types.JSONPatchType, bytes, metav1.PatchOptions{})
})

bytes, err := json.Marshal(patches)
gomega.Expect(err).NotTo(gomega.HaveOccurred())

patchedPod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Patch(context.TODO(), podName, types.JSONPatchType, bytes, metav1.PatchOptions{})
gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to patch pod.")
gomega.Expect(pod.Annotations[annotationName]).To(gomega.Equal(annotationValue))
gomega.Expect(patchedPod.Annotations[annotationName]).To(gomega.Equal(annotationValue))
}

// ParseQuantityOrDie parses quantity from string and dies with an error if
Expand Down Expand Up @@ -654,3 +671,36 @@ func anyContainsSubstring(arr []string, substr string) bool {
}
return false
}

// checkFeatureGateTestsEnabled check for enabled feature gates in the cluster used for the
// given VPA feature.
// Use this in a "beforeEach" call before any suites that use a featuregate.
func checkFeatureGateTestsEnabled(f *framework.Framework, feature featuregate.Feature, checkAdmission, checkUpdater bool) {
ginkgo.By(fmt.Sprintf("Checking %s cluster feature gate is on", feature))

if checkUpdater {
ginkgo.By(fmt.Sprintf("Checking %s VPA feature gate is enabled for updater", feature))

deploy, err := f.ClientSet.AppsV1().Deployments(VpaNamespace).Get(context.TODO(), "vpa-updater", metav1.GetOptions{})
gomega.Expect(err).NotTo(gomega.HaveOccurred())
gomega.Expect(deploy.Spec.Template.Spec.Containers).To(gomega.HaveLen(1))
vpaUpdaterPod := deploy.Spec.Template.Spec.Containers[0]
gomega.Expect(vpaUpdaterPod.Name).To(gomega.Equal("updater"))
if !anyContainsSubstring(vpaUpdaterPod.Args, fmt.Sprintf("%s=true", string(feature))) {
ginkgo.Skip(fmt.Sprintf("Skipping suite: %s feature gate is not enabled for the VPA updater", feature))
}
}

if checkAdmission {
ginkgo.By(fmt.Sprintf("Checking %s VPA feature gate is enabled for admission controller", feature))

deploy, err := f.ClientSet.AppsV1().Deployments(VpaNamespace).Get(context.TODO(), "vpa-admission-controller", metav1.GetOptions{})
gomega.Expect(err).NotTo(gomega.HaveOccurred())
gomega.Expect(deploy.Spec.Template.Spec.Containers).To(gomega.HaveLen(1))
vpaAdmissionPod := deploy.Spec.Template.Spec.Containers[0]
gomega.Expect(vpaAdmissionPod.Name).To(gomega.Equal("admission-controller"))
if !anyContainsSubstring(vpaAdmissionPod.Args, fmt.Sprintf("%s=true", string(feature))) {
ginkgo.Skip(fmt.Sprintf("Skipping suite: %s feature gate is not enabled for VPA admission controller", feature))
}
}
}
117 changes: 117 additions & 0 deletions vertical-pod-autoscaler/e2e/v1/full_vpa.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1"
"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/features"
"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/test"
"k8s.io/kubernetes/test/e2e/framework"
podsecurity "k8s.io/pod-security-admission/api"
Expand Down Expand Up @@ -347,6 +348,122 @@ var _ = FullVpaE2eDescribe("Pods under VPA with non-recognized recommender expli
})
})

var _ = FullVpaE2eDescribe("Pods under VPA with CPUStartupBoost", ginkgo.Label("FG:CPUStartupBoost"), func() {
var (
rc *ResourceConsumer
)
replicas := 3

ginkgo.AfterEach(func() {
rc.CleanUp()
})

f := framework.NewDefaultFramework("vertical-pod-autoscaling")
f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline

ginkgo.Describe("have CPU startup boost recommendation applied", func() {
ginkgo.BeforeEach(func() {
checkFeatureGateTestsEnabled(f, features.CPUStartupBoost, true, true)
waitForVpaWebhookRegistration(f)
})

ginkgo.It("to all containers of a pod", func() {
ns := f.Namespace.Name
ginkgo.By("Setting up a VPA CRD with CPUStartupBoost")
targetRef := &autoscaling.CrossVersionObjectReference{
APIVersion: "apps/v1",
Kind: "Deployment",
Name: "hamster",
}

containerName := GetHamsterContainerNameByIndex(0)
factor := int32(100)
vpaCRD := test.VerticalPodAutoscaler().
WithName("hamster-vpa").
WithNamespace(f.Namespace.Name).
WithTargetRef(targetRef).
WithUpdateMode(vpa_types.UpdateModeInPlaceOrRecreate).
WithContainer(containerName).
WithCPUStartupBoost(vpa_types.FactorStartupBoostType, &factor, nil, "10s").
Get()
InstallVPA(f, vpaCRD)

ginkgo.By("Setting up a hamster deployment")
rc = NewDynamicResourceConsumer("hamster", ns, KindDeployment,
replicas,
1, /*initCPUTotal*/
10, /*initMemoryTotal*/
1, /*initCustomMetric*/
initialCPU, /*cpuRequest*/
initialMemory, /*memRequest*/
f.ClientSet,
f.ScalesGetter)

// Pods should be created with boosted CPU (10m * 100 = 1000m)
err := waitForResourceRequestInRangeInPods(
f, pollTimeout, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceCPU,
ParseQuantityOrDie("900m"), ParseQuantityOrDie("1100m"))
gomega.Expect(err).NotTo(gomega.HaveOccurred())

// Pods should be scaled back down in-place after they become Ready and
// StartupBoost.CPU.Duration has elapsed
err = waitForResourceRequestInRangeInPods(
f, pollTimeout, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceCPU,
ParseQuantityOrDie(minimalCPULowerBound), ParseQuantityOrDie(minimalCPUUpperBound))
gomega.Expect(err).NotTo(gomega.HaveOccurred())
})

ginkgo.It("to a subset of containers in a pod", func() {
ns := f.Namespace.Name

ginkgo.By("Setting up a VPA CRD with CPUStartupBoost")
targetRef := &autoscaling.CrossVersionObjectReference{
APIVersion: "apps/v1",
Kind: "Deployment",
Name: "hamster",
}

containerName := GetHamsterContainerNameByIndex(0)
factor := int32(100)
vpaCRD := test.VerticalPodAutoscaler().
WithName("hamster-vpa").
WithNamespace(f.Namespace.Name).
WithTargetRef(targetRef).
WithUpdateMode(vpa_types.UpdateModeInPlaceOrRecreate).
WithContainer(containerName).
WithCPUStartupBoost(vpa_types.FactorStartupBoostType, &factor, nil, "10s").
Get()

InstallVPA(f, vpaCRD)

ginkgo.By("Setting up a hamster deployment")
rc = NewDynamicResourceConsumer("hamster", ns, KindDeployment,
replicas,
1, /*initCPUTotal*/
10, /*initMemoryTotal*/
1, /*initCustomMetric*/
initialCPU, /*cpuRequest*/
initialMemory, /*memRequest*/
f.ClientSet,
f.ScalesGetter)

// Pods should be created with boosted CPU (10m * 100 = 1000m)
err := waitForResourceRequestInRangeInPods(
f, pollTimeout, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceCPU,
ParseQuantityOrDie("900m"), ParseQuantityOrDie("1100m"))
gomega.Expect(err).NotTo(gomega.HaveOccurred())

// Pods should be scaled back down in-place after they become Ready and
// StartupBoost.CPU.Duration has elapsed
err = waitForResourceRequestInRangeInPods(
f, pollTimeout, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceCPU,
ParseQuantityOrDie(minimalCPULowerBound), ParseQuantityOrDie(minimalCPUUpperBound))
gomega.Expect(err).NotTo(gomega.HaveOccurred())
})
})

})

var _ = FullVpaE2eDescribe("OOMing pods under VPA", func() {
const replicas = 3

Expand Down
Loading
Loading