From 6d84870c59b478638d1dd2661633b8c50bbd37c5 Mon Sep 17 00:00:00 2001 From: Christian Melendez Date: Wed, 15 Oct 2025 15:47:07 +0000 Subject: [PATCH 1/4] Added features to the EKS script and added instructions to use it as a kubectl plugin --- README-EKS.md | 30 ++++++++-- eks-aperf.sh => kubectl-aperf | 105 ++++++++++++++++++++++++++++++---- 2 files changed, 119 insertions(+), 16 deletions(-) rename eks-aperf.sh => kubectl-aperf (63%) diff --git a/README-EKS.md b/README-EKS.md index 5fcec639..d4a563fc 100644 --- a/README-EKS.md +++ b/README-EKS.md @@ -88,10 +88,10 @@ Example node name: `ip-10-0-120-104.us-west-2.compute.internal` or `i-02a3f32795 #### 3b. Execute APerf Collection -Use the provided `eks-aperf.sh` script to run APerf on the selected node: +Use the provided `kubectl-aperf` script to run APerf on the selected node: ```bash -bash ./eks-aperf.sh \ +./kubectl-aperf \ --aperf_image="${APERF_ECRREPO}:latest" \ --node="ip-10-0-120-104.us-west-2.compute.internal" ``` @@ -111,7 +111,7 @@ bash ./eks-aperf.sh \ ```bash # Run APerf for 60 seconds with profiling enabled -bash ./eks-aperf.sh \ +./kubectl-aperf \ --aperf_image="${APERF_ECRREPO}:latest" \ --node="ip-10-0-120-104.us-west-2.compute.internal" \ --aperf_options="-p 60 --profile" \ @@ -122,7 +122,7 @@ bash ./eks-aperf.sh \ ```bash # Run APerf with custom CPU and memory settings -bash ./eks-aperf.sh \ +./kubectl-aperf \ --aperf_image="${APERF_ECRREPO}:latest" \ --node="ip-10-0-120-104.us-west-2.compute.internal" \ --cpu-request="2.0" \ @@ -133,7 +133,7 @@ bash ./eks-aperf.sh \ #### 3c. Collect Results -The `eks-aperf.sh` script will automatically run the following steps: +The `kubectl-aperf` script will automatically run the following steps: 1. **Pod Deployment**: Deploy a privileged pod on the specified node 2. **APerf Record**: Runs APerf record inside the pod with the specified options @@ -145,7 +145,7 @@ The APerf report will be downloaded as a compressed tarball file with a timestam Example of correct output execution of the script: ```bash -$ bash ./eks-aperf.sh --aperf_image="${APERF_ECRREPO}:latest" --namespace=aperf --node ip-10-0-120-104.us-west-2.compute.internal --aperf_options="-p 30 --profile" +$ ./kubectl-aperf --aperf_image="${APERF_ECRREPO}:latest" --namespace=aperf --node ip-10-0-120-104.us-west-2.compute.internal --aperf_options="-p 30 --profile" Tageted node instance type... m6g.8xlarge Check namespace security policy... Namespace 'aperf' has 'privileged' policy - privileged pods allowed. @@ -198,6 +198,24 @@ Done! - The pod is automatically cleaned up after execution +## Installing as a kubectl Plugin + +You can install `kubectl-aperf` as a kubectl plugin to run it as `kubectl aperf` instead of just `./kubectl-aperf`. + +To do so, run the following commands: + +```bash +sudo mv kubectl-aperf /usr/local/bin/ +kubectl plugin list +kubectl aperf --help +``` + +Now you can run it as: + +```bash +kubectl aperf --aperf_image="${APERF_ECRREPO}:latest" --node="ip-10-0-120-104.us-west-2.compute.internal" +``` + ## Known Limitations **Note**: The `--profile-java` option is not currently fully supported with this script. diff --git a/eks-aperf.sh b/kubectl-aperf similarity index 63% rename from eks-aperf.sh rename to kubectl-aperf index 1dfb7ce7..198ed872 100755 --- a/eks-aperf.sh +++ b/kubectl-aperf @@ -12,6 +12,8 @@ NAMESPACE="default" APERF_OPTIONS="" NODE_NAME="" APERF_IMAGE="" +REPORT_NAME="aperf_record" +OPEN_BROWSER=true SHOW_HELP=false CPU_REQUEST="1.0" MEMORY_REQUEST="1Gi" @@ -34,6 +36,8 @@ while [ $# -gt 0 ]; do --namespace) dest="NAMESPACE";; --aperf_options) dest="APERF_OPTIONS";; --aperf_image) dest="APERF_IMAGE";; + --report-name) dest="REPORT_NAME";; + --open-browser) dest="OPEN_BROWSER";; --cpu-request) dest="CPU_REQUEST";; --memory-request) dest="MEMORY_REQUEST";; --cpu-limit) dest="CPU_LIMIT";; @@ -68,6 +72,8 @@ if [ "$SHOW_HELP" = true ]; then echo " --node Required. The name of the Kubernetes node to run aperf on" echo " --namespace Optional. The Kubernetes namespace (default: '${NAMESPACE}')" echo " --aperf_options Optional. Options to pass to aperf (default: '${APERF_OPTIONS}')" + echo " --report-name Optional. Name for aperf record/report (default: '${REPORT_NAME}')" + echo " --open-browser Optional. Open report in browser (default: ${OPEN_BROWSER})" echo " --cpu-request Optional. CPU request (default: '${CPU_REQUEST}')" echo " --memory-request Optional. Memory request (default: '${MEMORY_REQUEST}')" echo " --cpu-limit Optional. CPU limit (default: '${CPU_LIMIT}')" @@ -93,6 +99,43 @@ fi POD_NAME="aperf-pod-${NODE_NAME//[.]/-}" +# Get node taints and generate tolerations +echo -e "${BOLD}Checking node taints...${NC}" +TAINTS=$(kubectl get node ${NODE_NAME} -o jsonpath='{.spec.taints[*]}' 2>/dev/null) + +TOLERATIONS="" +if [ -n "$TAINTS" ]; then + echo -e " ${YELLOW}Node has taints, adding tolerations to pod spec${NC}" + + # Parse taints and create tolerations YAML + TOLERATIONS=" tolerations:" + + # Get taints as JSON array and process each one + TAINT_COUNT=$(kubectl get node ${NODE_NAME} -o json | jq -r '.spec.taints | length' 2>/dev/null || echo "0") + + for ((i=0; i<$TAINT_COUNT; i++)); do + KEY=$(kubectl get node ${NODE_NAME} -o json | jq -r ".spec.taints[$i].key" 2>/dev/null) + VALUE=$(kubectl get node ${NODE_NAME} -o json | jq -r ".spec.taints[$i].value" 2>/dev/null) + EFFECT=$(kubectl get node ${NODE_NAME} -o json | jq -r ".spec.taints[$i].effect" 2>/dev/null) + + echo -e " Taint: ${KEY}=${VALUE}:${EFFECT}" + + TOLERATIONS="${TOLERATIONS} + - key: \"${KEY}\"" + + if [ "$VALUE" != "null" ] && [ -n "$VALUE" ]; then + TOLERATIONS="${TOLERATIONS} + value: \"${VALUE}\"" + fi + + TOLERATIONS="${TOLERATIONS} + effect: \"${EFFECT}\" + operator: \"Equal\"" + done +else + echo -e " ${GREEN}No taints found on node${NC}" +fi + # Create pod YAML as a variable POD_YAML=$(cat << EOF apiVersion: v1 @@ -104,6 +147,7 @@ metadata: spec: nodeSelector: kubernetes.io/hostname: "${NODE_NAME}" +${TOLERATIONS} containers: - name: aperf-runner image: ${APERF_IMAGE} @@ -115,13 +159,13 @@ spec: set -e echo -e "\nStarting Aperf recording execution..." - echo "Run: /usr/bin/aperf record -r aperf_record ${APERF_OPTIONS}" - sudo /usr/bin/aperf record -r aperf_record ${APERF_OPTIONS} + echo "Run: /usr/bin/aperf record -r ${REPORT_NAME} ${APERF_OPTIONS}" + sudo /usr/bin/aperf record -r ${REPORT_NAME} ${APERF_OPTIONS} echo "APerf record completed" echo -e "\nStarting Aperf report generation..." - echo "Run: /usr/bin/aperf report -r aperf_record -n aperf_report" - sudo /usr/bin/aperf report -r aperf_record -n aperf_report + echo "Run: /usr/bin/aperf report -r ${REPORT_NAME} -n ${REPORT_NAME}_report" + sudo /usr/bin/aperf report -r ${REPORT_NAME} -n ${REPORT_NAME}_report echo "APerf report generation completed" echo -e "\nWaiting for files to be copied..." @@ -174,10 +218,13 @@ fi # Show resource usage for pods on this node echo -e "${BOLD}Resource usage for pods on ${NODE_NAME}:${NC}" -rm /tmp/allpods.out 2> /dev/null; \ -kubectl top pods --all-namespaces > /tmp/allpods.out && \ -head -n 1 /tmp/allpods.out && \ -grep "$(kubectl get pods --all-namespaces --field-selector spec.nodeName=${NODE_NAME} -o jsonpath='{range .items[*]}{.metadata.name}{" "}{end}' | sed 's/[[:space:]]*$//' | sed 's/[[:space:]]/\\|/g')" /tmp/allpods.out --color=never +if kubectl top pods --all-namespaces > /tmp/allpods.out 2>/dev/null; then + head -n 1 /tmp/allpods.out + grep "$(kubectl get pods --all-namespaces --field-selector spec.nodeName=${NODE_NAME} -o jsonpath='{range .items[*]}{.metadata.name}{" "}{end}' | sed 's/[[:space:]]*$//' | sed 's/[[:space:]]/\\|/g')" /tmp/allpods.out --color=never || echo " No pods found on this node" + rm /tmp/allpods.out 2>/dev/null || true +else + echo " ${YELLOW}Note: kubectl top not available (metrics-server may not be installed)${NC}" +fi # Create APerf pod echo -e "\n${BOLD}Created pod configuration for node:${NC} ${NODE_NAME}${NC}" @@ -215,13 +262,51 @@ done kill $LOGS_PID 2>/dev/null || true # Copy files from pod to local directory -LOCAL_FILE="aperf_report_${POD_STARTTIME}.tar.gz" +LOCAL_FILE="${REPORT_NAME}_${POD_STARTTIME}.tar.gz" +EXTRACT_DIR="${REPORT_NAME}_${POD_STARTTIME}" echo -e "${NC}${BOLD}Aperf completed. Copying files from pod ${POD_NAME}...${NC}" -kubectl cp ${NAMESPACE}/${POD_NAME}:aperf_report.tar.gz ${LOCAL_FILE} +kubectl cp ${NAMESPACE}/${POD_NAME}:${REPORT_NAME}_report.tar.gz ${LOCAL_FILE} # Delete the pod after copying files echo -ne "${BOLD}Deleting pod to clean up resources...${NC} " kubectl delete pod ${POD_NAME} -n ${NAMESPACE} echo -e "${BOLD}${GREEN}Files copied to${NC} ${BLUE}${LOCAL_FILE}${NC}" + +# Extract the tar.gz file +echo -e "${BOLD}Extracting report files...${NC}" +mkdir -p "${EXTRACT_DIR}" +tar -xzf "${LOCAL_FILE}" -C "${EXTRACT_DIR}" +echo -e " ${GREEN}Extracted to${NC} ${BLUE}${EXTRACT_DIR}/${NC}" + +# Open index.html in browser if enabled +if [ "$OPEN_BROWSER" = true ]; then + INDEX_FILE="${EXTRACT_DIR}/${REPORT_NAME}_report/index.html" + + if [ -f "$INDEX_FILE" ]; then + echo -e "${BOLD}Opening report in browser...${NC}" + + # Detect OS and open browser accordingly + if [[ "$OSTYPE" == "darwin"* ]]; then + # macOS + open "$INDEX_FILE" + elif [[ "$OSTYPE" == "linux-gnu"* ]]; then + # Linux + if command -v xdg-open &> /dev/null; then + xdg-open "$INDEX_FILE" + elif command -v sensible-browser &> /dev/null; then + sensible-browser "$INDEX_FILE" + else + echo -e " ${YELLOW}Could not detect browser command. Please open manually:${NC} ${BLUE}${INDEX_FILE}${NC}" + fi + else + echo -e " ${YELLOW}Unsupported OS. Please open manually:${NC} ${BLUE}${INDEX_FILE}${NC}" + fi + else + echo -e " ${YELLOW}Warning: index.html not found at ${INDEX_FILE}${NC}" + echo -e " ${YELLOW}Extracted contents:${NC}" + ls -la "${EXTRACT_DIR}/" + fi +fi + echo -e "${BOLD}${GREEN}Done!${NC}" From 2bb1f640cb14082fdab6fc2644c3a7ae0ae3ff03 Mon Sep 17 00:00:00 2001 From: Christian Melendez Date: Fri, 17 Oct 2025 22:46:06 +0000 Subject: [PATCH 2/4] removed requests and limits as they're not needed --- kubectl-aperf | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/kubectl-aperf b/kubectl-aperf index 198ed872..72f8be6d 100755 --- a/kubectl-aperf +++ b/kubectl-aperf @@ -15,10 +15,6 @@ APERF_IMAGE="" REPORT_NAME="aperf_record" OPEN_BROWSER=true SHOW_HELP=false -CPU_REQUEST="1.0" -MEMORY_REQUEST="1Gi" -CPU_LIMIT="4.0" -MEMORY_LIMIT="4Gi" # Define color and formatting codes BOLD="\033[1m" @@ -38,10 +34,6 @@ while [ $# -gt 0 ]; do --aperf_image) dest="APERF_IMAGE";; --report-name) dest="REPORT_NAME";; --open-browser) dest="OPEN_BROWSER";; - --cpu-request) dest="CPU_REQUEST";; - --memory-request) dest="MEMORY_REQUEST";; - --cpu-limit) dest="CPU_LIMIT";; - --memory-limit) dest="MEMORY_LIMIT";; --help) SHOW_HELP=true shift @@ -74,10 +66,6 @@ if [ "$SHOW_HELP" = true ]; then echo " --aperf_options Optional. Options to pass to aperf (default: '${APERF_OPTIONS}')" echo " --report-name Optional. Name for aperf record/report (default: '${REPORT_NAME}')" echo " --open-browser Optional. Open report in browser (default: ${OPEN_BROWSER})" - echo " --cpu-request Optional. CPU request (default: '${CPU_REQUEST}')" - echo " --memory-request Optional. Memory request (default: '${MEMORY_REQUEST}')" - echo " --cpu-limit Optional. CPU limit (default: '${CPU_LIMIT}')" - echo " --memory-limit Optional. Memory limit (default: '${MEMORY_LIMIT}')" echo " --help Show this help message" exit 0 fi @@ -170,14 +158,6 @@ ${TOLERATIONS} echo -e "\nWaiting for files to be copied..." sleep 7200 - - resources: - requests: - memory: "${MEMORY_REQUEST}" - cpu: "${CPU_REQUEST}" - limits: - memory: "${MEMORY_LIMIT}" - cpu: "${CPU_LIMIT}" volumeMounts: - mountPath: /boot name: boot-volume From ddaf0b742cd8a3ca7115c5376de7b0ef8ef00eb3 Mon Sep 17 00:00:00 2001 From: Christian Melendez Date: Mon, 3 Nov 2025 13:07:26 +0000 Subject: [PATCH 3/4] Addressing comments from latest PR to support Windows, and fix formatting --- kubectl-aperf | 49 +++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/kubectl-aperf b/kubectl-aperf index 72f8be6d..8ba6ef44 100755 --- a/kubectl-aperf +++ b/kubectl-aperf @@ -87,13 +87,22 @@ fi POD_NAME="aperf-pod-${NODE_NAME//[.]/-}" +echo -e "${BOLD}Verifying node exists...${NC}" +if ! kubectl get node ${NODE_NAME} &>/dev/null; then + echo -e "${RED}Error: Node '${NODE_NAME}' not found in the cluster.${NC}" + echo -e "${YELLOW}Available nodes:${NC}" + kubectl get nodes -o custom-columns=NAME:.metadata.name --no-headers + exit 1 +fi +echo -e "${GREEN}Node '${NODE_NAME}' found${NC}" + # Get node taints and generate tolerations -echo -e "${BOLD}Checking node taints...${NC}" +echo -e "${BOLD}Checking node taints... " TAINTS=$(kubectl get node ${NODE_NAME} -o jsonpath='{.spec.taints[*]}' 2>/dev/null) TOLERATIONS="" if [ -n "$TAINTS" ]; then - echo -e " ${YELLOW}Node has taints, adding tolerations to pod spec${NC}" + echo -e "${YELLOW}Node has taints, adding tolerations to pod spec${NC}" # Parse taints and create tolerations YAML TOLERATIONS=" tolerations:" @@ -121,7 +130,7 @@ if [ -n "$TAINTS" ]; then operator: \"Equal\"" done else - echo -e " ${GREEN}No taints found on node${NC}" + echo -e "${GREEN}No taints found on node${NC}" fi # Create pod YAML as a variable @@ -197,13 +206,13 @@ else fi # Show resource usage for pods on this node -echo -e "${BOLD}Resource usage for pods on ${NODE_NAME}:${NC}" +echo -e "${BOLD}Resource usage for pods on ${NODE_NAME}: " if kubectl top pods --all-namespaces > /tmp/allpods.out 2>/dev/null; then head -n 1 /tmp/allpods.out grep "$(kubectl get pods --all-namespaces --field-selector spec.nodeName=${NODE_NAME} -o jsonpath='{range .items[*]}{.metadata.name}{" "}{end}' | sed 's/[[:space:]]*$//' | sed 's/[[:space:]]/\\|/g')" /tmp/allpods.out --color=never || echo " No pods found on this node" rm /tmp/allpods.out 2>/dev/null || true else - echo " ${YELLOW}Note: kubectl top not available (metrics-server may not be installed)${NC}" + echo "${YELLOW}Note: kubectl top not available (metrics-server may not be installed)${NC}" fi # Create APerf pod @@ -256,7 +265,24 @@ echo -e "${BOLD}${GREEN}Files copied to${NC} ${BLUE}${LOCAL_FILE}${NC}" # Extract the tar.gz file echo -e "${BOLD}Extracting report files...${NC}" mkdir -p "${EXTRACT_DIR}" -tar -xzf "${LOCAL_FILE}" -C "${EXTRACT_DIR}" + +if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "win32" || "$OSTYPE" == "cygwin" ]]; then + # Windows - use tar if available, otherwise try 7z or PowerShell + if command -v tar &> /dev/null; then + tar -xzf "${LOCAL_FILE}" -C "${EXTRACT_DIR}" + elif command -v 7z &> /dev/null; then + 7z x "${LOCAL_FILE}" -so | 7z x -si -ttar -o"${EXTRACT_DIR}" + elif command -v powershell.exe &> /dev/null; then + powershell.exe -Command "Expand-Archive -Path '${LOCAL_FILE}' -DestinationPath '${EXTRACT_DIR}' -Force" + else + echo -e " ${RED}Error: No extraction tool found. Please install tar, 7-Zip, or use PowerShell${NC}" + exit 1 + fi +else + # macOS/Linux + tar -xzf "${LOCAL_FILE}" -C "${EXTRACT_DIR}" +fi + echo -e " ${GREEN}Extracted to${NC} ${BLUE}${EXTRACT_DIR}/${NC}" # Open index.html in browser if enabled @@ -279,6 +305,17 @@ if [ "$OPEN_BROWSER" = true ]; then else echo -e " ${YELLOW}Could not detect browser command. Please open manually:${NC} ${BLUE}${INDEX_FILE}${NC}" fi + elif [[ "$OSTYPE" == "msys" || "$OSTYPE" == "win32" || "$OSTYPE" == "cygwin" ]]; then + # Windows + if command -v start &> /dev/null; then + start "$INDEX_FILE" + elif command -v cmd.exe &> /dev/null; then + cmd.exe /c start "$INDEX_FILE" + elif command -v powershell.exe &> /dev/null; then + powershell.exe -Command "Start-Process '${INDEX_FILE}'" + else + echo -e " ${YELLOW}Could not detect browser command. Please open manually:${NC} ${BLUE}${INDEX_FILE}${NC}" + fi else echo -e " ${YELLOW}Unsupported OS. Please open manually:${NC} ${BLUE}${INDEX_FILE}${NC}" fi From abe93ba3babaf0ec9e0180f5a7f33829b64e1d25 Mon Sep 17 00:00:00 2001 From: Christian Melendez Date: Wed, 5 Nov 2025 11:54:34 +0000 Subject: [PATCH 4/4] make the CPU and memory requests/limits optional --- kubectl-aperf | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/kubectl-aperf b/kubectl-aperf index 8ba6ef44..319963d6 100755 --- a/kubectl-aperf +++ b/kubectl-aperf @@ -15,6 +15,10 @@ APERF_IMAGE="" REPORT_NAME="aperf_record" OPEN_BROWSER=true SHOW_HELP=false +CPU_REQUEST="" +MEMORY_REQUEST="" +CPU_LIMIT="" +MEMORY_LIMIT="" # Define color and formatting codes BOLD="\033[1m" @@ -34,6 +38,10 @@ while [ $# -gt 0 ]; do --aperf_image) dest="APERF_IMAGE";; --report-name) dest="REPORT_NAME";; --open-browser) dest="OPEN_BROWSER";; + --cpu-request) dest="CPU_REQUEST";; + --memory-request) dest="MEMORY_REQUEST";; + --cpu-limit) dest="CPU_LIMIT";; + --memory-limit) dest="MEMORY_LIMIT";; --help) SHOW_HELP=true shift @@ -66,6 +74,10 @@ if [ "$SHOW_HELP" = true ]; then echo " --aperf_options Optional. Options to pass to aperf (default: '${APERF_OPTIONS}')" echo " --report-name Optional. Name for aperf record/report (default: '${REPORT_NAME}')" echo " --open-browser Optional. Open report in browser (default: ${OPEN_BROWSER})" + echo " --cpu-request Optional. CPU request (e.g., '1.0', '500m')" + echo " --memory-request Optional. Memory request (e.g., '1Gi', '512Mi')" + echo " --cpu-limit Optional. CPU limit (e.g., '1.0', '500m')" + echo " --memory-limit Optional. Memory limit (e.g., '1Gi', '512Mi')" echo " --help Show this help message" exit 0 fi @@ -133,6 +145,29 @@ else echo -e "${GREEN}No taints found on node${NC}" fi +RESOURCES="" +if [ -n "$CPU_REQUEST" ] || [ -n "$MEMORY_REQUEST" ] || [ -n "$CPU_LIMIT" ] || [ -n "$MEMORY_LIMIT" ]; then + RESOURCES=" resources:" + + if [ -n "$CPU_REQUEST" ] || [ -n "$MEMORY_REQUEST" ]; then + RESOURCES="${RESOURCES} + requests:" + [ -n "$MEMORY_REQUEST" ] && RESOURCES="${RESOURCES} + memory: \"${MEMORY_REQUEST}\"" + [ -n "$CPU_REQUEST" ] && RESOURCES="${RESOURCES} + cpu: \"${CPU_REQUEST}\"" + fi + + if [ -n "$CPU_LIMIT" ] || [ -n "$MEMORY_LIMIT" ]; then + RESOURCES="${RESOURCES} + limits:" + [ -n "$MEMORY_LIMIT" ] && RESOURCES="${RESOURCES} + memory: \"${MEMORY_LIMIT}\"" + [ -n "$CPU_LIMIT" ] && RESOURCES="${RESOURCES} + cpu: \"${CPU_LIMIT}\"" + fi +fi + # Create pod YAML as a variable POD_YAML=$(cat << EOF apiVersion: v1 @@ -167,6 +202,7 @@ ${TOLERATIONS} echo -e "\nWaiting for files to be copied..." sleep 7200 +${RESOURCES} volumeMounts: - mountPath: /boot name: boot-volume