add options for --max-devworkspaces and --delete-devworkspace-after-ready

Signed-off-by: Rohan Kumar <rohaan@redhat.com>

Author: rohanKanojia

.ci/openshift_e2e.sh

@@ -73,5 +73,6 @@ export CLEAN_UP_AFTER_SUITE="false"
 make test_e2e
 bumpLogs
 
-make test_load ARGS="--mode operator --max-vus 250 --separate-namespaces false --test-duration-minutes 25 --dwo-namespace devworkspace-controller --logs-dir ${ARTIFACT_DIR}/load-testing-logs"
-make uninstall
+make test_load ARGS="--mode operator --max-vus 500 --max-devworkspaces 1500 --delete-devworkspace-after-ready false --separate-namespaces false --test-duration-minutes 25 --dwo-namespace devworkspace-controller --logs-dir ${ARTIFACT_DIR}/load-testing-logs"
+
+#make uninstall

test/load/devworkspace_load_test.js

@@ -23,10 +23,12 @@ const inCluster = __ENV.IN_CLUSTER === 'true';
 const apiServer = inCluster ? `https://kubernetes.default.svc` : __ENV.KUBE_API;
 const token = inCluster ? open('/var/run/secrets/kubernetes.io/serviceaccount/token') : __ENV.KUBE_TOKEN;
 const useSeparateNamespaces = __ENV.SEPARATE_NAMESPACES === "true";
+const deleteDevWorkspaceAfterReady = __ENV.DELETE_DEVWORKSPACE_AFTER_READY === "true";
 const operatorNamespace = __ENV.DWO_NAMESPACE || 'openshift-operators';
 const externalDevWorkspaceLink = __ENV.DEVWORKSPACE_LINK || '';
 const shouldCreateAutomountResources = (__ENV.CREATE_AUTOMOUNT_RESOURCES || 'false') === 'true';
 const maxVUs = Number(__ENV.MAX_VUS || 50);
+const maxDevWorkspaces = Number(__ENV.MAX_DEVWORKSPACES || -1);
 const devWorkspaceReadyTimeout = Number(__ENV.DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS || 600);
 const autoMountConfigMapName = 'dwo-load-test-automount-configmap';
 const autoMountSecretName = 'dwo-load-test-automount-secret';
@@ -87,6 +89,12 @@ export function setup() {
 }
 
 export default function () {
+  if (maxDevWorkspaces > 0) {
+    const totalDevWorkspaces = getDevWorkspacesFromApiServer().length;
+    if (totalDevWorkspaces > maxDevWorkspaces) {
+      return;
+    }
+  }
   const vuId = __VU;
   const iteration = __ITER;
   const crName = `dw-test-${vuId}-${iteration}`;
@@ -104,7 +112,9 @@ export default function () {
     const devWorkspaceCreated = createNewDevWorkspace(namespace, vuId, iteration);
     if (devWorkspaceCreated) {
       waitUntilDevWorkspaceIsReady(vuId, crName, namespace);
-      deleteDevWorkspace(crName, namespace);
+      if (deleteDevWorkspaceAfterReady) {
+        deleteDevWorkspace(crName, namespace);
+      }
     }
   } catch (error) {
     console.error(`Load test for ${vuId}-${iteration} failed:`, error.message);
@@ -434,6 +444,23 @@ function downloadAndParseExternalWorkspace(externalDevWorkspaceLink) {
   return manifest;
 }
 
+function getDevWorkspacesFromApiServer() {
+  const basePath = useSeparateNamespaces
+      ? `${apiServer}/apis/workspace.devfile.io/v1alpha2/devworkspaces`
+      : `${apiServer}/apis/workspace.devfile.io/v1alpha2/namespaces/${loadTestNamespace}/devworkspaces`;
+
+  const url = `${basePath}?labelSelector=${labelKey}%3D${labelType}`;
+  const res = http.get(url, { headers });
+
+  if (res.status !== 200) {
+    console.error(`Failed to fetch DevWorkspaces: ${res.status} ${res.body}`);
+    return [];
+  }
+
+  const body = JSON.parse(res.body);
+  return body.items.map((dw) => dw.metadata.name);
+}
+
 function generateDevWorkspaceToCreate(vuId, iteration, namespace) {
   const name = `dw-test-${vuId}-${iteration}`;
   let devWorkspace = {};

test/load/runk6.sh

@@ -16,6 +16,8 @@ DEVWORKSPACE_LINK="https://gist.githubusercontent.com/rohanKanojia/ecf625afaf3fe
 MAX_VUS="100"
 DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS="1200"
 SEPARATE_NAMESPACES="false"
+DELETE_DEVWORKSPACE_AFTER_READY="true"
+MAX_DEVWORKSPACES="-1"
 CREATE_AUTOMOUNT_RESOURCES="false"
 LOGS_DIR="logs"
 TEST_DURATION_IN_MINUTES="25"
@@ -57,7 +59,9 @@ Usage: $0 [options]
 Options:
   --mode <operator|binary>                    Mode to run the script (default: operator)
   --max-vus <int>                             Number of virtual users for k6 (default: 100)
+  --max-devworkspaces <int>                   Maximum number of DevWorkspaces to create (by default, it's not specified)
   --separate-namespaces <true|false>          Use separate namespaces for workspaces (default: false)
+  --delete-devworkspace-after-ready           Delete DevWorkspace once it becomes Ready (default: true)
   --devworkspace-ready-timeout-seconds <int>  Timeout in seconds for workspace to become ready (default: 1200)
   --devworkspace-link <string>                DevWorkspace link (default: empty, opinionated DevWorkspace is created)
   --create-automount-resources <true|false>   Whether to create automount resources (default: false)
@@ -77,6 +81,10 @@ parse_arguments() {
         MAX_VUS="$2"; shift 2;;
       --separate-namespaces)
         SEPARATE_NAMESPACES="$2"; shift 2;;
+      --max-devworkspaces)
+        MAX_DEVWORKSPACES="$2"; shift 2;;
+      --delete-devworkspace-after-ready)
+        DELETE_DEVWORKSPACE_AFTER_READY="$2"; shift 2;;
       --devworkspace-ready-timeout-seconds)
         DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS="$2"; shift 2;;
       --devworkspace-link)
@@ -173,11 +181,31 @@ start_background_watchers() {
   kubectl get dw --watch --all-namespaces \
     >> "${LOGS_DIR}/${TIMESTAMP}_dw_watch.log" 2>&1 &
   PID_DW_WATCH=$!
+
+  echo "📄 Starting periodic failed DevWorkspaces report (every 10s)..."
+  (
+    while true; do
+      POLL_TIMESTAMP=$(date +%Y-%m-%d_%H-%M-%S)
+      kubectl get devworkspaces --all-namespaces -o json | jq -r '
+        ["namespace","name","phase","message"],
+        (.items[]
+          | select(.status.phase == "Failed")
+          | [
+              .metadata.namespace,
+              .metadata.name,
+              .status.phase,
+              (.status.message // "No message")
+            ])
+        | @csv' > "${LOGS_DIR}/dw_failure_report.csv"
+      sleep 10
+    done
+  ) &
+  PID_FAILED_DW_POLL=$!
 }
 
 stop_background_watchers() {
   echo "🛑 Stopping background watchers..."
-  kill "$PID_EVENTS_WATCH" "$PID_DW_WATCH" 2>/dev/null || true
+  kill "$PID_EVENTS_WATCH" "$PID_DW_WATCH" "$PID_FAILED_DW_POLL" 2>/dev/null || true
 }
 
 install_k6_operator() {
@@ -235,6 +263,10 @@ spec:
       value: '${TEST_DURATION_IN_MINUTES}'
     - name: DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS
       value: '${DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS}'
+    - name: DELETE_DEVWORKSPACE_AFTER_READY
+      value: '${DELETE_DEVWORKSPACE_AFTER_READY}'
+    - name: MAX_DEVWORKSPACES
+      value: '${MAX_DEVWORKSPACES}'
 EOF
 }
 
@@ -330,6 +362,8 @@ run_k6_binary_test() {
   MAX_VUS="${MAX_VUS}" \
   TEST_DURATION_IN_MINUTES="${TEST_DURATION_IN_MINUTES}" \
   DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS="${DEV_WORKSPACE_READY_TIMEOUT_IN_SECONDS}" \
+  DELETE_DEVWORKSPACE_AFTER_READY="${DELETE_DEVWORKSPACE_AFTER_READY}" \
+  MAX_DEVWORKSPACES="${MAX_DEVWORKSPACES}" \
   k6 run "${K6_SCRIPT}"
   exit_code=$?
   if [ $exit_code -ne 0 ]; then
@@ -338,4 +372,5 @@ run_k6_binary_test() {
   return 0
 }
 
+trap stop_background_watchers EXIT
 main "$@"