[Security Rules] Integrate OOM testing pipeline

.buildkite/scripts/common.sh

@@ -728,7 +728,7 @@ is_pr_affected() {
             return 1
         fi
         if ! is_supported_capability ; then
-            echo "[${package}] PR is not affected: capabilities not mached with the project (${SERVERLESS_PROJECT})"
+            echo "[${package}] PR is not affected: capabilities not matched with the project (${SERVERLESS_PROJECT})"
             return 1
         fi
         if [[ "${package}" == "fleet_server" ]]; then
@@ -763,10 +763,19 @@ is_pr_affected() {
     # Example:
     # https://buildkite.com/elastic/integrations/builds/25606
     # https://github.com/elastic/integrations/pull/13810
-    if git diff --name-only "${commit_merge}" "${to}" | grep -E -v '^(packages/|\.github/(CODEOWNERS|ISSUE_TEMPLATE|PULL_REQUEST_TEMPLATE|workflows/)|CODE_OF_CONDUCT\.md|README\.md|docs/|catalog-info\.yaml|\.buildkite/(pull-requests\.json|pipeline\.schedule-daily\.yml|pipeline\.schedule-weekly\.yml|pipeline\.backport\.yml))' > /dev/null; then
+    if git diff --name-only "${commit_merge}" "${to}" | grep -E -v '^(packages/|\.github/(CODEOWNERS|ISSUE_TEMPLATE|PULL_REQUEST_TEMPLATE|workflows/)|CODE_OF_CONDUCT\.md|README\.md|docs/|catalog-info\.yaml|\.buildkite/(pull-requests\.json|pipeline\.schedule-daily\.yml|pipeline\.schedule-weekly\.yml|pipeline\.backport\.yml|scripts/packages.+\.sh))' > /dev/null; then
         echo "[${package}] PR is affected: found non-package files"
         return 0
     fi
+    echoerr "[${package}] git-diff: check custom package checker script file (${commit_merge}..${to})"
+    # Avoid using "-q" in grep in this pipe, it could cause that some files updated are not detected due to SIGPIPE errors when "set -o pipefail"
+    # Example:
+    # https://buildkite.com/elastic/integrations/builds/25606
+    # https://github.com/elastic/integrations/pull/13810
+    if git diff --name-only "${commit_merge}" "${to}" | grep -E "^\.buildkite/scripts/packages/${package}.sh" > /dev/null; then
+        echo "[${package}] PR is affected: found package checker script changes"
+        return 0
+    fi
     echoerr "[${package}] git-diff: check package files (${commit_merge}..${to})"
     # Avoid using "-q" in grep in this pipe, it could cause that some files updated are not detected due to SIGPIPE errors when "set -o pipefail"
     # Example:
@@ -826,7 +835,8 @@ teardown_test_package() {
 }
 
 list_all_directories() {
-    find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort
+    # find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort
+    find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort | grep -E '^security_detection_engine$'
 }
 
 check_package() {

.buildkite/scripts/packages/security_detection_engine.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# Fetch active Kibana versions
+ACTIVE_KIBANA_VERSIONS=$(curl -sL https://raw.githubusercontent.com/elastic/kibana/main/versions.json | yq '.versions[].version')
+echo "Active Kibana versions: $(echo $ACTIVE_KIBANA_VERSIONS)"
+
+# Extract version spec from the manifest
+KIBANA_REQ=$(yq .conditions.kibana.version ./security_detection_engine/manifest.yml)
+echo "Kibana requirement from the security_detection_engine manifest: $KIBANA_REQ"
+
+# Dump a trivial Go program to filter by semver constrains
+TEMP_DIR=$(mktemp -d)
+SEMVER_FILTER_PATH="$TEMP_DIR/semver.go"
+
+cat <<'GO' > "$SEMVER_FILTER_PATH"
+package main
+
+import (
+	"fmt"
+	"os"
+	"github.com/Masterminds/semver/v3"
+)
+
+func main() {
+	c, _ := semver.NewConstraint(os.Args[1])
+	for _, s := range os.Args[2:] {
+		if v, _ := semver.NewVersion(s); c.Check(v) {
+			fmt.Println(s)
+		}
+	}
+}
+GO
+
+# Capture the "returned" array in STACK_VERSIONS
+read -r -a STACK_VERSIONS <<< $(go run "$SEMVER_FILTER_PATH" "$KIBANA_REQ" $ACTIVE_KIBANA_VERSIONS)
+
+# Trigger OOM testing pipeline for each stack version
+for STACK_VERSION in "${STACK_VERSIONS[@]}"
+do
+  echo "--- [security_detection_engine] Trigger OOM testing pipeline against $STACK_VERSION ECH"
+
+  cat <<YAML | buildkite-agent pipeline upload
+steps:
+  - key: 'run-oom-testing-$(echo "$STACK_VERSION" | sed 's/\./_/g')$BUILDKITE_BUILD_NUMBER'
+    label: ":elastic-cloud::bar_chart: [security_detection_engine] Test for OOM issues against ECH $STACK_VERSION"
+    trigger: "appex-qa-stateful-security-prebuilt-rules-ftr-oom-testing"
+    async: false
+    build:
+      message: "Test security_detection_engine package against $STACK_VERSION ($GITHUB_PR_BASE_OWNER/$GITHUB_PR_BASE_REPO, branch: $GITHUB_PR_BRANCH, commit: $BUILDKITE_COMMIT)"
+      env:
+        STACK_VERSION: $STACK_VERSION
+        ELASTIC_INTEGRATIONS_REPO_COMMIT: $BUILDKITE_COMMIT
+YAML
+done

.buildkite/scripts/test_one_package.sh

@@ -25,14 +25,27 @@ with_kubernetes
 
 use_elastic_package
 
+# Detect an absolute path to the individual package checker script.
+# This avoids issues caused by changing the working directory.
+current_folder=$(dirname "$(realpath $0)")
+custom_package_checker_script_path="$current_folder/packages/$package.sh"
+
+
 pushd packages > /dev/null
 exit_code=0
 if ! process_package "${package}" ; then
     # keep this message as a collapsed group in Buildkite, so it
     # is not hidden by the previous collapsed group.
     echo "--- [${package}] failed"
     exit_code=1
+elif [ -x "$custom_package_checker_script_path" ]; then
+  echo "--- [${package}] Run individual package checker"
+  "$custom_package_checker_script_path"
+else
+  echo "--- [${package}] Individual package checker $custom_package_checker_script_path is not found, continue..."
 fi
 popd > /dev/null
 
+pwd
+
 exit "${exit_code}"