From 644e2b2ab3e50ce1e24f64e4eaac38542c6419e0 Mon Sep 17 00:00:00 2001
From: Cas Donoghue <cas.donoghue@gmail.com>
Date: Tue, 4 Nov 2025 10:53:47 -0800
Subject: [PATCH 1/2] Use logstash-versions.yml for observabilitySRE stack
 versions (#18375)

* Use logstash-versions.yml for observabilitySRE stack versions

Previously the version of logstash checked in to the logstash repo was used to
determine stack version for smoke and acceptance testing of the observabilitySRE
artifact. Given the ES/Filebeat containers are subject to a unified release the
delay between a logstash version bump and those artifacts being available was
causing failures when the smoke/acceptance tests were run. In plugin testing the
way we deal with this is to ensure the logstash-versions.yml file is updated
when stack components are ready. This commit introduces the use of that pattern
to the observabilitySRE artifact testing. The important step will be to set the
STACK_VERSION to correspond to the desired stack version. For example the 9.2
logstash branch should look up `9.current`.

* track stream in versions.yml

* Finalize name/location for versions.yml mapping

* better comment

(cherry picked from commit d9fda379532e8c21f3c86af0bce8786c49ee73c0)
---
 .ci/updatecli/bump-logstash-version.yml | 10 ++++++++++
 .github/workflows/bump-logstash.yml     |  6 +++++-
 ci/observabilitySREacceptance_tests.sh  | 11 +++++++----
 ci/observabilitySREsmoke_tests.sh       |  8 ++++++--
 versions.yml                            |  3 +++
 5 files changed, 31 insertions(+), 7 deletions(-)

diff --git a/.ci/updatecli/bump-logstash-version.yml b/.ci/updatecli/bump-logstash-version.yml
index a100a19aeb8..1dc4186eca3 100644
--- a/.ci/updatecli/bump-logstash-version.yml
+++ b/.ci/updatecli/bump-logstash-version.yml
@@ -56,6 +56,16 @@ targets:
       key: $.logstash-core
       value: '{{ requiredEnv "LOGSTASH_VERSION" }}'
 
+  update_release_track:
+      name: Update logstash-release-track in versions.yml
+      kind: yaml
+      disablesourceinput: true
+      scmid: default
+      spec:
+        file: versions.yml
+        key: $.logstash-release-track
+        value: '{{ requiredEnv "LOGSTASH_RELEASE_TRACK" }}'
+
   update_gemfile_lock_dependency:
     name: Update logstash-core dependency in lockfile
     kind: file
diff --git a/.github/workflows/bump-logstash.yml b/.github/workflows/bump-logstash.yml
index 6679f1ed618..13756814cac 100644
--- a/.github/workflows/bump-logstash.yml
+++ b/.github/workflows/bump-logstash.yml
@@ -11,7 +11,10 @@ on:
         description: 'Logstash branch (example: 9.1)'
         required: true
         type: string
-
+      logstash_release_track:
+        description: 'Release track (example: 9.current, 9.previous, main)'
+        required: true
+        type: string
 jobs:
   bump:
     permissions:
@@ -29,3 +32,4 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           LOGSTASH_BRANCH: "${{ github.event.inputs.logstash_branch }}"
           LOGSTASH_VERSION: "${{ github.event.inputs.logstash_version }}"
+          LOGSTASH_RELEASE_TRACK: "${{ github.event.inputs.logstash_release_track }}"
diff --git a/ci/observabilitySREacceptance_tests.sh b/ci/observabilitySREacceptance_tests.sh
index b999ce1a064..24eb05769f9 100755
--- a/ci/observabilitySREacceptance_tests.sh
+++ b/ci/observabilitySREacceptance_tests.sh
@@ -2,9 +2,12 @@
 
 set -e
 
-QUALIFIED_VERSION="$(.buildkite/scripts/common/qualified-version.sh)"
-export OBSERVABILITY_SRE_IMAGE_VERSION="${OBSERVABILITY_SRE_IMAGE_VERSION:-$QUALIFIED_VERSION}"
-export ELASTICSEARCH_IMAGE_VERSION="${ELASTICSEARCH_IMAGE_VERSION:-$QUALIFIED_VERSION}"
-export FILEBEAT_IMAGE_VERSION="${FILEBEAT_IMAGE_VERSION:-$QUALIFIED_VERSION}"
+# Look up corresponding LOGSTASH_RELEASE_TRACK from versions.yml
+LOGSTASH_RELEASE_TRACK=$(ruby -ryaml -e "puts YAML.load_file('versions.yml')['logstash-release-track']")
+# Use logstash stream to find the corresponding stack verstion from logstash-versions.yml
+STACK_VERSION=$(ruby -ryaml -ropen-uri -e "puts YAML.load(URI.open('https://raw.githubusercontent.com/logstash-plugins/.ci/1.x/logstash-versions.yml'))['snapshots']['${LOGSTASH_RELEASE_TRACK}']")
+export OBSERVABILITY_SRE_IMAGE_VERSION="${OBSERVABILITY_SRE_IMAGE_VERSION:-$STACK_VERSION}"
+export ELASTICSEARCH_IMAGE_VERSION="${ELASTICSEARCH_IMAGE_VERSION:-$STACK_VERSION}"
+export FILEBEAT_IMAGE_VERSION="${FILEBEAT_IMAGE_VERSION:-$STACK_VERSION}"
 
 ./gradlew observabilitySREacceptanceTests --stacktrace
diff --git a/ci/observabilitySREsmoke_tests.sh b/ci/observabilitySREsmoke_tests.sh
index 744b2a79efa..bdd475b5dd1 100755
--- a/ci/observabilitySREsmoke_tests.sh
+++ b/ci/observabilitySREsmoke_tests.sh
@@ -3,8 +3,12 @@
 set -e
 
 QUALIFIED_VERSION="$(.buildkite/scripts/common/qualified-version.sh)"
-export ELASTICSEARCH_IMAGE_VERSION="${ELASTICSEARCH_IMAGE_VERSION:-$QUALIFIED_VERSION}"
-export FILEBEAT_IMAGE_VERSION="${FILEBEAT_IMAGE_VERSION:-$QUALIFIED_VERSION}"
+# Look up corresponding LOGSTASH_RELEASE_TRACK from versions.yml
+LOGSTASH_RELEASE_TRACK=$(ruby -ryaml -e "puts YAML.load_file('versions.yml')['logstash-release-track']")
+# Use logstash stream to find the corresponding stack verstion from logstash-versions.yml
+STACK_VERSION=$(ruby -ryaml -ropen-uri -e "puts YAML.load(URI.open('https://raw.githubusercontent.com/logstash-plugins/.ci/1.x/logstash-versions.yml'))['snapshots']['${LOGSTASH_RELEASE_TRACK}']")
+export ELASTICSEARCH_IMAGE_VERSION="${ELASTICSEARCH_IMAGE_VERSION:-$STACK_VERSION}"
+export FILEBEAT_IMAGE_VERSION="${FILEBEAT_IMAGE_VERSION:-$STACK_VERSION}"
 
 ./gradlew --stacktrace artifactDockerObservabilitySRE -PfedrampHighMode=true
 
diff --git a/versions.yml b/versions.yml
index c6437295160..81492096a92 100644
--- a/versions.yml
+++ b/versions.yml
@@ -3,6 +3,9 @@
 logstash: 8.19.7
 logstash-core: 8.19.7
 logstash-core-plugin-api: 2.1.16
+# Logstash release track corresponding to keys in https://github.com/logstash-plugins/.ci/blob/1.x/logstash-versions.yml
+# For example 9.current, 9.previous, etc
+logstash-release-track: main
 
 bundled_jdk:
   # for AdoptOpenJDK/OpenJDK jdk-14.0.1+7.1, the revision is 14.0.1 while the build is 7.1

From ac8d72fed8c755cfe16a1f6ad3929be0c5e20c89 Mon Sep 17 00:00:00 2001
From: Cas Donoghue <cas.donoghue@gmail.com>
Date: Tue, 4 Nov 2025 11:50:20 -0800
Subject: [PATCH 2/2] Update versions.yml

---
 versions.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/versions.yml b/versions.yml
index 81492096a92..32e3d035c5b 100644
--- a/versions.yml
+++ b/versions.yml
@@ -5,7 +5,7 @@ logstash-core: 8.19.7
 logstash-core-plugin-api: 2.1.16
 # Logstash release track corresponding to keys in https://github.com/logstash-plugins/.ci/blob/1.x/logstash-versions.yml
 # For example 9.current, 9.previous, etc
-logstash-release-track: main
+logstash-release-track: 8.current
 
 bundled_jdk:
   # for AdoptOpenJDK/OpenJDK jdk-14.0.1+7.1, the revision is 14.0.1 while the build is 7.1