From e215cb0867c935a2c45d5e31ecf52f5074a5b789 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Thu, 30 Oct 2025 12:12:33 +0530 Subject: [PATCH 01/46] ci: add e2e tests for docker and podman --- .github/workflows/ci-main.yaml | 99 ++++++++++++------- .../tasks/install_docker_debian.yml | 6 ++ ansible/tasks/docker/deploy_app.yml | 4 +- ansible/tasks/podman/deploy_app.yml | 4 +- 4 files changed, 73 insertions(+), 40 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index a2cdbb3280..d8ecbd5c2a 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -178,14 +178,13 @@ jobs: scanners: "vuln" test-container: + strategy: + matrix: + deployment_type: ["docker", "podman"] runs-on: ubuntu-latest needs: - meta - build_action - # runs all of the steps inside the specified container rather than on the VM host. - # Because of this the network configuration changes from host based network to a container network. - container: - image: python:3.9-buster services: splunk: @@ -200,42 +199,64 @@ jobs: SPLUNK_START_ARGS: --accept-license SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz - sc4s: - image: ${{ needs.meta.outputs.container_base }} - ports: - - 514:514 - - 601:601 - - 5614:5514 - - 5601:5601 - - 6000:6000 - - 6002:6002 - - 9000:9000 - env: - SC4S_DEST_SPLUNK_HEC_DEFAULT_URL: https://splunk:8088 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY: "no" - SC4S_DEST_SPLUNK_HEC_DEFAULT_HTTP_COMPRESSION: "yes" - SC4S_LISTEN_PFSENSE_FIREWALL_TCP_PORT: 6000 - SC4S_LISTEN_SIMPLE_TEST_ONE_TCP_PORT: 5514 - SC4S_LISTEN_SIMPLE_TEST_ONE_UDP_PORT: 5514 - SC4S_LISTEN_SIMPLE_TEST_TWO_TCP_PORT: 5601 - SC4S_LISTEN_SPECTRACOM_NTP_TCP_PORT: 6002 - SC4S_LISTEN_CISCO_ESA_TCP_PORT: 9000 - SC4S_LISTEN_RARITAN_DSX_TCP_PORT: 9001 - SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL: "yes" - SC4S_SOURCE_RICOH_SYSLOG_FIXHOST: "yes" - TEST_SC4S_ACTIVATE_EXAMPLES: "yes" - SC4S_DEBUG_CONTAINER: "yes" - SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG: "yes" - SC4S_NETAPP_ONTAP_NEW_FORMAT: "yes" - SC4S_USE_VPS_CACHE: "yes" - steps: - name: Checkout uses: actions/checkout@v4 with: submodules: false persist-credentials: false + - name: Install Ansible and other dependencies as python package + run: | + pip install ansible~=6.1.0 --no-cache-dir \ + && pip install pywinrm>=0.4.2 --no-cache-dir \ + && pip install ansible-lint>=6.0.0 --no-cache-dir \ + && pip install docker + - name: Configure Ansible Environment Variables + env: + ANSIBLE_CONFIG: ansible.cfg + ANSIBLE_HOST_KEY_CHECKING: False + run: | + echo "ANSIBLE_CONFIG is set to: $ANSIBLE_CONFIG" + echo "ANSIBLE_HOST_KEY_CHECKING is set to: $ANSIBLE_HOST_KEY_CHECKING" + - name: Update inventory file + run: | + cat << EOF > ansible/inventory/inventory.yaml + --- + all: + hosts: + children: + node: + hosts: + node_1: + ansible_host: 127.0.0.1 + ansible_connection: local + ansible_user: root + - name: Update env_file + run: | + echo "Updating ansible/inventory/inventory.yaml" + cat << EOF > ansible/resources/env_file + SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://127.0.0.1:8088 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no + + - name: Debug + run: | + whoami + uname -a + cat /etc/os-release + cat ansible/inventory/inventory.yaml + cat ansible/resources/env_file + docker ps + - name: Run Ansible Playbook + run: | + ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/${{ matrix.deployment_type }}.yml + - name: Debug 2 + run: | + whoami + uname -a + docker ps + systemctl status sc4s + docker images - name: Run tests run: | pip3 install poetry @@ -244,11 +265,17 @@ jobs: poetry run pytest -v --tb=long \ --splunk_type=external \ --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ - --splunk_host=splunk \ - --sc4s_host=sc4s \ + --splunk_host=127.0.0.1 \ + --sc4s_host=127.0.0.1 \ --junitxml=test-results/test.xml \ -n 14 \ -k 'not lite and not name_cache' + - name: artifact-test-results + uses: actions/upload-artifact@v4 + with: + name: test-results-xml + path: test-results/test.xml + if: ${{ !cancelled() }} test-ipv4-name-cache: runs-on: ubuntu-latest diff --git a/ansible/roles/install_docker/tasks/install_docker_debian.yml b/ansible/roles/install_docker/tasks/install_docker_debian.yml index 21a517907d..d3d434a774 100644 --- a/ansible/roles/install_docker/tasks/install_docker_debian.yml +++ b/ansible/roles/install_docker/tasks/install_docker_debian.yml @@ -23,10 +23,16 @@ repo: deb https://download.docker.com/linux/ubuntu focal stable state: present +- name: Check if 'docker' command exists + ansible.builtin.command: which docker + register: docker_command_check + ignore_errors: true + - name: Install docker-ce apt: name: docker-ce state: latest + when: docker_command_check.rc != 0 - name: Install Docker Module for Python pip: diff --git a/ansible/tasks/docker/deploy_app.yml b/ansible/tasks/docker/deploy_app.yml index 50462fac88..85ad163724 100644 --- a/ansible/tasks/docker/deploy_app.yml +++ b/ansible/tasks/docker/deploy_app.yml @@ -12,7 +12,7 @@ - name: Copying unit file on the server copy: - src: "/opt/ansible/resources/docker_sc4s.service" + src: "../../../ansible/resources/docker_sc4s.service" dest: "/lib/systemd/system/sc4s.service" owner: "{{ ansible_user }}" group: "{{ ansible_user }}" @@ -20,7 +20,7 @@ - name: Copying env_file file on the server copy: - src: "/opt/ansible/resources/env_file" + src: "../../../ansible/resources/env_file" dest: "/opt/sc4s/env_file" owner: "{{ ansible_user }}" group: "{{ ansible_user }}" diff --git a/ansible/tasks/podman/deploy_app.yml b/ansible/tasks/podman/deploy_app.yml index 07632d8eaa..f74fe85807 100644 --- a/ansible/tasks/podman/deploy_app.yml +++ b/ansible/tasks/podman/deploy_app.yml @@ -12,7 +12,7 @@ - name: Copying unit file on the server copy: - src: "/opt/ansible/resources/podman_sc4s.service" + src: "../../../ansible/resources/podman_sc4s.service" dest: "/lib/systemd/system/sc4s.service" owner: "{{ ansible_user }}" group: "{{ ansible_user }}" @@ -20,7 +20,7 @@ - name: Copying env_file file on the server copy: - src: "/opt/ansible/resources/env_file" + src: "../../../ansible/resources/env_file" dest: "/opt/sc4s/env_file" owner: "{{ ansible_user }}" group: "{{ ansible_user }}" From a895796be70e5c8d0c186b7fcf54ec5bd0c3d583 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Thu, 30 Oct 2025 14:07:45 +0530 Subject: [PATCH 02/46] chore: fix test failures in docker e2e --- .github/workflows/ci-main.yaml | 15 +++++++++++++++ ansible/resources/docker_sc4s.service | 7 +++++++ 2 files changed, 22 insertions(+) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index d8ecbd5c2a..b1540c41fb 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -238,6 +238,21 @@ jobs: SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://127.0.0.1:8088 SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630 SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no + SC4S_DEST_SPLUNK_HEC_DEFAULT_HTTP_COMPRESSION=yes + SC4S_LISTEN_PFSENSE_FIREWALL_TCP_PORT=6000 + SC4S_LISTEN_SIMPLE_TEST_ONE_TCP_PORT=5514 + SC4S_LISTEN_SIMPLE_TEST_ONE_UDP_PORT=5514 + SC4S_LISTEN_SIMPLE_TEST_TWO_TCP_PORT=5601 + SC4S_LISTEN_SPECTRACOM_NTP_TCP_PORT=6002 + SC4S_LISTEN_CISCO_ESA_TCP_PORT=9000 + SC4S_LISTEN_RARITAN_DSX_TCP_PORT=9001 + SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL=yes + SC4S_SOURCE_RICOH_SYSLOG_FIXHOST=yes + TEST_SC4S_ACTIVATE_EXAMPLES=yes + SC4S_DEBUG_CONTAINER=yes + SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG=yes + SC4S_NETAPP_ONTAP_NEW_FORMAT=yes + SC4S_USE_VPS_CACHE=yes - name: Debug run: | diff --git a/ansible/resources/docker_sc4s.service b/ansible/resources/docker_sc4s.service index 6b18a46cfd..bb6d490abd 100644 --- a/ansible/resources/docker_sc4s.service +++ b/ansible/resources/docker_sc4s.service @@ -38,6 +38,13 @@ ExecStart=/usr/bin/docker run \ -v "$SC4S_LOCAL_MOUNT" \ -v "$SC4S_ARCHIVE_MOUNT" \ -v "$SC4S_TLS_MOUNT" \ + -p 514:514 \ + -p 601:601 \ + -p 5614:5514 \ + -p 5601:5601 \ + -p 6000:6000 \ + -p 6002:6002 \ + -p 9000:9000 \ --env-file=/opt/sc4s/env_file \ --network host \ --name SC4S \ From d76404fb727a3e65670c8d12b58937b868bebe6c Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Thu, 30 Oct 2025 14:16:58 +0530 Subject: [PATCH 03/46] chore: make changes to fix issues in execution --- .github/workflows/ci-main.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index b1540c41fb..5c15ab86e2 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -270,8 +270,6 @@ jobs: whoami uname -a docker ps - systemctl status sc4s - docker images - name: Run tests run: | pip3 install poetry From 101d191f7ad606ff569c72eb389a2b2d6dbe608c Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Thu, 30 Oct 2025 14:27:41 +0530 Subject: [PATCH 04/46] chore: test --- .github/workflows/ci-main.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 5c15ab86e2..287d3be275 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -270,6 +270,7 @@ jobs: whoami uname -a docker ps + sleep 60 - name: Run tests run: | pip3 install poetry From d4ea706c39235fafed39ccac0605553df7dba043 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Thu, 30 Oct 2025 14:38:42 +0530 Subject: [PATCH 05/46] chore: test --- .github/workflows/ci-main.yaml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 287d3be275..2457e268fb 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -238,21 +238,6 @@ jobs: SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://127.0.0.1:8088 SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630 SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no - SC4S_DEST_SPLUNK_HEC_DEFAULT_HTTP_COMPRESSION=yes - SC4S_LISTEN_PFSENSE_FIREWALL_TCP_PORT=6000 - SC4S_LISTEN_SIMPLE_TEST_ONE_TCP_PORT=5514 - SC4S_LISTEN_SIMPLE_TEST_ONE_UDP_PORT=5514 - SC4S_LISTEN_SIMPLE_TEST_TWO_TCP_PORT=5601 - SC4S_LISTEN_SPECTRACOM_NTP_TCP_PORT=6002 - SC4S_LISTEN_CISCO_ESA_TCP_PORT=9000 - SC4S_LISTEN_RARITAN_DSX_TCP_PORT=9001 - SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL=yes - SC4S_SOURCE_RICOH_SYSLOG_FIXHOST=yes - TEST_SC4S_ACTIVATE_EXAMPLES=yes - SC4S_DEBUG_CONTAINER=yes - SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG=yes - SC4S_NETAPP_ONTAP_NEW_FORMAT=yes - SC4S_USE_VPS_CACHE=yes - name: Debug run: | From 29a1c5fbd0ba10562dfbea519132e2dc72540963 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Thu, 30 Oct 2025 14:46:50 +0530 Subject: [PATCH 06/46] chore: test --- .github/workflows/ci-main.yaml | 15 +++++++++++++++ ansible/resources/docker_sc4s.service | 7 ------- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 2457e268fb..287d3be275 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -238,6 +238,21 @@ jobs: SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://127.0.0.1:8088 SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630 SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no + SC4S_DEST_SPLUNK_HEC_DEFAULT_HTTP_COMPRESSION=yes + SC4S_LISTEN_PFSENSE_FIREWALL_TCP_PORT=6000 + SC4S_LISTEN_SIMPLE_TEST_ONE_TCP_PORT=5514 + SC4S_LISTEN_SIMPLE_TEST_ONE_UDP_PORT=5514 + SC4S_LISTEN_SIMPLE_TEST_TWO_TCP_PORT=5601 + SC4S_LISTEN_SPECTRACOM_NTP_TCP_PORT=6002 + SC4S_LISTEN_CISCO_ESA_TCP_PORT=9000 + SC4S_LISTEN_RARITAN_DSX_TCP_PORT=9001 + SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL=yes + SC4S_SOURCE_RICOH_SYSLOG_FIXHOST=yes + TEST_SC4S_ACTIVATE_EXAMPLES=yes + SC4S_DEBUG_CONTAINER=yes + SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG=yes + SC4S_NETAPP_ONTAP_NEW_FORMAT=yes + SC4S_USE_VPS_CACHE=yes - name: Debug run: | diff --git a/ansible/resources/docker_sc4s.service b/ansible/resources/docker_sc4s.service index bb6d490abd..6b18a46cfd 100644 --- a/ansible/resources/docker_sc4s.service +++ b/ansible/resources/docker_sc4s.service @@ -38,13 +38,6 @@ ExecStart=/usr/bin/docker run \ -v "$SC4S_LOCAL_MOUNT" \ -v "$SC4S_ARCHIVE_MOUNT" \ -v "$SC4S_TLS_MOUNT" \ - -p 514:514 \ - -p 601:601 \ - -p 5614:5514 \ - -p 5601:5601 \ - -p 6000:6000 \ - -p 6002:6002 \ - -p 9000:9000 \ --env-file=/opt/sc4s/env_file \ --network host \ --name SC4S \ From 2a6bc249e530259ab4e217061d43fd168e5a0d5c Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Mon, 3 Nov 2025 12:17:08 +0530 Subject: [PATCH 07/46] chore: update SC4S_IMAGE settings --- .github/workflows/ci-main.yaml | 16 +--------------- ansible/resources/docker_sc4s.service | 1 - ansible/resources/podman_sc4s.service | 1 - 3 files changed, 1 insertion(+), 17 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 287d3be275..6c8810d326 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -218,6 +218,7 @@ jobs: run: | echo "ANSIBLE_CONFIG is set to: $ANSIBLE_CONFIG" echo "ANSIBLE_HOST_KEY_CHECKING is set to: $ANSIBLE_HOST_KEY_CHECKING" + systemctl set-environment SC4S_IMAGE=${{ needs.meta.outputs.container_base }} - name: Update inventory file run: | cat << EOF > ansible/inventory/inventory.yaml @@ -253,24 +254,9 @@ jobs: SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG=yes SC4S_NETAPP_ONTAP_NEW_FORMAT=yes SC4S_USE_VPS_CACHE=yes - - - name: Debug - run: | - whoami - uname -a - cat /etc/os-release - cat ansible/inventory/inventory.yaml - cat ansible/resources/env_file - docker ps - name: Run Ansible Playbook run: | ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/${{ matrix.deployment_type }}.yml - - name: Debug 2 - run: | - whoami - uname -a - docker ps - sleep 60 - name: Run tests run: | pip3 install poetry diff --git a/ansible/resources/docker_sc4s.service b/ansible/resources/docker_sc4s.service index 6b18a46cfd..348c5e070d 100644 --- a/ansible/resources/docker_sc4s.service +++ b/ansible/resources/docker_sc4s.service @@ -8,7 +8,6 @@ Requires=docker.service WantedBy=multi-user.target [Service] -Environment="SC4S_IMAGE=ghcr.io/splunk/splunk-connect-for-syslog/container3:latest" # Required mount point for syslog-ng persist data (including disk buffer) Environment="SC4S_PERSIST_MOUNT=splunk-sc4s-var:/var/lib/syslog-ng" diff --git a/ansible/resources/podman_sc4s.service b/ansible/resources/podman_sc4s.service index 657f081a59..a671f24b13 100644 --- a/ansible/resources/podman_sc4s.service +++ b/ansible/resources/podman_sc4s.service @@ -7,7 +7,6 @@ After=NetworkManager.service network-online.target WantedBy=multi-user.target [Service] -Environment="SC4S_IMAGE=ghcr.io/splunk/splunk-connect-for-syslog/container3:latest" # Required mount point for syslog-ng persist data (including disk buffer) Environment="SC4S_PERSIST_MOUNT=splunk-sc4s-var:/var/lib/syslog-ng" From d74ab64d8c6791889b55228d66b3397f44d0f5b4 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Mon, 3 Nov 2025 12:22:05 +0530 Subject: [PATCH 08/46] chore: use sudo --- .github/workflows/ci-main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 6c8810d326..7b0c800580 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -218,7 +218,7 @@ jobs: run: | echo "ANSIBLE_CONFIG is set to: $ANSIBLE_CONFIG" echo "ANSIBLE_HOST_KEY_CHECKING is set to: $ANSIBLE_HOST_KEY_CHECKING" - systemctl set-environment SC4S_IMAGE=${{ needs.meta.outputs.container_base }} + sudo systemctl set-environment SC4S_IMAGE=${{ needs.meta.outputs.container_base }} - name: Update inventory file run: | cat << EOF > ansible/inventory/inventory.yaml From 66d4f2834095603c1f74b36c23f174f7bd69389c Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Mon, 3 Nov 2025 12:30:24 +0530 Subject: [PATCH 09/46] chore: test without setting ENV --- .github/workflows/ci-main.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 7b0c800580..0c321b662c 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -218,7 +218,6 @@ jobs: run: | echo "ANSIBLE_CONFIG is set to: $ANSIBLE_CONFIG" echo "ANSIBLE_HOST_KEY_CHECKING is set to: $ANSIBLE_HOST_KEY_CHECKING" - sudo systemctl set-environment SC4S_IMAGE=${{ needs.meta.outputs.container_base }} - name: Update inventory file run: | cat << EOF > ansible/inventory/inventory.yaml From ff12c40bbf60f5312c9f445e1129939c530d47e8 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Mon, 3 Nov 2025 12:52:06 +0530 Subject: [PATCH 10/46] chore: test named cache --- .github/workflows/ci-main.yaml | 119 ++++++++++++++------------------- 1 file changed, 52 insertions(+), 67 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 0c321b662c..7309efaa2f 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -218,6 +218,7 @@ jobs: run: | echo "ANSIBLE_CONFIG is set to: $ANSIBLE_CONFIG" echo "ANSIBLE_HOST_KEY_CHECKING is set to: $ANSIBLE_HOST_KEY_CHECKING" + sudo systemctl set-environment SC4S_IMAGE=${{ needs.meta.outputs.container_base }} - name: Update inventory file run: | cat << EOF > ansible/inventory/inventory.yaml @@ -276,19 +277,21 @@ jobs: path: test-results/test.xml if: ${{ !cancelled() }} - test-ipv4-name-cache: + test-name-cache: + strategy: + matrix: + deployment_type: ["docker", "podman"] + SC4S_IPV6_ENABLE: ["yes","no"] runs-on: ubuntu-latest needs: - meta - build_action - # runs all of the steps inside the specified container rather than on the VM host. - # Because of this the network configuration changes from host based network to a container network. - container: - image: python:3.9-buster + services: splunk: image: splunk/splunk:${{ fromJson(needs.meta.outputs.matrix_supportedSplunk)[0].version }} ports: + - 8000:8000 - 8088:8088 - 8089:8089 env: @@ -297,76 +300,52 @@ jobs: SPLUNK_START_ARGS: --accept-license SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz - sc4s: - image: ${{ needs.meta.outputs.container_base }} - ports: - - 514:514 - env: - SC4S_DEST_SPLUNK_HEC_DEFAULT_URL: https://splunk:8088 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY: "no" - SC4S_USE_NAME_CACHE: "yes" - SC4S_CLEAR_NAME_CACHE: "yes" - steps: - name: Checkout uses: actions/checkout@v4 with: submodules: false persist-credentials: false - - name: Run tests + - name: Install Ansible and other dependencies as python package run: | - pip3 install poetry - poetry install - mkdir -p test-results || true - poetry run pytest -v --tb=long \ - --splunk_type=external \ - --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ - --splunk_host=splunk \ - --sc4s_host=sc4s \ - --junitxml=test-results/test.xml \ - -n 1 \ - -m 'name_cache' - - test-ipv6-name-cache: - runs-on: ubuntu-latest - needs: - - meta - - build_action - # runs all of the steps inside the specified container rather than on the VM host. - # Because of this the network configuration changes from host based network to a container network. - container: - image: python:3.9-buster - services: - splunk: - image: splunk/splunk:${{ fromJson(needs.meta.outputs.matrix_supportedSplunk)[0].version }} - ports: - - 8088:8088 - - 8089:8089 - env: - SPLUNK_HEC_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 - SPLUNK_PASSWORD: Changed@11 - SPLUNK_START_ARGS: --accept-license - SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz - - sc4s: - image: ${{ needs.meta.outputs.container_base }} - ports: - - 514:514 + pip install ansible~=6.1.0 --no-cache-dir \ + && pip install pywinrm>=0.4.2 --no-cache-dir \ + && pip install ansible-lint>=6.0.0 --no-cache-dir \ + && pip install docker + - name: Configure Ansible Environment Variables env: - SC4S_DEST_SPLUNK_HEC_DEFAULT_URL: https://splunk:8088 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY: "no" + ANSIBLE_CONFIG: ansible.cfg + ANSIBLE_HOST_KEY_CHECKING: False + run: | + echo "ANSIBLE_CONFIG is set to: $ANSIBLE_CONFIG" + echo "ANSIBLE_HOST_KEY_CHECKING is set to: $ANSIBLE_HOST_KEY_CHECKING" + sudo systemctl set-environment SC4S_IMAGE=${{ needs.meta.outputs.container_base }} + - name: Update inventory file + run: | + cat << EOF > ansible/inventory/inventory.yaml + --- + all: + hosts: + children: + node: + hosts: + node_1: + ansible_host: 127.0.0.1 + ansible_connection: local + ansible_user: root + - name: Update env_file + run: | + echo "Updating ansible/inventory/inventory.yaml" + cat << EOF > ansible/resources/env_file + SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://127.0.0.1:8088 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no SC4S_USE_NAME_CACHE: "yes" SC4S_CLEAR_NAME_CACHE: "yes" - SC4S_IPV6_ENABLE: "yes" - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: false - persist-credentials: false + SC4S_IPV6_ENABLE: "${{ matrix.SC4S_IPV6_ENABLE }}" + - name: Run Ansible Playbook + run: | + ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/${{ matrix.deployment_type }}.yml - name: Run tests run: | pip3 install poetry @@ -375,11 +354,17 @@ jobs: poetry run pytest -v --tb=long \ --splunk_type=external \ --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ - --splunk_host=splunk \ - --sc4s_host=sc4s \ + --splunk_host=127.0.0.1 \ + --sc4s_host=127.0.0.1 \ --junitxml=test-results/test.xml \ -n 1 \ -m 'name_cache' + - name: artifact-test-results + uses: actions/upload-artifact@v4 + with: + name: test-results-xml + path: test-results/test.xml + if: ${{ !cancelled() }} test-healthcheck: runs-on: ubuntu-latest From 21364330ae621521177f744510f7771128da63ce Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Mon, 3 Nov 2025 12:53:33 +0530 Subject: [PATCH 11/46] chore: test --- .github/workflows/ci-main.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 7309efaa2f..e172d0dd3d 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -448,8 +448,7 @@ jobs: - meta - build_action - test-container - - test-ipv4-name-cache - - test-ipv6-name-cache + - test-name-cache - test-healthcheck - test-healthcheck-unit-tests - mike From efbe021d0f43f1d5c498fe0702d3b386d5ee9865 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Mon, 3 Nov 2025 13:39:02 +0530 Subject: [PATCH 12/46] chore: also update ci lite --- .github/workflows/ci-lite.yaml | 221 +++++++++++++++++---------------- 1 file changed, 116 insertions(+), 105 deletions(-) diff --git a/.github/workflows/ci-lite.yaml b/.github/workflows/ci-lite.yaml index 79c5066140..0cb31dcdbb 100644 --- a/.github/workflows/ci-lite.yaml +++ b/.github/workflows/ci-lite.yaml @@ -178,14 +178,13 @@ jobs: scanners: "vuln" test-container: + strategy: + matrix: + deployment_type: ["docker", "podman"] runs-on: ubuntu-latest needs: - meta - build_action - # runs all of the steps inside the specified container rather than on the VM host. - # Because of this the network configuration changes from host based network to a container network. - container: - image: python:3.9-buster services: splunk: @@ -200,41 +199,64 @@ jobs: SPLUNK_START_ARGS: --accept-license SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz - sc4s: - image: ${{ needs.meta.outputs.container_base }} - ports: - - 514:514 - - 601:601 - - 5614:5514 - - 5601:5601 - - 6000:6000 - - 6002:6002 - - 9000:9000 - env: - SC4S_DEST_SPLUNK_HEC_DEFAULT_URL: https://splunk:8088 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY: "no" - SC4S_DEST_SPLUNK_HEC_DEFAULT_HTTP_COMPRESSION: "yes" - SC4S_LISTEN_PFSENSE_FIREWALL_TCP_PORT: 6000 - SC4S_LISTEN_SIMPLE_TEST_ONE_TCP_PORT: 5514 - SC4S_LISTEN_SIMPLE_TEST_ONE_UDP_PORT: 5514 - SC4S_LISTEN_SIMPLE_TEST_TWO_TCP_PORT: 5601 - SC4S_LISTEN_SPECTRACOM_NTP_TCP_PORT: 6002 - SC4S_LISTEN_CISCO_ESA_TCP_PORT: 9000 - SC4S_LISTEN_RARITAN_DSX_TCP_PORT: 9001 - SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL: "yes" - SC4S_SOURCE_RICOH_SYSLOG_FIXHOST: "yes" - TEST_SC4S_ACTIVATE_EXAMPLES: "yes" - SC4S_DEBUG_CONTAINER: "yes" - SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG: "yes" - SC4S_NETAPP_ONTAP_NEW_FORMAT: "yes" - SC4S_USE_VPS_CACHE: "yes" steps: - name: Checkout uses: actions/checkout@v4 with: submodules: false persist-credentials: false + - name: Install Ansible and other dependencies as python package + run: | + pip install ansible~=6.1.0 --no-cache-dir \ + && pip install pywinrm>=0.4.2 --no-cache-dir \ + && pip install ansible-lint>=6.0.0 --no-cache-dir \ + && pip install docker + - name: Configure Ansible Environment Variables + env: + ANSIBLE_CONFIG: ansible.cfg + ANSIBLE_HOST_KEY_CHECKING: False + run: | + echo "ANSIBLE_CONFIG is set to: $ANSIBLE_CONFIG" + echo "ANSIBLE_HOST_KEY_CHECKING is set to: $ANSIBLE_HOST_KEY_CHECKING" + sudo systemctl set-environment SC4S_IMAGE=${{ needs.meta.outputs.container_base }} + - name: Update inventory file + run: | + cat << EOF > ansible/inventory/inventory.yaml + --- + all: + hosts: + children: + node: + hosts: + node_1: + ansible_host: 127.0.0.1 + ansible_connection: local + ansible_user: root + - name: Update env_file + run: | + echo "Updating ansible/inventory/inventory.yaml" + cat << EOF > ansible/resources/env_file + SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://127.0.0.1:8088 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no + SC4S_DEST_SPLUNK_HEC_DEFAULT_HTTP_COMPRESSION=yes + SC4S_LISTEN_PFSENSE_FIREWALL_TCP_PORT=6000 + SC4S_LISTEN_SIMPLE_TEST_ONE_TCP_PORT=5514 + SC4S_LISTEN_SIMPLE_TEST_ONE_UDP_PORT=5514 + SC4S_LISTEN_SIMPLE_TEST_TWO_TCP_PORT=5601 + SC4S_LISTEN_SPECTRACOM_NTP_TCP_PORT=6002 + SC4S_LISTEN_CISCO_ESA_TCP_PORT=9000 + SC4S_LISTEN_RARITAN_DSX_TCP_PORT=9001 + SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL=yes + SC4S_SOURCE_RICOH_SYSLOG_FIXHOST=yes + TEST_SC4S_ACTIVATE_EXAMPLES=yes + SC4S_DEBUG_CONTAINER=yes + SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG=yes + SC4S_NETAPP_ONTAP_NEW_FORMAT=yes + SC4S_USE_VPS_CACHE=yes + - name: Run Ansible Playbook + run: | + ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/${{ matrix.deployment_type }}.yml - name: Run tests run: | pip3 install poetry @@ -243,24 +265,32 @@ jobs: poetry run pytest -v --tb=long \ --splunk_type=external \ --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ - --splunk_host=splunk \ - --sc4s_host=sc4s \ + --splunk_host=127.0.0.1 \ + --sc4s_host=127.0.0.1 \ --junitxml=test-results/test.xml \ -n 14 -m "lite or addons" - - test-ipv4-name-cache: + - name: artifact-test-results + uses: actions/upload-artifact@v4 + with: + name: test-results-xml + path: test-results/test.xml + if: ${{ !cancelled() }} + + test-name-cache: + strategy: + matrix: + deployment_type: ["docker", "podman"] + SC4S_IPV6_ENABLE: ["yes","no"] runs-on: ubuntu-latest needs: - meta - build_action - # runs all of the steps inside the specified container rather than on the VM host. - # Because of this the network configuration changes from host based network to a container network. - container: - image: python:3.9-buster + services: splunk: image: splunk/splunk:${{ fromJson(needs.meta.outputs.matrix_supportedSplunk)[0].version }} ports: + - 8000:8000 - 8088:8088 - 8089:8089 env: @@ -269,76 +299,52 @@ jobs: SPLUNK_START_ARGS: --accept-license SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz - sc4s: - image: ${{ needs.meta.outputs.container_base }} - ports: - - 514:514 - env: - SC4S_DEST_SPLUNK_HEC_DEFAULT_URL: https://splunk:8088 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY: "no" - SC4S_USE_NAME_CACHE: "yes" - SC4S_CLEAR_NAME_CACHE: "yes" - steps: - name: Checkout uses: actions/checkout@v4 with: submodules: false persist-credentials: false - - name: Run tests + - name: Install Ansible and other dependencies as python package run: | - pip3 install poetry - poetry install - mkdir -p test-results || true - poetry run pytest -v --tb=long \ - --splunk_type=external \ - --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ - --splunk_host=splunk \ - --sc4s_host=sc4s \ - --junitxml=test-results/test.xml \ - -n 1 \ - -m 'name_cache' - - test-ipv6-name-cache: - runs-on: ubuntu-latest - needs: - - meta - - build_action - # runs all of the steps inside the specified container rather than on the VM host. - # Because of this the network configuration changes from host based network to a container network. - container: - image: python:3.9-buster - services: - splunk: - image: splunk/splunk:${{ fromJson(needs.meta.outputs.matrix_supportedSplunk)[0].version }} - ports: - - 8088:8088 - - 8089:8089 - env: - SPLUNK_HEC_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 - SPLUNK_PASSWORD: Changed@11 - SPLUNK_START_ARGS: --accept-license - SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz - - sc4s: - image: ${{ needs.meta.outputs.container_base }} - ports: - - 514:514 + pip install ansible~=6.1.0 --no-cache-dir \ + && pip install pywinrm>=0.4.2 --no-cache-dir \ + && pip install ansible-lint>=6.0.0 --no-cache-dir \ + && pip install docker + - name: Configure Ansible Environment Variables env: - SC4S_DEST_SPLUNK_HEC_DEFAULT_URL: https://splunk:8088 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY: "no" + ANSIBLE_CONFIG: ansible.cfg + ANSIBLE_HOST_KEY_CHECKING: False + run: | + echo "ANSIBLE_CONFIG is set to: $ANSIBLE_CONFIG" + echo "ANSIBLE_HOST_KEY_CHECKING is set to: $ANSIBLE_HOST_KEY_CHECKING" + sudo systemctl set-environment SC4S_IMAGE=${{ needs.meta.outputs.container_base }} + - name: Update inventory file + run: | + cat << EOF > ansible/inventory/inventory.yaml + --- + all: + hosts: + children: + node: + hosts: + node_1: + ansible_host: 127.0.0.1 + ansible_connection: local + ansible_user: root + - name: Update env_file + run: | + echo "Updating ansible/inventory/inventory.yaml" + cat << EOF > ansible/resources/env_file + SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://127.0.0.1:8088 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no SC4S_USE_NAME_CACHE: "yes" SC4S_CLEAR_NAME_CACHE: "yes" - SC4S_IPV6_ENABLE: "yes" - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: false - persist-credentials: false + SC4S_IPV6_ENABLE: "${{ matrix.SC4S_IPV6_ENABLE }}" + - name: Run Ansible Playbook + run: | + ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/${{ matrix.deployment_type }}.yml - name: Run tests run: | pip3 install poetry @@ -347,11 +353,17 @@ jobs: poetry run pytest -v --tb=long \ --splunk_type=external \ --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ - --splunk_host=splunk \ - --sc4s_host=sc4s \ + --splunk_host=127.0.0.1 \ + --sc4s_host=127.0.0.1 \ --junitxml=test-results/test.xml \ -n 1 \ -m 'name_cache' + - name: artifact-test-results + uses: actions/upload-artifact@v4 + with: + name: test-results-xml + path: test-results/test.xml + if: ${{ !cancelled() }} test-healthcheck: runs-on: ubuntu-latest @@ -412,8 +424,7 @@ jobs: - meta - build_action - test-container - - test-ipv4-name-cache - - test-ipv6-name-cache + - test-name-cache - test-healthcheck - test-healthcheck-unit-tests steps: From 4bc7c5d7f7b1c4ae22146789dc79cc13bdd9347a Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Mon, 3 Nov 2025 14:01:00 +0530 Subject: [PATCH 13/46] chore: test --- .github/workflows/ci-lite.yaml | 2 ++ .github/workflows/ci-main.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/ci-lite.yaml b/.github/workflows/ci-lite.yaml index 0cb31dcdbb..3019b154aa 100644 --- a/.github/workflows/ci-lite.yaml +++ b/.github/workflows/ci-lite.yaml @@ -182,6 +182,7 @@ jobs: matrix: deployment_type: ["docker", "podman"] runs-on: ubuntu-latest + continue-on-error: true needs: - meta - build_action @@ -282,6 +283,7 @@ jobs: deployment_type: ["docker", "podman"] SC4S_IPV6_ENABLE: ["yes","no"] runs-on: ubuntu-latest + continue-on-error: true needs: - meta - build_action diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index e172d0dd3d..951e0f6617 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -182,6 +182,7 @@ jobs: matrix: deployment_type: ["docker", "podman"] runs-on: ubuntu-latest + continue-on-error: true needs: - meta - build_action @@ -283,6 +284,7 @@ jobs: deployment_type: ["docker", "podman"] SC4S_IPV6_ENABLE: ["yes","no"] runs-on: ubuntu-latest + continue-on-error: true needs: - meta - build_action From 043382e48ef9108232024386e6c7775abccc4b31 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Mon, 3 Nov 2025 14:52:00 +0530 Subject: [PATCH 14/46] chore: Update artifacts name and use sed command to update SC4S image --- .github/workflows/ci-lite.yaml | 10 ++++++++-- .github/workflows/ci-main.yaml | 10 ++++++++-- ansible/resources/docker_sc4s.service | 1 + ansible/resources/podman_sc4s.service | 1 + 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-lite.yaml b/.github/workflows/ci-lite.yaml index 3019b154aa..62fed68fca 100644 --- a/.github/workflows/ci-lite.yaml +++ b/.github/workflows/ci-lite.yaml @@ -255,6 +255,9 @@ jobs: SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG=yes SC4S_NETAPP_ONTAP_NEW_FORMAT=yes SC4S_USE_VPS_CACHE=yes + - name: Update current SC4S image in unit file + run: | + sed -i 's/Environment=\"SC4S_IMAGE=ghcr.io\/splunk\/splunk-connect-for-syslog\/container3:latest"/Environment=\"SC4S_IMAGE=${{ needs.meta.outputs.container_base }}\"/' ansible/resources/${{ matrix.deployment_type }}_sc4s.service - name: Run Ansible Playbook run: | ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/${{ matrix.deployment_type }}.yml @@ -273,7 +276,7 @@ jobs: - name: artifact-test-results uses: actions/upload-artifact@v4 with: - name: test-results-xml + name: test-container-results_${{ matrix.deployment_type }}.xml path: test-results/test.xml if: ${{ !cancelled() }} @@ -344,6 +347,9 @@ jobs: SC4S_USE_NAME_CACHE: "yes" SC4S_CLEAR_NAME_CACHE: "yes" SC4S_IPV6_ENABLE: "${{ matrix.SC4S_IPV6_ENABLE }}" + - name: Update current SC4S image in unit file + run: | + sed -i 's/Environment=\"SC4S_IMAGE=ghcr.io\/splunk\/splunk-connect-for-syslog\/container3:latest"/Environment=\"SC4S_IMAGE=${{ needs.meta.outputs.container_base }}\"/' ansible/resources/${{ matrix.deployment_type }}_sc4s.service - name: Run Ansible Playbook run: | ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/${{ matrix.deployment_type }}.yml @@ -363,7 +369,7 @@ jobs: - name: artifact-test-results uses: actions/upload-artifact@v4 with: - name: test-results-xml + name: test-container-results_${{ matrix.deployment_type }}_IPv6_${{ matrix.SC4S_IPV6_ENABLE }}.xml path: test-results/test.xml if: ${{ !cancelled() }} diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 951e0f6617..c99fa78331 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -255,6 +255,9 @@ jobs: SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG=yes SC4S_NETAPP_ONTAP_NEW_FORMAT=yes SC4S_USE_VPS_CACHE=yes + - name: Update current SC4S image in unit file + run: | + sed -i 's/Environment=\"SC4S_IMAGE=ghcr.io\/splunk\/splunk-connect-for-syslog\/container3:latest"/Environment=\"SC4S_IMAGE=${{ needs.meta.outputs.container_base }}\"/' ansible/resources/${{ matrix.deployment_type }}_sc4s.service - name: Run Ansible Playbook run: | ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/${{ matrix.deployment_type }}.yml @@ -274,7 +277,7 @@ jobs: - name: artifact-test-results uses: actions/upload-artifact@v4 with: - name: test-results-xml + name: test-container-results_${{ matrix.deployment_type }}.xml path: test-results/test.xml if: ${{ !cancelled() }} @@ -345,6 +348,9 @@ jobs: SC4S_USE_NAME_CACHE: "yes" SC4S_CLEAR_NAME_CACHE: "yes" SC4S_IPV6_ENABLE: "${{ matrix.SC4S_IPV6_ENABLE }}" + - name: Update current SC4S image in unit file + run: | + sed -i 's/Environment=\"SC4S_IMAGE=ghcr.io\/splunk\/splunk-connect-for-syslog\/container3:latest"/Environment=\"SC4S_IMAGE=${{ needs.meta.outputs.container_base }}\"/' ansible/resources/${{ matrix.deployment_type }}_sc4s.service - name: Run Ansible Playbook run: | ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/${{ matrix.deployment_type }}.yml @@ -364,7 +370,7 @@ jobs: - name: artifact-test-results uses: actions/upload-artifact@v4 with: - name: test-results-xml + name: test-container-results_${{ matrix.deployment_type }}_IPv6_${{ matrix.SC4S_IPV6_ENABLE }}.xml path: test-results/test.xml if: ${{ !cancelled() }} diff --git a/ansible/resources/docker_sc4s.service b/ansible/resources/docker_sc4s.service index 348c5e070d..6b18a46cfd 100644 --- a/ansible/resources/docker_sc4s.service +++ b/ansible/resources/docker_sc4s.service @@ -8,6 +8,7 @@ Requires=docker.service WantedBy=multi-user.target [Service] +Environment="SC4S_IMAGE=ghcr.io/splunk/splunk-connect-for-syslog/container3:latest" # Required mount point for syslog-ng persist data (including disk buffer) Environment="SC4S_PERSIST_MOUNT=splunk-sc4s-var:/var/lib/syslog-ng" diff --git a/ansible/resources/podman_sc4s.service b/ansible/resources/podman_sc4s.service index a671f24b13..657f081a59 100644 --- a/ansible/resources/podman_sc4s.service +++ b/ansible/resources/podman_sc4s.service @@ -7,6 +7,7 @@ After=NetworkManager.service network-online.target WantedBy=multi-user.target [Service] +Environment="SC4S_IMAGE=ghcr.io/splunk/splunk-connect-for-syslog/container3:latest" # Required mount point for syslog-ng persist data (including disk buffer) Environment="SC4S_PERSIST_MOUNT=splunk-sc4s-var:/var/lib/syslog-ng" From e02bc89e848419ba861362742b2476ed07a45b16 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Mon, 3 Nov 2025 15:02:41 +0530 Subject: [PATCH 15/46] chore: test --- .github/workflows/ci-lite.yaml | 4 ++-- .github/workflows/ci-main.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-lite.yaml b/.github/workflows/ci-lite.yaml index 62fed68fca..74425d3a87 100644 --- a/.github/workflows/ci-lite.yaml +++ b/.github/workflows/ci-lite.yaml @@ -257,7 +257,7 @@ jobs: SC4S_USE_VPS_CACHE=yes - name: Update current SC4S image in unit file run: | - sed -i 's/Environment=\"SC4S_IMAGE=ghcr.io\/splunk\/splunk-connect-for-syslog\/container3:latest"/Environment=\"SC4S_IMAGE=${{ needs.meta.outputs.container_base }}\"/' ansible/resources/${{ matrix.deployment_type }}_sc4s.service + sed -i '' 's|Environment="SC4S_IMAGE=ghcr.io/splunk/splunk-connect-for-syslog/container3:latest"|Environment="SC4S_IMAGE=${{ needs.meta.outputs.container_base }}"|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service - name: Run Ansible Playbook run: | ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/${{ matrix.deployment_type }}.yml @@ -349,7 +349,7 @@ jobs: SC4S_IPV6_ENABLE: "${{ matrix.SC4S_IPV6_ENABLE }}" - name: Update current SC4S image in unit file run: | - sed -i 's/Environment=\"SC4S_IMAGE=ghcr.io\/splunk\/splunk-connect-for-syslog\/container3:latest"/Environment=\"SC4S_IMAGE=${{ needs.meta.outputs.container_base }}\"/' ansible/resources/${{ matrix.deployment_type }}_sc4s.service + sed -i '' 's|Environment="SC4S_IMAGE=ghcr.io/splunk/splunk-connect-for-syslog/container3:latest"|Environment="SC4S_IMAGE=${{ needs.meta.outputs.container_base }}"|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service - name: Run Ansible Playbook run: | ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/${{ matrix.deployment_type }}.yml diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index c99fa78331..50568c6030 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -257,7 +257,7 @@ jobs: SC4S_USE_VPS_CACHE=yes - name: Update current SC4S image in unit file run: | - sed -i 's/Environment=\"SC4S_IMAGE=ghcr.io\/splunk\/splunk-connect-for-syslog\/container3:latest"/Environment=\"SC4S_IMAGE=${{ needs.meta.outputs.container_base }}\"/' ansible/resources/${{ matrix.deployment_type }}_sc4s.service + sed -i '' 's|Environment="SC4S_IMAGE=ghcr.io/splunk/splunk-connect-for-syslog/container3:latest"|Environment="SC4S_IMAGE=${{ needs.meta.outputs.container_base }}"|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service - name: Run Ansible Playbook run: | ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/${{ matrix.deployment_type }}.yml @@ -350,7 +350,7 @@ jobs: SC4S_IPV6_ENABLE: "${{ matrix.SC4S_IPV6_ENABLE }}" - name: Update current SC4S image in unit file run: | - sed -i 's/Environment=\"SC4S_IMAGE=ghcr.io\/splunk\/splunk-connect-for-syslog\/container3:latest"/Environment=\"SC4S_IMAGE=${{ needs.meta.outputs.container_base }}\"/' ansible/resources/${{ matrix.deployment_type }}_sc4s.service + sed -i '' 's|Environment="SC4S_IMAGE=ghcr.io/splunk/splunk-connect-for-syslog/container3:latest"|Environment="SC4S_IMAGE=${{ needs.meta.outputs.container_base }}"|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service - name: Run Ansible Playbook run: | ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/${{ matrix.deployment_type }}.yml From 3dadded5314c6bf3b08a881799e8c81d0268a8e9 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Tue, 4 Nov 2025 11:14:57 +0530 Subject: [PATCH 16/46] chore: debug things --- .github/workflows/ci-main.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 50568c6030..fc27662b69 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -257,6 +257,8 @@ jobs: SC4S_USE_VPS_CACHE=yes - name: Update current SC4S image in unit file run: | + pwd + ls -la sed -i '' 's|Environment="SC4S_IMAGE=ghcr.io/splunk/splunk-connect-for-syslog/container3:latest"|Environment="SC4S_IMAGE=${{ needs.meta.outputs.container_base }}"|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service - name: Run Ansible Playbook run: | @@ -350,6 +352,8 @@ jobs: SC4S_IPV6_ENABLE: "${{ matrix.SC4S_IPV6_ENABLE }}" - name: Update current SC4S image in unit file run: | + pwd + ls -la sed -i '' 's|Environment="SC4S_IMAGE=ghcr.io/splunk/splunk-connect-for-syslog/container3:latest"|Environment="SC4S_IMAGE=${{ needs.meta.outputs.container_base }}"|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service - name: Run Ansible Playbook run: | From 4e795d56b1f2c69143fbc324866fa407c19981f9 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Tue, 4 Nov 2025 11:40:06 +0530 Subject: [PATCH 17/46] chore: test --- .github/workflows/ci-main.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index fc27662b69..f475034e16 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -259,6 +259,7 @@ jobs: run: | pwd ls -la + cat ansible/resources/${{ matrix.deployment_type }}_sc4s.service sed -i '' 's|Environment="SC4S_IMAGE=ghcr.io/splunk/splunk-connect-for-syslog/container3:latest"|Environment="SC4S_IMAGE=${{ needs.meta.outputs.container_base }}"|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service - name: Run Ansible Playbook run: | From f30915a88427847d6f77ce393db6f08913a17909 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Tue, 4 Nov 2025 11:50:30 +0530 Subject: [PATCH 18/46] chore: test --- .github/workflows/ci-main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index f475034e16..e312f73b7c 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -260,7 +260,7 @@ jobs: pwd ls -la cat ansible/resources/${{ matrix.deployment_type }}_sc4s.service - sed -i '' 's|Environment="SC4S_IMAGE=ghcr.io/splunk/splunk-connect-for-syslog/container3:latest"|Environment="SC4S_IMAGE=${{ needs.meta.outputs.container_base }}"|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service + sed -i 's|Environment="SC4S_IMAGE=ghcr.io/splunk/splunk-connect-for-syslog/container3:latest"|Environment="SC4S_IMAGE=${{ needs.meta.outputs.container_base }}"|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service - name: Run Ansible Playbook run: | ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/${{ matrix.deployment_type }}.yml From 9acb72e9376a35b62488470b691d3169389991d2 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Tue, 4 Nov 2025 11:57:54 +0530 Subject: [PATCH 19/46] chore: test --- .github/workflows/ci-lite.yaml | 4 ++-- .github/workflows/ci-main.yaml | 7 +------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci-lite.yaml b/.github/workflows/ci-lite.yaml index 74425d3a87..83182a6971 100644 --- a/.github/workflows/ci-lite.yaml +++ b/.github/workflows/ci-lite.yaml @@ -257,7 +257,7 @@ jobs: SC4S_USE_VPS_CACHE=yes - name: Update current SC4S image in unit file run: | - sed -i '' 's|Environment="SC4S_IMAGE=ghcr.io/splunk/splunk-connect-for-syslog/container3:latest"|Environment="SC4S_IMAGE=${{ needs.meta.outputs.container_base }}"|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service + sed -i 's|Environment="SC4S_IMAGE=ghcr.io/splunk/splunk-connect-for-syslog/container3:latest"|Environment="SC4S_IMAGE=${{ needs.meta.outputs.container_base }}"|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service - name: Run Ansible Playbook run: | ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/${{ matrix.deployment_type }}.yml @@ -349,7 +349,7 @@ jobs: SC4S_IPV6_ENABLE: "${{ matrix.SC4S_IPV6_ENABLE }}" - name: Update current SC4S image in unit file run: | - sed -i '' 's|Environment="SC4S_IMAGE=ghcr.io/splunk/splunk-connect-for-syslog/container3:latest"|Environment="SC4S_IMAGE=${{ needs.meta.outputs.container_base }}"|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service + sed -i 's|Environment="SC4S_IMAGE=ghcr.io/splunk/splunk-connect-for-syslog/container3:latest"|Environment="SC4S_IMAGE=${{ needs.meta.outputs.container_base }}"|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service - name: Run Ansible Playbook run: | ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/${{ matrix.deployment_type }}.yml diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index e312f73b7c..71a3adbfad 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -257,9 +257,6 @@ jobs: SC4S_USE_VPS_CACHE=yes - name: Update current SC4S image in unit file run: | - pwd - ls -la - cat ansible/resources/${{ matrix.deployment_type }}_sc4s.service sed -i 's|Environment="SC4S_IMAGE=ghcr.io/splunk/splunk-connect-for-syslog/container3:latest"|Environment="SC4S_IMAGE=${{ needs.meta.outputs.container_base }}"|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service - name: Run Ansible Playbook run: | @@ -353,9 +350,7 @@ jobs: SC4S_IPV6_ENABLE: "${{ matrix.SC4S_IPV6_ENABLE }}" - name: Update current SC4S image in unit file run: | - pwd - ls -la - sed -i '' 's|Environment="SC4S_IMAGE=ghcr.io/splunk/splunk-connect-for-syslog/container3:latest"|Environment="SC4S_IMAGE=${{ needs.meta.outputs.container_base }}"|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service + sed -i 's|Environment="SC4S_IMAGE=ghcr.io/splunk/splunk-connect-for-syslog/container3:latest"|Environment="SC4S_IMAGE=${{ needs.meta.outputs.container_base }}"|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service - name: Run Ansible Playbook run: | ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/${{ matrix.deployment_type }}.yml From 5c5a7a80e10300d73a01114f9180e6fb64c48d6e Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Tue, 4 Nov 2025 13:46:13 +0530 Subject: [PATCH 20/46] chore: Update ci enterprise --- .github/workflows/ci-enterprise.yaml | 220 ++++++++++++++++----------- .github/workflows/ci-lite.yaml | 2 +- .github/workflows/ci-main.yaml | 2 +- 3 files changed, 129 insertions(+), 95 deletions(-) diff --git a/.github/workflows/ci-enterprise.yaml b/.github/workflows/ci-enterprise.yaml index 10afd7a00e..7222bdbfbc 100644 --- a/.github/workflows/ci-enterprise.yaml +++ b/.github/workflows/ci-enterprise.yaml @@ -184,12 +184,15 @@ jobs: scanners: "vuln" test-container: + strategy: + matrix: + deployment_type: ["docker", "podman"] runs-on: ubuntu-latest + continue-on-error: true needs: - meta - build_action - container: - image: python:3.9-buster + services: splunk: image: splunk/splunk:${{ fromJson(needs.meta.outputs.matrix_supportedSplunk)[0].version }} @@ -202,41 +205,68 @@ jobs: SPLUNK_PASSWORD: Changed@11 SPLUNK_START_ARGS: --accept-license SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz - sc4s: - image: ${{ needs.meta.outputs.container_base }} - ports: - - 514:514 - - 601:601 - - 5614:5514 - - 5601:5601 - - 6000:6000 - - 6002:6002 - - 9000:9000 - env: - SC4S_DEST_SPLUNK_HEC_DEFAULT_URL: https://splunk:8088 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY: "no" - SC4S_DEST_SPLUNK_HEC_DEFAULT_HTTP_COMPRESSION: "yes" - SC4S_LISTEN_PFSENSE_FIREWALL_TCP_PORT: 6000 - SC4S_LISTEN_SIMPLE_TEST_ONE_TCP_PORT: 5514 - SC4S_LISTEN_SIMPLE_TEST_ONE_UDP_PORT: 5514 - SC4S_LISTEN_SIMPLE_TEST_TWO_TCP_PORT: 5601 - SC4S_LISTEN_SPECTRACOM_NTP_TCP_PORT: 6002 - SC4S_LISTEN_CISCO_ESA_TCP_PORT: 9000 - SC4S_LISTEN_RARITAN_DSX_TCP_PORT: 9001 - SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL: "yes" - SC4S_SOURCE_RICOH_SYSLOG_FIXHOST: "yes" - TEST_SC4S_ACTIVATE_EXAMPLES: "yes" - SC4S_DEBUG_CONTAINER: "yes" - SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG: "yes" - SC4S_NETAPP_ONTAP_NEW_FORMAT: "yes" - SC4S_USE_VPS_CACHE: "yes" + steps: - name: Checkout uses: actions/checkout@v4 with: submodules: false persist-credentials: false + - name: Install Ansible and other dependencies as python package + run: | + pip install ansible~=6.1.0 --no-cache-dir \ + && pip install pywinrm>=0.4.2 --no-cache-dir \ + && pip install ansible-lint>=6.0.0 --no-cache-dir \ + && pip install docker + - name: Configure Ansible Environment Variables + env: + ANSIBLE_CONFIG: ansible.cfg + ANSIBLE_HOST_KEY_CHECKING: False + run: | + echo "ANSIBLE_CONFIG is set to: $ANSIBLE_CONFIG" + echo "ANSIBLE_HOST_KEY_CHECKING is set to: $ANSIBLE_HOST_KEY_CHECKING" + sudo systemctl set-environment SC4S_IMAGE=${{ needs.meta.outputs.container_base }} + - name: Update inventory file + run: | + cat << EOF > ansible/inventory/inventory.yaml + --- + all: + hosts: + children: + node: + hosts: + node_1: + ansible_host: 127.0.0.1 + ansible_connection: local + ansible_user: root + - name: Update env_file + run: | + echo "Updating ansible/inventory/inventory.yaml" + cat << EOF > ansible/resources/env_file + SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://127.0.0.1:8088 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no + SC4S_DEST_SPLUNK_HEC_DEFAULT_HTTP_COMPRESSION=yes + SC4S_LISTEN_PFSENSE_FIREWALL_TCP_PORT=6000 + SC4S_LISTEN_SIMPLE_TEST_ONE_TCP_PORT=5514 + SC4S_LISTEN_SIMPLE_TEST_ONE_UDP_PORT=5514 + SC4S_LISTEN_SIMPLE_TEST_TWO_TCP_PORT=5601 + SC4S_LISTEN_SPECTRACOM_NTP_TCP_PORT=6002 + SC4S_LISTEN_CISCO_ESA_TCP_PORT=9000 + SC4S_LISTEN_RARITAN_DSX_TCP_PORT=9001 + SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL=yes + SC4S_SOURCE_RICOH_SYSLOG_FIXHOST=yes + TEST_SC4S_ACTIVATE_EXAMPLES=yes + SC4S_DEBUG_CONTAINER=yes + SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG=yes + SC4S_NETAPP_ONTAP_NEW_FORMAT=yes + SC4S_USE_VPS_CACHE=yes + - name: Update current SC4S image in unit file + run: | + sed -i 's|Environment="SC4S_IMAGE=ghcr.io/splunk/splunk-connect-for-syslog/container3:latest"|Environment="SC4S_IMAGE=${{ needs.meta.outputs.container_base }}"|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service + - name: Run Ansible Playbook + run: | + ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/${{ matrix.deployment_type }}.yml - name: Run tests run: | pip3 install poetry @@ -245,22 +275,34 @@ jobs: poetry run pytest -v --tb=long \ --splunk_type=external \ --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ - --splunk_host=splunk \ - --sc4s_host=sc4s \ + --splunk_host=127.0.0.1 \ + --sc4s_host=127.0.0.1 \ --junitxml=test-results/test.xml \ -n 14 \ -k 'not lite and not name_cache' - test-ipv4-name-cache: + - name: artifact-test-results + uses: actions/upload-artifact@v4 + with: + name: test-container-results_${{ matrix.deployment_type }}.xml + path: test-results/test.xml + if: ${{ !cancelled() }} + + test-name-cache: + strategy: + matrix: + deployment_type: ["docker", "podman"] + SC4S_IPV6_ENABLE: ["yes","no"] runs-on: ubuntu-latest + continue-on-error: true needs: - meta - build_action - container: - image: python:3.9-buster + services: splunk: image: splunk/splunk:${{ fromJson(needs.meta.outputs.matrix_supportedSplunk)[0].version }} ports: + - 8000:8000 - 8088:8088 - 8089:8089 env: @@ -268,70 +310,56 @@ jobs: SPLUNK_PASSWORD: Changed@11 SPLUNK_START_ARGS: --accept-license SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz - sc4s: - image: ${{ needs.meta.outputs.container_base }} - ports: - - 514:514 - env: - SC4S_DEST_SPLUNK_HEC_DEFAULT_URL: https://splunk:8088 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY: "no" - SC4S_USE_NAME_CACHE: "yes" - SC4S_CLEAR_NAME_CACHE: "yes" + steps: - name: Checkout uses: actions/checkout@v4 with: submodules: false persist-credentials: false - - name: Run tests + - name: Install Ansible and other dependencies as python package run: | - pip3 install poetry - poetry install - mkdir -p test-results || true - poetry run pytest -v --tb=long \ - --splunk_type=external \ - --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ - --splunk_host=splunk \ - --sc4s_host=sc4s \ - --junitxml=test-results/test.xml \ - -n 1 \ - -m 'name_cache' - test-ipv6-name-cache: - runs-on: ubuntu-latest - needs: - - meta - - build_action - container: - image: python:3.9-buster - services: - splunk: - image: splunk/splunk:${{ fromJson(needs.meta.outputs.matrix_supportedSplunk)[0].version }} - ports: - - 8088:8088 - - 8089:8089 - env: - SPLUNK_HEC_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 - SPLUNK_PASSWORD: Changed@11 - SPLUNK_START_ARGS: --accept-license - SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz - sc4s: - image: ${{ needs.meta.outputs.container_base }} - ports: - - 514:514 + pip install ansible~=6.1.0 --no-cache-dir \ + && pip install pywinrm>=0.4.2 --no-cache-dir \ + && pip install ansible-lint>=6.0.0 --no-cache-dir \ + && pip install docker + - name: Configure Ansible Environment Variables env: - SC4S_DEST_SPLUNK_HEC_DEFAULT_URL: https://splunk:8088 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY: "no" + ANSIBLE_CONFIG: ansible.cfg + ANSIBLE_HOST_KEY_CHECKING: False + run: | + echo "ANSIBLE_CONFIG is set to: $ANSIBLE_CONFIG" + echo "ANSIBLE_HOST_KEY_CHECKING is set to: $ANSIBLE_HOST_KEY_CHECKING" + sudo systemctl set-environment SC4S_IMAGE=${{ needs.meta.outputs.container_base }} + - name: Update inventory file + run: | + cat << EOF > ansible/inventory/inventory.yaml + --- + all: + hosts: + children: + node: + hosts: + node_1: + ansible_host: 127.0.0.1 + ansible_connection: local + ansible_user: root + - name: Update env_file + run: | + echo "Updating ansible/inventory/inventory.yaml" + cat << EOF > ansible/resources/env_file + SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://127.0.0.1:8088 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no SC4S_USE_NAME_CACHE: "yes" SC4S_CLEAR_NAME_CACHE: "yes" - SC4S_IPV6_ENABLE: "yes" - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: false - persist-credentials: false + SC4S_IPV6_ENABLE: "${{ matrix.SC4S_IPV6_ENABLE }}" + - name: Update current SC4S image in unit file + run: | + sed -i 's|Environment="SC4S_IMAGE=ghcr.io/splunk/splunk-connect-for-syslog/container3:latest"|Environment="SC4S_IMAGE=${{ needs.meta.outputs.container_base }}"|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service + - name: Run Ansible Playbook + run: | + ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/${{ matrix.deployment_type }}.yml - name: Run tests run: | pip3 install poetry @@ -340,11 +368,17 @@ jobs: poetry run pytest -v --tb=long \ --splunk_type=external \ --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ - --splunk_host=splunk \ - --sc4s_host=sc4s \ + --splunk_host=127.0.0.1 \ + --sc4s_host=127.0.0.1 \ --junitxml=test-results/test.xml \ -n 1 \ -m 'name_cache' + - name: artifact-test-results + uses: actions/upload-artifact@v4 + with: + name: test-name-cache-results_${{ matrix.deployment_type }}_IPv6_${{ matrix.SC4S_IPV6_ENABLE }}.xml + path: test-results/test.xml + if: ${{ !cancelled() }} release: name: Release runs-on: ubuntu-latest @@ -352,7 +386,7 @@ jobs: - meta - build_action - test-container - - test-ipv4-name-cache + - test-name-cache steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/ci-lite.yaml b/.github/workflows/ci-lite.yaml index 83182a6971..a477dba2b3 100644 --- a/.github/workflows/ci-lite.yaml +++ b/.github/workflows/ci-lite.yaml @@ -369,7 +369,7 @@ jobs: - name: artifact-test-results uses: actions/upload-artifact@v4 with: - name: test-container-results_${{ matrix.deployment_type }}_IPv6_${{ matrix.SC4S_IPV6_ENABLE }}.xml + name: test-name-cache-results_${{ matrix.deployment_type }}_IPv6_${{ matrix.SC4S_IPV6_ENABLE }}.xml path: test-results/test.xml if: ${{ !cancelled() }} diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 71a3adbfad..c403920499 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -370,7 +370,7 @@ jobs: - name: artifact-test-results uses: actions/upload-artifact@v4 with: - name: test-container-results_${{ matrix.deployment_type }}_IPv6_${{ matrix.SC4S_IPV6_ENABLE }}.xml + name: test-name-cache-results_${{ matrix.deployment_type }}_IPv6_${{ matrix.SC4S_IPV6_ENABLE }}.xml path: test-results/test.xml if: ${{ !cancelled() }} From 4573d8656ed5013b4a47c971f39f5e25c1975421 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Tue, 4 Nov 2025 15:06:23 +0530 Subject: [PATCH 21/46] chore: test --- tests/test_checkpoint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_checkpoint.py b/tests/test_checkpoint.py index 4cc6514c21..da89dadfcd 100644 --- a/tests/test_checkpoint.py +++ b/tests/test_checkpoint.py @@ -337,7 +337,7 @@ def test_checkpoint_splunk_os(record_property, setup_splunk, setup_sc4s, get_pid sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( - 'search earliest=-1m@m latest=+1m@m index=osnix "0x{{ pid }}" sourcetype="nix:syslog"' + 'search earliest=-1m@m latest=+1m@m index=* "0x{{ pid }}"' ) search = st.render(host=host, pid=pid) From 2f89bd0f526f71bff2ad94989f66589c79c24a90 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Wed, 5 Nov 2025 11:22:18 +0530 Subject: [PATCH 22/46] chore: test --- tests/test_checkpoint.py | 2 +- tests/test_name_cache.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_checkpoint.py b/tests/test_checkpoint.py index da89dadfcd..f214e0c3ec 100644 --- a/tests/test_checkpoint.py +++ b/tests/test_checkpoint.py @@ -337,7 +337,7 @@ def test_checkpoint_splunk_os(record_property, setup_splunk, setup_sc4s, get_pid sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( - 'search earliest=-1m@m latest=+1m@m index=* "0x{{ pid }}"' + 'search earliest=-1m@m latest=+1m@m index=osnix "0x{{ pid }}"' ) search = st.render(host=host, pid=pid) diff --git a/tests/test_name_cache.py b/tests/test_name_cache.py index bc68b395fb..0e2cd8094a 100644 --- a/tests/test_name_cache.py +++ b/tests/test_name_cache.py @@ -51,7 +51,7 @@ def test_name_cache(get_host_key, setup_splunk, setup_sc4s): time.sleep(1) # time to save the new cache entry epoch = send_message(template_no_host, setup_sc4s) - search = f'search _time="{epoch}" index=* host="{get_host_key}"' + search = f'search _time="{epoch}" index=*"' result_count, _ = splunk_single(setup_splunk, search) assert result_count == 1 From 48896600391301d4bd46c7f67fa3111c7402c3b2 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Wed, 5 Nov 2025 12:03:40 +0530 Subject: [PATCH 23/46] chore: test --- tests/test_checkpoint.py | 2 +- tests/test_name_cache.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_checkpoint.py b/tests/test_checkpoint.py index f214e0c3ec..4cc6514c21 100644 --- a/tests/test_checkpoint.py +++ b/tests/test_checkpoint.py @@ -337,7 +337,7 @@ def test_checkpoint_splunk_os(record_property, setup_splunk, setup_sc4s, get_pid sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( - 'search earliest=-1m@m latest=+1m@m index=osnix "0x{{ pid }}"' + 'search earliest=-1m@m latest=+1m@m index=osnix "0x{{ pid }}" sourcetype="nix:syslog"' ) search = st.render(host=host, pid=pid) diff --git a/tests/test_name_cache.py b/tests/test_name_cache.py index 0e2cd8094a..bc68b395fb 100644 --- a/tests/test_name_cache.py +++ b/tests/test_name_cache.py @@ -51,7 +51,7 @@ def test_name_cache(get_host_key, setup_splunk, setup_sc4s): time.sleep(1) # time to save the new cache entry epoch = send_message(template_no_host, setup_sc4s) - search = f'search _time="{epoch}" index=*"' + search = f'search _time="{epoch}" index=* host="{get_host_key}"' result_count, _ = splunk_single(setup_splunk, search) assert result_count == 1 From 3d1e3ef4c3cacf4f867d803b0aee3a5077bbb3c8 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest Date: Thu, 6 Nov 2025 11:45:15 +0530 Subject: [PATCH 24/46] chore: test --- ansible/tasks/docker/deploy_app.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ansible/tasks/docker/deploy_app.yml b/ansible/tasks/docker/deploy_app.yml index 85ad163724..03c4a7d284 100644 --- a/ansible/tasks/docker/deploy_app.yml +++ b/ansible/tasks/docker/deploy_app.yml @@ -18,6 +18,14 @@ group: "{{ ansible_user }}" mode: u=rw,g=rw +- name: Check current working directory on the target host + ansible.builtin.command: pwd + register: current_directory_output + +- name: Print the working directory + ansible.builtin.debug: + var: current_directory_output.stdout + - name: Copying env_file file on the server copy: src: "../../../ansible/resources/env_file" From 7bf5698580b43d9ad68cfd5a645debd613163e0c Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest <69460835+pdudhaiya-crest@users.noreply.github.com> Date: Tue, 18 Nov 2025 12:06:11 +0530 Subject: [PATCH 25/46] Update ci-main.yaml --- .github/workflows/ci-main.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index c403920499..aeb5a4f9ee 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -335,19 +335,19 @@ jobs: node: hosts: node_1: - ansible_host: 127.0.0.1 + ansible_host: 0.0.0.0 ansible_connection: local ansible_user: root - name: Update env_file run: | echo "Updating ansible/inventory/inventory.yaml" cat << EOF > ansible/resources/env_file - SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://127.0.0.1:8088 + SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://0.0.0.0:8088 SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630 SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no - SC4S_USE_NAME_CACHE: "yes" - SC4S_CLEAR_NAME_CACHE: "yes" - SC4S_IPV6_ENABLE: "${{ matrix.SC4S_IPV6_ENABLE }}" + SC4S_USE_NAME_CACHE=yes + SC4S_CLEAR_NAME_CACHE=yes + SC4S_IPV6_ENABLE=${{ matrix.SC4S_IPV6_ENABLE }} - name: Update current SC4S image in unit file run: | sed -i 's|Environment="SC4S_IMAGE=ghcr.io/splunk/splunk-connect-for-syslog/container3:latest"|Environment="SC4S_IMAGE=${{ needs.meta.outputs.container_base }}"|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service @@ -362,8 +362,8 @@ jobs: poetry run pytest -v --tb=long \ --splunk_type=external \ --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ - --splunk_host=127.0.0.1 \ - --sc4s_host=127.0.0.1 \ + --splunk_host=0.0.0.0 \ + --sc4s_host=0.0.0.0 \ --junitxml=test-results/test.xml \ -n 1 \ -m 'name_cache' From 8274cde9368e3037e41c21739991916849525e1b Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest <69460835+pdudhaiya-crest@users.noreply.github.com> Date: Tue, 18 Nov 2025 12:16:04 +0530 Subject: [PATCH 26/46] Update ci-main.yaml --- .github/workflows/ci-main.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index aeb5a4f9ee..c00428e2d5 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -342,7 +342,7 @@ jobs: run: | echo "Updating ansible/inventory/inventory.yaml" cat << EOF > ansible/resources/env_file - SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://0.0.0.0:8088 + SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://splunk:8088 SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630 SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no SC4S_USE_NAME_CACHE=yes @@ -362,8 +362,8 @@ jobs: poetry run pytest -v --tb=long \ --splunk_type=external \ --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ - --splunk_host=0.0.0.0 \ - --sc4s_host=0.0.0.0 \ + --splunk_host=splunk \ + --sc4s_host=SC4S \ --junitxml=test-results/test.xml \ -n 1 \ -m 'name_cache' From 3e7509fd9ea06c5560a8235576affce29d7d4eee Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest <69460835+pdudhaiya-crest@users.noreply.github.com> Date: Tue, 18 Nov 2025 14:20:53 +0530 Subject: [PATCH 27/46] Update ci-main.yaml --- .github/workflows/ci-main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index c00428e2d5..f718f16ff2 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -363,7 +363,7 @@ jobs: --splunk_type=external \ --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ --splunk_host=splunk \ - --sc4s_host=SC4S \ + --sc4s_host=sc4s \ --junitxml=test-results/test.xml \ -n 1 \ -m 'name_cache' From 1ba9c68877c61ba924595a0af1857d066023aa59 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest <69460835+pdudhaiya-crest@users.noreply.github.com> Date: Tue, 18 Nov 2025 14:21:34 +0530 Subject: [PATCH 28/46] Update docker_sc4s.service --- ansible/resources/docker_sc4s.service | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ansible/resources/docker_sc4s.service b/ansible/resources/docker_sc4s.service index 6b18a46cfd..0f15fbe8c0 100644 --- a/ansible/resources/docker_sc4s.service +++ b/ansible/resources/docker_sc4s.service @@ -40,7 +40,8 @@ ExecStart=/usr/bin/docker run \ -v "$SC4S_TLS_MOUNT" \ --env-file=/opt/sc4s/env_file \ --network host \ + --network-alias sc4s \ --name SC4S \ --rm $SC4S_IMAGE -Restart=on-failure \ No newline at end of file +Restart=on-failure From 2a3c8b8f208f002d7e32da2a51ae116b920bd40a Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest <69460835+pdudhaiya-crest@users.noreply.github.com> Date: Tue, 18 Nov 2025 14:37:59 +0530 Subject: [PATCH 29/46] Update docker_sc4s.service --- ansible/resources/docker_sc4s.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/resources/docker_sc4s.service b/ansible/resources/docker_sc4s.service index 0f15fbe8c0..6a7bc4d6a9 100644 --- a/ansible/resources/docker_sc4s.service +++ b/ansible/resources/docker_sc4s.service @@ -39,7 +39,7 @@ ExecStart=/usr/bin/docker run \ -v "$SC4S_ARCHIVE_MOUNT" \ -v "$SC4S_TLS_MOUNT" \ --env-file=/opt/sc4s/env_file \ - --network host \ + --network ${SC4S_NETWORK} \ --network-alias sc4s \ --name SC4S \ --rm $SC4S_IMAGE From 1eaf90b5cc12fa9e4ba508e14a838d89da130065 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest <69460835+pdudhaiya-crest@users.noreply.github.com> Date: Tue, 18 Nov 2025 14:45:38 +0530 Subject: [PATCH 30/46] Update ci-main.yaml --- .github/workflows/ci-main.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index f718f16ff2..2d56def38f 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -351,6 +351,9 @@ jobs: - name: Update current SC4S image in unit file run: | sed -i 's|Environment="SC4S_IMAGE=ghcr.io/splunk/splunk-connect-for-syslog/container3:latest"|Environment="SC4S_IMAGE=${{ needs.meta.outputs.container_base }}"|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service + - name: Update current SC4S image in unit file + run: | + sed -i 's|--network host|--network ${{ job.container.network }}"|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service - name: Run Ansible Playbook run: | ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/${{ matrix.deployment_type }}.yml From 6a0c6ea6c4c2bcf37400fb9dbf0aa977e117271b Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest <69460835+pdudhaiya-crest@users.noreply.github.com> Date: Tue, 18 Nov 2025 14:46:10 +0530 Subject: [PATCH 31/46] Update docker_sc4s.service --- ansible/resources/docker_sc4s.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/resources/docker_sc4s.service b/ansible/resources/docker_sc4s.service index 6a7bc4d6a9..0f15fbe8c0 100644 --- a/ansible/resources/docker_sc4s.service +++ b/ansible/resources/docker_sc4s.service @@ -39,7 +39,7 @@ ExecStart=/usr/bin/docker run \ -v "$SC4S_ARCHIVE_MOUNT" \ -v "$SC4S_TLS_MOUNT" \ --env-file=/opt/sc4s/env_file \ - --network ${SC4S_NETWORK} \ + --network host \ --network-alias sc4s \ --name SC4S \ --rm $SC4S_IMAGE From b1425e5f7dd68dac5c681658917ceace6ed54aaf Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest <69460835+pdudhaiya-crest@users.noreply.github.com> Date: Tue, 18 Nov 2025 14:49:40 +0530 Subject: [PATCH 32/46] Update ci-main.yaml --- .github/workflows/ci-main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 2d56def38f..a41d3b8b18 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -351,7 +351,7 @@ jobs: - name: Update current SC4S image in unit file run: | sed -i 's|Environment="SC4S_IMAGE=ghcr.io/splunk/splunk-connect-for-syslog/container3:latest"|Environment="SC4S_IMAGE=${{ needs.meta.outputs.container_base }}"|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service - - name: Update current SC4S image in unit file + - name: Update current SC4S network in unit file run: | sed -i 's|--network host|--network ${{ job.container.network }}"|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service - name: Run Ansible Playbook From 0947119cabf2e512cae6f3b4ef022794684f5829 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest <69460835+pdudhaiya-crest@users.noreply.github.com> Date: Tue, 18 Nov 2025 14:53:13 +0530 Subject: [PATCH 33/46] Update ci-main.yaml --- .github/workflows/ci-main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index a41d3b8b18..80a5cd7a5a 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -353,7 +353,7 @@ jobs: sed -i 's|Environment="SC4S_IMAGE=ghcr.io/splunk/splunk-connect-for-syslog/container3:latest"|Environment="SC4S_IMAGE=${{ needs.meta.outputs.container_base }}"|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service - name: Update current SC4S network in unit file run: | - sed -i 's|--network host|--network ${{ job.container.network }}"|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service + sed -i 's|--network host|--network ${{ job.container.network }}|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service - name: Run Ansible Playbook run: | ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/${{ matrix.deployment_type }}.yml From 9506440ff7cadf4f403d4fe954ecc934dc9e1590 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest <69460835+pdudhaiya-crest@users.noreply.github.com> Date: Tue, 18 Nov 2025 15:10:10 +0530 Subject: [PATCH 34/46] Update ci-main.yaml --- .github/workflows/ci-main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 80a5cd7a5a..4b9e789c8e 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -365,7 +365,7 @@ jobs: poetry run pytest -v --tb=long \ --splunk_type=external \ --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ - --splunk_host=splunk \ + --splunk_host=0.0.0.0 \ --sc4s_host=sc4s \ --junitxml=test-results/test.xml \ -n 1 \ From 3f519847127186b45967fac9b39b3ba39f72d927 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest <69460835+pdudhaiya-crest@users.noreply.github.com> Date: Tue, 18 Nov 2025 15:18:44 +0530 Subject: [PATCH 35/46] Update ci-main.yaml --- .github/workflows/ci-main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 4b9e789c8e..5e6e778893 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -365,7 +365,7 @@ jobs: poetry run pytest -v --tb=long \ --splunk_type=external \ --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ - --splunk_host=0.0.0.0 \ + --splunk_host=localhost \ --sc4s_host=sc4s \ --junitxml=test-results/test.xml \ -n 1 \ From f245a9273a0ae57d33297aacd1838a37cc78e6ec Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest <69460835+pdudhaiya-crest@users.noreply.github.com> Date: Wed, 19 Nov 2025 14:19:13 +0530 Subject: [PATCH 36/46] Update ci-main.yaml --- .github/workflows/ci-main.yaml | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 5e6e778893..4f49e02539 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -292,25 +292,15 @@ jobs: - meta - build_action - services: - splunk: - image: splunk/splunk:${{ fromJson(needs.meta.outputs.matrix_supportedSplunk)[0].version }} - ports: - - 8000:8000 - - 8088:8088 - - 8089:8089 - env: - SPLUNK_HEC_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 - SPLUNK_PASSWORD: Changed@11 - SPLUNK_START_ARGS: --accept-license - SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz - steps: - name: Checkout uses: actions/checkout@v4 with: submodules: false persist-credentials: false + - name: Setup Splunk + run: | + docker run --network host --network-alias splunk -p 8000:8000 -p 8088:8088 -p 8089:8089 -e "SPLUNK_HEC_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630" -e "SPLUNK_PASSWORD=Changed@11" -e "SPLUNK_START_ARGS=--accept-license" -e "SPLUNK_APPS_URL=https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz" -e GITHUB_ACTIONS=true -e CI=true splunk/splunk:9.4.0 - name: Install Ansible and other dependencies as python package run: | pip install ansible~=6.1.0 --no-cache-dir \ @@ -351,9 +341,9 @@ jobs: - name: Update current SC4S image in unit file run: | sed -i 's|Environment="SC4S_IMAGE=ghcr.io/splunk/splunk-connect-for-syslog/container3:latest"|Environment="SC4S_IMAGE=${{ needs.meta.outputs.container_base }}"|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service - - name: Update current SC4S network in unit file - run: | - sed -i 's|--network host|--network ${{ job.container.network }}|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service + # - name: Update current SC4S network in unit file + # run: | + # sed -i 's|--network host|--network ${{ job.container.network }}|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service - name: Run Ansible Playbook run: | ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/${{ matrix.deployment_type }}.yml From ef1197bdf609175b99b186a722f4277e21979a3c Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest <69460835+pdudhaiya-crest@users.noreply.github.com> Date: Wed, 19 Nov 2025 14:28:06 +0530 Subject: [PATCH 37/46] Update ci-main.yaml --- .github/workflows/ci-main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 4f49e02539..9aad584c07 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -300,7 +300,7 @@ jobs: persist-credentials: false - name: Setup Splunk run: | - docker run --network host --network-alias splunk -p 8000:8000 -p 8088:8088 -p 8089:8089 -e "SPLUNK_HEC_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630" -e "SPLUNK_PASSWORD=Changed@11" -e "SPLUNK_START_ARGS=--accept-license" -e "SPLUNK_APPS_URL=https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz" -e GITHUB_ACTIONS=true -e CI=true splunk/splunk:9.4.0 + docker run --network test_network --network-alias splunk -p 8000:8000 -p 8088:8088 -p 8089:8089 -e "SPLUNK_HEC_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630" -e "SPLUNK_PASSWORD=Changed@11" -e "SPLUNK_START_ARGS=--accept-license" -e "SPLUNK_APPS_URL=https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz" -e GITHUB_ACTIONS=true -e CI=true splunk/splunk:9.4.0 - name: Install Ansible and other dependencies as python package run: | pip install ansible~=6.1.0 --no-cache-dir \ From 911420335f41e51868e9605c6088508b8e688356 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest <69460835+pdudhaiya-crest@users.noreply.github.com> Date: Wed, 19 Nov 2025 14:28:33 +0530 Subject: [PATCH 38/46] Update docker_sc4s.service --- ansible/resources/docker_sc4s.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/resources/docker_sc4s.service b/ansible/resources/docker_sc4s.service index 0f15fbe8c0..4c973d0ebb 100644 --- a/ansible/resources/docker_sc4s.service +++ b/ansible/resources/docker_sc4s.service @@ -39,7 +39,7 @@ ExecStart=/usr/bin/docker run \ -v "$SC4S_ARCHIVE_MOUNT" \ -v "$SC4S_TLS_MOUNT" \ --env-file=/opt/sc4s/env_file \ - --network host \ + --network test_network \ --network-alias sc4s \ --name SC4S \ --rm $SC4S_IMAGE From a1f0ff42d62040492a19e22d56d80fc72e9f135b Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest <69460835+pdudhaiya-crest@users.noreply.github.com> Date: Wed, 19 Nov 2025 14:33:18 +0530 Subject: [PATCH 39/46] Update ci-main.yaml --- .github/workflows/ci-main.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 9aad584c07..091b7baf2b 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -298,6 +298,9 @@ jobs: with: submodules: false persist-credentials: false + - name: Setup Splunk + run: | + docker network create test_network - name: Setup Splunk run: | docker run --network test_network --network-alias splunk -p 8000:8000 -p 8088:8088 -p 8089:8089 -e "SPLUNK_HEC_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630" -e "SPLUNK_PASSWORD=Changed@11" -e "SPLUNK_START_ARGS=--accept-license" -e "SPLUNK_APPS_URL=https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz" -e GITHUB_ACTIONS=true -e CI=true splunk/splunk:9.4.0 From 6c8d7d6fa83b50290876d0f7a932421d66cbff93 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest <69460835+pdudhaiya-crest@users.noreply.github.com> Date: Wed, 19 Nov 2025 14:49:24 +0530 Subject: [PATCH 40/46] Update ci-main.yaml --- .github/workflows/ci-main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 091b7baf2b..6c6f6ee07c 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -303,7 +303,7 @@ jobs: docker network create test_network - name: Setup Splunk run: | - docker run --network test_network --network-alias splunk -p 8000:8000 -p 8088:8088 -p 8089:8089 -e "SPLUNK_HEC_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630" -e "SPLUNK_PASSWORD=Changed@11" -e "SPLUNK_START_ARGS=--accept-license" -e "SPLUNK_APPS_URL=https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz" -e GITHUB_ACTIONS=true -e CI=true splunk/splunk:9.4.0 + docker run -d --network test_network --network-alias splunk -p 8000:8000 -p 8088:8088 -p 8089:8089 -e "SPLUNK_HEC_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630" -e "SPLUNK_PASSWORD=Changed@11" -e "SPLUNK_START_ARGS=--accept-license" -e "SPLUNK_APPS_URL=https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz" -e GITHUB_ACTIONS=true -e CI=true splunk/splunk:9.4.0 - name: Install Ansible and other dependencies as python package run: | pip install ansible~=6.1.0 --no-cache-dir \ From a1e3eb8b0c44b57e61d0b6dc9d5ed0406d636b26 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest <69460835+pdudhaiya-crest@users.noreply.github.com> Date: Wed, 19 Nov 2025 14:55:19 +0530 Subject: [PATCH 41/46] Update ci-main.yaml --- .github/workflows/ci-main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 6c6f6ee07c..bd0d6cfed4 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -358,7 +358,7 @@ jobs: poetry run pytest -v --tb=long \ --splunk_type=external \ --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ - --splunk_host=localhost \ + --splunk_host=splunk \ --sc4s_host=sc4s \ --junitxml=test-results/test.xml \ -n 1 \ From 0752da84d838ea585b1f9c1c51b7a8ad17eecb74 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest <69460835+pdudhaiya-crest@users.noreply.github.com> Date: Wed, 19 Nov 2025 15:20:29 +0530 Subject: [PATCH 42/46] Update ci-main.yaml --- .github/workflows/ci-main.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index bd0d6cfed4..40282167ce 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -350,6 +350,10 @@ jobs: - name: Run Ansible Playbook run: | ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/${{ matrix.deployment_type }}.yml + - name: Update etc hosts + run: | + echo "127.0.0.1 splunk" >> /etc/hosts + echo "127.0.0.1 sc4s" >> /etc/hosts - name: Run tests run: | pip3 install poetry From 93ebb1b1eede5ec08bba8c0dddb20878ef517ff1 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest <69460835+pdudhaiya-crest@users.noreply.github.com> Date: Thu, 20 Nov 2025 11:14:12 +0530 Subject: [PATCH 43/46] Update ci-main.yaml --- .github/workflows/ci-main.yaml | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 40282167ce..ae4c5ce7b0 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -186,7 +186,6 @@ jobs: needs: - meta - build_action - services: splunk: image: splunk/splunk:${{ fromJson(needs.meta.outputs.matrix_supportedSplunk)[0].version }} @@ -199,7 +198,6 @@ jobs: SPLUNK_PASSWORD: Changed@11 SPLUNK_START_ARGS: --accept-license SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz - steps: - name: Checkout uses: actions/checkout@v4 @@ -291,19 +289,24 @@ jobs: needs: - meta - build_action - + services: + splunk: + image: splunk/splunk:${{ fromJson(needs.meta.outputs.matrix_supportedSplunk)[0].version }} + ports: + - 8000:8000 + - 8088:8088 + - 8089:8089 + env: + SPLUNK_HEC_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 + SPLUNK_PASSWORD: Changed@11 + SPLUNK_START_ARGS: --accept-license + SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz steps: - name: Checkout uses: actions/checkout@v4 with: submodules: false persist-credentials: false - - name: Setup Splunk - run: | - docker network create test_network - - name: Setup Splunk - run: | - docker run -d --network test_network --network-alias splunk -p 8000:8000 -p 8088:8088 -p 8089:8089 -e "SPLUNK_HEC_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630" -e "SPLUNK_PASSWORD=Changed@11" -e "SPLUNK_START_ARGS=--accept-license" -e "SPLUNK_APPS_URL=https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz" -e GITHUB_ACTIONS=true -e CI=true splunk/splunk:9.4.0 - name: Install Ansible and other dependencies as python package run: | pip install ansible~=6.1.0 --no-cache-dir \ @@ -352,8 +355,8 @@ jobs: ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/${{ matrix.deployment_type }}.yml - name: Update etc hosts run: | - echo "127.0.0.1 splunk" >> /etc/hosts - echo "127.0.0.1 sc4s" >> /etc/hosts + echo "192.168.1.100 localhost" >> /etc/hosts + cat /etc/hosts - name: Run tests run: | pip3 install poetry @@ -363,7 +366,7 @@ jobs: --splunk_type=external \ --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ --splunk_host=splunk \ - --sc4s_host=sc4s \ + --sc4s_host=192.168.1.100 \ --junitxml=test-results/test.xml \ -n 1 \ -m 'name_cache' From 33dd87948c0eca035d9a1a02b74b5c34b201785e Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest <69460835+pdudhaiya-crest@users.noreply.github.com> Date: Thu, 20 Nov 2025 11:30:14 +0530 Subject: [PATCH 44/46] Update ci-main.yaml --- .github/workflows/ci-main.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index ae4c5ce7b0..c972ee366f 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -353,10 +353,6 @@ jobs: - name: Run Ansible Playbook run: | ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/${{ matrix.deployment_type }}.yml - - name: Update etc hosts - run: | - echo "192.168.1.100 localhost" >> /etc/hosts - cat /etc/hosts - name: Run tests run: | pip3 install poetry @@ -366,7 +362,7 @@ jobs: --splunk_type=external \ --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ --splunk_host=splunk \ - --sc4s_host=192.168.1.100 \ + --sc4s_host=127.0.0.0 \ --junitxml=test-results/test.xml \ -n 1 \ -m 'name_cache' From 31cd5ce9dc53cf4f4bf638224ef5d7f5638910d5 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest <69460835+pdudhaiya-crest@users.noreply.github.com> Date: Thu, 20 Nov 2025 11:37:55 +0530 Subject: [PATCH 45/46] Update ci-main.yaml --- .github/workflows/ci-main.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index c972ee366f..3250425a52 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -338,7 +338,7 @@ jobs: run: | echo "Updating ansible/inventory/inventory.yaml" cat << EOF > ansible/resources/env_file - SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://splunk:8088 + SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://127.0.0.1:8088 SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630 SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no SC4S_USE_NAME_CACHE=yes @@ -361,7 +361,7 @@ jobs: poetry run pytest -v --tb=long \ --splunk_type=external \ --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ - --splunk_host=splunk \ + --splunk_host=127.0.0.1 \ --sc4s_host=127.0.0.0 \ --junitxml=test-results/test.xml \ -n 1 \ From 4f13e7b7c0c55436aad2777cc6ca844aa4bf4554 Mon Sep 17 00:00:00 2001 From: pdudhaiya-crest <69460835+pdudhaiya-crest@users.noreply.github.com> Date: Thu, 20 Nov 2025 11:43:57 +0530 Subject: [PATCH 46/46] Update ci-main.yaml --- .github/workflows/ci-main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 3250425a52..7dcdf23548 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -362,7 +362,7 @@ jobs: --splunk_type=external \ --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ --splunk_host=127.0.0.1 \ - --sc4s_host=127.0.0.0 \ + --sc4s_host=127.0.0.1 \ --junitxml=test-results/test.xml \ -n 1 \ -m 'name_cache'