diff --git a/.github/workflows/ci-enterprise.yaml b/.github/workflows/ci-enterprise.yaml index 10afd7a00e..7222bdbfbc 100644 --- a/.github/workflows/ci-enterprise.yaml +++ b/.github/workflows/ci-enterprise.yaml @@ -184,12 +184,15 @@ jobs: scanners: "vuln" test-container: + strategy: + matrix: + deployment_type: ["docker", "podman"] runs-on: ubuntu-latest + continue-on-error: true needs: - meta - build_action - container: - image: python:3.9-buster + services: splunk: image: splunk/splunk:${{ fromJson(needs.meta.outputs.matrix_supportedSplunk)[0].version }} @@ -202,41 +205,68 @@ jobs: SPLUNK_PASSWORD: Changed@11 SPLUNK_START_ARGS: --accept-license SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz - sc4s: - image: ${{ needs.meta.outputs.container_base }} - ports: - - 514:514 - - 601:601 - - 5614:5514 - - 5601:5601 - - 6000:6000 - - 6002:6002 - - 9000:9000 - env: - SC4S_DEST_SPLUNK_HEC_DEFAULT_URL: https://splunk:8088 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY: "no" - SC4S_DEST_SPLUNK_HEC_DEFAULT_HTTP_COMPRESSION: "yes" - SC4S_LISTEN_PFSENSE_FIREWALL_TCP_PORT: 6000 - SC4S_LISTEN_SIMPLE_TEST_ONE_TCP_PORT: 5514 - SC4S_LISTEN_SIMPLE_TEST_ONE_UDP_PORT: 5514 - SC4S_LISTEN_SIMPLE_TEST_TWO_TCP_PORT: 5601 - SC4S_LISTEN_SPECTRACOM_NTP_TCP_PORT: 6002 - SC4S_LISTEN_CISCO_ESA_TCP_PORT: 9000 - SC4S_LISTEN_RARITAN_DSX_TCP_PORT: 9001 - SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL: "yes" - SC4S_SOURCE_RICOH_SYSLOG_FIXHOST: "yes" - TEST_SC4S_ACTIVATE_EXAMPLES: "yes" - SC4S_DEBUG_CONTAINER: "yes" - SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG: "yes" - SC4S_NETAPP_ONTAP_NEW_FORMAT: "yes" - SC4S_USE_VPS_CACHE: "yes" + steps: - name: Checkout uses: actions/checkout@v4 with: submodules: false persist-credentials: false + - name: Install Ansible and other dependencies as python package + run: | + pip install ansible~=6.1.0 --no-cache-dir \ + && pip install pywinrm>=0.4.2 --no-cache-dir \ + && pip install ansible-lint>=6.0.0 --no-cache-dir \ + && pip install docker + - name: Configure Ansible Environment Variables + env: + ANSIBLE_CONFIG: ansible.cfg + ANSIBLE_HOST_KEY_CHECKING: False + run: | + echo "ANSIBLE_CONFIG is set to: $ANSIBLE_CONFIG" + echo "ANSIBLE_HOST_KEY_CHECKING is set to: $ANSIBLE_HOST_KEY_CHECKING" + sudo systemctl set-environment SC4S_IMAGE=${{ needs.meta.outputs.container_base }} + - name: Update inventory file + run: | + cat << EOF > ansible/inventory/inventory.yaml + --- + all: + hosts: + children: + node: + hosts: + node_1: + ansible_host: 127.0.0.1 + ansible_connection: local + ansible_user: root + - name: Update env_file + run: | + echo "Updating ansible/inventory/inventory.yaml" + cat << EOF > ansible/resources/env_file + SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://127.0.0.1:8088 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no + SC4S_DEST_SPLUNK_HEC_DEFAULT_HTTP_COMPRESSION=yes + SC4S_LISTEN_PFSENSE_FIREWALL_TCP_PORT=6000 + SC4S_LISTEN_SIMPLE_TEST_ONE_TCP_PORT=5514 + SC4S_LISTEN_SIMPLE_TEST_ONE_UDP_PORT=5514 + SC4S_LISTEN_SIMPLE_TEST_TWO_TCP_PORT=5601 + SC4S_LISTEN_SPECTRACOM_NTP_TCP_PORT=6002 + SC4S_LISTEN_CISCO_ESA_TCP_PORT=9000 + SC4S_LISTEN_RARITAN_DSX_TCP_PORT=9001 + SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL=yes + SC4S_SOURCE_RICOH_SYSLOG_FIXHOST=yes + TEST_SC4S_ACTIVATE_EXAMPLES=yes + SC4S_DEBUG_CONTAINER=yes + SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG=yes + SC4S_NETAPP_ONTAP_NEW_FORMAT=yes + SC4S_USE_VPS_CACHE=yes + - name: Update current SC4S image in unit file + run: | + sed -i 's|Environment="SC4S_IMAGE=ghcr.io/splunk/splunk-connect-for-syslog/container3:latest"|Environment="SC4S_IMAGE=${{ needs.meta.outputs.container_base }}"|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service + - name: Run Ansible Playbook + run: | + ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/${{ matrix.deployment_type }}.yml - name: Run tests run: | pip3 install poetry @@ -245,22 +275,34 @@ jobs: poetry run pytest -v --tb=long \ --splunk_type=external \ --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ - --splunk_host=splunk \ - --sc4s_host=sc4s \ + --splunk_host=127.0.0.1 \ + --sc4s_host=127.0.0.1 \ --junitxml=test-results/test.xml \ -n 14 \ -k 'not lite and not name_cache' - test-ipv4-name-cache: + - name: artifact-test-results + uses: actions/upload-artifact@v4 + with: + name: test-container-results_${{ matrix.deployment_type }}.xml + path: test-results/test.xml + if: ${{ !cancelled() }} + + test-name-cache: + strategy: + matrix: + deployment_type: ["docker", "podman"] + SC4S_IPV6_ENABLE: ["yes","no"] runs-on: ubuntu-latest + continue-on-error: true needs: - meta - build_action - container: - image: python:3.9-buster + services: splunk: image: splunk/splunk:${{ fromJson(needs.meta.outputs.matrix_supportedSplunk)[0].version }} ports: + - 8000:8000 - 8088:8088 - 8089:8089 env: @@ -268,70 +310,56 @@ jobs: SPLUNK_PASSWORD: Changed@11 SPLUNK_START_ARGS: --accept-license SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz - sc4s: - image: ${{ needs.meta.outputs.container_base }} - ports: - - 514:514 - env: - SC4S_DEST_SPLUNK_HEC_DEFAULT_URL: https://splunk:8088 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY: "no" - SC4S_USE_NAME_CACHE: "yes" - SC4S_CLEAR_NAME_CACHE: "yes" + steps: - name: Checkout uses: actions/checkout@v4 with: submodules: false persist-credentials: false - - name: Run tests + - name: Install Ansible and other dependencies as python package run: | - pip3 install poetry - poetry install - mkdir -p test-results || true - poetry run pytest -v --tb=long \ - --splunk_type=external \ - --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ - --splunk_host=splunk \ - --sc4s_host=sc4s \ - --junitxml=test-results/test.xml \ - -n 1 \ - -m 'name_cache' - test-ipv6-name-cache: - runs-on: ubuntu-latest - needs: - - meta - - build_action - container: - image: python:3.9-buster - services: - splunk: - image: splunk/splunk:${{ fromJson(needs.meta.outputs.matrix_supportedSplunk)[0].version }} - ports: - - 8088:8088 - - 8089:8089 - env: - SPLUNK_HEC_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 - SPLUNK_PASSWORD: Changed@11 - SPLUNK_START_ARGS: --accept-license - SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz - sc4s: - image: ${{ needs.meta.outputs.container_base }} - ports: - - 514:514 + pip install ansible~=6.1.0 --no-cache-dir \ + && pip install pywinrm>=0.4.2 --no-cache-dir \ + && pip install ansible-lint>=6.0.0 --no-cache-dir \ + && pip install docker + - name: Configure Ansible Environment Variables env: - SC4S_DEST_SPLUNK_HEC_DEFAULT_URL: https://splunk:8088 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY: "no" + ANSIBLE_CONFIG: ansible.cfg + ANSIBLE_HOST_KEY_CHECKING: False + run: | + echo "ANSIBLE_CONFIG is set to: $ANSIBLE_CONFIG" + echo "ANSIBLE_HOST_KEY_CHECKING is set to: $ANSIBLE_HOST_KEY_CHECKING" + sudo systemctl set-environment SC4S_IMAGE=${{ needs.meta.outputs.container_base }} + - name: Update inventory file + run: | + cat << EOF > ansible/inventory/inventory.yaml + --- + all: + hosts: + children: + node: + hosts: + node_1: + ansible_host: 127.0.0.1 + ansible_connection: local + ansible_user: root + - name: Update env_file + run: | + echo "Updating ansible/inventory/inventory.yaml" + cat << EOF > ansible/resources/env_file + SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://127.0.0.1:8088 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no SC4S_USE_NAME_CACHE: "yes" SC4S_CLEAR_NAME_CACHE: "yes" - SC4S_IPV6_ENABLE: "yes" - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: false - persist-credentials: false + SC4S_IPV6_ENABLE: "${{ matrix.SC4S_IPV6_ENABLE }}" + - name: Update current SC4S image in unit file + run: | + sed -i 's|Environment="SC4S_IMAGE=ghcr.io/splunk/splunk-connect-for-syslog/container3:latest"|Environment="SC4S_IMAGE=${{ needs.meta.outputs.container_base }}"|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service + - name: Run Ansible Playbook + run: | + ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/${{ matrix.deployment_type }}.yml - name: Run tests run: | pip3 install poetry @@ -340,11 +368,17 @@ jobs: poetry run pytest -v --tb=long \ --splunk_type=external \ --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ - --splunk_host=splunk \ - --sc4s_host=sc4s \ + --splunk_host=127.0.0.1 \ + --sc4s_host=127.0.0.1 \ --junitxml=test-results/test.xml \ -n 1 \ -m 'name_cache' + - name: artifact-test-results + uses: actions/upload-artifact@v4 + with: + name: test-name-cache-results_${{ matrix.deployment_type }}_IPv6_${{ matrix.SC4S_IPV6_ENABLE }}.xml + path: test-results/test.xml + if: ${{ !cancelled() }} release: name: Release runs-on: ubuntu-latest @@ -352,7 +386,7 @@ jobs: - meta - build_action - test-container - - test-ipv4-name-cache + - test-name-cache steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/ci-lite.yaml b/.github/workflows/ci-lite.yaml index 79c5066140..a477dba2b3 100644 --- a/.github/workflows/ci-lite.yaml +++ b/.github/workflows/ci-lite.yaml @@ -178,14 +178,14 @@ jobs: scanners: "vuln" test-container: + strategy: + matrix: + deployment_type: ["docker", "podman"] runs-on: ubuntu-latest + continue-on-error: true needs: - meta - build_action - # runs all of the steps inside the specified container rather than on the VM host. - # Because of this the network configuration changes from host based network to a container network. - container: - image: python:3.9-buster services: splunk: @@ -200,41 +200,67 @@ jobs: SPLUNK_START_ARGS: --accept-license SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz - sc4s: - image: ${{ needs.meta.outputs.container_base }} - ports: - - 514:514 - - 601:601 - - 5614:5514 - - 5601:5601 - - 6000:6000 - - 6002:6002 - - 9000:9000 - env: - SC4S_DEST_SPLUNK_HEC_DEFAULT_URL: https://splunk:8088 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY: "no" - SC4S_DEST_SPLUNK_HEC_DEFAULT_HTTP_COMPRESSION: "yes" - SC4S_LISTEN_PFSENSE_FIREWALL_TCP_PORT: 6000 - SC4S_LISTEN_SIMPLE_TEST_ONE_TCP_PORT: 5514 - SC4S_LISTEN_SIMPLE_TEST_ONE_UDP_PORT: 5514 - SC4S_LISTEN_SIMPLE_TEST_TWO_TCP_PORT: 5601 - SC4S_LISTEN_SPECTRACOM_NTP_TCP_PORT: 6002 - SC4S_LISTEN_CISCO_ESA_TCP_PORT: 9000 - SC4S_LISTEN_RARITAN_DSX_TCP_PORT: 9001 - SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL: "yes" - SC4S_SOURCE_RICOH_SYSLOG_FIXHOST: "yes" - TEST_SC4S_ACTIVATE_EXAMPLES: "yes" - SC4S_DEBUG_CONTAINER: "yes" - SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG: "yes" - SC4S_NETAPP_ONTAP_NEW_FORMAT: "yes" - SC4S_USE_VPS_CACHE: "yes" steps: - name: Checkout uses: actions/checkout@v4 with: submodules: false persist-credentials: false + - name: Install Ansible and other dependencies as python package + run: | + pip install ansible~=6.1.0 --no-cache-dir \ + && pip install pywinrm>=0.4.2 --no-cache-dir \ + && pip install ansible-lint>=6.0.0 --no-cache-dir \ + && pip install docker + - name: Configure Ansible Environment Variables + env: + ANSIBLE_CONFIG: ansible.cfg + ANSIBLE_HOST_KEY_CHECKING: False + run: | + echo "ANSIBLE_CONFIG is set to: $ANSIBLE_CONFIG" + echo "ANSIBLE_HOST_KEY_CHECKING is set to: $ANSIBLE_HOST_KEY_CHECKING" + sudo systemctl set-environment SC4S_IMAGE=${{ needs.meta.outputs.container_base }} + - name: Update inventory file + run: | + cat << EOF > ansible/inventory/inventory.yaml + --- + all: + hosts: + children: + node: + hosts: + node_1: + ansible_host: 127.0.0.1 + ansible_connection: local + ansible_user: root + - name: Update env_file + run: | + echo "Updating ansible/inventory/inventory.yaml" + cat << EOF > ansible/resources/env_file + SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://127.0.0.1:8088 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no + SC4S_DEST_SPLUNK_HEC_DEFAULT_HTTP_COMPRESSION=yes + SC4S_LISTEN_PFSENSE_FIREWALL_TCP_PORT=6000 + SC4S_LISTEN_SIMPLE_TEST_ONE_TCP_PORT=5514 + SC4S_LISTEN_SIMPLE_TEST_ONE_UDP_PORT=5514 + SC4S_LISTEN_SIMPLE_TEST_TWO_TCP_PORT=5601 + SC4S_LISTEN_SPECTRACOM_NTP_TCP_PORT=6002 + SC4S_LISTEN_CISCO_ESA_TCP_PORT=9000 + SC4S_LISTEN_RARITAN_DSX_TCP_PORT=9001 + SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL=yes + SC4S_SOURCE_RICOH_SYSLOG_FIXHOST=yes + TEST_SC4S_ACTIVATE_EXAMPLES=yes + SC4S_DEBUG_CONTAINER=yes + SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG=yes + SC4S_NETAPP_ONTAP_NEW_FORMAT=yes + SC4S_USE_VPS_CACHE=yes + - name: Update current SC4S image in unit file + run: | + sed -i 's|Environment="SC4S_IMAGE=ghcr.io/splunk/splunk-connect-for-syslog/container3:latest"|Environment="SC4S_IMAGE=${{ needs.meta.outputs.container_base }}"|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service + - name: Run Ansible Playbook + run: | + ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/${{ matrix.deployment_type }}.yml - name: Run tests run: | pip3 install poetry @@ -243,24 +269,33 @@ jobs: poetry run pytest -v --tb=long \ --splunk_type=external \ --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ - --splunk_host=splunk \ - --sc4s_host=sc4s \ + --splunk_host=127.0.0.1 \ + --sc4s_host=127.0.0.1 \ --junitxml=test-results/test.xml \ -n 14 -m "lite or addons" - - test-ipv4-name-cache: + - name: artifact-test-results + uses: actions/upload-artifact@v4 + with: + name: test-container-results_${{ matrix.deployment_type }}.xml + path: test-results/test.xml + if: ${{ !cancelled() }} + + test-name-cache: + strategy: + matrix: + deployment_type: ["docker", "podman"] + SC4S_IPV6_ENABLE: ["yes","no"] runs-on: ubuntu-latest + continue-on-error: true needs: - meta - build_action - # runs all of the steps inside the specified container rather than on the VM host. - # Because of this the network configuration changes from host based network to a container network. - container: - image: python:3.9-buster + services: splunk: image: splunk/splunk:${{ fromJson(needs.meta.outputs.matrix_supportedSplunk)[0].version }} ports: + - 8000:8000 - 8088:8088 - 8089:8089 env: @@ -269,76 +304,55 @@ jobs: SPLUNK_START_ARGS: --accept-license SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz - sc4s: - image: ${{ needs.meta.outputs.container_base }} - ports: - - 514:514 - env: - SC4S_DEST_SPLUNK_HEC_DEFAULT_URL: https://splunk:8088 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY: "no" - SC4S_USE_NAME_CACHE: "yes" - SC4S_CLEAR_NAME_CACHE: "yes" - steps: - name: Checkout uses: actions/checkout@v4 with: submodules: false persist-credentials: false - - name: Run tests + - name: Install Ansible and other dependencies as python package run: | - pip3 install poetry - poetry install - mkdir -p test-results || true - poetry run pytest -v --tb=long \ - --splunk_type=external \ - --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ - --splunk_host=splunk \ - --sc4s_host=sc4s \ - --junitxml=test-results/test.xml \ - -n 1 \ - -m 'name_cache' - - test-ipv6-name-cache: - runs-on: ubuntu-latest - needs: - - meta - - build_action - # runs all of the steps inside the specified container rather than on the VM host. - # Because of this the network configuration changes from host based network to a container network. - container: - image: python:3.9-buster - services: - splunk: - image: splunk/splunk:${{ fromJson(needs.meta.outputs.matrix_supportedSplunk)[0].version }} - ports: - - 8088:8088 - - 8089:8089 - env: - SPLUNK_HEC_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 - SPLUNK_PASSWORD: Changed@11 - SPLUNK_START_ARGS: --accept-license - SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz - - sc4s: - image: ${{ needs.meta.outputs.container_base }} - ports: - - 514:514 + pip install ansible~=6.1.0 --no-cache-dir \ + && pip install pywinrm>=0.4.2 --no-cache-dir \ + && pip install ansible-lint>=6.0.0 --no-cache-dir \ + && pip install docker + - name: Configure Ansible Environment Variables env: - SC4S_DEST_SPLUNK_HEC_DEFAULT_URL: https://splunk:8088 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY: "no" + ANSIBLE_CONFIG: ansible.cfg + ANSIBLE_HOST_KEY_CHECKING: False + run: | + echo "ANSIBLE_CONFIG is set to: $ANSIBLE_CONFIG" + echo "ANSIBLE_HOST_KEY_CHECKING is set to: $ANSIBLE_HOST_KEY_CHECKING" + sudo systemctl set-environment SC4S_IMAGE=${{ needs.meta.outputs.container_base }} + - name: Update inventory file + run: | + cat << EOF > ansible/inventory/inventory.yaml + --- + all: + hosts: + children: + node: + hosts: + node_1: + ansible_host: 127.0.0.1 + ansible_connection: local + ansible_user: root + - name: Update env_file + run: | + echo "Updating ansible/inventory/inventory.yaml" + cat << EOF > ansible/resources/env_file + SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://127.0.0.1:8088 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no SC4S_USE_NAME_CACHE: "yes" SC4S_CLEAR_NAME_CACHE: "yes" - SC4S_IPV6_ENABLE: "yes" - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: false - persist-credentials: false + SC4S_IPV6_ENABLE: "${{ matrix.SC4S_IPV6_ENABLE }}" + - name: Update current SC4S image in unit file + run: | + sed -i 's|Environment="SC4S_IMAGE=ghcr.io/splunk/splunk-connect-for-syslog/container3:latest"|Environment="SC4S_IMAGE=${{ needs.meta.outputs.container_base }}"|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service + - name: Run Ansible Playbook + run: | + ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/${{ matrix.deployment_type }}.yml - name: Run tests run: | pip3 install poetry @@ -347,11 +361,17 @@ jobs: poetry run pytest -v --tb=long \ --splunk_type=external \ --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ - --splunk_host=splunk \ - --sc4s_host=sc4s \ + --splunk_host=127.0.0.1 \ + --sc4s_host=127.0.0.1 \ --junitxml=test-results/test.xml \ -n 1 \ -m 'name_cache' + - name: artifact-test-results + uses: actions/upload-artifact@v4 + with: + name: test-name-cache-results_${{ matrix.deployment_type }}_IPv6_${{ matrix.SC4S_IPV6_ENABLE }}.xml + path: test-results/test.xml + if: ${{ !cancelled() }} test-healthcheck: runs-on: ubuntu-latest @@ -412,8 +432,7 @@ jobs: - meta - build_action - test-container - - test-ipv4-name-cache - - test-ipv6-name-cache + - test-name-cache - test-healthcheck - test-healthcheck-unit-tests steps: diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index a2cdbb3280..7dcdf23548 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -178,15 +178,14 @@ jobs: scanners: "vuln" test-container: + strategy: + matrix: + deployment_type: ["docker", "podman"] runs-on: ubuntu-latest + continue-on-error: true needs: - meta - build_action - # runs all of the steps inside the specified container rather than on the VM host. - # Because of this the network configuration changes from host based network to a container network. - container: - image: python:3.9-buster - services: splunk: image: splunk/splunk:${{ fromJson(needs.meta.outputs.matrix_supportedSplunk)[0].version }} @@ -199,43 +198,67 @@ jobs: SPLUNK_PASSWORD: Changed@11 SPLUNK_START_ARGS: --accept-license SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz - - sc4s: - image: ${{ needs.meta.outputs.container_base }} - ports: - - 514:514 - - 601:601 - - 5614:5514 - - 5601:5601 - - 6000:6000 - - 6002:6002 - - 9000:9000 - env: - SC4S_DEST_SPLUNK_HEC_DEFAULT_URL: https://splunk:8088 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY: "no" - SC4S_DEST_SPLUNK_HEC_DEFAULT_HTTP_COMPRESSION: "yes" - SC4S_LISTEN_PFSENSE_FIREWALL_TCP_PORT: 6000 - SC4S_LISTEN_SIMPLE_TEST_ONE_TCP_PORT: 5514 - SC4S_LISTEN_SIMPLE_TEST_ONE_UDP_PORT: 5514 - SC4S_LISTEN_SIMPLE_TEST_TWO_TCP_PORT: 5601 - SC4S_LISTEN_SPECTRACOM_NTP_TCP_PORT: 6002 - SC4S_LISTEN_CISCO_ESA_TCP_PORT: 9000 - SC4S_LISTEN_RARITAN_DSX_TCP_PORT: 9001 - SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL: "yes" - SC4S_SOURCE_RICOH_SYSLOG_FIXHOST: "yes" - TEST_SC4S_ACTIVATE_EXAMPLES: "yes" - SC4S_DEBUG_CONTAINER: "yes" - SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG: "yes" - SC4S_NETAPP_ONTAP_NEW_FORMAT: "yes" - SC4S_USE_VPS_CACHE: "yes" - steps: - name: Checkout uses: actions/checkout@v4 with: submodules: false persist-credentials: false + - name: Install Ansible and other dependencies as python package + run: | + pip install ansible~=6.1.0 --no-cache-dir \ + && pip install pywinrm>=0.4.2 --no-cache-dir \ + && pip install ansible-lint>=6.0.0 --no-cache-dir \ + && pip install docker + - name: Configure Ansible Environment Variables + env: + ANSIBLE_CONFIG: ansible.cfg + ANSIBLE_HOST_KEY_CHECKING: False + run: | + echo "ANSIBLE_CONFIG is set to: $ANSIBLE_CONFIG" + echo "ANSIBLE_HOST_KEY_CHECKING is set to: $ANSIBLE_HOST_KEY_CHECKING" + sudo systemctl set-environment SC4S_IMAGE=${{ needs.meta.outputs.container_base }} + - name: Update inventory file + run: | + cat << EOF > ansible/inventory/inventory.yaml + --- + all: + hosts: + children: + node: + hosts: + node_1: + ansible_host: 127.0.0.1 + ansible_connection: local + ansible_user: root + - name: Update env_file + run: | + echo "Updating ansible/inventory/inventory.yaml" + cat << EOF > ansible/resources/env_file + SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://127.0.0.1:8088 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no + SC4S_DEST_SPLUNK_HEC_DEFAULT_HTTP_COMPRESSION=yes + SC4S_LISTEN_PFSENSE_FIREWALL_TCP_PORT=6000 + SC4S_LISTEN_SIMPLE_TEST_ONE_TCP_PORT=5514 + SC4S_LISTEN_SIMPLE_TEST_ONE_UDP_PORT=5514 + SC4S_LISTEN_SIMPLE_TEST_TWO_TCP_PORT=5601 + SC4S_LISTEN_SPECTRACOM_NTP_TCP_PORT=6002 + SC4S_LISTEN_CISCO_ESA_TCP_PORT=9000 + SC4S_LISTEN_RARITAN_DSX_TCP_PORT=9001 + SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL=yes + SC4S_SOURCE_RICOH_SYSLOG_FIXHOST=yes + TEST_SC4S_ACTIVATE_EXAMPLES=yes + SC4S_DEBUG_CONTAINER=yes + SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG=yes + SC4S_NETAPP_ONTAP_NEW_FORMAT=yes + SC4S_USE_VPS_CACHE=yes + - name: Update current SC4S image in unit file + run: | + sed -i 's|Environment="SC4S_IMAGE=ghcr.io/splunk/splunk-connect-for-syslog/container3:latest"|Environment="SC4S_IMAGE=${{ needs.meta.outputs.container_base }}"|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service + - name: Run Ansible Playbook + run: | + ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/${{ matrix.deployment_type }}.yml - name: Run tests run: | pip3 install poetry @@ -244,25 +267,33 @@ jobs: poetry run pytest -v --tb=long \ --splunk_type=external \ --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ - --splunk_host=splunk \ - --sc4s_host=sc4s \ + --splunk_host=127.0.0.1 \ + --sc4s_host=127.0.0.1 \ --junitxml=test-results/test.xml \ -n 14 \ -k 'not lite and not name_cache' + - name: artifact-test-results + uses: actions/upload-artifact@v4 + with: + name: test-container-results_${{ matrix.deployment_type }}.xml + path: test-results/test.xml + if: ${{ !cancelled() }} - test-ipv4-name-cache: + test-name-cache: + strategy: + matrix: + deployment_type: ["docker", "podman"] + SC4S_IPV6_ENABLE: ["yes","no"] runs-on: ubuntu-latest + continue-on-error: true needs: - meta - build_action - # runs all of the steps inside the specified container rather than on the VM host. - # Because of this the network configuration changes from host based network to a container network. - container: - image: python:3.9-buster services: splunk: image: splunk/splunk:${{ fromJson(needs.meta.outputs.matrix_supportedSplunk)[0].version }} ports: + - 8000:8000 - 8088:8088 - 8089:8089 env: @@ -270,77 +301,58 @@ jobs: SPLUNK_PASSWORD: Changed@11 SPLUNK_START_ARGS: --accept-license SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz - - sc4s: - image: ${{ needs.meta.outputs.container_base }} - ports: - - 514:514 - env: - SC4S_DEST_SPLUNK_HEC_DEFAULT_URL: https://splunk:8088 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY: "no" - SC4S_USE_NAME_CACHE: "yes" - SC4S_CLEAR_NAME_CACHE: "yes" - steps: - name: Checkout uses: actions/checkout@v4 with: submodules: false persist-credentials: false - - name: Run tests + - name: Install Ansible and other dependencies as python package run: | - pip3 install poetry - poetry install - mkdir -p test-results || true - poetry run pytest -v --tb=long \ - --splunk_type=external \ - --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ - --splunk_host=splunk \ - --sc4s_host=sc4s \ - --junitxml=test-results/test.xml \ - -n 1 \ - -m 'name_cache' - - test-ipv6-name-cache: - runs-on: ubuntu-latest - needs: - - meta - - build_action - # runs all of the steps inside the specified container rather than on the VM host. - # Because of this the network configuration changes from host based network to a container network. - container: - image: python:3.9-buster - services: - splunk: - image: splunk/splunk:${{ fromJson(needs.meta.outputs.matrix_supportedSplunk)[0].version }} - ports: - - 8088:8088 - - 8089:8089 - env: - SPLUNK_HEC_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 - SPLUNK_PASSWORD: Changed@11 - SPLUNK_START_ARGS: --accept-license - SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz - - sc4s: - image: ${{ needs.meta.outputs.container_base }} - ports: - - 514:514 + pip install ansible~=6.1.0 --no-cache-dir \ + && pip install pywinrm>=0.4.2 --no-cache-dir \ + && pip install ansible-lint>=6.0.0 --no-cache-dir \ + && pip install docker + - name: Configure Ansible Environment Variables env: - SC4S_DEST_SPLUNK_HEC_DEFAULT_URL: https://splunk:8088 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY: "no" - SC4S_USE_NAME_CACHE: "yes" - SC4S_CLEAR_NAME_CACHE: "yes" - SC4S_IPV6_ENABLE: "yes" - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: false - persist-credentials: false + ANSIBLE_CONFIG: ansible.cfg + ANSIBLE_HOST_KEY_CHECKING: False + run: | + echo "ANSIBLE_CONFIG is set to: $ANSIBLE_CONFIG" + echo "ANSIBLE_HOST_KEY_CHECKING is set to: $ANSIBLE_HOST_KEY_CHECKING" + sudo systemctl set-environment SC4S_IMAGE=${{ needs.meta.outputs.container_base }} + - name: Update inventory file + run: | + cat << EOF > ansible/inventory/inventory.yaml + --- + all: + hosts: + children: + node: + hosts: + node_1: + ansible_host: 0.0.0.0 + ansible_connection: local + ansible_user: root + - name: Update env_file + run: | + echo "Updating ansible/inventory/inventory.yaml" + cat << EOF > ansible/resources/env_file + SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://127.0.0.1:8088 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no + SC4S_USE_NAME_CACHE=yes + SC4S_CLEAR_NAME_CACHE=yes + SC4S_IPV6_ENABLE=${{ matrix.SC4S_IPV6_ENABLE }} + - name: Update current SC4S image in unit file + run: | + sed -i 's|Environment="SC4S_IMAGE=ghcr.io/splunk/splunk-connect-for-syslog/container3:latest"|Environment="SC4S_IMAGE=${{ needs.meta.outputs.container_base }}"|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service + # - name: Update current SC4S network in unit file + # run: | + # sed -i 's|--network host|--network ${{ job.container.network }}|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service + - name: Run Ansible Playbook + run: | + ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/${{ matrix.deployment_type }}.yml - name: Run tests run: | pip3 install poetry @@ -349,11 +361,17 @@ jobs: poetry run pytest -v --tb=long \ --splunk_type=external \ --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ - --splunk_host=splunk \ - --sc4s_host=sc4s \ + --splunk_host=127.0.0.1 \ + --sc4s_host=127.0.0.1 \ --junitxml=test-results/test.xml \ -n 1 \ -m 'name_cache' + - name: artifact-test-results + uses: actions/upload-artifact@v4 + with: + name: test-name-cache-results_${{ matrix.deployment_type }}_IPv6_${{ matrix.SC4S_IPV6_ENABLE }}.xml + path: test-results/test.xml + if: ${{ !cancelled() }} test-healthcheck: runs-on: ubuntu-latest @@ -437,8 +455,7 @@ jobs: - meta - build_action - test-container - - test-ipv4-name-cache - - test-ipv6-name-cache + - test-name-cache - test-healthcheck - test-healthcheck-unit-tests - mike diff --git a/ansible/resources/docker_sc4s.service b/ansible/resources/docker_sc4s.service index 6b18a46cfd..4c973d0ebb 100644 --- a/ansible/resources/docker_sc4s.service +++ b/ansible/resources/docker_sc4s.service @@ -39,8 +39,9 @@ ExecStart=/usr/bin/docker run \ -v "$SC4S_ARCHIVE_MOUNT" \ -v "$SC4S_TLS_MOUNT" \ --env-file=/opt/sc4s/env_file \ - --network host \ + --network test_network \ + --network-alias sc4s \ --name SC4S \ --rm $SC4S_IMAGE -Restart=on-failure \ No newline at end of file +Restart=on-failure diff --git a/ansible/roles/install_docker/tasks/install_docker_debian.yml b/ansible/roles/install_docker/tasks/install_docker_debian.yml index 21a517907d..d3d434a774 100644 --- a/ansible/roles/install_docker/tasks/install_docker_debian.yml +++ b/ansible/roles/install_docker/tasks/install_docker_debian.yml @@ -23,10 +23,16 @@ repo: deb https://download.docker.com/linux/ubuntu focal stable state: present +- name: Check if 'docker' command exists + ansible.builtin.command: which docker + register: docker_command_check + ignore_errors: true + - name: Install docker-ce apt: name: docker-ce state: latest + when: docker_command_check.rc != 0 - name: Install Docker Module for Python pip: diff --git a/ansible/tasks/docker/deploy_app.yml b/ansible/tasks/docker/deploy_app.yml index 50462fac88..03c4a7d284 100644 --- a/ansible/tasks/docker/deploy_app.yml +++ b/ansible/tasks/docker/deploy_app.yml @@ -12,15 +12,23 @@ - name: Copying unit file on the server copy: - src: "/opt/ansible/resources/docker_sc4s.service" + src: "../../../ansible/resources/docker_sc4s.service" dest: "/lib/systemd/system/sc4s.service" owner: "{{ ansible_user }}" group: "{{ ansible_user }}" mode: u=rw,g=rw +- name: Check current working directory on the target host + ansible.builtin.command: pwd + register: current_directory_output + +- name: Print the working directory + ansible.builtin.debug: + var: current_directory_output.stdout + - name: Copying env_file file on the server copy: - src: "/opt/ansible/resources/env_file" + src: "../../../ansible/resources/env_file" dest: "/opt/sc4s/env_file" owner: "{{ ansible_user }}" group: "{{ ansible_user }}" diff --git a/ansible/tasks/podman/deploy_app.yml b/ansible/tasks/podman/deploy_app.yml index 07632d8eaa..f74fe85807 100644 --- a/ansible/tasks/podman/deploy_app.yml +++ b/ansible/tasks/podman/deploy_app.yml @@ -12,7 +12,7 @@ - name: Copying unit file on the server copy: - src: "/opt/ansible/resources/podman_sc4s.service" + src: "../../../ansible/resources/podman_sc4s.service" dest: "/lib/systemd/system/sc4s.service" owner: "{{ ansible_user }}" group: "{{ ansible_user }}" @@ -20,7 +20,7 @@ - name: Copying env_file file on the server copy: - src: "/opt/ansible/resources/env_file" + src: "../../../ansible/resources/env_file" dest: "/opt/sc4s/env_file" owner: "{{ ansible_user }}" group: "{{ ansible_user }}"