chore(ci): reorganize integration test files

.github/workflows/changes.yml

@@ -324,7 +324,7 @@ jobs:
 
       # creates a yaml file that contains the filters for each integration,
       # extracted from the output of the `vdev int ci-paths` command, which
-      # sources the paths from the scripts/integration/.../test.yaml files
+      # sources the paths from the tests/integration/.../config/test.yaml files
       - name: Create filter rules for integrations
         run: vdev int ci-paths > int_test_filters.yaml
 

src/nats.rs

@@ -266,7 +266,7 @@ mod tests {
         parse_auth(
             r#"
             strategy = "credentials_file"
-            credentials_file.path = "tests/data/nats/nats.creds"
+            credentials_file.path = "tests/integration/nats/data/nats.creds"
         "#,
         )
         .unwrap();

src/sinks/gcp_chronicle/chronicle_unstructured.rs

@@ -718,10 +718,9 @@ mod integration_tests {
         trace_init();
 
         let log_type = random_string(10);
-        let (sink, healthcheck) =
-            config_build(&log_type, "/home/vector/scripts/integration/gcp/auth.json")
-                .await
-                .expect("Building sink failed");
+        let (sink, healthcheck) = config_build(&log_type, "tests/integration/gcp/config/auth.json")
+            .await
+            .expect("Building sink failed");
 
         healthcheck.await.expect("Health check failed");
 
@@ -749,11 +748,7 @@ mod integration_tests {
 
         let log_type = random_string(10);
         // Test with an auth file that doesnt match the public key sent to the dummy chronicle server.
-        let sink = config_build(
-            &log_type,
-            "/home/vector/scripts/integration/gcp/invalidauth.json",
-        )
-        .await;
+        let sink = config_build(&log_type, "tests/integration/gcp/config/invalidauth.json").await;
 
         assert!(sink.is_err())
     }
@@ -765,10 +760,9 @@ mod integration_tests {
         // The chronicle-emulator we are testing against is setup so a `log_type` of "INVALID"
         // will return a `400 BAD_REQUEST`.
         let log_type = "INVALID";
-        let (sink, healthcheck) =
-            config_build(log_type, "/home/vector/scripts/integration/gcp/auth.json")
-                .await
-                .expect("Building sink failed");
+        let (sink, healthcheck) = config_build(log_type, "tests/integration/gcp/config/auth.json")
+            .await
+            .expect("Building sink failed");
 
         healthcheck.await.expect("Health check failed");
 

src/sinks/nats/integration_tests.rs

@@ -252,7 +252,7 @@ async fn nats_tls_valid() {
     conf.tls = Some(TlsEnableableConfig {
         enabled: Some(true),
         options: TlsConfig {
-            ca_file: Some("tests/data/nats/rootCA.pem".into()),
+            ca_file: Some("tests/integration/nats/data/rootCA.pem".into()),
             ..Default::default()
         },
     });
@@ -293,9 +293,9 @@ async fn nats_tls_client_cert_valid() {
     conf.tls = Some(TlsEnableableConfig {
         enabled: Some(true),
         options: TlsConfig {
-            ca_file: Some("tests/data/nats/rootCA.pem".into()),
-            crt_file: Some("tests/data/nats/nats-client.pem".into()),
-            key_file: Some("tests/data/nats/nats-client.key".into()),
+            ca_file: Some("tests/integration/nats/data/rootCA.pem".into()),
+            crt_file: Some("tests/integration/nats/data/nats-client.pem".into()),
+            key_file: Some("tests/integration/nats/data/nats-client.key".into()),
             ..Default::default()
         },
     });
@@ -319,7 +319,7 @@ async fn nats_tls_client_cert_invalid() {
     conf.tls = Some(TlsEnableableConfig {
         enabled: Some(true),
         options: TlsConfig {
-            ca_file: Some("tests/data/nats/rootCA.pem".into()),
+            ca_file: Some("tests/integration/nats/data/rootCA.pem".into()),
             ..Default::default()
         },
     });
@@ -343,13 +343,13 @@ async fn nats_tls_jwt_auth_valid() {
     conf.tls = Some(TlsEnableableConfig {
         enabled: Some(true),
         options: TlsConfig {
-            ca_file: Some("tests/data/nats/rootCA.pem".into()),
+            ca_file: Some("tests/integration/nats/data/rootCA.pem".into()),
             ..Default::default()
         },
     });
     conf.auth = Some(NatsAuthConfig::CredentialsFile {
         credentials_file: NatsAuthCredentialsFile {
-            path: "tests/data/nats/nats.creds".into(),
+            path: "tests/integration/nats/data/nats.creds".into(),
         },
     });
 
@@ -372,13 +372,13 @@ async fn nats_tls_jwt_auth_invalid() {
     conf.tls = Some(TlsEnableableConfig {
         enabled: Some(true),
         options: TlsConfig {
-            ca_file: Some("tests/data/nats/rootCA.pem".into()),
+            ca_file: Some("tests/integration/nats/data/rootCA.pem".into()),
             ..Default::default()
         },
     });
     conf.auth = Some(NatsAuthConfig::CredentialsFile {
         credentials_file: NatsAuthCredentialsFile {
-            path: "tests/data/nats/nats-bad.creds".into(),
+            path: "tests/integration/nats/data/nats-bad.creds".into(),
         },
     });
 

src/sources/amqp.rs

@@ -599,7 +599,7 @@ pub mod test {
     }
 }
 
-/// Integration tests use the docker compose files in `scripts/integration/docker-compose.amqp.yml`.
+/// Integration tests use the docker compose files in `tests/integration/docker-compose.amqp.yml`.
 #[cfg(feature = "amqp-integration-tests")]
 #[cfg(test)]
 mod integration_test {

src/sources/http_client/integration_tests.rs

@@ -244,7 +244,7 @@ async fn tls_invalid_ca() {
         headers: HashMap::new(),
         method: HttpMethod::Get,
         tls: Some(TlsConfig {
-            ca_file: Some("tests/data/http-client/certs/invalid-ca-cert.pem".into()),
+            ca_file: Some("tests/integration/http-client/data/certs/invalid-ca-cert.pem".into()),
             ..Default::default()
         }),
         auth: None,

src/sources/logstash.rs

@@ -966,8 +966,12 @@ mod integration_tests {
                 Some(TlsEnableableConfig {
                     enabled: Some(true),
                     options: TlsConfig {
-                        crt_file: Some("tests/data/host.docker.internal.crt".into()),
-                        key_file: Some("tests/data/host.docker.internal.key".into()),
+                        crt_file: Some(
+                            "tests/integration/shared/data/host.docker.internal.crt".into(),
+                        ),
+                        key_file: Some(
+                            "tests/integration/shared/data/host.docker.internal.key".into(),
+                        ),
                         ..Default::default()
                     },
                 }),

src/sources/nats/integration_tests.rs

@@ -280,7 +280,7 @@ async fn nats_tls_valid() {
     conf.tls = Some(TlsEnableableConfig {
         enabled: Some(true),
         options: TlsConfig {
-            ca_file: Some("tests/data/nats/rootCA.pem".into()),
+            ca_file: Some("tests/integration/nats/data/rootCA.pem".into()),
             ..Default::default()
         },
     });
@@ -317,9 +317,9 @@ async fn nats_tls_client_cert_valid() {
     conf.tls = Some(TlsEnableableConfig {
         enabled: Some(true),
         options: TlsConfig {
-            ca_file: Some("tests/data/nats/rootCA.pem".into()),
-            crt_file: Some("tests/data/nats/nats-client.pem".into()),
-            key_file: Some("tests/data/nats/nats-client.key".into()),
+            ca_file: Some("tests/integration/nats/data/rootCA.pem".into()),
+            crt_file: Some("tests/integration/nats/data/nats-client.pem".into()),
+            key_file: Some("tests/integration/nats/data/nats-client.key".into()),
             ..Default::default()
         },
     });
@@ -341,7 +341,7 @@ async fn nats_tls_client_cert_invalid() {
     conf.tls = Some(TlsEnableableConfig {
         enabled: Some(true),
         options: TlsConfig {
-            ca_file: Some("tests/data/nats/rootCA.pem".into()),
+            ca_file: Some("tests/integration/nats/data/rootCA.pem".into()),
             ..Default::default()
         },
     });
@@ -363,13 +363,13 @@ async fn nats_tls_jwt_auth_valid() {
     conf.tls = Some(TlsEnableableConfig {
         enabled: Some(true),
         options: TlsConfig {
-            ca_file: Some("tests/data/nats/rootCA.pem".into()),
+            ca_file: Some("tests/integration/nats/data/rootCA.pem".into()),
             ..Default::default()
         },
     });
     conf.auth = Some(NatsAuthConfig::CredentialsFile {
         credentials_file: NatsAuthCredentialsFile {
-            path: "tests/data/nats/nats.creds".into(),
+            path: "tests/integration/nats/data/nats.creds".into(),
         },
     });
 
@@ -390,13 +390,13 @@ async fn nats_tls_jwt_auth_invalid() {
     conf.tls = Some(TlsEnableableConfig {
         enabled: Some(true),
         options: TlsConfig {
-            ca_file: Some("tests/data/nats/rootCA.pem".into()),
+            ca_file: Some("tests/integration/nats/data/rootCA.pem".into()),
             ..Default::default()
         },
     });
     conf.auth = Some(NatsAuthConfig::CredentialsFile {
         credentials_file: NatsAuthCredentialsFile {
-            path: "tests/data/nats/nats-bad.creds".into(),
+            path: "tests/integration/nats/data/nats-bad.creds".into(),
         },
     });
 

scripts/integration/amqp/compose.yaml → tests/integration/amqp/config/compose.yaml

@@ -12,7 +12,7 @@ services:
       - RABBITMQ_SSL_CACERTFILE=/code/tests/data/ca/intermediate_server/certs/ca-chain.cert.pem
       - RABBITMQ_SSL_FAIL_IF_NO_PEER_CERT=false
     volumes:
-      - ../../..:/code
+      - ../../../..:/code
 
 networks:
   default:

scripts/integration/aws/test.yaml → tests/integration/aws/config/test.yaml

@@ -29,4 +29,4 @@ paths:
 - "src/sinks/aws_*/**"
 - "src/sinks/util/**"
 - "src/transforms/aws_*"
-- "scripts/integration/aws/**"
+- "tests/integration/aws/**"

scripts/integration/axiom/test.yaml → tests/integration/axiom/config/test.yaml

@@ -18,4 +18,4 @@ matrix:
 paths:
 - "src/sinks/axiom.rs"
 - "src/sinks/util/**"
-- "scripts/integration/axiom/**"
+- "tests/integration/axiom/**"

scripts/integration/azure/test.yaml → tests/integration/azure/config/test.yaml

@@ -16,4 +16,4 @@ matrix:
 paths:
 - "src/sinks/azure_**"
 - "src/sinks/util/**"
-- "scripts/integration/azure/**"
+- "tests/integration/azure/**"

scripts/integration/clickhouse/test.yaml → tests/integration/clickhouse/config/test.yaml

@@ -14,4 +14,4 @@ matrix:
 paths:
 - "src/sinks/clickhouse/**"
 - "src/sinks/util/**"
-- "scripts/integration/clickhouse/**"
+- "tests/integration/clickhouse/**"

scripts/integration/databend/compose.yaml → tests/integration/databend/config/compose.yaml

@@ -5,7 +5,7 @@ services:
     image: docker.io/minio/minio
     command: server /data
     volumes:
-      - ../../../tests/data/databend/miniodat/:/data
+      - ../data/miniodat/:/data
   databend:
     image: docker.io/datafuselabs/databend:${CONFIG_VERSION}
     environment:

scripts/integration/databend/test.yaml → tests/integration/databend/config/test.yaml

@@ -15,4 +15,4 @@ matrix:
 paths:
 - "src/sinks/databend/**"
 - "src/sinks/util/**"
-- "scripts/integration/databend/**"
+- "tests/integration/databend/**"

scripts/integration/datadog-agent/compose.yaml → tests/integration/datadog-agent/config/compose.yaml

@@ -16,7 +16,7 @@ services:
     - DD_HOSTNAME=datadog-agent
     - DD_SERIALIZER_COMPRESSOR_KIND=zstd
     volumes:
-    - ../../../tests/data/datadog-agent/conf.yaml:/etc/datadog-agent/conf.d/test.d/conf.yaml
+    - ../data/conf.yaml:/etc/datadog-agent/conf.d/test.d/conf.yaml
   datadog-trace-agent:
     image: docker.io/datadog/agent:7.31.0
     environment:

scripts/integration/datadog-agent/test.yaml → tests/integration/datadog-agent/config/test.yaml

@@ -19,4 +19,4 @@ paths:
 - "src/common/datadog.rs"
 - "src/internal_events/datadog_*"
 - "src/sources/datadog_agent/**"
-- "scripts/integration/datadog-agent/**"
+- "tests/integration/datadog-agent/**"

scripts/integration/datadog-logs/test.yaml → tests/integration/datadog-logs/config/test.yaml

@@ -17,4 +17,4 @@ paths:
 - "src/internal_events/datadog_*"
 - "src/sinks/datadog/logs/**"
 - "src/sinks/util/**"
-- "scripts/integration/datadog-logs/**"
+- "tests/integration/datadog-logs/**"

scripts/integration/datadog-metrics/test.yaml → tests/integration/datadog-metrics/config/test.yaml

@@ -17,4 +17,4 @@ paths:
 - "src/internal_events/datadog_*"
 - "src/sinks/datadog/metrics/**"
 - "src/sinks/util/**"
-- "scripts/integration/datadog-metrics/**"
+- "tests/integration/datadog-metrics/**"

scripts/integration/datadog-traces/test.yaml → tests/integration/datadog-traces/config/test.yaml

@@ -21,4 +21,4 @@ paths:
 - "src/internal_events/datadog_*"
 - "src/sinks/datadog/traces/**"
 - "src/sinks/util/**"
-- "scripts/integration/datadog-traces/**"
+- "tests/integration/datadog-traces/**"

scripts/integration/dnstap/compose.yaml → tests/integration/dnstap/config/compose.yaml

@@ -3,7 +3,7 @@ version: "3"
 services:
   dnstap:
     build:
-      context: ../../../tests/data/dnstap
+      context: ../data
     container_name: vector_dnstap
     hostname: ns.example.com
     volumes:

scripts/integration/dnstap/test.yaml → tests/integration/dnstap/config/test.yaml

@@ -19,4 +19,4 @@ paths:
 - "src/internal_events/dnstap.rs"
 - "src/sources/dnstap/**"
 - "src/sources/util/**"
-- "scripts/integration/dnstap/**"
+- "tests/integration/dnstap/**"

scripts/integration/docker-logs/test.yaml → tests/integration/docker-logs/config/test.yaml

@@ -16,4 +16,4 @@ paths:
 - "src/internal_events/docker_logs.rs"
 - "src/sources/docker_logs/**"
 - "src/sources/util/**"
-- "scripts/integration/docker-logs/**"
+- "tests/integration/docker-logs/**"

scripts/integration/elasticsearch/compose.yaml → tests/integration/elasticsearch/config/compose.yaml

@@ -24,7 +24,7 @@ services:
     - ELASTIC_PASSWORD=vector
     - ES_JAVA_OPTS=-Xms400m -Xmx400m
     volumes:
-    - ../../../tests/data/ca:/usr/share/elasticsearch/config/certs:ro
+    - ../../../data/ca:/usr/share/elasticsearch/config/certs:ro
 
 networks:
   default:

scripts/integration/elasticsearch/test.yaml → tests/integration/elasticsearch/config/test.yaml

@@ -18,4 +18,4 @@ matrix:
 paths:
 - "src/sinks/elasticsearch/**"
 - "src/sinks/util/**"
-- "scripts/integration/elasticsearch/**"
+- "tests/integration/elasticsearch/**"

scripts/integration/eventstoredb/compose.yaml → tests/integration/eventstoredb/config/compose.yaml

@@ -5,7 +5,7 @@ services:
     image: docker.io/eventstore/eventstore:${CONFIG_VERSION}
     command: --insecure --stats-period-sec=1
     volumes:
-    - ../../../tests/data:/etc/vector:ro
+    - ../../../data:/etc/vector:ro
 
 networks:
   default:

scripts/integration/eventstoredb/test.yaml → tests/integration/eventstoredb/config/test.yaml

@@ -12,4 +12,4 @@ paths:
 - "src/internal_events/eventstoredb_metrics.rs"
 - "src/sources/eventstoredb_metrics/**"
 - "src/sources/util/**"
-- "scripts/integration/eventstoredb/**"
+- "tests/integration/eventstoredb/**"

scripts/integration/fluent/test.yaml → tests/integration/fluent/config/test.yaml

@@ -17,4 +17,4 @@ paths:
 - "src/internal_events/fluent.rs"
 - "src/sources/fluent/**"
 - "src/sources/util/**"
-- "scripts/integration/fluent/**"
+- "tests/integration/fluent/**"

scripts/integration/gcp/test.yaml → tests/integration/gcp/config/test.yaml

@@ -20,4 +20,4 @@ paths:
 - "src/sinks/gcp/**"
 - "src/sinks/util/**"
 - "src/gcp.rs"
-- "scripts/integration/gcp/**"
+- "tests/integration/gcp/**"

scripts/integration/greptimedb/test.yaml → tests/integration/greptimedb/config/test.yaml

@@ -18,4 +18,4 @@ matrix:
 # expressions are evaluated using https://github.com/micromatch/picomatch
 paths:
 - "src/sinks/greptimedb/**"
-- "scripts/integration/greptimedb/**"
+- "tests/integration/greptimedb/**"

scripts/integration/http-client/compose.yaml → tests/integration/http-client/config/compose.yaml

@@ -6,7 +6,7 @@ services:
     command:
     - /data
     volumes:
-    - ../../../tests/data/http-client/serve:/data
+    - ../data/serve:/data
   dufs-auth:
     image: docker.io/sigoden/dufs:${CONFIG_VERSION}
     command:
@@ -16,7 +16,7 @@ services:
     - basic
     - /data
     volumes:
-    - ../../../tests/data/http-client/serve:/data
+    - ../data/serve:/data
   dufs-https:
     image: docker.io/sigoden/dufs:${CONFIG_VERSION}
     command:
@@ -26,9 +26,9 @@ services:
     - /certs/ca.key.pem
     - /data
     volumes:
-    - ../../../tests/data/http-client/serve:/data
-    - ../../../tests/data/ca/intermediate_server/certs/dufs-https-chain.cert.pem:/certs/ca.cert.pem
-    - ../../../tests/data/ca/intermediate_server/private/dufs-https.key.pem:/certs/ca.key.pem
+    - ../data/serve:/data
+    - ../../../data/ca/intermediate_server/certs/dufs-https-chain.cert.pem:/certs/ca.cert.pem
+    - ../../../data/ca/intermediate_server/private/dufs-https.key.pem:/certs/ca.key.pem
 
 networks:
   default: