chore(ci): reorganize integration test files (vectordotdev#24108)

* chore(ci): reorganize integration test files

* revert nats data move

* fix paths to data

* fix more paths

* amqp fix

* Fix logstash int tests shared files paths

* Rename scripts/integration -> tests/integration

* scripts/integration -> tests/integration in source files

* Fix scripts->tests

* Fix gcp paths

* Fix http-client pem path

* move nats data

---------

Co-authored-by: Thomas <thomas.schneider@datadoghq.com>

Author: 2 people

.github/workflows/changes.yml

@@ -324,7 +324,7 @@ jobs:
 
       # creates a yaml file that contains the filters for each integration,
       # extracted from the output of the `vdev int ci-paths` command, which
-      # sources the paths from the scripts/integration/.../test.yaml files
+      # sources the paths from the tests/integration/.../config/test.yaml files
       - name: Create filter rules for integrations
         run: vdev int ci-paths > int_test_filters.yaml
 

src/nats.rs

@@ -266,7 +266,7 @@ mod tests {
         parse_auth(
             r#"
             strategy = "credentials_file"
-            credentials_file.path = "tests/data/nats/nats.creds"
+            credentials_file.path = "tests/integration/nats/data/nats.creds"
         "#,
         )
         .unwrap();

src/sinks/gcp_chronicle/chronicle_unstructured.rs

@@ -718,10 +718,9 @@ mod integration_tests {
         trace_init();
 
         let log_type = random_string(10);
-        let (sink, healthcheck) =
-            config_build(&log_type, "/home/vector/scripts/integration/gcp/auth.json")
-                .await
-                .expect("Building sink failed");
+        let (sink, healthcheck) = config_build(&log_type, "tests/integration/gcp/config/auth.json")
+            .await
+            .expect("Building sink failed");
 
         healthcheck.await.expect("Health check failed");
 
@@ -749,11 +748,7 @@ mod integration_tests {
 
         let log_type = random_string(10);
         // Test with an auth file that doesnt match the public key sent to the dummy chronicle server.
-        let sink = config_build(
-            &log_type,
-            "/home/vector/scripts/integration/gcp/invalidauth.json",
-        )
-        .await;
+        let sink = config_build(&log_type, "tests/integration/gcp/config/invalidauth.json").await;
 
         assert!(sink.is_err())
     }
@@ -765,10 +760,9 @@ mod integration_tests {
         // The chronicle-emulator we are testing against is setup so a `log_type` of "INVALID"
         // will return a `400 BAD_REQUEST`.
         let log_type = "INVALID";
-        let (sink, healthcheck) =
-            config_build(log_type, "/home/vector/scripts/integration/gcp/auth.json")
-                .await
-                .expect("Building sink failed");
+        let (sink, healthcheck) = config_build(log_type, "tests/integration/gcp/config/auth.json")
+            .await
+            .expect("Building sink failed");
 
         healthcheck.await.expect("Health check failed");
 

src/sinks/nats/integration_tests.rs

@@ -252,7 +252,7 @@ async fn nats_tls_valid() {
     conf.tls = Some(TlsEnableableConfig {
         enabled: Some(true),
         options: TlsConfig {
-            ca_file: Some("tests/data/nats/rootCA.pem".into()),
+            ca_file: Some("tests/integration/nats/data/rootCA.pem".into()),
             ..Default::default()
         },
     });
@@ -293,9 +293,9 @@ async fn nats_tls_client_cert_valid() {
     conf.tls = Some(TlsEnableableConfig {
         enabled: Some(true),
         options: TlsConfig {
-            ca_file: Some("tests/data/nats/rootCA.pem".into()),
-            crt_file: Some("tests/data/nats/nats-client.pem".into()),
-            key_file: Some("tests/data/nats/nats-client.key".into()),
+            ca_file: Some("tests/integration/nats/data/rootCA.pem".into()),
+            crt_file: Some("tests/integration/nats/data/nats-client.pem".into()),
+            key_file: Some("tests/integration/nats/data/nats-client.key".into()),
             ..Default::default()
         },
     });
@@ -319,7 +319,7 @@ async fn nats_tls_client_cert_invalid() {
     conf.tls = Some(TlsEnableableConfig {
         enabled: Some(true),
         options: TlsConfig {
-            ca_file: Some("tests/data/nats/rootCA.pem".into()),
+            ca_file: Some("tests/integration/nats/data/rootCA.pem".into()),
             ..Default::default()
         },
     });
@@ -343,13 +343,13 @@ async fn nats_tls_jwt_auth_valid() {
     conf.tls = Some(TlsEnableableConfig {
         enabled: Some(true),
         options: TlsConfig {
-            ca_file: Some("tests/data/nats/rootCA.pem".into()),
+            ca_file: Some("tests/integration/nats/data/rootCA.pem".into()),
             ..Default::default()
         },
     });
     conf.auth = Some(NatsAuthConfig::CredentialsFile {
         credentials_file: NatsAuthCredentialsFile {
-            path: "tests/data/nats/nats.creds".into(),
+            path: "tests/integration/nats/data/nats.creds".into(),
         },
     });
 
@@ -372,13 +372,13 @@ async fn nats_tls_jwt_auth_invalid() {
     conf.tls = Some(TlsEnableableConfig {
         enabled: Some(true),
         options: TlsConfig {
-            ca_file: Some("tests/data/nats/rootCA.pem".into()),
+            ca_file: Some("tests/integration/nats/data/rootCA.pem".into()),
             ..Default::default()
         },
     });
     conf.auth = Some(NatsAuthConfig::CredentialsFile {
         credentials_file: NatsAuthCredentialsFile {
-            path: "tests/data/nats/nats-bad.creds".into(),
+            path: "tests/integration/nats/data/nats-bad.creds".into(),
         },
     });
 

src/sources/amqp.rs

@@ -618,7 +618,7 @@ pub mod test {
     }
 }
 
-/// Integration tests use the docker compose files in `scripts/integration/docker-compose.amqp.yml`.
+/// Integration tests use the docker compose files in `tests/integration/docker-compose.amqp.yml`.
 #[cfg(feature = "amqp-integration-tests")]
 #[cfg(test)]
 mod integration_test {

src/sources/http_client/integration_tests.rs

@@ -244,7 +244,7 @@ async fn tls_invalid_ca() {
         headers: HashMap::new(),
         method: HttpMethod::Get,
         tls: Some(TlsConfig {
-            ca_file: Some("tests/data/http-client/certs/invalid-ca-cert.pem".into()),
+            ca_file: Some("tests/integration/http-client/data/certs/invalid-ca-cert.pem".into()),
             ..Default::default()
         }),
         auth: None,

src/sources/logstash.rs

@@ -966,8 +966,12 @@ mod integration_tests {
                 Some(TlsEnableableConfig {
                     enabled: Some(true),
                     options: TlsConfig {
-                        crt_file: Some("tests/data/host.docker.internal.crt".into()),
-                        key_file: Some("tests/data/host.docker.internal.key".into()),
+                        crt_file: Some(
+                            "tests/integration/shared/data/host.docker.internal.crt".into(),
+                        ),
+                        key_file: Some(
+                            "tests/integration/shared/data/host.docker.internal.key".into(),
+                        ),
                         ..Default::default()
                     },
                 }),

src/sources/nats/integration_tests.rs

@@ -280,7 +280,7 @@ async fn nats_tls_valid() {
     conf.tls = Some(TlsEnableableConfig {
         enabled: Some(true),
         options: TlsConfig {
-            ca_file: Some("tests/data/nats/rootCA.pem".into()),
+            ca_file: Some("tests/integration/nats/data/rootCA.pem".into()),
             ..Default::default()
         },
     });
@@ -317,9 +317,9 @@ async fn nats_tls_client_cert_valid() {
     conf.tls = Some(TlsEnableableConfig {
         enabled: Some(true),
         options: TlsConfig {
-            ca_file: Some("tests/data/nats/rootCA.pem".into()),
-            crt_file: Some("tests/data/nats/nats-client.pem".into()),
-            key_file: Some("tests/data/nats/nats-client.key".into()),
+            ca_file: Some("tests/integration/nats/data/rootCA.pem".into()),
+            crt_file: Some("tests/integration/nats/data/nats-client.pem".into()),
+            key_file: Some("tests/integration/nats/data/nats-client.key".into()),
             ..Default::default()
         },
     });
@@ -341,7 +341,7 @@ async fn nats_tls_client_cert_invalid() {
     conf.tls = Some(TlsEnableableConfig {
         enabled: Some(true),
         options: TlsConfig {
-            ca_file: Some("tests/data/nats/rootCA.pem".into()),
+            ca_file: Some("tests/integration/nats/data/rootCA.pem".into()),
             ..Default::default()
         },
     });
@@ -363,13 +363,13 @@ async fn nats_tls_jwt_auth_valid() {
     conf.tls = Some(TlsEnableableConfig {
         enabled: Some(true),
         options: TlsConfig {
-            ca_file: Some("tests/data/nats/rootCA.pem".into()),
+            ca_file: Some("tests/integration/nats/data/rootCA.pem".into()),
             ..Default::default()
         },
     });
     conf.auth = Some(NatsAuthConfig::CredentialsFile {
         credentials_file: NatsAuthCredentialsFile {
-            path: "tests/data/nats/nats.creds".into(),
+            path: "tests/integration/nats/data/nats.creds".into(),
         },
     });
 
@@ -390,13 +390,13 @@ async fn nats_tls_jwt_auth_invalid() {
     conf.tls = Some(TlsEnableableConfig {
         enabled: Some(true),
         options: TlsConfig {
-            ca_file: Some("tests/data/nats/rootCA.pem".into()),
+            ca_file: Some("tests/integration/nats/data/rootCA.pem".into()),
             ..Default::default()
         },
     });
     conf.auth = Some(NatsAuthConfig::CredentialsFile {
         credentials_file: NatsAuthCredentialsFile {
-            path: "tests/data/nats/nats-bad.creds".into(),
+            path: "tests/integration/nats/data/nats-bad.creds".into(),
         },
     });