diff --git a/.github/workflows/Integration.yml b/.github/workflows/Integration.yml index 03ec295e..3e968800 100644 --- a/.github/workflows/Integration.yml +++ b/.github/workflows/Integration.yml @@ -11,6 +11,8 @@ permissions: env: CARGO_TERM_COLOR: always + SLACK_TEST_WEBHOOK: ${{ secrets.SLACK_TEST_WEBHOOK }} + SLACK_TEST_BOT_TOKEN: ${{ secrets.SLACK_TEST_BOT_TOKEN }} jobs: ubuntu-cranelift: diff --git a/.gitignore b/.gitignore index 15e6d744..43431f27 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,4 @@ /runtime/target /modules/target /target -/plaid-stl/target \ No newline at end of file +/plaid-stl/target diff --git a/modules/Cargo.lock b/modules/Cargo.lock index 552a0ac7..7f372762 100644 --- a/modules/Cargo.lock +++ b/modules/Cargo.lock @@ -86,6 +86,15 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "example_github_graphql" +version = "0.1.0" +dependencies = [ + "plaid_stl", + "serde", + "serde_json", +] + [[package]] name = "getrandom" version = "0.2.15" @@ -180,7 +189,7 @@ checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] name = "plaid_stl" -version = "0.23.2" +version = "0.25.0" dependencies = [ "base64 0.13.1", "chrono", @@ -424,6 +433,15 @@ dependencies = [ "serde_json", ] +[[package]] +name = "test_slack" +version = "0.1.0" +dependencies = [ + "plaid_stl", + "serde", + "serde_json", +] + [[package]] name = "test_sshcerts_usage" version = "0.1.0" diff --git a/modules/Cargo.toml b/modules/Cargo.toml index 4992a6b0..6f6021a2 100644 --- a/modules/Cargo.toml +++ b/modules/Cargo.toml @@ -3,6 +3,8 @@ resolver = "2" members = [ + "examples/example_github_graphql", + "tests/test_crashtest", "tests/test_db", "tests/test_fileopen", @@ -17,6 +19,7 @@ members = [ "tests/test_time", "tests/test_shared_db_rule_1", "tests/test_shared_db_rule_2", + "tests/test_slack", ] [profile.release] diff --git a/modules/examples/example_github_graphql/Cargo.toml b/modules/examples/example_github_graphql/Cargo.toml new file mode 100644 index 00000000..ecf5ceab --- /dev/null +++ b/modules/examples/example_github_graphql/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "example_github_graphql" +description = "Example to show how to use the GitHub GraphQL API with Plaid." +version = "0.1.0" +edition = "2021" + +[dependencies] +plaid_stl = { path = "../../../runtime/plaid-stl" } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1" + +[lib] +crate-type = ["cdylib"] diff --git a/modules/examples/example_github_graphql/src/lib.rs b/modules/examples/example_github_graphql/src/lib.rs new file mode 100644 index 00000000..430d75e2 --- /dev/null +++ b/modules/examples/example_github_graphql/src/lib.rs @@ -0,0 +1,171 @@ +use plaid_stl::{entrypoint_with_source_and_response, github, messages::LogSource, plaid}; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use std::{collections::HashMap, error::Error, fmt::Display}; + +#[derive(Debug)] +enum Errors { + BadSender = 1, + BadConfiguration, + BadAuthentication, + NoOrganization, + NetworkFailure, + UnknownFailure, +} + +impl Display for Errors { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self) + } +} + +impl Error for Errors {} + +// This is configured in the runtime configuration within the api.toml +// (because it is part of the GitHub API configuration) +const GITHUB_GRAPHQL_QUERY: &str = "saml_identities"; + +entrypoint_with_source_and_response!(); + +#[derive(Deserialize)] +struct PageInfo { + #[serde(rename = "hasNextPage")] + has_next_page: bool, + #[serde(rename = "endCursor")] + end_cursor: String, +} + +#[derive(Deserialize)] +struct SamlIdentity { + #[serde(rename = "nameId")] + name_id: Option, +} + +#[derive(Deserialize)] +struct User { + login: String, +} + +#[derive(Deserialize)] +struct Node { + user: Option, + #[serde(rename = "samlIdentity")] + saml_identity: SamlIdentity, +} + +#[derive(Deserialize)] +struct ExternalIdentities { + #[serde(rename = "pageInfo")] + page_info: PageInfo, + nodes: Vec, +} + +#[derive(Serialize)] +struct ReturnData { + found_users: HashMap, + emails_no_users: Vec, + users_no_emails: Vec, +} + +fn simple_const_time_compare(a: &str, b: &str) -> bool { + // Compare lengths first to short-circuit if they are not equal + if a.len() != b.len() { + return false; + } + // Compare each character in constant time + a.chars() + .zip(b.chars()) + .fold(0, |acc, (x, y)| acc | (x as u8 ^ y as u8)) + == 0 +} + +fn main(_: String, source: LogSource) -> Result, Errors> { + // This module should only be called from a webhook POST request + match source { + LogSource::WebhookGet(_) => {} + _ => return Err(Errors::BadSender), + }; + + let true_auth_token = plaid::get_secrets("organization_fetch_auth_token") + .map_err(|_| Errors::BadConfiguration)?; + + let provided_auth_token = + plaid::get_headers("Authorization").map_err(|_| Errors::BadAuthentication)?; + + if !simple_const_time_compare(&true_auth_token, &provided_auth_token) { + return Err(Errors::BadAuthentication); + } + + let organization = + plaid::get_query_params("organization").map_err(|_| Errors::NoOrganization)?; + + // Hold all the results as a mapping of user ID to email + let mut found_users: HashMap = HashMap::new(); + let mut emails_no_users = Vec::new(); + let mut users_no_emails = Vec::new(); + let mut previous_cursor = String::new(); + // We need to make multiple requests because there could be more than 100 users in + // an organization, and the GraphQL API returns a maximum of 100 results. + loop { + let variables = [ + ("organization".to_string(), organization.clone()), + ("cursor".to_string(), previous_cursor.clone()), + ] + .into(); + + match github::make_graphql_query(GITHUB_GRAPHQL_QUERY, variables) { + Ok(result) => { + let result: Value = serde_json::from_str(&result).unwrap(); + + let ext_ident = result + .get("data") + .and_then(|v| v.get("organization")) + .and_then(|v| v.get("samlIdentityProvider")) + .and_then(|v| v.get("externalIdentities")); + + let ext_ident: ExternalIdentities = + serde_json::from_value(ext_ident.unwrap().clone()).map_err(|e| { + plaid::print_debug_string(&format!( + "Failed to deserialize external identities: {e}", + )); + Errors::NetworkFailure + })?; + + for user in ext_ident.nodes { + match (user.user, user.saml_identity.name_id) { + (Some(u), Some(email)) => { + // We have both a user and an email + found_users.insert(u.login, email); + } + (Some(u), None) => { + // We have a user but no email + users_no_emails.push(u.login); + } + (None, Some(email)) => { + // We have an email but no user + emails_no_users.push(email); + } + (None, None) => { + // Neither user nor email + continue; + } + } + } + if !ext_ident.page_info.has_next_page { + return Ok(Some( + serde_json::to_string(&ReturnData { + found_users, + emails_no_users, + users_no_emails, + }) + .unwrap(), + )); + } + previous_cursor = ext_ident.page_info.end_cursor; + } + _ => return Err(Errors::UnknownFailure), + } + } +} diff --git a/modules/tests/test_db/harness/harness.sh b/modules/tests/test_db/harness/harness.sh index ffd78b26..291a538e 100755 --- a/modules/tests/test_db/harness/harness.sh +++ b/modules/tests/test_db/harness/harness.sh @@ -16,50 +16,50 @@ RH_PID=$! sleep 2 # Call the webhook -curl -d "get:some_key" http://$PLAID_LOCATION/webhook/$URL; sleep 0.5 -curl -d "insert:my_key:first_value" http://$PLAID_LOCATION/webhook/$URL; sleep 0.5 -curl -d "get:some_key" http://$PLAID_LOCATION/webhook/$URL; sleep 0.5 -curl -d "get:my_key" http://$PLAID_LOCATION/webhook/$URL; sleep 0.5 -curl -d "insert:my_key:second_value" http://$PLAID_LOCATION/webhook/$URL; sleep 0.5 -curl -d "get:some_key" http://$PLAID_LOCATION/webhook/$URL; sleep 0.5 -curl -d "get:my_key" http://$PLAID_LOCATION/webhook/$URL; sleep 0.5 -curl -d "delete:my_key" http://$PLAID_LOCATION/webhook/$URL; sleep 0.5 -curl -d "get:my_key" http://$PLAID_LOCATION/webhook/$URL; sleep 0.5 -curl -d "delete:another_key" http://$PLAID_LOCATION/webhook/$URL; sleep 0.5 -curl -d "get:another_key" http://$PLAID_LOCATION/webhook/$URL; sleep 0.5 +curl -d "get:some_key" http://$PLAID_LOCATION/webhook/$URL +curl -d "insert:my_key:first_value" http://$PLAID_LOCATION/webhook/$URL +curl -d "get:some_key" http://$PLAID_LOCATION/webhook/$URL +curl -d "get:my_key" http://$PLAID_LOCATION/webhook/$URL +curl -d "insert:my_key:second_value" http://$PLAID_LOCATION/webhook/$URL +curl -d "get:some_key" http://$PLAID_LOCATION/webhook/$URL +curl -d "get:my_key" http://$PLAID_LOCATION/webhook/$URL +curl -d "delete:my_key" http://$PLAID_LOCATION/webhook/$URL +curl -d "get:my_key" http://$PLAID_LOCATION/webhook/$URL +curl -d "delete:another_key" http://$PLAID_LOCATION/webhook/$URL +curl -d "get:another_key" http://$PLAID_LOCATION/webhook/$URL # At this point the DB is empty -curl -d "insert:my_key:first_value" http://$PLAID_LOCATION/webhook/$URL; sleep 0.5 +curl -d "insert:my_key:first_value" http://$PLAID_LOCATION/webhook/$URL # too many bytes for the configured storage limit -curl -d "insert:a_key:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" http://$PLAID_LOCATION/webhook/$URL; sleep 0.5 +curl -d "insert:a_key:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" http://$PLAID_LOCATION/webhook/$URL # empty because insertion failed -curl -d "get:a_key" http://$PLAID_LOCATION/webhook/$URL; sleep 0.5 +curl -d "get:a_key" http://$PLAID_LOCATION/webhook/$URL # this is within the limit, so it's fine -curl -d "insert:a_key:a" http://$PLAID_LOCATION/webhook/$URL; sleep 0.5 +curl -d "insert:a_key:a" http://$PLAID_LOCATION/webhook/$URL # returns "a" -curl -d "get:a_key" http://$PLAID_LOCATION/webhook/$URL; sleep 0.5 -curl -d "delete:my_key" http://$PLAID_LOCATION/webhook/$URL; sleep 0.5 -curl -d "delete:a_key" http://$PLAID_LOCATION/webhook/$URL; sleep 0.5 +curl -d "get:a_key" http://$PLAID_LOCATION/webhook/$URL +curl -d "delete:my_key" http://$PLAID_LOCATION/webhook/$URL +curl -d "delete:a_key" http://$PLAID_LOCATION/webhook/$URL # now the DB is empty, so we can insert the long key/value pair -curl -d "insert:a_key:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" http://$PLAID_LOCATION/webhook/$URL; sleep 0.5 -curl -d "get:a_key" http://$PLAID_LOCATION/webhook/$URL; sleep 0.5 -curl -d "delete:a_key" http://$PLAID_LOCATION/webhook/$URL; sleep 0.5 +curl -d "insert:a_key:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" http://$PLAID_LOCATION/webhook/$URL +curl -d "get:a_key" http://$PLAID_LOCATION/webhook/$URL +curl -d "delete:a_key" http://$PLAID_LOCATION/webhook/$URL # the DB is empty -curl -d "insert:my_key:a" http://$PLAID_LOCATION/webhook/$URL; sleep 0.5 -curl -d "insert:my_new_key:b" http://$PLAID_LOCATION/webhook/$URL; sleep 0.5 -curl -d "insert:a_key:c" http://$PLAID_LOCATION/webhook/$URL; sleep 0.5 -curl -d "list_keys:all:my_key|my_new_key|a_key" http://$PLAID_LOCATION/webhook/$URL; sleep 0.5 -curl -d "list_keys:prefix:my:my_key|my_new_key" http://$PLAID_LOCATION/webhook/$URL; sleep 0.5 +curl -d "insert:my_key:a" http://$PLAID_LOCATION/webhook/$URL +curl -d "insert:my_new_key:b" http://$PLAID_LOCATION/webhook/$URL +curl -d "insert:a_key:c" http://$PLAID_LOCATION/webhook/$URL +curl -d "list_keys:all:my_key|my_new_key|a_key" http://$PLAID_LOCATION/webhook/$URL +curl -d "list_keys:prefix:my:my_key|my_new_key" http://$PLAID_LOCATION/webhook/$URL curl -d "delete:my_key" http://$PLAID_LOCATION/webhook/$URL curl -d "delete:my_new_key" http://$PLAID_LOCATION/webhook/$URL -curl -d "delete:a_key" http://$PLAID_LOCATION/webhook/$URL; sleep 0.5 +curl -d "delete:a_key" http://$PLAID_LOCATION/webhook/$URL # the DB is empty -curl -d "insert:some_key:a" http://$PLAID_LOCATION/webhook/$URL; sleep 0.5 -curl -d "insert_check_returned_data:some_key:a" http://$PLAID_LOCATION/webhook/$URL; sleep 0.5 -curl -d "delete:some_key" http://$PLAID_LOCATION/webhook/$URL; sleep 0.5 +curl -d "insert:some_key:a" http://$PLAID_LOCATION/webhook/$URL +curl -d "insert_check_returned_data:some_key:a" http://$PLAID_LOCATION/webhook/$URL +curl -d "delete:some_key" http://$PLAID_LOCATION/webhook/$URL # the DB is empty -curl -d "insert:some_key:a" http://$PLAID_LOCATION/webhook/$URL; sleep 0.5 -curl -d "delete_check_returned_data:some_key:a" http://$PLAID_LOCATION/webhook/$URL; sleep 0.5 -curl -d "insert:some_key:some_value" http://$PLAID_LOCATION/webhook/$URL; sleep 0.5 +curl -d "insert:some_key:a" http://$PLAID_LOCATION/webhook/$URL +curl -d "delete_check_returned_data:some_key:a" http://$PLAID_LOCATION/webhook/$URL +curl -d "insert:some_key:some_value" http://$PLAID_LOCATION/webhook/$URL curl -d "delete_check_returned_data:some_key:some_value" http://$PLAID_LOCATION/webhook/$URL # the DB is empty diff --git a/modules/tests/test_slack/Cargo.toml b/modules/tests/test_slack/Cargo.toml new file mode 100644 index 00000000..5b8a0aa9 --- /dev/null +++ b/modules/tests/test_slack/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "test_slack" +description = "Test rule to check the Slack API works" +version = "0.1.0" +edition = "2021" + +[dependencies] +plaid_stl = { path = "../../../runtime/plaid-stl" } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1" + +[lib] +crate-type = ["cdylib"] diff --git a/modules/tests/test_slack/harness/harness.sh b/modules/tests/test_slack/harness/harness.sh new file mode 100755 index 00000000..f55ab398 --- /dev/null +++ b/modules/tests/test_slack/harness/harness.sh @@ -0,0 +1,41 @@ +#!/bin/bash +set -e +# If GITHUB_ACTIONS is not set, skip because Plaid won't be running +# with the Slack API properly configured +if [ -z "$GITHUB_ACTIONS" ]; then + echo "Not running in GitHub Actions, skipping Slack tests" + exit 0 +fi + +if [ -z "$SLACK_TEST_WEBHOOK" ] || [ -z "$SLACK_TEST_BOT_TOKEN" ]; then + echo "Slack secrets are not available, skipping Slack tests" + exit 0 +fi + +URL="testslack" +FILE="received_data.$URL.txt" + +# Start the webhook +$REQUEST_HANDLER > $FILE & +if [ $? -ne 0 ]; then + echo "SlackTest: Failed to start request handler" + rm $FILE + exit 1 +fi + +RH_PID=$! + + + +# Call the webhook +OUTPUT=$(curl -XPOST -d 'slack_input' http://$PLAID_LOCATION/webhook/$URL) +sleep 2 +kill $RH_PID 2>&1 > /dev/null + +echo -e "OK\nOK\nOK\nOK\nOK" > expected.txt +diff expected.txt $FILE +RESULT=$? + +rm -f $FILE expected.txt + +exit $RESULT diff --git a/modules/tests/test_slack/src/lib.rs b/modules/tests/test_slack/src/lib.rs new file mode 100644 index 00000000..8a55fb3c --- /dev/null +++ b/modules/tests/test_slack/src/lib.rs @@ -0,0 +1,64 @@ +use std::collections::HashMap; + +use plaid_stl::network::make_named_request; +use plaid_stl::slack::{self, get_presence, post_message, post_text_to_webhook, user_info}; +use plaid_stl::{entrypoint_with_source, messages::LogSource, plaid}; + +entrypoint_with_source!(); + +fn main(log: String, _: LogSource) -> Result<(), i32> { + plaid::print_debug_string(&format!("Testing slack APIs with log: {log}")); + + if let Err(_) = post_text_to_webhook("test_webhook", "Testing this makes it to slack") { + plaid::print_debug_string("Failed to post to slack"); + panic!("Couldn't post to slack") + } + + make_named_request("test-response", "OK", HashMap::new()).unwrap(); + + let user_id = slack::get_id_from_email("plaid-testing", "mitchell@confurious.io") + .unwrap_or_else(|_| { + plaid::print_debug_string("Failed to get user ID from email"); + panic!("Couldn't get user ID from email") + }); + + make_named_request("test-response", "OK", HashMap::new()).unwrap(); + + if let Err(_) = post_message( + "plaid-testing", + &user_id, + "Testing that this goes directly to obelisk", + ) { + plaid::print_debug_string("Failed to send Slack message"); + panic!("Couldn't send Slack message") + } + + make_named_request("test-response", "OK", HashMap::new()).unwrap(); + + match get_presence("plaid-testing", &user_id) { + Ok(presence) => { + plaid::print_debug_string(&format!("Got user presence as: {}", presence.presence)) + } + Err(_) => { + plaid::print_debug_string("Failed to get user presence"); + panic!("Couldn't get user presence") + } + } + + make_named_request("test-response", "OK", HashMap::new()).unwrap(); + + match user_info("plaid-testing", &user_id) { + Ok(info) => plaid::print_debug_string(&format!( + "Got user info. Status is: [{}]. TZ is: [{}]", + info.user.profile.status_text, info.user.tz_label + )), + Err(_) => { + plaid::print_debug_string("Failed to get user presence"); + panic!("Couldn't get user presence") + } + } + + make_named_request("test-response", "OK", HashMap::new()).unwrap(); + + Ok(()) +} diff --git a/runtime/Cargo.lock b/runtime/Cargo.lock index 57ff8511..ad91b02d 100644 --- a/runtime/Cargo.lock +++ b/runtime/Cargo.lock @@ -23,17 +23,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" -[[package]] -name = "aes" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" -dependencies = [ - "cfg-if", - "cipher", - "cpufeatures", -] - [[package]] name = "ahash" version = "0.8.11" @@ -234,9 +223,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4471bef4c22a06d2c7a1b6492493d3fdf24a805323109d6874f9c94d5906ac14" +checksum = "687bc16bc431a8533fe0097c7f0182874767f920989d7260950172ae8e3c4465" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -269,9 +258,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.5.6" +version = "1.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aff45ffe35196e593ea3b9dd65b320e51e2dda95aff4390bc459e461d09c6ad" +checksum = "4f6c68419d8ba16d9a7463671593c54f81ba58cab466e9b759418da606dcc2e2" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -285,7 +274,6 @@ dependencies = [ "fastrand", "http 0.2.12", "http-body 0.4.6", - "once_cell", "percent-encoding", "pin-project-lite", "tracing", @@ -315,6 +303,28 @@ dependencies = [ "tracing", ] +[[package]] +name = "aws-sdk-identitystore" +version = "1.73.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e50bd32b8b1e0408f6fbf482c603bcf50aed895f4ab21e82d8b1f34592b7a303" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "regex-lite", + "tracing", +] + [[package]] name = "aws-sdk-kms" version = "1.65.0" @@ -433,9 +443,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.3.0" +version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69d03c3c05ff80d54ff860fe38c726f6f494c639ae975203a101335f223386db" +checksum = "ddfb9021f581b71870a17eac25b52335b82211cdc092e02b6876b2bcefa61666" dependencies = [ "aws-credential-types", "aws-smithy-http", @@ -447,7 +457,6 @@ dependencies = [ "hmac", "http 0.2.12", "http 1.3.1", - "once_cell", "percent-encoding", "sha2", "time", @@ -467,9 +476,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.62.0" +version = "0.62.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5949124d11e538ca21142d1fba61ab0a2a2c1bc3ed323cdb3e4b878bfb83166" +checksum = "99335bec6cdc50a346fda1437f9fefe33abf8c99060739a546a16457f2862ca9" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", @@ -479,7 +488,6 @@ dependencies = [ "http 0.2.12", "http 1.3.1", "http-body 0.4.6", - "once_cell", "percent-encoding", "pin-project-lite", "pin-utils", @@ -516,21 +524,20 @@ dependencies = [ [[package]] name = "aws-smithy-json" -version = "0.61.3" +version = "0.61.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92144e45819cae7dc62af23eac5a038a58aa544432d2102609654376a900bd07" +checksum = "a16e040799d29c17412943bdbf488fd75db04112d0c0d4b9290bacf5ae0014b9" dependencies = [ "aws-smithy-types", ] [[package]] name = "aws-smithy-observability" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445d065e76bc1ef54963db400319f1dd3ebb3e0a74af20f7f7630625b0cc7cc0" +checksum = "9364d5989ac4dd918e5cc4c4bdcc61c9be17dcd2586ea7f69e348fc7c6cab393" dependencies = [ "aws-smithy-runtime-api", - "once_cell", ] [[package]] @@ -545,9 +552,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.8.1" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0152749e17ce4d1b47c7747bdfec09dac1ccafdcbc741ebf9daa2a373356730f" +checksum = "14302f06d1d5b7d333fd819943075b13d27c7700b414f574c3c35859bfb55d5e" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -561,7 +568,6 @@ dependencies = [ "http 1.3.1", "http-body 0.4.6", "http-body 1.0.1", - "once_cell", "pin-project-lite", "pin-utils", "tokio", @@ -570,9 +576,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.7.4" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3da37cf5d57011cb1753456518ec76e31691f1f474b73934a284eb2a1c76510f" +checksum = "bd8531b6d8882fd8f48f82a9754e682e29dd44cff27154af51fa3eb730f59efb" dependencies = [ "aws-smithy-async", "aws-smithy-types", @@ -587,9 +593,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.3.0" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836155caafba616c0ff9b07944324785de2ab016141c3550bd1c07882f8cee8f" +checksum = "d498595448e43de7f4296b7b7a18a8a02c61ec9349128c80a368f7c3b4ab11a8" dependencies = [ "base64-simd", "bytes", @@ -622,9 +628,9 @@ dependencies = [ [[package]] name = "aws-types" -version = "1.3.6" +version = "1.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3873f8deed8927ce8d04487630dc9ff73193bab64742a61d050e57a68dec4125" +checksum = "8a322fec39e4df22777ed3ad8ea868ac2f94cd15e1a55f6ee8d8d6305057689a" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -849,25 +855,6 @@ dependencies = [ "either", ] -[[package]] -name = "bzip2" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" -dependencies = [ - "bzip2-sys", -] - -[[package]] -name = "bzip2-sys" -version = "0.1.13+1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" -dependencies = [ - "cc", - "pkg-config", -] - [[package]] name = "cc" version = "1.2.18" @@ -915,16 +902,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "cipher" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" -dependencies = [ - "crypto-common", - "inout", -] - [[package]] name = "clang-sys" version = "1.8.1" @@ -1174,21 +1151,6 @@ version = "0.110.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56b08621c00321efcfa3eee6a3179adc009e21ea8d24ca7adc3c326184bc3f48" -[[package]] -name = "crc" -version = "3.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" -dependencies = [ - "crc-catalog", -] - -[[package]] -name = "crc-catalog" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" - [[package]] name = "crc32fast" version = "1.4.2" @@ -1329,12 +1291,6 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "575f75dfd25738df5b91b8e43e14d44bda14637a58fae779fd2b064f8bf3e010" -[[package]] -name = "deflate64" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da692b8d1080ea3045efaab14434d40468c3d8657e42abddfffca87b428f4c1b" - [[package]] name = "der" version = "0.7.9" @@ -1391,6 +1347,27 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "derive_more" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", + "unicode-xid", +] + [[package]] name = "digest" version = "0.10.7" @@ -1783,7 +1760,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" dependencies = [ "fallible-iterator", - "indexmap 2.9.0", + "indexmap", "stable_deref_trait", ] @@ -1822,7 +1799,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.9.0", + "indexmap", "slab", "tokio", "tokio-util", @@ -1841,19 +1818,13 @@ dependencies = [ "futures-core", "futures-sink", "http 1.3.1", - "indexmap 2.9.0", + "indexmap", "slab", "tokio", "tokio-util", "tracing", ] -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - [[package]] name = "hashbrown" version = "0.13.2" @@ -2354,16 +2325,6 @@ dependencies = [ "icu_properties", ] -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", -] - [[package]] name = "indexmap" version = "2.9.0" @@ -2399,15 +2360,6 @@ dependencies = [ "syn 2.0.100", ] -[[package]] -name = "inout" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" -dependencies = [ - "generic-array", -] - [[package]] name = "instant" version = "0.1.13" @@ -2636,6 +2588,12 @@ dependencies = [ "zip", ] +[[package]] +name = "libunwind" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6639b70a7ce854b79c70d7e83f16b5dc0137cc914f3d7d03803b513ecc67ac" + [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -2706,33 +2664,23 @@ dependencies = [ ] [[package]] -name = "lzma-rs" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" -dependencies = [ - "byteorder", - "crc", -] - -[[package]] -name = "lzma-sys" -version = "0.1.20" +name = "mach2" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" +checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709" dependencies = [ - "cc", "libc", - "pkg-config", ] [[package]] -name = "mach2" -version = "0.4.2" +name = "macho-unwind-info" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709" +checksum = "bb4bdc8b0ce69932332cf76d24af69c3a155242af95c226b2ab6c2e371ed1149" dependencies = [ - "libc", + "thiserror 2.0.12", + "zerocopy 0.8.24", + "zerocopy-derive 0.8.24", ] [[package]] @@ -2924,15 +2872,6 @@ dependencies = [ "libm", ] -[[package]] -name = "object" -version = "0.30.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03b4680b86d9cfafba8fc491dc9b6df26b68cf40e9e6cd73909194759a63c385" -dependencies = [ - "memchr", -] - [[package]] name = "object" version = "0.32.2" @@ -2942,7 +2881,7 @@ dependencies = [ "crc32fast", "flate2", "hashbrown 0.14.5", - "indexmap 2.9.0", + "indexmap", "memchr", "ruzstd", ] @@ -3100,16 +3039,6 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" -[[package]] -name = "pbkdf2" -version = "0.12.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" -dependencies = [ - "digest", - "hmac", -] - [[package]] name = "pem" version = "1.1.1" @@ -3144,6 +3073,48 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand 0.8.5", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project" version = "1.1.10" @@ -3205,12 +3176,13 @@ checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" [[package]] name = "plaid" -version = "0.23.2" +version = "0.25.0" dependencies = [ "alkali", "async-trait", "aws-config", "aws-sdk-dynamodb", + "aws-sdk-identitystore", "aws-sdk-kms", "aws-sdk-secretsmanager", "base64 0.13.1", @@ -3254,7 +3226,7 @@ dependencies = [ [[package]] name = "plaid_stl" -version = "0.23.2" +version = "0.25.0" dependencies = [ "base64 0.13.1", "chrono", @@ -3748,7 +3720,7 @@ dependencies = [ "bytecheck 0.8.1", "bytes", "hashbrown 0.15.2", - "indexmap 2.9.0", + "indexmap", "munge", "ptr_meta 0.3.0", "rancor", @@ -4010,7 +3982,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "58c4eb8a81997cf040a091d1f7e1938aeab6749d3a0dfa73af43cdc32393483d" dependencies = [ "byteorder", - "derive_more", + "derive_more 0.99.19", "twox-hash", ] @@ -4142,9 +4114,9 @@ dependencies = [ [[package]] name = "serde-wasm-bindgen" -version = "0.4.5" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3b4c031cd0d9014307d82b8abf653c0290fbdaeb4c02d00c63cf52f728628bf" +checksum = "8302e169f0eddcc139c70f139d19d6467353af16f9fce27e8c30158036a1e16b" dependencies = [ "js-sys", "serde", @@ -4277,6 +4249,12 @@ dependencies = [ "time", ] +[[package]] +name = "siphasher" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" + [[package]] name = "slab" version = "0.4.9" @@ -5112,17 +5090,20 @@ dependencies = [ [[package]] name = "wasmer" -version = "5.0.4" +version = "6.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "998dea47d6bb6a8fc7dd8a17b13bf8de277e007229f270c67105cb67fc2d9657" +checksum = "f25dccc6251837449135914ee1978731c2c3df9fc727088eb7e098736c0f15d1" dependencies = [ "bindgen 0.70.1", "bytes", "cfg-if", "cmake", - "indexmap 1.9.3", + "derive_more 1.0.0", + "idna_adapter", + "indexmap", "js-sys", "more-asserts", + "paste", "rustc-demangle", "serde", "serde-wasm-bindgen", @@ -5139,25 +5120,24 @@ dependencies = [ "wasmer-derive", "wasmer-types", "wasmer-vm", + "wasmparser", "windows-sys 0.59.0", - "xz", - "zip", ] [[package]] name = "wasmer-compiler" -version = "5.0.4" +version = "6.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "082f48ba006cce2358f6c63d8dba732c9d12ba5833008c9ff2944290eb72c3dc" +checksum = "6f35baeb0d5b20710b5b9c59477dbf813b1ac53da33ee46cb22f8c4190e3986e" dependencies = [ "backtrace", "bytes", "cfg-if", "enum-iterator", "enumset", - "lazy_static", "leb128", "libc", + "macho-unwind-info", "memmap2", "more-asserts", "object 0.32.2", @@ -5177,9 +5157,9 @@ dependencies = [ [[package]] name = "wasmer-compiler-cranelift" -version = "5.0.4" +version = "6.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0272ad7daf59d43419cda9ae58b9cf8df6a115c8632fbb91c600892dff52f7a8" +checksum = "6d657a96003ce3f54a3cbbf681fcd782b983f9362c97cfbbe243cbf66790e004" dependencies = [ "cranelift-codegen", "cranelift-entity", @@ -5197,23 +5177,24 @@ dependencies = [ [[package]] name = "wasmer-compiler-llvm" -version = "5.0.4" +version = "6.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a8f2200f7bb3dd1cfd6d1f454b41cb92c8e73f2045764057703af13d8852a5b" +checksum = "7ae6f59f49a3177ad35896a1ff3348a3af557b7a8d8cd63999fee1beaf90bb37" dependencies = [ "byteorder", "cc", "inkwell", "itertools 0.10.5", - "lazy_static", "libc", - "object 0.30.4", + "object 0.32.2", + "phf", "rayon", "regex", "rustc_version", "semver", "smallvec", "target-lexicon", + "tracing", "wasmer-compiler", "wasmer-types", "wasmer-vm", @@ -5221,9 +5202,9 @@ dependencies = [ [[package]] name = "wasmer-derive" -version = "5.0.4" +version = "6.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87b16fa0b2199083143705698ea9fc2ffd0328d7061f54e0e20ccd0ec2020466" +checksum = "62b57be80a67de03c2a02d697bfd763e097546b11f0020cf9930ebaa4f8cf965" dependencies = [ "proc-macro-error2", "proc-macro2", @@ -5233,9 +5214,9 @@ dependencies = [ [[package]] name = "wasmer-middlewares" -version = "5.0.4" +version = "6.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "371b38abbba1a0fb747bee025d3a40c776842be4a2052b7a3ca18bd2db80669d" +checksum = "e61dce6fcf320ab506a9cbf6f12255ccc894e93c5d7a530f14cc7717e81a3c94" dependencies = [ "wasmer", "wasmer-types", @@ -5244,16 +5225,16 @@ dependencies = [ [[package]] name = "wasmer-types" -version = "5.0.4" +version = "6.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41b0b9b3242c1a6269e544401b1741a56502a5f79db8e1b318cacf1b34b2690d" +checksum = "1b8424d15f5c19a8df972fc9367d75ba3b87af63b279208d50a32e9a298d944b" dependencies = [ "bytecheck 0.6.12", "enum-iterator", "enumset", "getrandom 0.2.15", "hex", - "indexmap 2.9.0", + "indexmap", "more-asserts", "rkyv", "sha2", @@ -5264,9 +5245,9 @@ dependencies = [ [[package]] name = "wasmer-vm" -version = "5.0.4" +version = "6.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92c7f8dbaceb0ab7901702e3c2bb43b09d2635aaa5ad39679c6560bcb9acde7c" +checksum = "faabfffefc6fc350bb5b07301f05ba604a18c3d2d97c6354183f15792577056d" dependencies = [ "backtrace", "cc", @@ -5276,9 +5257,9 @@ dependencies = [ "dashmap", "enum-iterator", "fnv", - "indexmap 2.9.0", - "lazy_static", + "indexmap", "libc", + "libunwind", "mach2", "memoffset", "more-asserts", @@ -5291,15 +5272,11 @@ dependencies = [ [[package]] name = "wasmparser" -version = "0.216.1" +version = "0.224.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cc7c63191ae61c70befbe6045b9be65ef2082fa89421a386ae172cb1e08e92d" +checksum = "04f17a5917c2ddd3819e84c661fae0d6ba29d7b9c1f0e96c708c65a9c4188e11" dependencies = [ - "ahash", "bitflags 2.9.0", - "hashbrown 0.14.5", - "indexmap 2.9.0", - "semver", ] [[package]] @@ -5661,24 +5638,6 @@ version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" -[[package]] -name = "xz" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c887690ff2a2e233e8e49633461521f98ec57fbff9d59a884c9a4f04ec1da34" -dependencies = [ - "xz2", -] - -[[package]] -name = "xz2" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" -dependencies = [ - "lzma-sys", -] - [[package]] name = "yasna" version = "0.5.2" @@ -5821,26 +5780,13 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1dcb24d0152526ae49b9b96c1dcf71850ca1e0b882e4e28ed898a93c41334744" dependencies = [ - "aes", "arbitrary", - "bzip2", - "constant_time_eq 0.3.1", "crc32fast", "crossbeam-utils", - "deflate64", "flate2", - "getrandom 0.3.2", - "hmac", - "indexmap 2.9.0", - "lzma-rs", + "indexmap", "memchr", - "pbkdf2", - "sha1", - "time", - "xz2", - "zeroize", "zopfli", - "zstd", ] [[package]] @@ -5856,31 +5802,3 @@ dependencies = [ "once_cell", "simd-adler32", ] - -[[package]] -name = "zstd" -version = "0.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" -dependencies = [ - "zstd-safe", -] - -[[package]] -name = "zstd-safe" -version = "7.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" -dependencies = [ - "zstd-sys", -] - -[[package]] -name = "zstd-sys" -version = "2.0.15+zstd.1.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237" -dependencies = [ - "cc", - "pkg-config", -] diff --git a/runtime/plaid-stl/Cargo.toml b/runtime/plaid-stl/Cargo.toml index b45934ab..614aec3d 100644 --- a/runtime/plaid-stl/Cargo.toml +++ b/runtime/plaid-stl/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "plaid_stl" -version = "0.23.2" +version = "0.25.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/runtime/plaid-stl/src/datetime.rs b/runtime/plaid-stl/src/datetime.rs index 4e91dc5f..c2f98a90 100644 --- a/runtime/plaid-stl/src/datetime.rs +++ b/runtime/plaid-stl/src/datetime.rs @@ -1,4 +1,4 @@ -use chrono::{NaiveDate, Utc, DateTime}; +use chrono::{DateTime, NaiveDate, Utc}; use serde::{Deserialize, Deserializer}; // Custom deserializer from a ISO 8601 string to a DateTime @@ -8,15 +8,15 @@ where { let s = String::deserialize(deserializer)?; - Ok( - DateTime::parse_from_rfc3339(&s) - .map_err(serde::de::Error::custom)? - .with_timezone(&Utc) - ) + Ok(DateTime::parse_from_rfc3339(&s) + .map_err(serde::de::Error::custom)? + .with_timezone(&Utc)) } // Custom deserializer from a possibly empty ISO 8601 string to an optional DateTime -pub fn deserialize_option_rfc3339_timestamp<'de, D>(deserializer: D) -> Result>, D::Error> +pub fn deserialize_option_rfc3339_timestamp<'de, D>( + deserializer: D, +) -> Result>, D::Error> where D: Deserializer<'de>, { @@ -30,13 +30,13 @@ where // If the value is an empty string, we assume the default value has been returned instead of // throwing parsing errors if s.is_empty() { - return Ok(None) + return Ok(None); } Ok(Some( DateTime::parse_from_rfc3339(&s) .map_err(serde::de::Error::custom)? - .with_timezone(&Utc) + .with_timezone(&Utc), )) } @@ -55,10 +55,11 @@ where // If the value is an empty string, we assume the default value has been returned instead of // throwing parsing errors if s.is_empty() { - return Ok(None) + return Ok(None); } // Parse the "YYYY-MM-DD" string into a NaiveDate - Ok(Some(NaiveDate::parse_from_str(&s, "%Y-%m-%d").map_err(serde::de::Error::custom)?)) + Ok(Some( + NaiveDate::parse_from_str(&s, "%Y-%m-%d").map_err(serde::de::Error::custom)?, + )) } - diff --git a/runtime/plaid-stl/src/github/mod.rs b/runtime/plaid-stl/src/github/mod.rs index 7fa52c65..ba609532 100644 --- a/runtime/plaid-stl/src/github/mod.rs +++ b/runtime/plaid-stl/src/github/mod.rs @@ -1713,3 +1713,48 @@ pub fn delete_deploy_key( Ok(()) } + +/// Request reviewers for a pull request. +#[derive(Serialize, Deserialize)] +pub struct PullRequestRequestReviewers { + pub owner: String, + pub repo: String, + pub pull_number: u64, + pub reviewers: Vec, + pub team_reviewers: Vec, +} + +/// Delete a deploy key with given ID from a given repository. +/// For more details, see https://docs.github.com/en/rest/deploy-keys/deploy-keys?apiVersion=2022-11-28#delete-a-deploy-key +pub fn pull_request_request_reviewers( + owner: impl Display, + repo: impl Display, + pull_request: u64, + reviewers: &[impl Display], + team_reviewers: &[impl Display], +) -> Result<(), PlaidFunctionError> { + extern "C" { + new_host_function!(github, pull_request_request_reviewers); + } + + let params = PullRequestRequestReviewers { + owner: owner.to_string(), + repo: repo.to_string(), + pull_number: pull_request, + reviewers: reviewers.iter().map(|s| s.to_string()).collect(), + team_reviewers: team_reviewers.iter().map(|s| s.to_string()).collect(), + }; + + let params = serde_json::to_string(¶ms).unwrap(); + let res = unsafe { + github_pull_request_request_reviewers(params.as_bytes().as_ptr(), params.as_bytes().len()) + }; + + // There was an error with the Plaid system. Maybe the API is not + // configured. + if res < 0 { + return Err(res.into()); + } + + Ok(()) +} diff --git a/runtime/plaid-stl/src/network/mod.rs b/runtime/plaid-stl/src/network/mod.rs index 537dacae..b4fad99a 100644 --- a/runtime/plaid-stl/src/network/mod.rs +++ b/runtime/plaid-stl/src/network/mod.rs @@ -58,14 +58,14 @@ pub fn make_named_request_with_buf_size( request_name: String, body: String, variables: HashMap, - headers: Option> + headers: Option>, } let request = MakeRequestRequest { request_name: name.to_owned(), body: body.to_owned(), variables, - headers + headers, }; let request = serde_json::to_string(&request).unwrap(); @@ -112,5 +112,11 @@ pub fn make_named_request_with_headers( variables: HashMap, headers: HashMap, ) -> Result { - return make_named_request_with_buf_size(name, body, variables, Some(headers), RETURN_BUFFER_SIZE); + return make_named_request_with_buf_size( + name, + body, + variables, + Some(headers), + RETURN_BUFFER_SIZE, + ); } diff --git a/runtime/plaid-stl/src/slack/mod.rs b/runtime/plaid-stl/src/slack/mod.rs index ff15f0b0..119e17f9 100644 --- a/runtime/plaid-stl/src/slack/mod.rs +++ b/runtime/plaid-stl/src/slack/mod.rs @@ -1,6 +1,6 @@ use std::collections::HashMap; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use crate::PlaidFunctionError; @@ -22,7 +22,7 @@ fn slack_format(msg: &str) -> String { struct SlackText { text: String, } - + let new_message = SlackText { text: msg.to_string(), }; @@ -87,6 +87,13 @@ pub fn post_raw_text_to_webhook(name: &str, log: &str) -> Result<(), i32> { Ok(()) } +/// Data to be sent to the Plaid runtime for posting a message +#[derive(Serialize, Deserialize)] +pub struct PostMessage { + pub bot: String, + pub body: String, +} + pub fn post_message(bot: &str, channel: &str, text: &str) -> Result<(), PlaidFunctionError> { extern "C" { new_host_function!(slack, post_message); @@ -97,11 +104,11 @@ pub fn post_message(bot: &str, channel: &str, text: &str) -> Result<(), PlaidFun text: text.to_owned(), }; - let mut params: HashMap<&'static str, String> = HashMap::new(); - params.insert("bot", bot.to_string()); - params.insert("body", serde_json::to_string(&message).unwrap()); - - let params = serde_json::to_string(¶ms).unwrap(); + let params = serde_json::to_string(&PostMessage { + bot: bot.to_string(), + body: serde_json::to_string(&message).unwrap(), + }) + .unwrap(); let res = unsafe { slack_post_message(params.as_ptr(), params.len()) }; @@ -126,11 +133,11 @@ pub fn post_message_with_blocks( blocks: text.to_owned(), }; - let mut params: HashMap<&'static str, String> = HashMap::new(); - params.insert("bot", bot.to_string()); - params.insert("body", serde_json::to_string(&message).unwrap()); - - let params = serde_json::to_string(¶ms).unwrap(); + let params = serde_json::to_string(&PostMessage { + bot: bot.to_string(), + body: serde_json::to_string(&message).unwrap(), + }) + .unwrap(); let res = unsafe { slack_post_message(params.as_ptr(), params.len()) }; @@ -141,16 +148,23 @@ pub fn post_message_with_blocks( Ok(()) } +/// Data to be sent to the runtime to open a view +#[derive(Serialize, Deserialize)] +pub struct ViewOpen { + pub bot: String, + pub body: String, +} + pub fn views_open(bot: &str, view: &str) -> Result<(), PlaidFunctionError> { extern "C" { new_host_function!(slack, views_open); } - let mut params: HashMap<&'static str, &str> = HashMap::new(); - params.insert("bot", bot); - params.insert("body", view); - - let params = serde_json::to_string(¶ms).unwrap(); + let params = serde_json::to_string(&ViewOpen { + bot: bot.to_string(), + body: view.to_string(), + }) + .unwrap(); let res = unsafe { slack_views_open(params.as_ptr(), params.len()) }; @@ -161,7 +175,14 @@ pub fn views_open(bot: &str, view: &str) -> Result<(), PlaidFunctionError> { Ok(()) } -/// Get a Slack user's ID from their email address +/// Data to be sent to the runtime for getting a user's ID from their email address +#[derive(Serialize, Deserialize)] +pub struct GetIdFromEmail { + pub bot: String, + pub email: String, +} + +/// Get a user's ID from their email address pub fn get_id_from_email(bot: &str, email: &str) -> Result { extern "C" { new_host_function_with_error_buffer!(slack, get_id_from_email); @@ -169,11 +190,11 @@ pub fn get_id_from_email(bot: &str, email: &str) -> Result = HashMap::new(); - params.insert("bot", bot); - params.insert("email", email); - - let params = serde_json::to_string(¶ms).unwrap(); + let params = serde_json::to_string(&GetIdFromEmail { + bot: bot.to_string(), + email: email.to_string(), + }) + .unwrap(); let res = unsafe { slack_get_id_from_email( @@ -195,3 +216,133 @@ pub fn get_id_from_email(bot: &str, email: &str) -> Result Result { + extern "C" { + new_host_function_with_error_buffer!(slack, get_presence); + } + const RETURN_BUFFER_SIZE: usize = 32 * 1024; // 32 KiB + let mut return_buffer = vec![0; RETURN_BUFFER_SIZE]; + + let params = serde_json::to_string(&GetPresence { + bot: bot.to_string(), + id: id.to_string(), + }) + .unwrap(); + + let res = unsafe { + slack_get_presence( + params.as_bytes().as_ptr(), + params.as_bytes().len(), + return_buffer.as_mut_ptr(), + RETURN_BUFFER_SIZE, + ) + }; + + if res < 0 { + return Err(res.into()); + } + + return_buffer.truncate(res as usize); + // This should be safe because unless the Plaid runtime is expressly trying + // to mess with us, this came from a String in the API module. + let res = String::from_utf8(return_buffer).unwrap(); + + // This should only happen if the Slack API returns a different structure + // than expected. Which would be odd because to get here the runtime + // successfully parsed the response. + serde_json::from_str(&res).map_err(|_| PlaidFunctionError::Unknown) +} + +#[derive(Serialize, Deserialize)] +pub struct SlackUser { + pub id: String, + pub team_id: String, + pub name: String, + pub deleted: bool, + pub color: String, + pub real_name: String, + pub tz: String, + pub tz_label: String, + pub tz_offset: i32, + pub profile: SlackUserProfile, + pub is_admin: bool, + pub is_owner: bool, + pub is_primary_owner: bool, + pub is_restricted: bool, + pub is_ultra_restricted: bool, + pub is_bot: bool, + pub is_app_user: bool, + pub updated: i32, +} + +#[derive(Serialize, Deserialize)] +pub struct SlackUserProfile { + pub status_text: String, + pub status_emoji: String, +} + +/// Data to be sent to the runtime for getting a user's presence status +#[derive(Serialize, Deserialize)] +pub struct UserInfo { + pub bot: String, + pub id: String, +} + +#[derive(Serialize, Deserialize)] +pub struct UserInfoResponse { + pub ok: bool, + pub user: SlackUser, +} + +/// Get a user's info from their ID +pub fn user_info(bot: &str, id: &str) -> Result { + extern "C" { + new_host_function_with_error_buffer!(slack, user_info); + } + const RETURN_BUFFER_SIZE: usize = 32 * 1024; // 32 KiB + let mut return_buffer = vec![0; RETURN_BUFFER_SIZE]; + + let params = serde_json::to_string(&GetPresence { + bot: bot.to_string(), + id: id.to_string(), + }) + .unwrap(); + + let res = unsafe { + slack_user_info( + params.as_bytes().as_ptr(), + params.as_bytes().len(), + return_buffer.as_mut_ptr(), + RETURN_BUFFER_SIZE, + ) + }; + + if res < 0 { + return Err(res.into()); + } + + return_buffer.truncate(res as usize); + // This should be safe because unless the Plaid runtime is expressly trying + // to mess with us, this came from a String in the API module. + let res = String::from_utf8(return_buffer).unwrap(); + + // This should only happen if the Slack API returns a different structure + // than expected. Which would be odd because to get here the runtime + // successfully parsed the response. + serde_json::from_str(&res).map_err(|_| PlaidFunctionError::Unknown) +} diff --git a/runtime/plaid-stl/src/splunk/mod.rs b/runtime/plaid-stl/src/splunk/mod.rs index 09385a74..e6416707 100644 --- a/runtime/plaid-stl/src/splunk/mod.rs +++ b/runtime/plaid-stl/src/splunk/mod.rs @@ -4,30 +4,27 @@ use serde::Serialize; use crate::PlaidFunctionError; -pub fn post_log(hec_name: &str, log: T) -> Result<(), PlaidFunctionError> -where T: Serialize { +pub fn post_log(hec_name: &str, log: T) -> Result<(), PlaidFunctionError> +where + T: Serialize, +{ extern "C" { new_host_function!(splunk, post_hec); } let data = serde_json::to_string(&log).map_err(|_| PlaidFunctionError::InternalApiError)?; - let mut params:HashMap<&'static str, String> = HashMap::new(); + let mut params: HashMap<&'static str, String> = HashMap::new(); params.insert("hec_name", hec_name.to_string()); params.insert("log", data); let params = serde_json::to_string(¶ms).unwrap(); - let res = unsafe { - splunk_post_hec( - params.as_ptr(), - params.len(), - ) - }; + let res = unsafe { splunk_post_hec(params.as_ptr(), params.len()) }; if res < 0 { - return Err(res.into()) + return Err(res.into()); } Ok(()) -} \ No newline at end of file +} diff --git a/runtime/plaid/Cargo.toml b/runtime/plaid/Cargo.toml index 39ee400f..ed65e5a4 100644 --- a/runtime/plaid/Cargo.toml +++ b/runtime/plaid/Cargo.toml @@ -1,13 +1,13 @@ [package] name = "plaid" -version = "0.23.2" +version = "0.25.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [features] default = ["aws", "sled", "cranelift"] -aws = ["dep:aws-sdk-kms", "dep:aws-sdk-dynamodb"] +aws = ["dep:aws-sdk-kms", "dep:aws-sdk-dynamodb", "dep:aws-sdk-identitystore"] cranelift = ["wasmer/cranelift"] llvm = ["wasmer/llvm"] @@ -16,6 +16,7 @@ alkali = "0.3.0" async-trait = "0.1.56" aws-config = "1.5.5" aws-sdk-dynamodb = { version = "1.69.0", optional = true } +aws-sdk-identitystore = { version = "1.73.0", optional = true } aws-sdk-kms = { version = "1.41.0", optional = true } aws-sdk-secretsmanager = "1.57.0" base64 = "0.13" @@ -65,8 +66,8 @@ uuid = { version = "1", features = ["v4"] } url = "2.5.2" urlencoding = "2.1.3" warp = { version = "0.3", features = ["tls"] } -wasmer = { version = "5", default-features = false } -wasmer-middlewares = "5" +wasmer = { version = "6", default-features = false } +wasmer-middlewares = "6" [[example]] name = "github-tailer" diff --git a/runtime/plaid/resources/config/apis.toml b/runtime/plaid/resources/config/apis.toml new file mode 100644 index 00000000..fd6d8e01 --- /dev/null +++ b/runtime/plaid/resources/config/apis.toml @@ -0,0 +1,190 @@ +[apis."general"] +[apis."general".network.web_requests] +[apis."general"."network"."web_requests"."test-response"] +verb = "post" +uri = "https://localhost:8998/response" +return_body = true +return_code = true +allowed_rules = [ + "test_time.wasm", + "test_logback.wasm", + "test_db.wasm", + "test_fileopen.wasm", + "test_random.wasm", + "test_mnr.wasm", + "test_get_everything.wasm", + "test_shared_db_rule_1.wasm", + "test_shared_db_rule_2.wasm", + "test_slack.wasm", +] +root_certificate = """ +{plaid-secret{integration-test-root-ca}} +""" +[apis."general"."network"."web_requests"."test-response"."headers"] +testheader = "Some data here" + +[apis."general"."network"."web_requests"."test-response-mnr"] +verb = "post" +uri = "https://localhost:8998/testmnr" +return_body = true +return_code = true +allowed_rules = ["test_mnr.wasm"] +root_certificate = """ +{plaid-secret{integration-test-root-ca}} +""" +[apis."general"."network"."web_requests"."test-response-mnr"."headers"] + +[apis."general"."network"."web_requests"."test-response-mnr-headers"] +verb = "post" +uri = "https://localhost:8998/testmnr/headers" +return_body = true +return_code = true +allowed_rules = ["test_mnr.wasm"] +root_certificate = """ +{plaid-secret{integration-test-root-ca}} +""" +[apis."general"."network"."web_requests"."test-response-mnr-headers"."headers"] +first_header = "first_value" + +[apis."general"."network"."web_requests"."test-response-mnr-vars"] +verb = "post" +uri = "https://localhost:8998/testmnr/{variable}" +return_body = true +return_code = true +allowed_rules = ["test_mnr.wasm"] +root_certificate = """ +{plaid-secret{integration-test-root-ca}} +""" +[apis."general"."network"."web_requests"."test-response-mnr-vars"."headers"] + +[apis."general"."network"."web_requests"."google_test"] +verb = "get" +uri = "https://www.google.com/" +return_body = true +return_code = true +allowed_rules = ["testing_test.wasm"] +[apis."general"."network"."web_requests"."google_test"."headers"] + +[apis."general"."network"."web_requests"."testmode_allow"] +verb = "get" +uri = "https://captive.apple.com/" +return_body = true +return_code = true +allowed_rules = ["test_testmode.wasm"] +available_in_test_mode = true +[apis."general"."network"."web_requests"."testmode_allow"."headers"] + +[apis."general"."network"."web_requests"."testmode_deny"] +verb = "get" +uri = "https://captive.apple.com/" +return_body = true +return_code = true +allowed_rules = ["test_testmode.wasm"] +available_in_test_mode = false +[apis."general"."network"."web_requests"."testmode_deny"."headers"] + +# [apis."general"."network"."web_requests"."list_deploy_keys"] +# verb = "get" +# uri = "https://api.github.com/repos/{owner}/{repo}/keys" +# return_body = true +# return_code = true +# allowed_rules = ["testing_test.wasm"] +# [apis."general"."network"."web_requests"."list_deploy_keys"."headers"] +# Authorization = "Bearer github_pat_11AAS..." +# "X-GitHub-Api-Version" = "2022-11-28" +# "Accept" = "application/vnd.github+json" +# "User-Agent" = "Plaid/0.10" + +# [apis."general"."network"."web_requests"."create_deploy_key"] +# verb = "post" +# uri = "https://api.github.com/repos/{owner}/{repo}/keys" +# return_body = false +# return_code = true +# allowed_rules = ["testing_test.wasm"] +# [apis."general"."network"."web_requests"."create_deploy_key"."headers"] +# Authorization = "Bearer github_pat_11AAS..." +# "X-GitHub-Api-Version" = "2022-11-28" +# "Accept" = "application/vnd.github+json" +# "User-Agent" = "Plaid/0.10" + +# [apis."github"] +# token = "" +# [apis."github".graphql_queries] + +[apis."slack"] +[apis."slack"."webhooks"] +test_webhook = "{plaid-secret{test-webhook-secret}}" +[apis."slack"."bot_tokens"] +"plaid-testing" = "{plaid-secret{test-slack-bot-token}}" + +[apis."web"] +[apis."web".keys] +[apis."web".keys."5d313aea523d41569469e4abd72028d2"] +# To generate the ECDSA256 key PEM, run the following commands: +# openssl ecparam -genkey -name prime256v1 -out ec-params.key +# openssl pkcs8 -topk8 -nocrypt -in ec-params.key -out private-key.pem +# openssl ec -in ec-params.pem -pubout + +# This is the example private key from JWT.io. Do not use it in production. +private_key = """ +-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgevZzL1gdAFr88hb2 +OF/2NxApJCzGCEDdfSp6VQO30hyhRANCAAQRWz+jn65BtOMvdyHKcvjBeBSDZH2r +1RTwjmYSi9R/zpBnuQ4EiMnCqfMPWiZqB4QdbAd0E7oH50VpuZ1P087G +-----END PRIVATE KEY----- +""" + +allowed_rules = ["testing_test.wasm"] + +[apis."yubikey"] +client_id = 99999 +secret_key = "" + +# KMS USING IAM + +# [apis."aws"] +# [apis."aws"."kms".authentication] +# [apis."aws"."kms".key_configuration] +# "some_key_id" = ["testing_test.wasm", "another_rule.wasm"] + +# KMS USING API KEY + +# [apis."aws"] +# [apis."aws"."kms"] +# [apis."aws"."kms".authentication] +# access_key_id = "asdf" +# secret_access_key = "asdf" +# region = "asdf" +# [apis."aws"."kms".key_configuration] +# "some_key_id" = ["testing_test.wasm", "another_rule.wasm"] + +#[apis."github"] +#[apis."github".authentication] +#logbacks_allowed = "Unlimited" +#app_id = 0 +#installation_id = 0 +#private_key = """{plaid-secret{plaid-github-app-private-key}}""" + +# [apis."github"."graphql_queries"] +# saml_identities = """ +# query($organization: String!, $cursor: String) { +# organization(login: $organization) { +# samlIdentityProvider { +# externalIdentities(first: 100, after: $cursor) { +# pageInfo { +# hasNextPage +# endCursor +# } +# nodes { +# user { +# login +# } +# samlIdentity { +# nameId +# } +# } +# } +# } +# } +# } +# """ diff --git a/runtime/plaid/resources/config/data.toml b/runtime/plaid/resources/config/data.toml new file mode 100644 index 00000000..03dc8664 --- /dev/null +++ b/runtime/plaid/resources/config/data.toml @@ -0,0 +1,26 @@ +[data] + +# [data.websocket] +# [data.websocket.websockets] +# [data.websocket."websockets"."demo_rpc_call"] +# log_type = "testing" +# [data.websocket."websockets"."demo_rpc_call".uris] +# simplystaking = "wss://some_websocket" +# [data.websocket."websockets"."demo_rpc_call".message_config] +# message = "{ \"id\": 1, \"jsonrpc\": \"2.0\", \"method\": \"eth_getBlockByNumber\", \"params\": [ \"finalized\", false ]}" +# sleep_duration = 100 ## This means the message is sent every 0.1 seconds +# [data.websocket."websockets"."demo_rpc_call"."headers"] + +# [data.github] +# org = "" +# log_type = "Web" # Can be one of Web, Git, All + +# Authentication using FPAT +# [data.github.authentication] +# token = "" + +# Authentication using GitHub App Creds +## [data.github.authentication] +## app_id = 1234 +## installation_id = 1234 +## private_key = "" diff --git a/runtime/plaid/resources/config/executor.toml b/runtime/plaid/resources/config/executor.toml new file mode 100644 index 00000000..3ad19997 --- /dev/null +++ b/runtime/plaid/resources/config/executor.toml @@ -0,0 +1,18 @@ +[executor] +execution_threads = 2 + +[executor.dedicated_threads."test_db"] +num_threads = 1 + +[executor.dedicated_threads."my_other_logtype"] +num_threads = 2 +log_queue_size = 1024 + +[executor.dedicated_threads."testing"] +num_threads = 1 + +[executor.dedicated_threads."testing2"] +num_threads = 2 + +[executor.dedicated_threads."testing4"] +num_threads = 4 diff --git a/runtime/plaid/resources/config/loading.toml b/runtime/plaid/resources/config/loading.toml new file mode 100644 index 00000000..5f506b39 --- /dev/null +++ b/runtime/plaid/resources/config/loading.toml @@ -0,0 +1,105 @@ +# Configure the loading and linking system. This configures how much +# computation is allocated for module invocations and manual remapping +# of module names to log types which is useful is a synthetic logtype +# contains underscores. +[loading] +module_dir = "../compiled_modules/" +lru_cache_size = 200 +compiler_backend = "cranelift" + +test_mode = true +test_mode_exemptions = [ + "test_crashtest.wasm", + "test_db.wasm", + "test_fileopen.wasm", + "test_get_everything.wasm", + "test_logback.wasm", + "test_mnr.wasm", + "test_persistent_response.wasm", + "test_random.wasm", + "test_regex.wasm", + "test_sshcerts_usage.wasm", + "test_time.wasm", + "test_shared_db_rule_1.wasm", + "test_shared_db_rule_2.wasm", + "test_slack.wasm", +] + + +[loading.persistent_response_size] +"test_persistent_response.wasm" = 1024 +"test_sshcerts_usage.wasm" = 1024 +"test_logback.wasm" = 1024 +"test_mnr.wasm" = 1024 +"test_get_everything.wasm" = 1024 +"example_github_graphql.wasm" = 100_000 + +[loading.universal_accessory_data] +"key_1" = "value_1" +"key_2" = "value_2" +"key_4" = "value_4_ignored" + +[loading.accessory_data_log_type_overrides.test_geteverything] +"key_1" = "value_1_intermediate" +"key_4" = "value_4" + +[loading.accessory_data_file_overrides."test_get_everything.wasm"] +"key_1" = "value_1_new" +"key_3" = "value_3" + +[loading.secrets] +[loading.secrets."testing"] +"test_secret" = "{plaid-secret{test-secret}}" + +[loading.secrets."test_geteverything"] +"my_secret" = "verySecureSecret" + +[loading.secrets."example_github_graphql"] +"organization_fetch_auth_token" = "{plaid-secret{example-auth-token}}" + +[loading.log_type_overrides] +"test_crashtest.wasm" = "crashtest" +"test_db.wasm" = "test_db" +"test_fileopen.wasm" = "test_fileopen" +"test_get_everything.wasm" = "test_geteverything" +"test_logback.wasm" = "test_logback" +"test_mnr.wasm" = "test_mnr" +"test_persistent_response.wasm" = "prtest" +"test_random.wasm" = "test_random" +"test_regex.wasm" = "test_regex" +"test_sshcerts_usage.wasm" = "test_sshcerts" +"test_testmode.wasm" = "testmode" +"test_time.wasm" = "time" +"test_shared_db_rule_1.wasm" = "test_shareddb_1" +"test_shared_db_rule_2.wasm" = "test_shareddb_2" +"test_slack.wasm" = "test_slack" +"example_github_graphql.wasm" = "example_github_graphql" + +# Configure the computation amount. See the loader module for more +# information on how computation cost is calculated. +[loading.computation_amount] +default = 55_000_000 +[loading.computation_amount.log_type] +okta = 9_000_000 +[loading.computation_amount.module_overrides] +"example_rule.wasm" = 5_000_000 +"test_shared_db_rule_1.wasm" = 1_000_000_000 +"test_shared_db_rule_2.wasm" = 1_000_000_000 + +[loading.memory_page_count] +default = 300 +[loading.memory_page_count.log_type] +okta = 200 +[loading.memory_page_count.module_overrides] +"example_rule.wasm" = 50 +"test_crashtest.wasm" = 150 + +[loading.storage_size] +default = "Unlimited" +[loading.storage_size.log_type] +[loading.storage_size.module_overrides] +"test_db.wasm" = { Limited = 50 } + +[loading.module_signing] +authorized_signers = ["{plaid-secret{public-key}}"] +signatures_required = 1 diff --git a/runtime/plaid/resources/config/logging.toml b/runtime/plaid/resources/config/logging.toml new file mode 100644 index 00000000..0efbef30 --- /dev/null +++ b/runtime/plaid/resources/config/logging.toml @@ -0,0 +1,3 @@ +# Configure the logging system. In this case we only configure the +# stdout logger +[logging."stdout"] diff --git a/runtime/plaid/resources/config/performance_monitoring.toml b/runtime/plaid/resources/config/performance_monitoring.toml new file mode 100644 index 00000000..4c78d95d --- /dev/null +++ b/runtime/plaid/resources/config/performance_monitoring.toml @@ -0,0 +1,6 @@ +# Uncomment this to run with performance monitoring enabled +[performance_monitoring] + +# This is an optional field. If no path is provided, the resulting +# file is written to /runtime/performance-monitoring/metrics.txt +# output_file_path = "../somedirectory/file.txt" diff --git a/runtime/plaid/resources/config/storage.toml b/runtime/plaid/resources/config/storage.toml new file mode 100644 index 00000000..813cc811 --- /dev/null +++ b/runtime/plaid/resources/config/storage.toml @@ -0,0 +1,16 @@ +[storage.shared_dbs."shared_db_1"] +size_limit = { Limited = 50 } +r = ["test_shared_db_rule_1.wasm"] +rw = ["test_shared_db_rule_2.wasm"] + +[storage.db] +sled_path = "/tmp/sled" + +# [storage.db] +# table_name = "test-plaid" + +# [storage.db.authentication] +# access_key_id = "value here" +# secret_access_key = "value here" +# session_token = "value here" +# region = "value here" diff --git a/runtime/plaid/resources/config/webhooks.toml b/runtime/plaid/resources/config/webhooks.toml new file mode 100644 index 00000000..ad58b1a7 --- /dev/null +++ b/runtime/plaid/resources/config/webhooks.toml @@ -0,0 +1,132 @@ +[webhooks."internal"] +listen_address = "0.0.0.0:4554" + +# Webhooks for tests +[webhooks."internal".webhooks."timetest"] +log_type = "time" +headers = [] + +[webhooks."internal".webhooks."persistentresponsetest"] +log_type = "prtest" +logbacks_allowed = "Unlimited" +headers = ["x-forwarded-for"] +[webhooks."internal".webhooks."persistentresponsetest".get_mode] +response_mode = "rule:test_persistent_response.wasm" +[webhooks."internal".webhooks."persistentresponsetest".get_mode.caching_mode] +type = "None" + +[webhooks."internal".webhooks."testsshcerts"] +log_type = "test_sshcerts" +headers = [] +[webhooks."internal".webhooks."testsshcerts".get_mode] +response_mode = "rule:test_sshcerts_usage.wasm" +[webhooks."internal".webhooks."testsshcerts".get_mode.caching_mode] +type = "None" + +[webhooks."internal".webhooks."crashtest"] +log_type = "crashtest" +headers = [] + +[webhooks."internal".webhooks."testlogback"] +log_type = "test_logback" +logbacks_allowed = { Limited = 1 } +headers = [] +[webhooks."internal".webhooks."testlogback".get_mode] +response_mode = "rule:test_logback.wasm" +[webhooks."internal".webhooks."testlogback".get_mode.caching_mode] +type = "None" + +[webhooks."internal".webhooks."testdb"] +log_type = "test_db" +headers = [] + +[webhooks."internal".webhooks."testfileopen"] +log_type = "test_fileopen" +headers = [] + +[webhooks."internal".webhooks."testshareddb_1"] +log_type = "test_shareddb_1" +headers = [] + +[webhooks."internal".webhooks."testshareddb_2"] +log_type = "test_shareddb_2" +headers = [] + +[webhooks."internal".webhooks."testrandom"] +log_type = "test_random" +headers = [] + +[webhooks."internal".webhooks."testmnr"] +log_type = "test_mnr" +headers = [] + +[webhooks."internal".webhooks."testregex"] +log_type = "test_regex" +headers = [] + + +[webhooks."internal".webhooks."examplegithub_graphql"] +log_type = "none" +headers = ["Authorization"] +[webhooks."internal".webhooks."examplegithub_graphql".get_mode] +response_mode = "rule:example_github_graphql.wasm" +[webhooks."internal".webhooks."examplegithub_graphql".get_mode.caching_mode] +# Note that this needs to be none as any other value will cause a security issue. This is because +# caching is handled at the runtime level, thus not giving the rule a chance to deny returning +# the cached data. +type = "None" + + +[webhooks."internal".webhooks."testgeteverything"] +log_type = "test_geteverything" +headers = ["Authorization", "my_secret"] +[webhooks."internal".webhooks."testgeteverything".get_mode] +response_mode = "rule:test_get_everything.wasm" +[webhooks."internal".webhooks."testgeteverything".get_mode.caching_mode] +type = "None" + +[webhooks."internal".webhooks."testmode"] +log_type = "testmode" +headers = [] + +[webhooks."internal".webhooks."testslack"] +log_type = "test_slack" +headers = [] + +# End webhooks for tests + +# Additional webhook examples +[webhooks."internal".webhooks."FFFFA"] +log_type = "testing" +headers = ["x-forwarded-for"] +[webhooks."internal".webhooks."FFFFA".get_mode] +response_mode = "static:this is just static data to return to the caller" + +[webhooks."internal".webhooks."LOADTEST1"] +log_type = "testing" +headers = ["notalegitheader", "reallynotlegit"] + +[webhooks."internal".webhooks."LOADTEST2"] +log_type = "testing2" +headers = ["notalegitheader", "reallynotlegit"] + +[webhooks."internal".webhooks."LOADTEST4"] +log_type = "testing4" +headers = ["notalegitheader", "reallynotlegit"] + +[webhooks."internal".webhooks."FFFFB"] +log_type = "testing" +headers = ["x-forwarded-for"] +[webhooks."internal".webhooks."FFFFB".get_mode] +response_mode = "rule:testing_test.wasm" +[webhooks."internal".webhooks."FFFFB".get_mode.caching_mode] +type = "Timed" +validity = 10 + +[webhooks."external"] +listen_address = "0.0.0.0:4556" +[webhooks."external".webhooks."AAAA"] +log_type = "testing" +headers = ["notalegitheader", "reallynotlegit"] +[webhooks."external".webhooks."AAAA".get_mode] +response_mode = "facebook:somelongstring" diff --git a/runtime/plaid/resources/docker/Dockerfile b/runtime/plaid/resources/docker/Dockerfile index bca4d9d2..25d69755 100644 --- a/runtime/plaid/resources/docker/Dockerfile +++ b/runtime/plaid/resources/docker/Dockerfile @@ -20,4 +20,4 @@ COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ COPY --from=builder /build/target/release/plaid /plaid RUN useradd -m plaiduser USER plaiduser -CMD [ "/plaid", "--config", "/config/plaid.toml", "--secrets", "/config/secrets.json" ] +CMD [ "/plaid", "--config", "/config/config", "--secrets", "/config/secrets.json" ] diff --git a/runtime/plaid/resources/docker/musl/Dockerfile.aarch64 b/runtime/plaid/resources/docker/musl/Dockerfile.aarch64 index e4c00f5b..b5cd65df 100644 --- a/runtime/plaid/resources/docker/musl/Dockerfile.aarch64 +++ b/runtime/plaid/resources/docker/musl/Dockerfile.aarch64 @@ -14,4 +14,4 @@ FROM scratch AS runtime COPY --from=alpine /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ COPY --from=builder /build/target/aarch64-unknown-linux-musl/release/plaid /plaid USER 1000 -CMD [ "/plaid", "--config", "/config/plaid.toml", "--secrets", "/config/secrets.json" ] +CMD [ "/plaid", "--config", "/config/config", "--secrets", "/config/secrets.json" ] diff --git a/runtime/plaid/resources/docker/musl/Dockerfile.amd64 b/runtime/plaid/resources/docker/musl/Dockerfile.amd64 index 0a88ecb0..ac1466ab 100644 --- a/runtime/plaid/resources/docker/musl/Dockerfile.amd64 +++ b/runtime/plaid/resources/docker/musl/Dockerfile.amd64 @@ -14,4 +14,4 @@ FROM scratch AS runtime COPY --from=alpine /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ COPY --from=builder /build/target/x86_64-unknown-linux-musl/release/plaid /plaid USER 1000 -CMD [ "/plaid", "--config", "/config/plaid.toml", "--secrets", "/config/secrets.json" ] +CMD [ "/plaid", "--config", "/config/config", "--secrets", "/config/secrets.json" ] diff --git a/runtime/plaid/resources/plaid.toml b/runtime/plaid/resources/plaid.toml deleted file mode 100644 index f5cbfa52..00000000 --- a/runtime/plaid/resources/plaid.toml +++ /dev/null @@ -1,415 +0,0 @@ -execution_threads = 2 - -# Uncomment this to run with performance monitoring enabled -[performance_monitoring] - -# This is an optional field. If no path is provided, the resulting -# file is written to /runtime/performance-monitoring/metrics.txt -# output_file_path = "../somedirectory/file.txt" - -[storage.shared_dbs."shared_db_1"] -size_limit = { Limited = 50 } -r = ["test_shared_db_rule_1.wasm"] -rw = ["test_shared_db_rule_2.wasm"] - -[storage.db] -sled_path = "/tmp/sled" - -# [storage.db] -# table_name = "test-plaid" - -# [storage.db.authentication] -# access_key_id = "value here" -# secret_access_key = "value here" -# session_token = "value here" -# region = "value here" - -# Configure the logging system. In this case we only configure the -# stdout logger -[logging."stdout"] - - -# Configure the loading and linking system. This configures how much -# computation is allocated for module invocations and manual remapping -# of module names to log types which is useful is a synthetic logtype -# contains underscores. -[loading] -module_dir = "../compiled_modules/" -lru_cache_size = 200 -compiler_backend = "cranelift" - -test_mode = true -test_mode_exemptions = [ - "test_crashtest.wasm", - "test_db.wasm", - "test_fileopen.wasm", - "test_get_everything.wasm", - "test_logback.wasm", - "test_mnr.wasm", - "test_persistent_response.wasm", - "test_random.wasm", - "test_regex.wasm", - "test_sshcerts_usage.wasm", - "test_time.wasm", - "test_shared_db_rule_1.wasm", - "test_shared_db_rule_2.wasm", -] - - -# Uncomment this to require that all rules be signed by an authorized signer -[loading.module_signing] -authorized_signers = [ - "{plaid-secret{public-key}}", -] -signatures_required = 1 - -[loading.persistent_response_size] -"test_persistent_response.wasm" = 1024 -"test_sshcerts_usage.wasm" = 1024 -"test_logback.wasm" = 1024 -"test_mnr.wasm" = 1024 -"test_get_everything.wasm" = 1024 - -[loading.universal_accessory_data] -"key_1" = "value_1" -"key_2" = "value_2" -"key_4" = "value_4_ignored" - -[loading.accessory_data_log_type_overrides.test_geteverything] -"key_1" = "value_1_intermediate" -"key_4" = "value_4" - -[loading.accessory_data_file_overrides."test_get_everything.wasm"] -"key_1" = "value_1_new" -"key_3" = "value_3" - -[loading.secrets] -[loading.secrets."testing"] -"test_secret" = "{plaid-secret{test-secret}}" - -[loading.secrets."test_geteverything"] -"my_secret" = "verySecureSecret" - -[loading.log_type_overrides] -"test_crashtest.wasm" = "crashtest" -"test_db.wasm" = "test_db" -"test_fileopen.wasm" = "test_fileopen" -"test_get_everything.wasm" = "test_geteverything" -"test_logback.wasm" = "test_logback" -"test_mnr.wasm" = "test_mnr" -"test_persistent_response.wasm" = "prtest" -"test_random.wasm" = "test_random" -"test_regex.wasm" = "test_regex" -"test_sshcerts_usage.wasm" = "test_sshcerts" -"test_testmode.wasm" = "testmode" -"test_time.wasm" = "time" -"test_shared_db_rule_1.wasm" = "test_shareddb_1" -"test_shared_db_rule_2.wasm" = "test_shareddb_2" - -# Configure the computation amount. See the loader module for more -# information on how computation cost is calculated. -[loading.computation_amount] -default = 55_000_000 -[loading.computation_amount.log_type] -okta = 9_000_000 -[loading.computation_amount.module_overrides] -"example_rule.wasm" = 5_000_000 -"test_shared_db_rule_1.wasm" = 1_000_000_000 -"test_shared_db_rule_2.wasm" = 1_000_000_000 - -[loading.memory_page_count] -default = 300 -[loading.memory_page_count.log_type] -okta = 200 -[loading.memory_page_count.module_overrides] -"example_rule.wasm" = 50 -"test_crashtest.wasm" = 150 - -[loading.storage_size] -default = "Unlimited" -[loading.storage_size.log_type] -[loading.storage_size.module_overrides] -"test_db.wasm" = { Limited = 50 } - - -# [apis."okta"] -# token = "" -# domain = "" - -[apis."general"] -[apis."general".network.web_requests] -[apis."general"."network"."web_requests"."test-response"] -verb = "post" -uri = "http://localhost:8998/response" -return_body = true -return_code = true -allowed_rules = [ - "test_time.wasm", - "test_logback.wasm", - "test_db.wasm", - "test_fileopen.wasm", - "test_random.wasm", - "test_mnr.wasm", - "test_get_everything.wasm", - "test_shared_db_rule_1.wasm", - "test_shared_db_rule_2.wasm", -] -[apis."general"."network"."web_requests"."test-response"."headers"] -testheader = "Some data here" - -[apis."general"."network"."web_requests"."test-response-mnr"] -verb = "post" -uri = "http://localhost:8998/testmnr" -return_body = true -return_code = true -allowed_rules = ["test_mnr.wasm"] -[apis."general"."network"."web_requests"."test-response-mnr"."headers"] - -[apis."general"."network"."web_requests"."test-response-mnr-headers"] -verb = "post" -uri = "http://localhost:8998/testmnr/headers" -return_body = true -return_code = true -allowed_rules = ["test_mnr.wasm"] -[apis."general"."network"."web_requests"."test-response-mnr-headers"."headers"] -first_header = "first_value" - -[apis."general"."network"."web_requests"."test-response-mnr-vars"] -verb = "post" -uri = "http://localhost:8998/testmnr/{variable}" -return_body = true -return_code = true -allowed_rules = ["test_mnr.wasm"] -[apis."general"."network"."web_requests"."test-response-mnr-vars"."headers"] - -[apis."general"."network"."web_requests"."google_test"] -verb = "get" -uri = "https://www.google.com/" -return_body = true -return_code = true -allowed_rules = ["testing_test.wasm"] -[apis."general"."network"."web_requests"."google_test"."headers"] - -[apis."general"."network"."web_requests"."testmode_allow"] -verb = "get" -uri = "https://captive.apple.com/" -return_body = true -return_code = true -allowed_rules = ["test_testmode.wasm"] -available_in_test_mode = true -[apis."general"."network"."web_requests"."testmode_allow"."headers"] - -[apis."general"."network"."web_requests"."testmode_deny"] -verb = "get" -uri = "https://captive.apple.com/" -return_body = true -return_code = true -allowed_rules = ["test_testmode.wasm"] -available_in_test_mode = false -[apis."general"."network"."web_requests"."testmode_deny"."headers"] - -# [apis."general"."network"."web_requests"."list_deploy_keys"] -# verb = "get" -# uri = "https://api.github.com/repos/{owner}/{repo}/keys" -# return_body = true -# return_code = true -# allowed_rules = ["testing_test.wasm"] -# [apis."general"."network"."web_requests"."list_deploy_keys"."headers"] -# Authorization = "Bearer github_pat_11AAS..." -# "X-GitHub-Api-Version" = "2022-11-28" -# "Accept" = "application/vnd.github+json" -# "User-Agent" = "Plaid/0.10" - -# [apis."general"."network"."web_requests"."create_deploy_key"] -# verb = "post" -# uri = "https://api.github.com/repos/{owner}/{repo}/keys" -# return_body = false -# return_code = true -# allowed_rules = ["testing_test.wasm"] -# [apis."general"."network"."web_requests"."create_deploy_key"."headers"] -# Authorization = "Bearer github_pat_11AAS..." -# "X-GitHub-Api-Version" = "2022-11-28" -# "Accept" = "application/vnd.github+json" -# "User-Agent" = "Plaid/0.10" - -# [apis."github"] -# token = "" -# [apis."github".graphql_queries] - -[apis."slack"] -[apis."slack"."webhooks"] -[apis."slack"."bot_tokens"] - -[apis."web"] -[apis."web".keys] -[apis."web".keys."5d313aea523d41569469e4abd72028d2"] -# To generate the ECDSA256 key PEM, run the following commands: -# openssl ecparam -genkey -name prime256v1 -out ec-params.key -# openssl pkcs8 -topk8 -nocrypt -in ec-params.key -out private-key.pem -# openssl ec -in ec-params.pem -pubout - -# This is the example private key from JWT.io. Do not use it in production. -private_key = """ ------BEGIN PRIVATE KEY----- -MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgevZzL1gdAFr88hb2 -OF/2NxApJCzGCEDdfSp6VQO30hyhRANCAAQRWz+jn65BtOMvdyHKcvjBeBSDZH2r -1RTwjmYSi9R/zpBnuQ4EiMnCqfMPWiZqB4QdbAd0E7oH50VpuZ1P087G ------END PRIVATE KEY----- -""" - -allowed_rules = ["testing_test.wasm"] - -[apis."yubikey"] -client_id = 99999 -secret_key = "" - -# KMS USING IAM - -# [apis."aws"] -# [apis."aws"."kms".authentication] -# [apis."aws"."kms".key_configuration] -# "some_key_id" = ["testing_test.wasm", "another_rule.wasm"] - -# KMS USING API KEY - -# [apis."aws"] -# [apis."aws"."kms"] -# [apis."aws"."kms".authentication] -# access_key_id = "asdf" -# secret_access_key = "asdf" -# region = "asdf" -# [apis."aws"."kms".key_configuration] -# "some_key_id" = ["testing_test.wasm", "another_rule.wasm"] - - -[data] - -# [data.websocket] -# [data.websocket.websockets] -# [data.websocket."websockets"."demo_rpc_call"] -# log_type = "testing" -# [data.websocket."websockets"."demo_rpc_call".uris] -# simplystaking = "wss://some_websocket" -# [data.websocket."websockets"."demo_rpc_call".message_config] -# message = "{ \"id\": 1, \"jsonrpc\": \"2.0\", \"method\": \"eth_getBlockByNumber\", \"params\": [ \"finalized\", false ]}" -# sleep_duration = 100 ## This means the message is sent every 0.1 seconds -# [data.websocket."websockets"."demo_rpc_call"."headers"] - -# [data.github] -# org = "" -# log_type = "Web" # Can be one of Web, Git, All - -# Authentication using FPAT -# [data.github.authentication] -# token = "" - -# Authentication using GitHub App Creds -## [data.github.authentication] -## app_id = 1234 -## installation_id = 1234 -## private_key = "" - -[webhooks."internal"] -listen_address = "0.0.0.0:4554" - -# Webhooks for tests -[webhooks."internal".webhooks."timetest"] -log_type = "time" -headers = [] - -[webhooks."internal".webhooks."persistentresponsetest"] -log_type = "prtest" -logbacks_allowed = "Unlimited" -headers = ["x-forwarded-for"] -[webhooks."internal".webhooks."persistentresponsetest".get_mode] -response_mode = "rule:test_persistent_response.wasm" -[webhooks."internal".webhooks."persistentresponsetest".get_mode.caching_mode] -type = "None" - -[webhooks."internal".webhooks."testsshcerts"] -log_type = "test_sshcerts" -headers = [] -[webhooks."internal".webhooks."testsshcerts".get_mode] -response_mode = "rule:test_sshcerts_usage.wasm" -[webhooks."internal".webhooks."testsshcerts".get_mode.caching_mode] -type = "None" - -[webhooks."internal".webhooks."crashtest"] -log_type = "crashtest" -headers = [] - -[webhooks."internal".webhooks."testlogback"] -log_type = "test_logback" -logbacks_allowed = { Limited = 1 } -headers = [] -[webhooks."internal".webhooks."testlogback".get_mode] -response_mode = "rule:test_logback.wasm" -[webhooks."internal".webhooks."testlogback".get_mode.caching_mode] -type = "None" - -[webhooks."internal".webhooks."testdb"] -log_type = "test_db" -headers = [] - -[webhooks."internal".webhooks."testfileopen"] -log_type = "test_fileopen" -headers = [] - -[webhooks."internal".webhooks."testshareddb_1"] -log_type = "test_shareddb_1" -headers = [] - -[webhooks."internal".webhooks."testshareddb_2"] -log_type = "test_shareddb_2" -headers = [] - -[webhooks."internal".webhooks."testrandom"] -log_type = "test_random" -headers = [] - -[webhooks."internal".webhooks."testmnr"] -log_type = "test_mnr" -headers = [] - -[webhooks."internal".webhooks."testregex"] -log_type = "test_regex" -headers = [] - - -[webhooks."internal".webhooks."testgeteverything"] -log_type = "test_geteverything" -headers = ["Authorization", "my_secret"] -[webhooks."internal".webhooks."testgeteverything".get_mode] -response_mode = "rule:test_get_everything.wasm" -[webhooks."internal".webhooks."testgeteverything".get_mode.caching_mode] -type = "None" - -[webhooks."internal".webhooks."testmode"] -log_type = "testmode" -headers = [] - -# End webhooks for tests - -# Additional webhook examples -[webhooks."internal".webhooks."FFFFA"] -log_type = "testing" -headers = ["x-forwarded-for"] -[webhooks."internal".webhooks."FFFFA".get_mode] -response_mode = "static:this is just static data to return to the caller" - -[webhooks."internal".webhooks."FFFFB"] -log_type = "testing" -headers = ["x-forwarded-for"] -[webhooks."internal".webhooks."FFFFB".get_mode] -response_mode = "rule:testing_test.wasm" -[webhooks."internal".webhooks."FFFFB".get_mode.caching_mode] -type = "Timed" -validity = 10 - -[webhooks."external"] -listen_address = "0.0.0.0:4556" -[webhooks."external".webhooks."AAAA"] -log_type = "testing" -headers = ["notalegitheader", "reallynotlegit"] -[webhooks."external".webhooks."AAAA".get_mode] -response_mode = "facebook:somelongstring" diff --git a/runtime/plaid/resources/secrets.example.toml b/runtime/plaid/resources/secrets.example.toml index 6cc75843..40652c66 100644 --- a/runtime/plaid/resources/secrets.example.toml +++ b/runtime/plaid/resources/secrets.example.toml @@ -1,2 +1,6 @@ "test-secret" = "This is a test secret" "public-key" = "{CI_PUBLIC_KEY_PLACEHOLDER}" +"test-webhook-secret" = "{CI_SLACK_TEST_WEBHOOK}" +"test-slack-bot-token" = "{CI_SLACK_TEST_BOT_TOKEN}" +"integration-test-root-ca" = """{CI_CERTIFICATE_PLACEHOLDER}""" +"example-auth-token" = "475e52cf9c4c1a62e1de7392598d42bab026b5cd2d86bfa33cfb9716de958739" diff --git a/runtime/plaid/src/apis/aws/iam.rs b/runtime/plaid/src/apis/aws/iam.rs new file mode 100644 index 00000000..b0cf1765 --- /dev/null +++ b/runtime/plaid/src/apis/aws/iam.rs @@ -0,0 +1,190 @@ +use aws_sdk_identitystore::Client; +use serde::Deserialize; + +use crate::{apis::ApiError, get_aws_sdk_config, AwsAuthentication}; + +/// Defines configuration for the Iam API +#[derive(Deserialize)] +pub struct IamConfig { + /// This can either be: + /// - `IAM`: Uses the IAM role assigned to the instance or environment. + /// - `ApiKey`: Uses explicit credentials, including an access key ID, secret access key, and region. + authentication: AwsAuthentication, + /// The unique identifier of the Identity Store + identity_store_id: String, +} + +/// Represents the Iam API that handles all requests to IAM +pub struct Iam { + /// The underlying IAM client used to interact with the IAM API. + client: Client, + /// The unique identifier of the Identity Store + identity_store_id: String, +} + +impl Iam { + /// Creates a new instance of `Iam` + pub async fn new(config: IamConfig) -> Self { + let sdk_config = get_aws_sdk_config(config.authentication).await; + let client = aws_sdk_identitystore::Client::new(&sdk_config); + + Self { + client, + identity_store_id: config.identity_store_id, + } + } + + /// Retrieve a user's ID given their username. + #[allow(deprecated)] // for the .filters() call + async fn get_user_id_by_username(&self, user_name: &str) -> Result { + let filter = aws_sdk_identitystore::types::Filter::builder() + .attribute_path("UserName") + .attribute_value(user_name) + .build() + .map_err(|e| ApiError::IamError(format!("Could not create filter for user: {e}")))?; + + let user_id = self + .client + .list_users() + .identity_store_id(&self.identity_store_id) + .filters(filter) + .send() + .await + .map_err(|e| ApiError::IamError(format!("Could not list users with filter: {e}")))? + .users() + .first() + .map(|u| u.user_id().to_string()) + .ok_or(ApiError::IamError("Could not get user ID".to_string()))?; + Ok(user_id) + } + + /// Retrieve a group's ID given its name. + #[allow(deprecated)] // for the .filters() call + async fn get_group_id_by_name(&self, group_name: &str) -> Result { + let filter = aws_sdk_identitystore::types::Filter::builder() + .attribute_path("DisplayName") + .attribute_value(group_name) + .build() + .map_err(|e| ApiError::IamError(format!("Could not create filter for group: {e}")))?; + + let group_id = self + .client + .list_groups() + .identity_store_id(&self.identity_store_id) + .filters(filter) + .send() + .await + .map_err(|e| ApiError::IamError(format!("Could not list groups: {e}")))? + .groups + .first() + .map(|g| g.group_id().to_string()) + .ok_or(ApiError::IamError("Could not get group ID".to_string()))?; + Ok(group_id) + } + + /// Create a user in an AWS Identity Store. + pub async fn create_user(&self, user_name: &str, display_name: &str) -> Result<(), ApiError> { + self.client + .create_user() + .identity_store_id(&self.identity_store_id) + .user_name(user_name) + .display_name(display_name) + .send() + .await + .map(|_| ()) // if it's OK, we don't care about the output + .map_err(|e| ApiError::IamError(format!("Could not create user {user_name}: {e}"))) + } + + /// Delete a user from an AWS Identity Store. + pub async fn delete_user(&self, user_name: &str) -> Result<(), ApiError> { + let user_id = self.get_user_id_by_username(user_name).await?; + self.client + .delete_user() + .identity_store_id(&self.identity_store_id) + .user_id(user_id) + .send() + .await + .map(|_| ()) // if it's OK, we don't care about the output + .map_err(|e| ApiError::IamError(format!("Could not delete user: {e}"))) + } + + /// Add a user to a group + pub async fn add_user_to_group( + &self, + user_name: &str, + group_name: &str, + ) -> Result<(), ApiError> { + let user_id = self.get_user_id_by_username(user_name).await?; + let member_id = aws_sdk_identitystore::types::MemberId::UserId(user_id.to_string()); + let group_id = self.get_group_id_by_name(group_name).await?; + + self.client + .create_group_membership() + .identity_store_id(&self.identity_store_id) + .group_id(group_id) + .member_id(member_id) + .send() + .await + .map(|_| ()) // if it's OK, we don't care about the output + .map_err(|e| { + ApiError::IamError(format!( + "Could not assign user {user_id} to group {group_name}: {e}" + )) + }) + } + + /// Remove a user from a group + pub async fn remove_user_from_group( + &self, + user_name: &str, + group_name: &str, + ) -> Result<(), ApiError> { + let user_id = self.get_user_id_by_username(user_name).await?; + let group_id = self.get_group_id_by_name(group_name).await?; + + // Get all memberships for the given group + let memberships = self + .client + .list_group_memberships() + .identity_store_id(&self.identity_store_id) + .group_id(group_id) + .send() + .await + .map_err(|e| { + ApiError::IamError(format!( + "Could not list group memberships for group {group_name}: {e}" + )) + })? + .group_memberships; + + // Find the user's membership in the given group + if let Some(membership) = memberships.into_iter().find(|m| { + m.member_id().map_or(false, |id| { + id.as_user_id().unwrap_or(&String::new()).as_str() == user_id + }) + }) { + let membership_id = membership + .membership_id() + .ok_or(ApiError::IamError(format!( + "Membership ID missing for user {user_name} in group {group_name}" + )))?; + + // Finally, delete the user's membership for the given group + self.client + .delete_group_membership() + .identity_store_id(&self.identity_store_id) + .membership_id(membership_id) + .send() + .await + .map(|_| ()) // if it's OK, we don't care about the output + .map_err(|e| { + ApiError::IamError(format!( + "Could not delete memberships for user {user_name} in group {group_name}: {e}" + )) + }) + } else { + // The user is not a member of the given group: do nothing + return Ok(()); + } + } +} diff --git a/runtime/plaid/src/apis/aws/mod.rs b/runtime/plaid/src/apis/aws/mod.rs index ae7f59d3..c4f2bcdf 100644 --- a/runtime/plaid/src/apis/aws/mod.rs +++ b/runtime/plaid/src/apis/aws/mod.rs @@ -1,6 +1,7 @@ use kms::{Kms, KmsConfig}; use serde::Deserialize; +pub mod iam; pub mod kms; /// The entire configuration of AWS APIs implemented in Plaid diff --git a/runtime/plaid/src/apis/general/logback.rs b/runtime/plaid/src/apis/general/logback.rs index a6a21922..e6c914c5 100644 --- a/runtime/plaid/src/apis/general/logback.rs +++ b/runtime/plaid/src/apis/general/logback.rs @@ -8,7 +8,14 @@ impl General { /// type. You need to be very careful when allowing modules to use this /// because it can be used to trigger other rules with greater access than /// the calling module has. - pub fn log_back(&self, type_: &str, log: &[u8], module: &str, delay: u64, logbacks_allowed: LogbacksAllowed) -> bool { + pub fn log_back( + &self, + type_: &str, + log: &[u8], + module: &str, + delay: u64, + logbacks_allowed: LogbacksAllowed, + ) -> bool { let msg = Message::new( type_.to_string(), log.to_vec(), @@ -16,15 +23,10 @@ impl General { logbacks_allowed, ); - // If the delay is zero, we can get the log through much faster without - // waiting for the data collector to find it, buffer it, and finally - // enqueue it on the Message channel by doing it ourselves. - if delay == 0 { - self.log_sender.send(msg).is_ok() - } else { - self.delayed_log_sender - .send(DelayedMessage::new(delay, msg)) - .is_ok() - } + // Send the message to the dedicated channel, from where it will + // be picked up by the dedicated data generator. + self.delayed_log_sender + .send(DelayedMessage::new(delay, msg)) + .is_ok() } } diff --git a/runtime/plaid/src/apis/general/mod.rs b/runtime/plaid/src/apis/general/mod.rs index 1127afcb..dbbaa242 100644 --- a/runtime/plaid/src/apis/general/mod.rs +++ b/runtime/plaid/src/apis/general/mod.rs @@ -8,16 +8,16 @@ use reqwest::Client; use ring::rand::SystemRandom; use serde::Deserialize; -use std::time::Duration; +use std::{collections::HashMap, time::Duration}; -use crate::{data::DelayedMessage, executor::Message}; +use crate::data::DelayedMessage; use super::default_timeout_seconds; #[derive(Deserialize)] pub struct GeneralConfig { /// Configuration for network requests - network: network::Config, + pub network: network::Config, /// The number of seconds until an external API request times out. /// If no value is provided, the result of `default_timeout_seconds()` will be used. #[serde(default = "default_timeout_seconds")] @@ -28,32 +28,65 @@ pub struct General { /// General Plaid configuration config: GeneralConfig, /// Client to make requests with - client: Client, - /// Sender object for messages - log_sender: Sender, + clients: Clients, /// Sender object for messages that must be processed with a delay delayed_log_sender: Sender, /// Secure random generator system_random: SystemRandom, } -impl General { - pub fn new( - config: GeneralConfig, - log_sender: Sender, - delayed_log_sender: Sender, - ) -> Self { - let client = reqwest::Client::builder() - .timeout(Duration::from_secs(config.api_timeout_seconds)) +/// Holds the default HTTP client plus any named clients with per-request customizations. +pub struct Clients { + /// The default `Client` used for requests without custom timeouts or certificates. + default: Client, + /// Named `Client` instances configured with custom timeouts or root certificates. + specialized: HashMap, +} + +impl Clients { + fn new(config: &GeneralConfig) -> Self { + let default_timeout_duration = Duration::from_secs(config.api_timeout_seconds); + let default = reqwest::Client::builder() + .timeout(default_timeout_duration) .build() .unwrap(); + let specialized = config + .network + .web_requests + .iter() + .filter_map(|(name, req)| { + if req.timeout.is_some() || req.root_certificate.is_some() { + let mut builder = reqwest::Client::builder() + .timeout(req.timeout.unwrap_or(default_timeout_duration)); + + if let Some(ca) = req.root_certificate.clone() { + builder = builder.add_root_certificate(ca); + } + + let client = builder.build().unwrap(); + Some((name.clone(), client)) + } else { + None + } + }) + .collect::>(); + + Self { + default, + specialized, + } + } +} + +impl General { + pub fn new(config: GeneralConfig, delayed_log_sender: Sender) -> Self { + let clients = Clients::new(&config); let system_random = SystemRandom::new(); Self { config, - client, - log_sender, + clients, delayed_log_sender, system_random, } diff --git a/runtime/plaid/src/apis/general/network.rs b/runtime/plaid/src/apis/general/network.rs index 0effd502..ab9cb8ff 100644 --- a/runtime/plaid/src/apis/general/network.rs +++ b/runtime/plaid/src/apis/general/network.rs @@ -1,8 +1,8 @@ -use std::{collections::HashMap, sync::Arc}; +use std::{collections::HashMap, sync::Arc, time::Duration}; use crate::loader::PlaidModule; -use reqwest::header::HeaderMap; -use serde::{Deserialize, Serialize}; +use reqwest::{header::HeaderMap, Certificate, Client}; +use serde::{de, Deserialize, Serialize}; use crate::apis::ApiError; @@ -10,7 +10,7 @@ use super::General; #[derive(Deserialize)] pub struct Config { - web_requests: HashMap, + pub web_requests: HashMap, } /// Request to make a web request @@ -18,7 +18,7 @@ pub struct Config { struct MakeRequestRequest { /// Body of the request body: String, - /// Name of the request - defined in plaid.toml + /// Name of the request - defined in the configuration request_name: String, /// Variables to include in the request. Variables take the place of an idenfitifer in the request URI variables: HashMap, @@ -41,6 +41,15 @@ pub struct Request { return_body: bool, /// Flag to return the code from the request return_code: bool, + /// Optional root TLS certificate to use for this request. + /// When set, the request will be sent via a special HTTP client configured with this certificate. + #[serde(default, deserialize_with = "certificate_deserializer")] + pub root_certificate: Option, + /// Optional per‐request timeout. + /// When set, the request will be sent via a special HTTP client configured with this timeout; + /// if unset, the default timeout from the API config is used. + #[serde(default, deserialize_with = "duration_deserializer")] + pub timeout: Option, /// Rules allowed to use this request allowed_rules: Vec, /// Headers to include in the request @@ -51,6 +60,39 @@ pub struct Request { available_in_test_mode: bool, } +/// Deserialize a non‐zero timeout (1–255 seconds) into a `Duration`, erroring on 0. +fn duration_deserializer<'de, D>(deserializer: D) -> Result, D::Error> +where + D: de::Deserializer<'de>, +{ + let duration = Option::::deserialize(deserializer)?; + match duration { + None => Ok(None), + Some(0) => Err(de::Error::custom( + "Invalid timeout duration provided. Acceptable values are between 1 and 255 seconds", + )), + Some(secs) => Ok(Some(Duration::from_secs(secs as u64))), + } +} + +/// Deserialize a PEM‐encoded string into a `Certificate`, erroring on parse failure. +fn certificate_deserializer<'de, D>(deserializer: D) -> Result, D::Error> +where + D: de::Deserializer<'de>, +{ + let pem = Option::::deserialize(deserializer)?; + match pem { + None => Ok(None), + Some(pem) => { + let cert = Certificate::from_pem(pem.as_bytes()).map_err(|e| { + serde::de::Error::custom(format!("Invalid certificate provided. Error: {e}")) + })?; + + Ok(Some(cert)) + } + } +} + /// Data returned by a request. #[derive(Serialize)] struct ReturnData { @@ -82,7 +124,8 @@ impl General { let auth = request.get("auth"); let request_builder = self - .client + .clients + .default .post(url) .header("Content-Type", "application/json; charset=utf-8"); @@ -159,12 +202,13 @@ impl General { uri = uri.replace(format!("{{{}}}", replacement.0).as_str(), replacement.1); } + let client = self.get_client(&request_name); let request_builder = match request_specification.verb.as_str() { - "delete" => self.client.delete(&uri), - "get" => self.client.get(&uri), - "patch" => self.client.patch(&uri), - "post" => self.client.post(&uri), - "put" => self.client.put(&uri), + "delete" => client.delete(&uri), + "get" => client.get(&uri), + "patch" => client.patch(&uri), + "post" => client.post(&uri), + "put" => client.put(&uri), // Not sure we want to support head //"head" => self.client.head(&request_specification.uri), _ => return Err(ApiError::BadRequest), @@ -204,4 +248,12 @@ impl General { Err(e) => Err(ApiError::NetworkError(e)), } } + + fn get_client(&self, mnr: &str) -> &Client { + if let Some(client) = self.clients.specialized.get(mnr) { + client + } else { + &self.clients.default + } + } } diff --git a/runtime/plaid/src/apis/github/mod.rs b/runtime/plaid/src/apis/github/mod.rs index 776fb91f..57b71714 100644 --- a/runtime/plaid/src/apis/github/mod.rs +++ b/runtime/plaid/src/apis/github/mod.rs @@ -6,6 +6,7 @@ mod environments; mod graphql; mod members; mod pats; +mod pull_requests; mod repos; mod secrets; mod teams; diff --git a/runtime/plaid/src/apis/github/pull_requests.rs b/runtime/plaid/src/apis/github/pull_requests.rs new file mode 100644 index 00000000..aff4b73a --- /dev/null +++ b/runtime/plaid/src/apis/github/pull_requests.rs @@ -0,0 +1,67 @@ +use plaid_stl::github::PullRequestRequestReviewers; +use serde::Serialize; + +use super::Github; +use crate::{ + apis::{github::GitHubError, ApiError}, + loader::PlaidModule, +}; +use std::sync::Arc; + +impl Github { + /// Add reviewers to a pull request + pub async fn pull_request_request_reviewers( + &self, + params: &str, + module: Arc, + ) -> Result { + #[derive(Serialize)] + struct RequestReviewers { + reviewers: Vec, + team_reviewers: Vec, + } + let request: PullRequestRequestReviewers = + serde_json::from_str(params).map_err(|_| ApiError::BadRequest)?; + + // Parse out all the parameters from our parameter string + let owner = self.validate_org(&request.owner)?; + let repo = self.validate_repository_name(&request.repo)?; + let pull_number = request.pull_number; + + for reviewer in &request.reviewers { + self.validate_username(&reviewer)?; + } + + for team in &request.team_reviewers { + self.validate_team_slug(&team)?; + } + + info!("Requesting reviews from users: [{}] and teams: [{}] on [{owner}/{repo}/{pull_number}] org on behalf of {module}", request.reviewers.join(", "), request.team_reviewers.join(", ")); + + let address = format!("/repos/{owner}/{repo}/pulls/{pull_number}/requested_reviewers"); + + let body = RequestReviewers { + reviewers: request.reviewers.clone(), + team_reviewers: request.team_reviewers.clone(), + }; + + match self.make_generic_post_request(address, body, module).await { + Ok((status, Ok(_))) => { + if status == 201 { + Ok(true) + } else if status == 404 { + Ok(false) + } else if status == 422 { + warn!("Some of the reviewers or teams are not collaborators on this repository. Context: [{owner}/{repo}/{pull_number}]. Users: [{}] and teams: [{}]", request.reviewers.join(", "), request.team_reviewers.join(", ")); + Ok(false) + } else { + Err(ApiError::GitHubError(GitHubError::UnexpectedStatusCode( + status, + ))) + } + } + Ok((_, Err(e))) => Err(e), + Err(e) => Err(e), + } + } +} diff --git a/runtime/plaid/src/apis/github/repos.rs b/runtime/plaid/src/apis/github/repos.rs index 47b405ba..e1e8c16c 100644 --- a/runtime/plaid/src/apis/github/repos.rs +++ b/runtime/plaid/src/apis/github/repos.rs @@ -148,7 +148,9 @@ impl Github { let page = self.validate_pint(request.get("page").ok_or(ApiError::BadRequest)?)?; info!("Fetching files for Pull Request Nr {pull_request} from [{organization}/{repository_name}] on behalf of {module}"); - let address = format!("/repos/{organization}/{repository_name}/pulls/{pull_request}/files?page={page}"); + let address = format!( + "/repos/{organization}/{repository_name}/pulls/{pull_request}/files?page={page}" + ); match self.make_generic_get_request(address, module).await { Ok((status, Ok(body))) => { @@ -386,10 +388,13 @@ impl Github { let request: HashMap<&str, &str> = serde_json::from_str(params).map_err(|_| ApiError::BadRequest)?; - let username = self.validate_username(request.get("usename").ok_or(ApiError::BadRequest)?)?; - let repository_name = - self.validate_repository_name(request.get("repository_name").ok_or(ApiError::BadRequest)?)?; - let pull_request = self.validate_pint(request.get("pull_request").ok_or(ApiError::BadRequest)?)?; + let username = + self.validate_username(request.get("usename").ok_or(ApiError::BadRequest)?)?; + let repository_name = self.validate_repository_name( + request.get("repository_name").ok_or(ApiError::BadRequest)?, + )?; + let pull_request = + self.validate_pint(request.get("pull_request").ok_or(ApiError::BadRequest)?)?; let comment = request.get("comment").ok_or(ApiError::BadRequest)?; info!("Commenting on Pull Request [{pull_request}] in repo [{repository_name}] on behalf of {module}"); @@ -397,7 +402,7 @@ impl Github { #[derive(Serialize)] struct Body<'a> { - body: &'a str + body: &'a str, } match self diff --git a/runtime/plaid/src/apis/mod.rs b/runtime/plaid/src/apis/mod.rs index f3020fe6..65b938ab 100644 --- a/runtime/plaid/src/apis/mod.rs +++ b/runtime/plaid/src/apis/mod.rs @@ -31,7 +31,7 @@ use tokio::runtime::Runtime; use web::{Web, WebConfig}; use yubikey::{Yubikey, YubikeyConfig}; -use crate::{data::DelayedMessage, executor::Message}; +use crate::data::DelayedMessage; use self::rustica::{Rustica, RusticaConfig}; @@ -90,14 +90,12 @@ pub enum ApiError { YubikeyError(yubikey::YubikeyError), WebError(web::WebError), TestMode, + #[cfg(feature = "aws")] + IamError(String), } impl Api { - pub async fn new( - config: ApiConfigs, - log_sender: Sender, - delayed_log_sender: Sender, - ) -> Self { + pub async fn new(config: ApiConfigs, delayed_log_sender: Sender) -> Self { #[cfg(feature = "aws")] let aws = match config.aws { Some(aws) => Some(Aws::new(aws).await), @@ -105,7 +103,7 @@ impl Api { }; let general = match config.general { - Some(gc) => Some(General::new(gc, log_sender, delayed_log_sender)), + Some(gc) => Some(General::new(gc, delayed_log_sender)), _ => None, }; diff --git a/runtime/plaid/src/apis/okta/mod.rs b/runtime/plaid/src/apis/okta/mod.rs index f03e7a49..65e57bc8 100644 --- a/runtime/plaid/src/apis/okta/mod.rs +++ b/runtime/plaid/src/apis/okta/mod.rs @@ -1,7 +1,10 @@ mod groups; mod users; -use jwt_simple::{claims::Claims, prelude::{Duration as JwtDuration, RS256KeyPair, RSAKeyPairLike}}; +use jwt_simple::{ + claims::Claims, + prelude::{Duration as JwtDuration, RS256KeyPair, RSAKeyPairLike}, +}; use reqwest::Client; use serde::{de, Deserialize, Serialize}; @@ -16,7 +19,7 @@ enum Authentication { /// Authenticate to the Okta API via an auth token ApiKey { /// The authentication token - token: String + token: String, }, /// Authenticate to the Okta API as an Okta app OktaApp { @@ -24,8 +27,8 @@ enum Authentication { client_id: String, /// Private key for the Okta app #[serde(deserialize_with = "private_key_deserializer")] - private_key: RS256KeyPair - } + private_key: RS256KeyPair, + }, } /// Custom deserialize for an Okta app's private key. @@ -37,7 +40,8 @@ where D: de::Deserializer<'de>, { let pem_key = String::deserialize(deserializer)?; - Ok(RS256KeyPair::from_pem(&pem_key).map_err(|_| de::Error::custom("Could not deserialize app's private key")))? + Ok(RS256KeyPair::from_pem(&pem_key) + .map_err(|_| de::Error::custom("Could not deserialize app's private key")))? } /// Configuration for Plaid's Okta API @@ -69,14 +73,14 @@ pub enum OktaError { AuthenticationFailure, BadPrivateKey, JwtSignatureFailure, - BadJsonResponse + BadJsonResponse, } /// Which operation we will execute through the API. This is used to determine /// the OAuth 2.0 scope to include in the request for an access token. pub enum OktaOperation { GetUserInfo, - RemoveUserFromGroup + RemoveUserFromGroup, } impl OktaOperation { @@ -86,7 +90,7 @@ impl OktaOperation { // https://developer.okta.com/docs/api/openapi/okta-management/management/tag/User/#tag/User/operation/getUser Self::GetUserInfo => "okta.users.read", // https://developer.okta.com/docs/api/openapi/okta-management/management/tag/Group/#tag/Group/operation/unassignUserFromGroup - Self::RemoveUserFromGroup => "okta.groups.manage" + Self::RemoveUserFromGroup => "okta.groups.manage", } } } @@ -105,7 +109,10 @@ impl Okta { pub async fn get_authorization_header(&self, op: &OktaOperation) -> Result { match &self.config.authentication { Authentication::ApiKey { token } => Ok(format!("SSWS {}", token)), - Authentication::OktaApp { client_id, private_key } => { + Authentication::OktaApp { + client_id, + private_key, + } => { let access_token = self.get_access_token(op, client_id, private_key).await?; Ok(format!("Bearer {}", access_token)) } @@ -119,40 +126,56 @@ impl Okta { .with_issuer(client_id) .with_audience(format!("https://{}/oauth2/v1/token", self.config.domain)) .with_subject(client_id); - Ok(private_key.sign(claims).map_err(|_| OktaError::JwtSignatureFailure)?) + Ok(private_key + .sign(claims) + .map_err(|_| OktaError::JwtSignatureFailure)?) } /// Obtain an access token from Okta, in exchange for a properly constructed JWT. - async fn get_access_token(&self, op: &OktaOperation, client_id: &str, private_key: &RS256KeyPair) -> Result { + async fn get_access_token( + &self, + op: &OktaOperation, + client_id: &str, + private_key: &RS256KeyPair, + ) -> Result { // For more details, see https://developer.okta.com/docs/guides/implement-oauth-for-okta-serviceapp/main/#get-an-access-token #[derive(Serialize)] struct Form<'a> { grant_type: &'a str, scope: &'a str, client_assertion_type: &'a str, - client_assertion: &'a str + client_assertion: &'a str, } #[derive(Deserialize)] #[allow(dead_code)] - struct AccessTokenResponse{ + struct AccessTokenResponse { token_type: String, expires_in: u32, access_token: String, - scope: String + scope: String, } let form = Form { grant_type: "client_credentials", scope: op.to_okta_scope(), client_assertion_type: "urn:ietf:params:oauth:client-assertion-type:jwt-bearer", - client_assertion: &self.get_jwt(client_id, private_key)? + client_assertion: &self.get_jwt(client_id, private_key)?, }; - let req = self.client.post(format!("https://{}/oauth2/v1/token", self.config.domain)) + let req = self + .client + .post(format!("https://{}/oauth2/v1/token", self.config.domain)) .header("Accept", "application/json") .form(&form); - let res = req.send().await.map_err(|_| OktaError::AuthenticationFailure)?; - let access_token = res.json::().await.map_err(|_| OktaError::BadJsonResponse)?.access_token; + let res = req + .send() + .await + .map_err(|_| OktaError::AuthenticationFailure)?; + let access_token = res + .json::() + .await + .map_err(|_| OktaError::BadJsonResponse)? + .access_token; Ok(access_token) } } diff --git a/runtime/plaid/src/apis/slack/api.rs b/runtime/plaid/src/apis/slack/api.rs index 0479eadb..be948da5 100644 --- a/runtime/plaid/src/apis/slack/api.rs +++ b/runtime/plaid/src/apis/slack/api.rs @@ -1,6 +1,10 @@ -use std::{collections::HashMap, sync::Arc}; +use std::sync::Arc; -use serde::{Deserialize, Serialize}; +use plaid_stl::slack::{ + GetIdFromEmail, GetPresence, GetPresenceResponse, PostMessage, UserInfo, UserInfoResponse, + ViewOpen, +}; +use reqwest::{Client, RequestBuilder}; use crate::{ apis::{slack::SlackError, ApiError}, @@ -10,138 +14,193 @@ use crate::{ use super::Slack; enum Apis { - PostMessage, - ViewsOpen, - LookupByEmail, + PostMessage(plaid_stl::slack::PostMessage), + ViewsOpen(plaid_stl::slack::ViewOpen), + LookupByEmail(plaid_stl::slack::GetIdFromEmail), + GetPresence(plaid_stl::slack::GetPresence), + UserInfo(plaid_stl::slack::UserInfo), } -impl std::fmt::Display for Apis { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match &self { - Self::PostMessage => write!(f, "chat.postMessage"), - Self::ViewsOpen => write!(f, "views.open"), - Self::LookupByEmail => write!(f, "users.lookupByEmail"), +const SLACK_API_URL: &str = "https://slack.com/api/"; +type Result = std::result::Result; + +impl Apis { + fn build_request(&self, client: &Client) -> RequestBuilder { + match self { + Self::PostMessage(p) => client + .post(format!("{SLACK_API_URL}{api}", api = "chat.postMessage")) + .body(p.body.clone()) + .header("Content-Type", "application/json; charset=utf-8"), + Self::ViewsOpen(p) => client + .post(format!("{SLACK_API_URL}{api}", api = "view.open")) + .body(p.body.clone()) + .header("Content-Type", "application/json; charset=utf-8"), + Self::LookupByEmail(p) => client.get(format!( + "{SLACK_API_URL}{api}?email={email}", + api = "users.lookupByEmail", + email = p.email, + )), + Self::GetPresence(p) => client.get(format!( + "{SLACK_API_URL}{api}?user={user}", + api = "users.getPresence", + user = p.id, + )), + Self::UserInfo(p) => client.get(format!( + "{SLACK_API_URL}{api}?user={user}", + api = "users.info", + user = p.id, + )), } } } -/// Slack user profile as returned by https://api.slack.com/methods/users.lookupByEmail -#[derive(Serialize, Deserialize)] -struct SlackUserProfile { - user: SlackUser, -} - -#[derive(Serialize, Deserialize)] -struct SlackUser { - id: String, +impl std::fmt::Display for Apis { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::PostMessage(_) => write!(f, "PostMessage"), + Self::ViewsOpen(_) => write!(f, "ViewsOpen"), + Self::LookupByEmail(_) => write!(f, "LookupByEmail"), + Self::GetPresence(_) => write!(f, "GetPresence"), + Self::UserInfo(_) => write!(f, "UserInfo"), + } + } } impl Slack { /// Get token for a bot, if present - fn get_token(&self, bot: &str) -> Result { - let token = self.config.bot_tokens.get(bot).ok_or(())?; - Ok(format!("Bearer {token}")) + fn get_token(&self, bot: &str) -> Result<&String> { + self.config + .bot_tokens + .get(bot) + .ok_or(ApiError::SlackError(SlackError::UnknownBot( + bot.to_string(), + ))) } - /// Make a call to the Slack API. Depending on which API we are calling, a GET or a POST are executed. - async fn call_slack(&self, params: &str, api: Apis) -> Result { - let request: HashMap = - serde_json::from_str(params).map_err(|_| ApiError::BadRequest)?; - - let bot = request - .get("bot") - .ok_or(ApiError::MissingParameter("bot".to_string()))? - .to_string(); - - let token = self.get_token(&bot).map_err(|_| { - error!("A module tried to call api {api} for a bot that didn't exist: {bot}"); - ApiError::SlackError(SlackError::UnknownBot(bot.to_string())) - })?; - - info!("Calling {api} for bot: {bot}"); - match api { - Apis::PostMessage | Apis::ViewsOpen => { - // It's a POST call - let body = request - .get("body") - .ok_or(ApiError::MissingParameter("body".to_string()))? - .to_string(); - match self - .client - .post(format!("https://slack.com/api/{api}")) - .header("Authorization", token) - .header("Content-Type", "application/json; charset=utf-8") - .body(body) - .send() - .await - { - Ok(r) => { - let status = r.status(); - if status == 200 { - return Ok("".to_string()); - } - let response = r.text().await; - error!("Slack data returned: {}", response.unwrap_or_default()); - - return Err(ApiError::SlackError(SlackError::UnexpectedStatusCode( - status.as_u16(), - ))); - } - Err(e) => return Err(ApiError::NetworkError(e)), - } - } - Apis::LookupByEmail => { - // It's a GET call - let email = request - .get("email") - .ok_or(ApiError::MissingParameter("email".to_string()))? - .to_string(); - match self - .client - .get(format!("https://slack.com/api/{api}?email={email}")) - .header("Authorization", token) - .send() - .await - { - Ok(r) => { - let status = r.status(); - if status == 200 { - let response = r.json::().await.map_err(|_| { - ApiError::SlackError(SlackError::UnexpectedPayload( - "could not deserialize to Slack user profile".to_string(), - )) - })?; - return Ok(response.user.id); - } - error!("Failed to retrieve user's Slack ID"); - return Err(ApiError::SlackError(SlackError::UnexpectedStatusCode( - status.as_u16(), - ))); - } - Err(e) => return Err(ApiError::NetworkError(e)), - } - } - } + /// Make a call to the Slack API + async fn call_slack( + &self, + bot: String, + api: Apis, + module: Arc, + ) -> Result<(u16, String)> { + let r = api + .build_request(&self.client) + .header("Authorization", format!("Bearer {}", self.get_token(&bot)?)); + + info!("Calling [{api}] using bot: [{bot}] on behalf of: [{module}]"); + let resp = r.send().await.map_err(|e| ApiError::NetworkError(e))?; + let status = resp.status(); + let response = resp.text().await.unwrap_or_default(); + trace!("Slack returned: {status}: {response}"); + Ok((status.as_u16(), response)) } /// Open an arbitrary view for a configured bot. The view contents is defined by the caller but the bot /// must be configured in Plaid. - pub async fn views_open(&self, params: &str, _: Arc) -> Result { - self.call_slack(params, Apis::ViewsOpen).await.map(|_| 0) + pub async fn views_open(&self, params: &str, module: Arc) -> Result { + let p: ViewOpen = serde_json::from_str(params).map_err(|_| ApiError::BadRequest)?; + match self + .call_slack(p.bot.clone(), Apis::ViewsOpen(p), module) + .await + { + Ok((200, _)) => Ok(0), + Ok((status, _)) => Err(ApiError::SlackError(SlackError::UnexpectedStatusCode( + status, + ))), + Err(e) => Err(e), + } } /// Call the Slack postMessage API. The message and location are defined by the module but the bot /// must be configured in Plaid. - pub async fn post_message(&self, params: &str, _: Arc) -> Result { - self.call_slack(params, Apis::PostMessage).await.map(|_| 0) + pub async fn post_message(&self, params: &str, module: Arc) -> Result { + let p: PostMessage = serde_json::from_str(params).map_err(|_| ApiError::BadRequest)?; + match self + .call_slack(p.bot.clone(), Apis::PostMessage(p), module) + .await + { + Ok((200, _)) => Ok(0), + Ok((status, _)) => Err(ApiError::SlackError(SlackError::UnexpectedStatusCode( + status, + ))), + Err(e) => Err(e), + } } /// Calls the Slack API to retrieve a user's Slack ID from their email address pub async fn get_id_from_email( &self, params: &str, - _: Arc, - ) -> Result { - self.call_slack(params, Apis::LookupByEmail).await + module: Arc, + ) -> Result { + let p: GetIdFromEmail = serde_json::from_str(params).map_err(|_| ApiError::BadRequest)?; + match self + .call_slack(p.bot.clone(), Apis::LookupByEmail(p), module) + .await + { + Ok((200, response)) => { + let response: UserInfoResponse = serde_json::from_str(&response).map_err(|e| { + ApiError::SlackError(SlackError::UnexpectedPayload(e.to_string())) + })?; + Ok(response.user.id) + } + Ok((status, _)) => Err(ApiError::SlackError(SlackError::UnexpectedStatusCode( + status, + ))), + Err(e) => Err(e), + } + } + + /// Get a user's presence status from their ID + pub async fn get_presence(&self, params: &str, module: Arc) -> Result { + let p: GetPresence = serde_json::from_str(params).map_err(|_| ApiError::BadRequest)?; + match self + .call_slack(p.bot.clone(), Apis::GetPresence(p), module) + .await + { + Ok((200, response)) => { + let gp_response: GetPresenceResponse = + serde_json::from_str(&response).map_err(|e| { + ApiError::SlackError(SlackError::UnexpectedPayload(e.to_string())) + })?; + if !gp_response.ok { + return Err(ApiError::SlackError(SlackError::UnexpectedPayload( + response, + ))); + } + Ok(response) + } + Ok((status, _)) => Err(ApiError::SlackError(SlackError::UnexpectedStatusCode( + status, + ))), + Err(e) => Err(e), + } + } + + /// Get a user's info from their ID + pub async fn user_info(&self, params: &str, module: Arc) -> Result { + let p: UserInfo = serde_json::from_str(params).map_err(|_| ApiError::BadRequest)?; + match self + .call_slack(p.bot.clone(), Apis::UserInfo(p), module) + .await + { + Ok((200, response)) => { + let up_response: UserInfoResponse = + serde_json::from_str(&response).map_err(|e| { + ApiError::SlackError(SlackError::UnexpectedPayload(e.to_string())) + })?; + if !up_response.ok { + return Err(ApiError::SlackError(SlackError::UnexpectedPayload( + response, + ))); + } + Ok(response) + } + Ok((status, _)) => Err(ApiError::SlackError(SlackError::UnexpectedStatusCode( + status, + ))), + Err(e) => Err(e), + } } } diff --git a/runtime/plaid/src/bin/plaid.rs b/runtime/plaid/src/bin/plaid.rs index 38efb860..8631fbdc 100644 --- a/runtime/plaid/src/bin/plaid.rs +++ b/runtime/plaid/src/bin/plaid.rs @@ -2,7 +2,12 @@ extern crate log; use performance::ModulePerformanceMetadata; -use plaid::{config::{CachingMode, GetMode, ResponseMode, WebhookServerConfiguration}, loader::PlaidModule, logging::Logger, *}; +use plaid::{ + config::{CachingMode, GetMode, ResponseMode, WebhookServerConfiguration}, + loader::PlaidModule, + logging::Logger, + *, +}; use apis::Api; use data::Data; @@ -12,10 +17,17 @@ use storage::Storage; use tokio::{signal, sync::RwLock, task::JoinSet}; use tokio_util::sync::CancellationToken; -use std::{collections::HashMap, convert::Infallible, net::SocketAddr, pin::Pin, sync::Arc, time::{SystemTime, UNIX_EPOCH}}; +use std::{ + collections::HashMap, + convert::Infallible, + net::SocketAddr, + pin::Pin, + sync::Arc, + time::{SystemTime, UNIX_EPOCH}, +}; -use crossbeam_channel::{bounded, TrySendError}; -use warp::{hyper::body::Bytes, path, Filter, http::HeaderMap}; +use crossbeam_channel::TrySendError; +use warp::{http::HeaderMap, hyper::body::Bytes, path, Filter}; #[tokio::main] async fn main() -> Result<(), Box> { @@ -24,7 +36,12 @@ async fn main() -> Result<(), Box> { info!("Reading configuration"); let config = config::configure()?; - let (log_sender, log_receiver) = bounded(config.log_queue_size); + + // Create thread pools for log execution + let exec_thread_pools = thread_pools::ExecutionThreadPools::new(&config.executor); + + // For convenience, keep a reference to the log_sender for the general channel, so that we can quickly clone it around + let log_sender = &exec_thread_pools.general_pool.sender; info!("Starting logging subsystem"); let (els, _logging_handler) = Logger::start(config.logging); @@ -32,34 +49,47 @@ async fn main() -> Result<(), Box> { // Create the storage system if one is configured let storage = match config.storage { - Some(config) => Some(Arc::new(Storage::new(config).await?)), - None => None, - }; - - match storage { - None => info!("No persistent storage system configured; unexecuted log backs will be lost on shutdown"), - Some(ref storage) => { + Some(config) => { info!("Storage system configured"); - match &storage.shared_dbs { + let s = Arc::new(Storage::new(config).await?); + match &s.shared_dbs { None => info!("No shared DBs configured"), Some(dbs) => { - info!("Configured shared DBs: {:?}", dbs.keys().collect::>()); + info!( + "Configured shared DBs: {:?}", + dbs.keys().collect::>() + ); } } + Some(s) } - } + None => { + info!("No persistent storage system configured; unexecuted log backs will be lost on shutdown"); + None + } + }; + + // The internal system always gets a storage: if we don't have a persistent one, we create an in-memory one + let internal_storage = match &storage { + Some(s) => s.clone(), + None => Arc::new(Storage::new_in_memory()), + }; // This sender provides an internal route to sending logs. This is what // powers the logback functions. - - let delayed_log_sender = Data::start(config.data, log_sender.clone(), storage.clone(), els.clone()) - .await - .expect("The data system failed to start") - .unwrap(); + let delayed_log_sender = Data::start( + config.data, + log_sender.clone(), + internal_storage.clone(), + els.clone(), + ) + .await + .expect("The data system failed to start") + .unwrap(); info!("Configurating APIs for Modules"); // Create the API that powers all the wrapped calls that modules can make - let api = Api::new(config.apis, log_sender.clone(), delayed_log_sender).await; + let api = Api::new(config.apis, delayed_log_sender).await; // Create an Arc so all the handlers have access to our API object let api = Arc::new(api); @@ -68,7 +98,9 @@ async fn main() -> Result<(), Box> { let cancellation_token = CancellationToken::new(); let ct = cancellation_token.clone(); tokio::spawn(async move { - signal::ctrl_c().await.expect("Failed to listen for shutdown signal"); + signal::ctrl_c() + .await + .expect("Failed to listen for shutdown signal"); info!("Shutdown signal received, sending cancellation notice to all listening tasks."); ct.cancel(); }); @@ -77,15 +109,15 @@ async fn main() -> Result<(), Box> { Some(perf) => { warn!("Plaid is running with performance monitoring enabled - this is NOT recommended for production deployments. Metadata about rule execution will be logged to a channel that aggregates and reports metrics."); let (sender, rx) = crossbeam_channel::bounded::(4096); - + let token = cancellation_token.clone(); let handle = tokio::task::spawn(async move { perf.start(rx, token).await; - }); + }); (Some(sender), Some(handle)) - }, - None => (None, None) + } + None => (None, None), }; info!("Loading all the modules"); @@ -93,38 +125,50 @@ async fn main() -> Result<(), Box> { let modules = Arc::new(loader::load(config.loading, storage.clone()).await.unwrap()); let modules_by_name = Arc::new(modules.get_modules()); + // Print information about the threads we are starting info!( - "Starting the execution threads of which {} were requested", - config.execution_threads + "Starting {} execution threads for general execution. Log queue size = {}", + exec_thread_pools.general_pool.num_threads, + exec_thread_pools + .general_pool + .sender + .capacity() + .unwrap_or_default() ); + for (log_type, tp) in &exec_thread_pools.dedicated_pools { + let thread_or_threads = if tp.num_threads == 1 { + "thread" + } else { + "threads" + }; + info!("Starting {} {thread_or_threads} dedicated to log type [{log_type}]. Log queue size = {}", tp.num_threads, tp.sender.capacity().unwrap_or_default()); + } // Create the executor that will handle all the logs that come in and immediate // requests for handling some configured get requests. let executor = Executor::new( - log_receiver, + exec_thread_pools.clone(), modules.get_channels(), api, storage, - config.execution_threads, els.clone(), - performance_sender.clone() + performance_sender.clone(), ); - let _executor = Arc::new(executor); + let executor = Arc::new(executor); info!("Configured Webhook Servers"); - let webhook_server_post_log_sender = log_sender.clone(); let webhook_servers: Vec>>> = config .webhooks .into_iter() .map(|(server_name, config)| { - let webhook_server_post_log_sender = webhook_server_post_log_sender.clone(); let server_address: SocketAddr = config .listen_address .parse() .expect("A server had an invalid address"); let webhooks = config.webhooks.clone(); + let exec = executor.clone(); let post_route = warp::post() .and(warp::body::content_length_limit(1024 * 256)) .and(path!("webhook" / String)) @@ -155,7 +199,7 @@ async fn main() -> Result<(), Box> { } // Webhook exists, buffer log - if let Err(e) = webhook_server_post_log_sender.try_send(message) { + if let Err(e) = exec.execute_webhook_message(message) { match e { TrySendError::Full(_) => error!("Queue Full! [{}] log dropped!", webhook_configuration.log_type), // TODO: Have this actually cause Plaid to exit @@ -175,11 +219,13 @@ async fn main() -> Result<(), Box> { let get_route = warp::get() .and(path!("webhook" / String)) .and(warp::query::>()) + .and(warp::body::bytes()) + .and(warp::header::headers_cloned()) .and(with(webhook_config.clone())) .and(with(modules_by_name.clone())) .and(with(get_cache.clone())) .and(with(webhook_server_get_log_sender.clone())) - .and_then(|webhook: String, query: HashMap, webhook_config: Arc, modules: Arc>>, get_cache: Arc>>, log_sender: crossbeam_channel::Sender| async move { + .and_then(|webhook: String, query: HashMap, body: Bytes, headers: HeaderMap, webhook_config: Arc, modules: Arc>>, get_cache: Arc>>, log_sender: crossbeam_channel::Sender| async move { if let Some(webhook_configuration) = webhook_config.webhooks.get(&webhook) { match &webhook_configuration.get_mode { // Note that CacheMode is elided here as there is no caching for static data @@ -248,7 +294,7 @@ async fn main() -> Result<(), Box> { false }, } - }, + } }; // If the webhook has a label, use that as the source, otherwise use the webhook address @@ -262,16 +308,22 @@ async fn main() -> Result<(), Box> { let (response_send, response_recv) = tokio::sync::oneshot::channel(); // Construct a message to send to the rule - let message = Message::new_detailed( + let mut message = Message::new_detailed( name.to_string(), - String::new().into_bytes(), + body.to_vec(), source, logbacks_allowed, - HashMap::new(), query.into_iter().map(|(k, v)| (k, v.into_bytes())).collect(), Some(response_send), Some(rule.clone())); + // Configure headers + for requested_header in webhook_configuration.headers.iter() { + if let Some(value) = headers.get(requested_header) { + message.headers.insert(requested_header.to_string(), value.as_bytes().to_vec()); + } + } + // Put the message into the standard message queue if let Err(e) = log_sender.try_send(message) { match e { diff --git a/runtime/plaid/src/bin/request_handler.rs b/runtime/plaid/src/bin/request_handler.rs index e7e7d3fe..6beedff5 100644 --- a/runtime/plaid/src/bin/request_handler.rs +++ b/runtime/plaid/src/bin/request_handler.rs @@ -1,3 +1,5 @@ +use std::fs; + use warp::Filter; #[tokio::main] @@ -52,7 +54,18 @@ async fn main() { }, ); + let cert = fs::read("/tmp/plaid_config/server.pem").expect("failed to read server.pem"); + let key = fs::read("/tmp/plaid_config/server.key").expect("failed to read server.key"); + // Start the server on 127.0.0.1:8998 - let routes = post_route.or(mnr_vars_route).or(mnr_headers_route).or(mnr_route); - warp::serve(routes).run(([127, 0, 0, 1], 8998)).await; + let routes = post_route + .or(mnr_vars_route) + .or(mnr_headers_route) + .or(mnr_route); + warp::serve(routes) + .tls() + .cert(cert) + .key(key) + .run(([127, 0, 0, 1], 8998)) + .await; } diff --git a/runtime/plaid/src/bin/secrets_manager/file_to_aws.rs b/runtime/plaid/src/bin/secrets_manager/file_to_aws.rs index c483254a..f266cd94 100644 --- a/runtime/plaid/src/bin/secrets_manager/file_to_aws.rs +++ b/runtime/plaid/src/bin/secrets_manager/file_to_aws.rs @@ -30,29 +30,87 @@ pub async fn file_to_aws( // Upload secrets to SM for secret in secrets { println!("Uploading {}...", secret.name); - match sm_client - .create_secret() - .name(secret.name.clone()) - .kms_key_id(kms_key_id.to_string()) - .secret_string(secret.value) - .force_overwrite_replica_secret(overwrite) + + // If we don't want to overwrite existing secrets, just try to create and continue on + if !overwrite { + create_secret(&sm_client, &secret, &kms_key_id).await; + continue; + } + + // If in overwrite mode, first get the ARN of the secret from Secrets Manager. If we cannot find an ARN + // for the secret, we'll just create the secret and continue + let secret_id = match sm_client + .get_secret_value() + .secret_id(&secret.name) .send() .await { - Ok(_) => {} + Ok(response) => { + let Some(arn) = response.arn else { + eprintln!( + "No ARN present in response from Secrets Manager for {}. Skipping...", + secret.name + ); + continue; + }; + arn + } Err(e) => { let err = e.into_service_error(); - // If it fails because a secret is already there, just log it but don't fail. - // Otherwise it is a real failure. - if err.is_resource_exists_exception() { - println!("Secret with name {} already exists in Secrets Manager. Skipping (NOT overwriting) it...", secret.name); - } else { - panic!( - "Error while uploading secrets to AWS Secrets Manager: {}", - err + if err.is_resource_not_found_exception() { + println!( + "No existing secret found for {}. Creating a new one...", + &secret.name ); + create_secret(&sm_client, &secret, &kms_key_id).await; + continue; } + + eprintln!( + "Failed to get ARN of {} from Secrets Manager and cannot update its value. Error: {err}", + &secret.name + ); + continue; } + }; + + let response = sm_client + .update_secret() + .secret_id(secret_id) + .kms_key_id(kms_key_id.to_string()) + .secret_string(secret.value) + .send() + .await; + + if let Err(e) = response { + eprintln!( + "Failed to update secret value for {}. Error: {e}", + secret.name + ) + } + } +} + +async fn create_secret(client: &Client, secret: &PlaidSecret, kms_key_id: &impl Display) { + let response = client + .create_secret() + .name(&secret.name) + .kms_key_id(kms_key_id.to_string()) + .secret_string(&secret.value) + .send() + .await; + + if let Err(e) = response { + let err = e.into_service_error(); + // If it fails because a secret is already there, just log it but don't fail. + // Otherwise it is a real failure. + if err.is_resource_exists_exception() { + println!("Secret with name {} already exists in Secrets Manager. Skipping (NOT overwriting) it...", secret.name); + } else { + panic!( + "Error while uploading {} to AWS Secrets Manager. Error: {err}", + secret.name + ); } } } diff --git a/runtime/plaid/src/config.rs b/runtime/plaid/src/config.rs index e06fc6be..d010f60b 100644 --- a/runtime/plaid/src/config.rs +++ b/runtime/plaid/src/config.rs @@ -4,6 +4,7 @@ use plaid_stl::messages::LogbacksAllowed; use ring::digest::{self, digest}; use serde::{de, Deserialize}; use std::collections::HashMap; +use std::path::PathBuf; use crate::performance::PerformanceMonitoring; @@ -98,15 +99,17 @@ pub struct WebhookServerConfiguration { pub webhooks: HashMap, } -/// The full configuration of Plaid +/// Configuration for a thread pool / channel dedicated to a log type #[derive(Deserialize)] -pub struct Configuration { - /// How APIs are configured. These APIs are accessible to modules - /// so they can take advantage of Plaid abstractions - pub apis: ApiConfigs, - /// Data generators. These are systems that pull data directly rather - /// than waiting for data to come in via Webhook - pub data: DataConfig, +pub struct DedicatedThreadsConfig { + pub num_threads: u8, + #[serde(default = "default_log_queue_size")] + pub log_queue_size: usize, +} + +/// The configuration for the executor system +#[derive(Deserialize)] +pub struct ExecutorConfig { /// How many threads should be used for executing modules when logs come in /// /// Modules do not get more than one thread, this just means that modules can @@ -115,6 +118,23 @@ pub struct Configuration { /// The maximum number of logs in the queue to be processed at once #[serde(default = "default_log_queue_size")] pub log_queue_size: usize, + /// Number of threads dedicated to specific log types. + /// This is a mapping {log type --> num threads}. + #[serde(default)] + pub dedicated_threads: HashMap, +} + +/// The full configuration of Plaid +#[derive(Deserialize)] +pub struct Configuration { + /// How APIs are configured. These APIs are accessible to modules + /// so they can take advantage of Plaid abstractions + pub apis: ApiConfigs, + /// Data generators. These are systems that pull data directly rather + /// than waiting for data to come in via Webhook + pub data: DataConfig, + /// The executor subsystem. + pub executor: ExecutorConfig, /// Configuration for how Plaid monitors rule performance. When enabled, /// Plaid outputs a metrics file with performance metadata for all /// rules than have been run at least once. @@ -208,9 +228,9 @@ pub fn configure() -> Result { .about("Write security rules in anything that compiles to WASM, run them with only the access they need.") .arg( Arg::new("config") - .help("Path to the configuration toml file") + .help("Path to the folder with configuration toml files") .long("config") - .default_value("./plaid/resources/plaid.toml") + .default_value("./plaid/resources/config") ) .arg( Arg::new("secrets") @@ -219,28 +239,51 @@ pub fn configure() -> Result { .default_value("./plaid/private-resources/secrets.toml") ).get_matches(); - let config_path = matches.get_one::("config").unwrap(); + let config_folder = matches.get_one::("config").unwrap(); let secrets_path = matches.get_one::("secrets").unwrap(); - read_and_interpolate(config_path, secrets_path, false) + read_and_interpolate(config_folder, secrets_path, false) } -/// Reads a configuration file and a secrets file, interpolates the secrets into the configuration, -/// and parses the result into a `Configuration` struct. +/// Reads configuration files from a given folder and a secrets file, concatenates the config files into one, +/// interpolates the secrets into the configuration, and parses the result into a `Configuration` struct. pub fn read_and_interpolate( - config_path: &str, + config_folder: &str, secrets_path: &str, show_config: bool, ) -> Result { - // Read the configuration file - let mut config = match std::fs::read_to_string(config_path) { - Ok(config) => config, + // Read the configuration files from a given folder, and concatenate them into one + let mut config = String::new(); + + let entries = match std::fs::read_dir(config_folder) { + Ok(x) => x, Err(e) => { - error!("Encountered file error when trying to read configuration!. Error: {e}"); + error!("Encountered error when trying to read configuration folder! Error: {e}"); return Err(ConfigurationError::FileError); } }; + let mut paths: Vec = entries + .filter_map(Result::ok) + .map(|dir_entry| dir_entry.path()) + .filter(|path| path.is_file()) + .collect(); + paths.sort_by(|a, b| { + a.file_name() + .and_then(|a_name| b.file_name().map(|b_name| a_name.cmp(b_name))) + .unwrap_or(std::cmp::Ordering::Equal) + }); + + for path in paths { + match std::fs::read_to_string(&path) { + Ok(content) => config.push_str(&content), + Err(e) => { + error!("Encountered error when trying to read configuration! Error: {e}"); + return Err(ConfigurationError::FileError); + } + }; + } + // Read the secrets file and parse into a TOML object let secrets = std::fs::read_to_string(secrets_path) .map_err(|e| { @@ -278,12 +321,12 @@ pub fn read_and_interpolate( let config: Configuration = match toml::from_str(&config) { Ok(config) => config, Err(e) => { - error!("Encountered parsing error while reading configuration with interpolated secrets!. Error: {e}"); + error!("Encountered parsing error while reading configuration with interpolated secrets! Error: {e}"); return Err(ConfigurationError::ParsingError); } }; - if config.execution_threads == 0 { + if config.executor.execution_threads == 0 { return Err(ConfigurationError::ExecutionThreadsInvalid); } diff --git a/runtime/plaid/src/data/internal/mod.rs b/runtime/plaid/src/data/internal/mod.rs index 799c1e44..ec46c6d0 100644 --- a/runtime/plaid/src/data/internal/mod.rs +++ b/runtime/plaid/src/data/internal/mod.rs @@ -16,9 +16,6 @@ use super::DataError; const LOGBACK_NS: &str = "logback_internal"; -#[derive(Deserialize, Default)] -pub struct InternalConfig {} - #[derive(Serialize, Deserialize)] pub struct DelayedMessage { pub delay: u64, @@ -47,62 +44,56 @@ impl std::cmp::PartialOrd for DelayedMessage { impl std::cmp::Ord for DelayedMessage { fn cmp(&self, other: &Self) -> std::cmp::Ordering { - if self.delay < other.delay { - std::cmp::Ordering::Less - } else { - std::cmp::Ordering::Greater - } + self.delay.cmp(&other.delay) } } pub struct Internal { - #[allow(dead_code)] - config: InternalConfig, log_heap: BinaryHeap>, sender: Sender, receiver: Receiver, internal_sender: Sender, - storage: Option>, + storage: Arc, +} + +/// Fill the log heap with the content read from the DB +async fn fill_heap_from_db( + storage: Arc, +) -> Result>, DataError> { + let mut log_heap = BinaryHeap::new(); + + let previous_logs = storage + .fetch_all(LOGBACK_NS, None) + .await + .map_err(|e| DataError::StorageError(e))?; + + for (key, value) in previous_logs { + // We want to extract from the DB a message and a delay. + let (message, delay) = match serde_json::from_slice::(&value) { + Ok(item) => (item.message, item.delay), + Err(e) => { + warn!( + "Skipping log in storage system which could not be deserialized [{e}]: {:X?}", + key + ); + continue; + } + }; + log_heap.push(Reverse(DelayedMessage { delay, message })); + } + + Ok(log_heap) } impl Internal { pub async fn new( - config: InternalConfig, log_sender: Sender, - storage: Option>, + storage: Arc, ) -> Result { let (internal_sender, receiver) = bounded(4096); - - let mut log_heap = BinaryHeap::new(); - - if let Some(storage) = &storage { - let previous_logs = storage - .fetch_all(LOGBACK_NS, None) - .await - .map_err(|e| DataError::StorageError(e))?; - - for (key, value) in previous_logs { - // We want to extract from the DB a message and a delay. - // First, we try deserializing the new format, where the DB key is a message ID, and the value is the serialized DelayedMessage - let (message, delay) = match serde_json::from_slice::(&value) { - Ok(item) => { - // Everything OK, we were deserializing a logback in the "new" format - (item.message, item.delay) - } - Err(e) => { - warn!( - "Skipping log in storage system which could not be deserialized [{e}]: {:X?}", - key - ); - continue; - } - }; - log_heap.push(Reverse(DelayedMessage { delay, message })); - } - } + let log_heap = fill_heap_from_db(storage.clone()).await?; Ok(Self { - config, log_heap, sender: log_sender, receiver, @@ -122,29 +113,39 @@ impl Internal { .expect("Time went backwards") .as_secs(); - // Pull all logs off the channel, set their time, and put them on the heap. + // Pull all logs off the channel, set their time, and store them in the DB. + // Then discard the current content of the heap, pull the content of the DB + // and use it to fill the heap. // - // If persistence is available, we will also set them there in case the system - // reboots + // This ensures that modifications which are made out-of-band to the DB are + // reflected in the heap's content. + + // First, receive everything from the channel and write to DB while let Ok(mut log) = self.receiver.try_recv() { log.delay += current_time; - if let Some(storage) = &self.storage { - // Prepare what will be stored in the DB by serializing the DelayedMessage - if let Ok(db_item) = serde_json::to_vec(&log) { - if let Err(e) = storage - .insert(LOGBACK_NS.to_string(), log.message.id.clone(), db_item) - .await - { - error!("Storage system could not persist a message: {e}"); - } + // Prepare what will be stored in the DB by serializing the DelayedMessage + if let Ok(db_item) = serde_json::to_vec(&log) { + if let Err(e) = self + .storage + .insert(LOGBACK_NS.to_string(), log.message.id.clone(), db_item) + .await + { + error!("Storage system could not persist a message: {e}"); } } - - // Put the log into the in-memory heap - self.log_heap.push(Reverse(log)); } + // TODO the following part will be executed only by the instance that processes logbacks + + // Then, overwrite the current heap with the content read from the DB + self.log_heap = fill_heap_from_db(self.storage.clone()) + .await + .map_err(|e| format!("{e:?}"))?; + + // Now the heap is reflecting the content of the DB: we can look at it + // and see if something should be executed. + while let Some(heap_top) = self.log_heap.peek() { let heap_top = &heap_top.0; @@ -157,16 +158,13 @@ impl Internal { } let log = self.log_heap.pop().unwrap(); - if let Some(storage) = &self.storage { - // Delete the logback from the storage because we are about to send it for processing. - // According to the new format, the key is the ID inside the DelayedMessage's message field - match storage.delete(LOGBACK_NS, &log.0.message.id).await { - Ok(None) => { - error!("We tried to delete a log back message that wasn't persisted") - } - Ok(Some(_)) => (), - Err(e) => error!("Error removing persisted log: {e}"), + // Delete the logback from the storage because we are about to send it for processing. + match self.storage.delete(LOGBACK_NS, &log.0.message.id).await { + Ok(None) => { + error!("We tried to delete a log back message that wasn't persisted") } + Ok(Some(_)) => (), + Err(e) => error!("Error removing persisted log: {e}"), } self.sender.send(log.0.message).unwrap(); } diff --git a/runtime/plaid/src/data/mod.rs b/runtime/plaid/src/data/mod.rs index 5cb14beb..e1b8ed8f 100644 --- a/runtime/plaid/src/data/mod.rs +++ b/runtime/plaid/src/data/mod.rs @@ -29,7 +29,6 @@ pub use self::internal::DelayedMessage; pub struct DataConfig { github: Option, okta: Option, - internal: Option, interval: Option, websocket: Option, } @@ -58,7 +57,7 @@ impl DataInternal { async fn new( config: DataConfig, logger: Sender, - storage: Option>, + storage: Arc, els: Logger, ) -> Result { let github = config @@ -69,19 +68,7 @@ impl DataInternal { .okta .map(|okta| okta::Okta::new(okta, logger.clone())); - let internal = match config.internal { - Some(internal) => { - internal::Internal::new(internal, logger.clone(), storage.clone()).await - } - None => { - internal::Internal::new( - internal::InternalConfig::default(), - logger.clone(), - storage.clone(), - ) - .await - } - }; + let internal = internal::Internal::new(logger.clone(), storage.clone()).await; let interval = config .interval @@ -105,7 +92,7 @@ impl Data { pub async fn start( config: DataConfig, sender: Sender, - storage: Option>, + storage: Arc, els: Logger, ) -> Result>, DataError> { let di = DataInternal::new(config, sender, storage, els).await?; diff --git a/runtime/plaid/src/executor/mod.rs b/runtime/plaid/src/executor/mod.rs index ca34e64e..5b1bfef5 100644 --- a/runtime/plaid/src/executor/mod.rs +++ b/runtime/plaid/src/executor/mod.rs @@ -1,3 +1,5 @@ +pub mod thread_pools; + use crate::apis::Api; use crate::functions::{ @@ -8,7 +10,8 @@ use crate::logging::{Logger, LoggingError}; use crate::performance::ModulePerformanceMetadata; use crate::storage::Storage; -use crossbeam_channel::{Receiver, Sender}; +use crossbeam_channel::{Receiver, Sender, TrySendError}; +use thread_pools::ExecutionThreadPools; use tokio::sync::oneshot::Sender as OneShotSender; use plaid_stl::messages::{LogSource, LogbacksAllowed}; @@ -87,7 +90,6 @@ impl Message { data: Vec, source: LogSource, logbacks_allowed: LogbacksAllowed, - headers: HashMap>, query_params: HashMap>, response_sender: Option>>, module: Option>, @@ -96,7 +98,7 @@ impl Message { id: uuid::Uuid::new_v4().to_string(), type_, data, - headers, + headers: HashMap::new(), query_params, source, logbacks_allowed, @@ -146,6 +148,7 @@ pub struct Env { /// The executor that processes messages pub struct Executor { _handles: Vec>>, + thread_pools: ExecutionThreadPools, } /// Errors encountered by the executor while trying to execute a module @@ -409,24 +412,6 @@ fn process_message_with_module( els: Logger, performance_mode: Option>, ) -> Result<(), ExecutorError> { - // For every module that operates on that log type - // Mark this rule as currently being processed by locking the mutex - // This lock will be dropped at the end of the iteration so we don't - // need to handle unlocking it - let _lock = match module.concurrency_unsafe { - Some(ref mutex) => match mutex.lock() { - Ok(guard) => Some(guard), - Err(p_err) => { - error!( - "Lock was poisoned on [{}]. Clearing and continuing: {p_err}.", - module.name - ); - mutex.clear_poison(); - mutex.lock().ok() - } - }, - None => None, - }; // TODO @obelisk: This will quietly swallow locking errors on the persistent response // This will eventually be caught if something tries to update the response but I don't // know if that's good enough. @@ -606,18 +591,19 @@ fn determine_error( impl Executor { pub fn new( - receiver: Receiver, + thread_pools: ExecutionThreadPools, modules: HashMap>>, api: Arc, storage: Option>, - execution_threads: u8, els: Logger, performance_monitoring_mode: Option>, ) -> Self { let mut _handles = vec![]; - for i in 0..execution_threads { - info!("Starting Execution Thread {i}"); - let receiver = receiver.clone(); + + // General processing + for i in 0..thread_pools.general_pool.num_threads { + info!("Starting Execution Thread {i} Dedicated to General Processing"); + let receiver = thread_pools.general_pool.receiver.clone(); let api = api.clone(); let storage = storage.clone(); let modules = modules.clone(); @@ -628,6 +614,44 @@ impl Executor { })); } - Self { _handles } + // Dedicated processing + for (log_type, thread_pool) in &thread_pools.dedicated_pools { + for i in 0..thread_pool.num_threads { + info!("Starting Execution Thread {i} Dedicated to {log_type}"); + let receiver = thread_pool.receiver.clone(); + let api = api.clone(); + let storage = storage.clone(); + let modules = modules.clone(); + let els = els.clone(); + let performance_sender = performance_monitoring_mode.clone(); + _handles.push(thread::spawn(move || { + execution_loop(receiver, modules, api, storage, els, performance_sender) + })); + } + } + Self { + _handles, + thread_pools, + } + } + + /// Execute a message coming from a webhook, by sending it to the appropriate thread pool. + /// That will be the thread pool dedicated to the message's type, if one exists, or the + /// default thread pool for general execution. + pub fn execute_webhook_message( + self: &Self, + message: Message, + ) -> Result<(), TrySendError> { + let sender = match self.thread_pools.dedicated_pools.get(&message.type_) { + Some(tp) => { + // We have a dedicated thread pool for this type: send the log to that channel + &tp.sender + } + None => { + // We have no dedicated thread pool for this type: just send to the general one. + &self.thread_pools.general_pool.sender + } + }; + sender.try_send(message) } } diff --git a/runtime/plaid/src/executor/thread_pools.rs b/runtime/plaid/src/executor/thread_pools.rs new file mode 100644 index 00000000..ffbe6b19 --- /dev/null +++ b/runtime/plaid/src/executor/thread_pools.rs @@ -0,0 +1,64 @@ +use std::collections::HashMap; + +use crossbeam_channel::{bounded, Receiver, Sender}; + +use crate::config::ExecutorConfig; + +use super::Message; + +/// A pool of threads to process logs +#[derive(Clone)] +pub struct ThreadPool { + pub num_threads: u8, + pub sender: Sender, + pub receiver: Receiver, +} + +impl ThreadPool { + /// Create a new thread pool with the given number of threads, operating + /// on a channel with the given size limit. + pub fn new(num_threads: u8, queue_size: usize) -> Self { + let (sender, receiver) = bounded(queue_size); + ThreadPool { + num_threads, + sender, + receiver, + } + } +} + +/// A struct that keeps track of all Plaid's thread pools +#[derive(Clone)] +pub struct ExecutionThreadPools { + /// Thread pool for general processing, i.e., for processing logs + /// which do not have a dedicated thread pool. + pub general_pool: ThreadPool, + /// Thread pools dedicated to specific log types. + /// Mapping { log_type --> thread_pool } + pub dedicated_pools: HashMap, +} + +impl ExecutionThreadPools { + /// Create a new ExecutionThreadPools object by initializing only the thread + /// pool for general processing. Other thread pools, if present, must be + /// added separately by inserting into the `dedicated_pools` map. + pub fn new(executor_config: &ExecutorConfig) -> Self { + // If we are dedicating threads to specific log types, create their channels and add them to the map + let dedicated_pools: HashMap = executor_config + .dedicated_threads + .iter() + .map(|(logtype, config)| { + let tp = ThreadPool::new(config.num_threads, config.log_queue_size); + (logtype.clone(), tp) + }) + .collect(); + + ExecutionThreadPools { + general_pool: ThreadPool::new( + executor_config.execution_threads, + executor_config.log_queue_size, + ), + dedicated_pools, + } + } +} diff --git a/runtime/plaid/src/functions/api.rs b/runtime/plaid/src/functions/api.rs index b1b699f3..77d25c15 100644 --- a/runtime/plaid/src/functions/api.rs +++ b/runtime/plaid/src/functions/api.rs @@ -322,6 +322,11 @@ impl_new_function_with_error_buffer!(github, add_users_to_org_copilot, DISALLOW_ impl_new_function_with_error_buffer!(github, remove_users_from_org_copilot, DISALLOW_IN_TEST_MODE); impl_new_function_with_error_buffer!(github, get_custom_properties_values, ALLOW_IN_TEST_MODE); impl_new_function!(github, comment_on_pull_request, DISALLOW_IN_TEST_MODE); +impl_new_function!( + github, + pull_request_request_reviewers, + DISALLOW_IN_TEST_MODE +); // GitHub Functions only available with GitHub App authentication impl_new_function!(github, review_fpat_requests_for_org, DISALLOW_IN_TEST_MODE); @@ -375,6 +380,8 @@ impl_new_function!(slack, post_message, ALLOW_IN_TEST_MODE); impl_new_function_with_error_buffer!(slack, get_id_from_email, ALLOW_IN_TEST_MODE); impl_new_function!(slack, post_to_arbitrary_webhook, ALLOW_IN_TEST_MODE); impl_new_function!(slack, post_to_named_webhook, ALLOW_IN_TEST_MODE); +impl_new_function_with_error_buffer!(slack, get_presence, ALLOW_IN_TEST_MODE); +impl_new_function_with_error_buffer!(slack, user_info, ALLOW_IN_TEST_MODE); // Splunk Functions impl_new_function!(splunk, post_hec, ALLOW_IN_TEST_MODE); @@ -598,6 +605,9 @@ pub fn to_api_function( "github_delete_deploy_key" => { Function::new_typed_with_env(&mut store, &env, github_delete_deploy_key) } + "github_pull_request_request_reviewers" => { + Function::new_typed_with_env(&mut store, &env, github_pull_request_request_reviewers) + } // Slack Calls "slack_post_to_named_webhook" => { @@ -611,6 +621,8 @@ pub fn to_api_function( "slack_get_id_from_email" => { Function::new_typed_with_env(&mut store, &env, slack_get_id_from_email) } + "slack_get_presence" => Function::new_typed_with_env(&mut store, &env, slack_get_presence), + "slack_user_info" => Function::new_typed_with_env(&mut store, &env, slack_user_info), // General Calls "general_simple_json_post_request" => { diff --git a/runtime/plaid/src/functions/cache.rs b/runtime/plaid/src/functions/cache.rs index aa4baa2a..48488217 100644 --- a/runtime/plaid/src/functions/cache.rs +++ b/runtime/plaid/src/functions/cache.rs @@ -4,7 +4,6 @@ use crate::{executor::Env, functions::FunctionErrors}; use super::{get_memory, safely_get_string, safely_write_data_back}; - /// Store data in the cache system if one is configured pub fn insert( env: FunctionEnvMut, @@ -27,7 +26,10 @@ pub fn insert( let memory_view = match get_memory(&env, &store) { Ok(memory_view) => memory_view, Err(e) => { - error!("{}: Memory error in cache_insert: {:?}", env_data.module.name, e); + error!( + "{}: Memory error in cache_insert: {:?}", + env_data.module.name, e + ); return FunctionErrors::CouldNotGetAdequateMemory as i32; } }; @@ -35,7 +37,10 @@ pub fn insert( let key = match safely_get_string(&memory_view, key_buf, key_buf_len) { Ok(s) => s, Err(e) => { - error!("{}: Key error in cache_insert: {:?}", env_data.module.name, e); + error!( + "{}: Key error in cache_insert: {:?}", + env_data.module.name, e + ); return FunctionErrors::ParametersNotUtf8 as i32; } }; @@ -44,7 +49,10 @@ pub fn insert( let value = match safely_get_string(&memory_view, value_buf, value_buf_len) { Ok(d) => d, Err(e) => { - error!("{}: Value error in cache_insert: {:?}", env_data.module.name, e); + error!( + "{}: Value error in cache_insert: {:?}", + env_data.module.name, e + ); return FunctionErrors::CouldNotGetAdequateMemory as i32; } }; @@ -100,7 +108,10 @@ pub fn get( let memory_view = match get_memory(&env, &store) { Ok(memory_view) => memory_view, Err(e) => { - error!("{}: Memory error in cache_get: {:?}", env_data.module.name, e); + error!( + "{}: Memory error in cache_get: {:?}", + env_data.module.name, e + ); return FunctionErrors::CouldNotGetAdequateMemory as i32; } }; @@ -123,7 +134,10 @@ pub fn get( ) { Ok(x) => x, Err(e) => { - error!("{}: Data write error in cache_get: {:?}", env_data.module.name, e); + error!( + "{}: Data write error in cache_get: {:?}", + env_data.module.name, e + ); e as i32 } } diff --git a/runtime/plaid/src/functions/internal.rs b/runtime/plaid/src/functions/internal.rs index f5a1f079..b720c618 100644 --- a/runtime/plaid/src/functions/internal.rs +++ b/runtime/plaid/src/functions/internal.rs @@ -28,7 +28,11 @@ pub fn print_debug_string(env: FunctionEnvMut, log_buffer: WasmPtr, log let message = match safely_get_string(&memory_view, log_buffer, log_buffer_size) { Ok(s) => s, Err(e) => { - error!("{}: Error in print_debug_string: {:?}", env.data().module.name, e); + error!( + "{}: Error in print_debug_string: {:?}", + env.data().module.name, + e + ); return; } }; @@ -59,7 +63,11 @@ pub fn set_error_context( let message = match safely_get_string(&memory_view, context_buffer, context_buffer_size) { Ok(s) => s, Err(e) => { - error!("{}: Error in set_error_context: {:?}", env.data().module.name, e); + error!( + "{}: Error in set_error_context: {:?}", + env.data().module.name, + e + ); return; } }; @@ -178,7 +186,10 @@ pub fn log_back_detailed( let memory_view = match get_memory(&env, &store) { Ok(memory_view) => memory_view, Err(e) => { - error!("{}: Memory error in log_back: {:?}", env_data.module.name, e); + error!( + "{}: Memory error in log_back: {:?}", + env_data.module.name, e + ); return 1; } }; @@ -204,7 +215,13 @@ pub fn log_back_detailed( api.clone().runtime.block_on(async move { match api.general.as_ref() { Some(general) => { - if general.log_back(&type_, &log, &env_data.module.name, delay as u64, assigned_budget) { + if general.log_back( + &type_, + &log, + &env_data.module.name, + delay as u64, + assigned_budget, + ) { 0 } else { 1 diff --git a/runtime/plaid/src/functions/memory.rs b/runtime/plaid/src/functions/memory.rs index a6b86ef1..e953a40f 100644 --- a/runtime/plaid/src/functions/memory.rs +++ b/runtime/plaid/src/functions/memory.rs @@ -7,7 +7,10 @@ use super::FunctionErrors; /// When a host function is executing we need to be able to access the guest's memory /// for read and write operations. This safely gets those from the environment and /// handles all failure cases. -pub fn get_memory<'a>(env: &FunctionEnvMut, store: &'a StoreRef) -> Result, FunctionErrors> { +pub fn get_memory<'a>( + env: &FunctionEnvMut, + store: &'a StoreRef, +) -> Result, FunctionErrors> { // Fetch the store and memory which make up the needed components // of the execution environment. let memory = match &env.data().memory { @@ -27,19 +30,27 @@ pub fn get_memory<'a>(env: &FunctionEnvMut, store: &'a StoreRef) -> Result< /// Safely get a string from the guest's memory. This function will take a pointer provided by the /// guest, then use a built in function to read the string. -pub fn safely_get_string(memory_view: &MemoryView, data_buffer: WasmPtr, buffer_size: u32) -> Result { +pub fn safely_get_string( + memory_view: &MemoryView, + data_buffer: WasmPtr, + buffer_size: u32, +) -> Result { match data_buffer.read_utf8_string(&memory_view, buffer_size as u32) { Ok(s) => Ok(s), Err(_) => { error!("Failed to read the log message from the guest's memory"); Err(FunctionErrors::ParametersNotUtf8) - }, + } } } /// Safely get a Vec from the guest's memory. This function will take a pointer provided by the /// guest, then do a bounds checked read. -pub fn safely_get_memory(memory_view: &MemoryView, data_buffer: WasmPtr, buffer_size: u32) -> Result, FunctionErrors> { +pub fn safely_get_memory( + memory_view: &MemoryView, + data_buffer: WasmPtr, + buffer_size: u32, +) -> Result, FunctionErrors> { let mut buffer = vec![0; buffer_size as usize]; memory_view .read(data_buffer.offset().into(), &mut buffer) @@ -49,9 +60,14 @@ pub fn safely_get_memory(memory_view: &MemoryView, data_buffer: WasmPtr, buf } /// Safely write data back to the guest's memory. This function will take a pointer provided -/// to it by the guest, do some bounds checking, and then write the data back into the guest's +/// to it by the guest, do some bounds checking, and then write the data back into the guest's /// memory. It will return the number of bytes written or an error if the buffer is too small. -pub fn safely_write_data_back(memory_view: &MemoryView, data: &[u8], data_buffer: WasmPtr, buffer_size: u32) -> Result { +pub fn safely_write_data_back( + memory_view: &MemoryView, + data: &[u8], + data_buffer: WasmPtr, + buffer_size: u32, +) -> Result { if buffer_size == 0 { return Ok(data.len() as i32); } @@ -64,7 +80,6 @@ pub fn safely_write_data_back(memory_view: &MemoryView, data: &[u8], data_buffer .slice(&memory_view, data.len() as u32) .map_err(|_| FunctionErrors::CouldNotGetAdequateMemory)?; - for i in 0..data.len() { if let Err(_) = values.index(i as u64).write(data[i]) { return Err(FunctionErrors::FailedToWriteGuestMemory); diff --git a/runtime/plaid/src/functions/response.rs b/runtime/plaid/src/functions/response.rs index ef9ac323..774e73ca 100644 --- a/runtime/plaid/src/functions/response.rs +++ b/runtime/plaid/src/functions/response.rs @@ -4,7 +4,6 @@ use crate::{executor::Env, functions::FunctionErrors}; use super::{get_memory, safely_get_string, safely_write_data_back}; - /// Implement a way for a module to get the existing response. This would have been /// set by previous invocations of the module and allows an additional basic form of state. pub fn get_response( diff --git a/runtime/plaid/src/loader/limits.rs b/runtime/plaid/src/loader/limits.rs index 9b59b369..83da1027 100644 --- a/runtime/plaid/src/loader/limits.rs +++ b/runtime/plaid/src/loader/limits.rs @@ -1,13 +1,15 @@ use std::ptr::NonNull; use wasmer::{ - vm::{self, MemoryError, MemoryStyle, TableStyle, VMMemoryDefinition, VMTableDefinition}, - MemoryType, Pages, TableType, Tunables, + sys::{ + vm::{VMMemory, VMMemoryDefinition, VMTable, VMTableDefinition}, + Tunables, + }, + MemoryError, MemoryStyle, MemoryType, Pages, TableStyle, TableType, }; // This is to be able to set the tunables - /// A custom tunables that allows you to set a memory limit. /// /// After adjusting the memory limits, it delegates all other logic @@ -84,7 +86,7 @@ impl Tunables for LimitingTunables { &self, ty: &MemoryType, style: &MemoryStyle, - ) -> Result { + ) -> Result { let adjusted = self.adjust_memory(ty); self.validate_memory(&adjusted)?; self.base.create_host_memory(&adjusted, style) @@ -98,7 +100,7 @@ impl Tunables for LimitingTunables { ty: &MemoryType, style: &MemoryStyle, vm_definition_location: NonNull, - ) -> Result { + ) -> Result { let adjusted = self.adjust_memory(ty); self.validate_memory(&adjusted)?; self.base @@ -108,7 +110,7 @@ impl Tunables for LimitingTunables { /// Create a table owned by the host given a [`TableType`] and a [`TableStyle`]. /// /// Delegated to base. - fn create_host_table(&self, ty: &TableType, style: &TableStyle) -> Result { + fn create_host_table(&self, ty: &TableType, style: &TableStyle) -> Result { self.base.create_host_table(ty, style) } @@ -120,7 +122,7 @@ impl Tunables for LimitingTunables { ty: &TableType, style: &TableStyle, vm_definition_location: NonNull, - ) -> Result { + ) -> Result { self.base.create_vm_table(ty, style, vm_definition_location) } -} \ No newline at end of file +} diff --git a/runtime/plaid/src/loader/mod.rs b/runtime/plaid/src/loader/mod.rs index be442f2e..9a66988f 100644 --- a/runtime/plaid/src/loader/mod.rs +++ b/runtime/plaid/src/loader/mod.rs @@ -13,18 +13,22 @@ use std::collections::HashMap; use std::fmt::{Display, Formatter}; use std::fs::{self}; use std::num::NonZeroUsize; -use std::sync::{Arc, Mutex, RwLock}; +use std::sync::{Arc, RwLock}; use utils::{ cost_function, get_module_computation_limit, get_module_page_count, get_module_persistent_storage_limit, read_and_configure_secrets, read_and_parse_modules, }; +use wasmer::sys::{NativeEngineExt, Target}; + +use wasmer::sys::CompilerConfig; + #[cfg(feature = "llvm")] -use wasmer::LLVM; +use wasmer::sys::LLVM; #[cfg(feature = "cranelift")] -use wasmer::Cranelift; +use wasmer::sys::Cranelift; -use wasmer::{sys::BaseTunables, CompilerConfig, Engine, Module, NativeEngineExt, Pages, Target}; +use wasmer::{sys::BaseTunables, Engine, Module, Pages}; use wasmer_middlewares::Metering; use crate::storage::Storage; @@ -101,10 +105,6 @@ pub enum CompilerBackend { pub struct Configuration { /// Where to load modules from pub module_dir: String, - /// A list of case-insensitive rule names that **cannot** be executed in parallel. We assume that rules can be executed - /// in parallel unless otherwise noted. Rules that cannot execute in parallel wait until the - /// executor is finished processing a rule before beginning their own execution. - pub single_threaded_rules: Option>, /// What the log type of a module should be if it's not the first part of the filename pub log_type_overrides: HashMap, /// How much computation a module is allowed to do @@ -267,12 +267,6 @@ pub struct PlaidModule { pub cache: Option>>>, /// See the PersistentResponse type. pub persistent_response: Option, - /// Indicates whether the module is safe for concurrent execution. - /// - /// - If `None`, the module can be executed concurrently without any restrictions. - /// - If `Some`, the module is marked as unsafe for concurrent execution, and the `Mutex<()>` - /// is used to ensure mutual exclusion, preventing multiple threads from executing it simultaneously. - pub concurrency_unsafe: Option>, /// If the module is in test mode, meaning it should not be allowed to cause side effects pub test_mode: bool, } @@ -306,7 +300,6 @@ impl PlaidModule { storage: Option>, module_bytes: Vec, log_type: &str, - concurrency_unsafe: Option>, test_mode: bool, compiler_backend: &CompilerBackend, ) -> Result { @@ -360,7 +353,7 @@ impl PlaidModule { }; let storage_current = Arc::new(RwLock::new(storage_current_bytes)); - info!("Name: [{filename}] Computation Limit: [{computation_limit}] Memory Limit: [{page_limit} pages] Storage: [{storage_current_bytes}/{storage_limit} bytes used] Log Type: [{log_type}]. Concurrency Safe: [{}] Test Mode: [{test_mode}]", concurrency_unsafe.is_none()); + info!("Name: [{filename}] Computation Limit: [{computation_limit}] Memory Limit: [{page_limit} pages] Storage: [{storage_current_bytes}/{storage_limit} bytes used] Log Type: [{log_type}]. Test Mode: [{test_mode}]"); for import in module.imports() { info!("\tImport: {}", import.name()); } @@ -377,7 +370,6 @@ impl PlaidModule { secrets: None, cache: None, persistent_response: None, - concurrency_unsafe, test_mode, }) } @@ -468,18 +460,6 @@ pub async fn load( type_[0].to_string() }; - // Check if this rule can be executed in parallel - let concurrency_safe = config.single_threaded_rules.as_ref().map_or(None, |rules| { - if rules - .iter() - .any(|rule| rule.eq_ignore_ascii_case(&filename)) - { - Some(Mutex::new(())) - } else { - None - } - }); - // Default is the global test mode. Then if the module is in the exemptions specification // we will disable test mode for that module. let test_mode = config.test_mode && !config.test_mode_exemptions.contains(&filename); @@ -493,7 +473,6 @@ pub async fn load( storage.clone(), module_bytes, &type_, - concurrency_safe, test_mode, &config.compiler_backend, ) diff --git a/runtime/plaid/src/loader/utils.rs b/runtime/plaid/src/loader/utils.rs index c6d7bb94..e91a0fc0 100644 --- a/runtime/plaid/src/loader/utils.rs +++ b/runtime/plaid/src/loader/utils.rs @@ -1,9 +1,10 @@ +use wasmer::sys::wasmparser::Operator; + use super::errors::Errors; use super::{LimitValue, LimitableAmount, LimitedAmount}; use std::collections::HashMap; use std::fs::DirEntry; -use wasmer::wasmparser::Operator; const CALL_COST: u64 = 10; diff --git a/runtime/plaid/src/logging/webhook.rs b/runtime/plaid/src/logging/webhook.rs index 980b98cf..72bc9d39 100644 --- a/runtime/plaid/src/logging/webhook.rs +++ b/runtime/plaid/src/logging/webhook.rs @@ -26,7 +26,6 @@ pub struct WebhookLogger { config: Config, } - impl WebhookLogger { /// Implement the new function for the Splunk logger. This converts /// the configuration struct into a type that can handle sending @@ -35,7 +34,8 @@ impl WebhookLogger { // I don't think this can fail with our settings so we do an unwrap let client = reqwest::Client::builder() .timeout(Duration::from_secs(config.timeout.into())) - .build().unwrap(); + .build() + .unwrap(); Self { runtime: handle, @@ -53,14 +53,16 @@ impl PlaidLogger for WebhookLogger { fn send_log(&self, log: &WrappedLog) -> Result<(), LoggingError> { let data = match serde_json::to_string(&log) { Ok(json) => json, - Err(e) => return Err(LoggingError::SerializationError(e.to_string())) + Err(e) => return Err(LoggingError::SerializationError(e.to_string())), }; - let res = self.client.post(&self.config.url) + let res = self + .client + .post(&self.config.url) .header("Content-Type", "application/x-www-form-urlencoded") .header("Content-Length", data.len()) .body(data); - + let res = if let Some(auth) = &self.config.auth_header { res.header("Authorization", auth) } else { diff --git a/runtime/plaid/src/storage/dynamodb/mod.rs b/runtime/plaid/src/storage/dynamodb/mod.rs index 8773c93b..6acaa1e8 100644 --- a/runtime/plaid/src/storage/dynamodb/mod.rs +++ b/runtime/plaid/src/storage/dynamodb/mod.rs @@ -44,6 +44,10 @@ impl DynamoDb { #[async_trait] impl StorageProvider for DynamoDb { + fn is_persistent(&self) -> bool { + true + } + async fn insert( &self, namespace: String, diff --git a/runtime/plaid/src/storage/in_memory/mod.rs b/runtime/plaid/src/storage/in_memory/mod.rs new file mode 100644 index 00000000..8ac56471 --- /dev/null +++ b/runtime/plaid/src/storage/in_memory/mod.rs @@ -0,0 +1,118 @@ +//! This module provides a way for Plaid to use an in-memory store as a DB. Note - This storage is not persisted across reboots. + +use super::{StorageError, StorageProvider}; +use async_trait::async_trait; +use std::{collections::HashMap, sync::Arc}; +use tokio::sync::RwLock; + +pub struct InMemoryDb { + db: Arc>>>>, +} + +impl InMemoryDb { + pub fn new() -> Result { + Ok(Self { + db: Arc::new(RwLock::new(HashMap::new())), + }) + } +} + +#[async_trait] +impl StorageProvider for InMemoryDb { + fn is_persistent(&self) -> bool { + false + } + + async fn insert( + &self, + namespace: String, + key: String, + value: Vec, + ) -> Result>, StorageError> { + let mut db = self.db.write().await; + let ns = db.entry(namespace).or_default(); + Ok(ns.insert(key, value)) + } + + async fn get(&self, namespace: &str, key: &str) -> Result>, StorageError> { + let db = self.db.read().await; + Ok(db.get(namespace).and_then(|ns| ns.get(key).cloned())) + } + + async fn delete(&self, namespace: &str, key: &str) -> Result>, StorageError> { + let mut db = self.db.write().await; + if let Some(ns) = db.get_mut(namespace) { + Ok(ns.remove(key)) + } else { + Ok(None) + } + } + + async fn list_keys( + &self, + namespace: &str, + prefix: Option<&str>, + ) -> Result, StorageError> { + let keys = self + .db + .read() + .await + .get(namespace) + .map(|ns| { + ns.keys() + .filter(|k| prefix.map_or(true, |p| k.starts_with(p))) + .cloned() + .collect() + }) + .unwrap_or_default(); + Ok(keys) + } + + async fn fetch_all( + &self, + namespace: &str, + prefix: Option<&str>, + ) -> Result)>, StorageError> { + let db = self.db.read().await; + let values = db + .get(namespace) + .map(|ns| { + ns.iter() + .filter(|(k, _)| prefix.map_or(true, |p| k.starts_with(p))) + .map(|(k, v)| (k.clone(), v.clone())) + .collect() + }) + .unwrap_or_default(); + Ok(values) + } + + async fn get_namespace_byte_size(&self, namespace: &str) -> Result { + let all = self.fetch_all(namespace, None).await?; + let mut counter = 0u64; + for item in all { + // Count bytes for keys and values + counter += item.0.as_bytes().len() as u64 + item.1.len() as u64; + } + Ok(counter) + } + + async fn apply_migration( + &self, + namespace: &str, + f: Box) -> (String, Vec) + Send + Sync>, + ) -> Result<(), StorageError> { + // Get all the data for this namespace + let data = self.fetch_all(namespace, None).await?; + // For each key/value pair, perform the migration... + for (key, value) in data { + // Apply the transformation and obtain a new key and new value + let (new_key, new_value) = f(key.clone(), value); + // Delete the old entry because we are about to insert the new one + self.delete(namespace, &key).await?; + // And insert the new pair + self.insert(namespace.to_string(), new_key, new_value) + .await?; + } + Ok(()) + } +} diff --git a/runtime/plaid/src/storage/mod.rs b/runtime/plaid/src/storage/mod.rs index 5ef5ca5a..54d7fdbc 100644 --- a/runtime/plaid/src/storage/mod.rs +++ b/runtime/plaid/src/storage/mod.rs @@ -11,7 +11,10 @@ pub mod dynamodb; #[cfg(feature = "sled")] pub mod sled; +pub mod in_memory; + use futures_util::future::join_all; +use in_memory::InMemoryDb; use serde::Deserialize; use crate::loader::LimitValue; @@ -93,6 +96,9 @@ impl std::error::Error for StorageError {} /// Defines the basic methods that all storage providers must offer. #[async_trait] pub trait StorageProvider { + /// Return whether this storage provider is backed by persistent storage. + /// If not, it means the data only lives in memory and is lost in case of a reboot. + fn is_persistent(&self) -> bool; /// Insert a new key pair into the storage provider async fn insert( &self, @@ -137,6 +143,13 @@ pub trait StorageProvider { } impl Storage { + pub fn new_in_memory() -> Self { + Self { + database: Box::new(InMemoryDb::new().unwrap()), + shared_dbs: None, + } + } + pub async fn new(config: Config) -> Result { // Try building a database from the values in the config let database: Box = match config.db { diff --git a/runtime/plaid/src/storage/sled/mod.rs b/runtime/plaid/src/storage/sled/mod.rs index 24b8cd05..ccca6375 100644 --- a/runtime/plaid/src/storage/sled/mod.rs +++ b/runtime/plaid/src/storage/sled/mod.rs @@ -29,6 +29,10 @@ impl Sled { #[async_trait] impl StorageProvider for Sled { + fn is_persistent(&self) -> bool { + true + } + async fn insert( &self, namespace: String, diff --git a/testing/integration.sh b/testing/integration.sh index 6b75e39b..3d7fa554 100755 --- a/testing/integration.sh +++ b/testing/integration.sh @@ -1,8 +1,14 @@ #!/bin/bash -# Build all of the Plaid workspace -PLATFORM=$(uname -a) -CONFIG_PATH="plaid/resources/plaid.toml" +# Set up all the variables we need to run the integration tests +CONFIG_PATH="runtime/plaid/resources/config" +CONFIG_WORKING_PATH="/tmp/plaid_config/configs" + +SECRET_PATH="runtime/plaid/resources/secrets.example.toml" +SECRET_WORKING_PATH="/tmp/plaid_config/secrets.example.toml" + +export REQUEST_HANDLER=$(pwd)/runtime/target/release/request_handler + # Compiler should be passed in as the first argument if [ -z "$1" ]; then @@ -11,6 +17,13 @@ if [ -z "$1" ]; then fi echo "Testing runtime with compiler: $1" +# Set up the working directory +rm -rf $CONFIG_WORKING_PATH +mkdir -p $CONFIG_WORKING_PATH + +# Copy the configuration and secrets to the tmp directory +cp -r $CONFIG_PATH/* $CONFIG_WORKING_PATH +cp $SECRET_PATH $SECRET_WORKING_PATH # On macOS, we need to install a brew provided version of LLVM # so that we can compile WASM binaries. @@ -19,8 +32,6 @@ if uname | grep -q Darwin; then PATH="/opt/homebrew/opt/llvm/bin:$PATH" fi -export REQUEST_HANDLER=$(pwd)/runtime/target/release/request_handler - echo "Building All Plaid Modules" cd modules cargo build --all --release @@ -34,14 +45,69 @@ cp -r modules/target/wasm32-unknown-unknown/release/test_*.wasm compiled_modules ssh-keygen -t ed25519 -f plaidrules_key_ed25519 -N "" public_key=$(cat plaidrules_key_ed25519.pub | awk '{printf "%s %s %s", $1, $2, $3}') +# Generate a self-signed cert to test MNRs with +openssl genrsa -out ca.key 4096 + +# Generate a self-signed CA cert with CA:TRUE +openssl req -x509 -new -nodes \ + -key ca.key \ + -days 1 \ + -subj "/CN=My Test CA" \ + -addext "basicConstraints = CA:TRUE,pathlen:1" \ + -out ca.pem + +# Generate a server key + CSR +openssl genrsa -out server.key 4096 +openssl req -new -key server.key \ + -subj "/CN=localhost" \ + -out server.csr + +# Create extfile for leaf cert +cat > san.cnf <