Implement appsec jwt (#3352)

Author: estringana

appsec/src/helper/action.hpp

@@ -28,7 +28,7 @@ struct action {
 
 struct event {
     bool keep = false;
-    std::vector<std::string> data;
+    std::vector<std::string> triggers; // json fragments
     std::vector<action> actions;
 };
 } // namespace dds

appsec/src/helper/client.cpp

@@ -276,7 +276,7 @@ std::shared_ptr<typename T::response> client::publish(
                 extra_record_action.parameters = {};
                 response->actions.push_back(extra_record_action);
             }
-            response->triggers = std::move(res->events);
+            response->triggers = std::move(res->triggers);
             response->force_keep = res->force_keep;
 
             DD_STDLOG(DD_STDLOG_ATTACK_DETECTED);

appsec/src/helper/engine.cpp

@@ -97,7 +97,7 @@ std::optional<engine::result> engine::context::publish(
         DD_STDLOG(DD_STDLOG_IG_DATA_PUSHED, entry.key());
     }
 
-    event event_;
+    event event;
 
     auto common =
         std::atomic_load_explicit(&common_, std::memory_order_acquire);
@@ -113,25 +113,30 @@ std::optional<engine::result> engine::context::publish(
         }
         try {
             const auto &listener = it->second;
-            listener->call(data, event_, rasp_rule);
+            listener->call(data, event, rasp_rule);
         } catch (std::exception &e) {
             SPDLOG_ERROR("subscriber failed: {}", e.what());
         }
     }
 
-    if (event_.actions.empty() && event_.data.empty()) {
+    // force_keep indicates to the extension that it should change the
+    // priority of the span. We set it to true if libddwaf tells us to AND
+    // if the limiter allows it.
+    // XXX: the limiter should probably only be invoked once per request
+    const bool force_keep = event.keep && limiter_.allow();
+
+    if (!force_keep && event.actions.empty() && event.triggers.empty()) {
         return std::nullopt;
     }
 
-    const bool force_keep = event_.keep && limiter_.allow();
-    dds::engine::result res{{}, std::move(event_.data), force_keep};
-    // Currently the only action the extension can perform is block
-    if (event_.actions.empty()) {
+    // no actions, but we have json fragments in triggers. Add a record action
+    if (event.actions.empty()) {
         action record = {dds::action_type::record, {}};
-        res.actions.emplace_back(std::move(record));
+        event.actions.emplace_back(std::move(record));
     }
 
-    for (auto const &action : event_.actions) {
+    dds::engine::result res{{}, std::move(event.triggers), force_keep};
+    for (auto const &action : event.actions) {
         dds::action new_action;
         new_action.type = action.type;
         new_action.parameters.insert(

appsec/src/helper/engine.hpp

@@ -38,7 +38,7 @@ class engine {
 
     struct result {
         std::vector<dds::action> actions;
-        std::vector<std::string> events;
+        std::vector<std::string> triggers; // json fragments
         bool force_keep;
     };
 

appsec/src/helper/json_helper.cpp

@@ -132,9 +132,11 @@ void json_to_object(ddwaf_object *object, T &doc)
     }
     case rapidjson::kNumberType: {
         if (doc.IsInt64()) {
-            ddwaf_object_string_from_signed(object, doc.GetInt64());
+            ddwaf_object_signed(object, doc.GetInt64());
         } else if (doc.IsUint64()) {
-            ddwaf_object_string_from_unsigned(object, doc.GetUint64());
+            ddwaf_object_unsigned(object, doc.GetUint64());
+        } else if (doc.IsDouble()) {
+            ddwaf_object_float(object, doc.GetDouble());
         }
         break;
     }

appsec/src/helper/parameter_base.hpp

@@ -89,6 +89,10 @@ class parameter_base : public ddwaf_object {
     {
         return ddwaf_object::type != DDWAF_OBJ_INVALID;
     }
+    [[nodiscard]] bool is_float() const noexcept
+    {
+        return ddwaf_object::type == DDWAF_OBJ_FLOAT;
+    }
     // NOLINTNEXTLINE
     operator ddwaf_object *() noexcept { return this; }
 
@@ -120,7 +124,13 @@ class parameter_base : public ddwaf_object {
         }
         return intValue;
     }
-
+    explicit operator double() const
+    {
+        if (!is_float()) {
+            throw bad_cast("parameter not a float");
+        }
+        return f64;
+    }
     explicit operator bool() const
     {
         if (!is_boolean()) {

appsec/src/helper/subscriber/waf.cpp

@@ -206,7 +206,7 @@ void format_waf_result(
         if (events != nullptr) {
             const parameter_view events_pv{*events};
             for (const auto &event_pv : events_pv) {
-                event.data.emplace_back(parameter_to_json(event_pv));
+                event.triggers.emplace_back(parameter_to_json(event_pv));
             }
         }
     } catch (const std::exception &e) {
@@ -563,19 +563,20 @@ void instance::listener::call(
         // This converts the events to JSON which is already done in the
         // switch below so it's slightly inefficient, albeit since it's only
         // done on debug, we can live with it...
+        std::string events_json;
         if (events != nullptr) {
-            DD_STDLOG(DD_STDLOG_AFTER_WAF,
-                parameter_to_json(parameter_view{*events}), duration / millis);
+            events_json = parameter_to_json(parameter_view{*events});
+            DD_STDLOG(DD_STDLOG_AFTER_WAF, events_json, duration / millis);
         }
-        SPDLOG_DEBUG(
-            "Waf response: code {} - actions {} - attributes {} - keep {}",
-            fmt::underlying(code),
+        SPDLOG_DEBUG("Waf response: code {} - keep {} - duration {} - timeout "
+                     "{} - actions {} - attributes {} - events {}",
+            fmt::underlying(code), event.keep, duration / millis, timeout,
             actions != nullptr ? parameter_to_json(parameter_view{*actions})
                                : "",
             attributes != nullptr
                 ? parameter_to_json(parameter_view{*attributes})
                 : "",
-            event.keep);
+            events_json);
     } else {
         run_waf();
     }
@@ -618,12 +619,40 @@ void instance::listener::call(
     }
     if (attributes != nullptr) {
         const parameter_view attributes_pv{*attributes};
-        for (const auto &derivative : attributes_pv) {
+        for (const parameter_view &derivative : attributes_pv) {
             if (derivative.key().starts_with("_dd.appsec.s.")) {
-                attributes_.emplace(
-                    derivative.key(), parameter_to_json(derivative));
+                std::string json_derivative = parameter_to_json(derivative);
+                if (json_derivative.length() > max_plain_schema_allowed) {
+                    auto encoded = compress(json_derivative);
+                    if (encoded) {
+                        json_derivative = base64_encode(encoded.value(), false);
+                    }
+                }
+                if (json_derivative.length() <= max_schema_size) {
+                    meta_attributes_.emplace(
+                        derivative.key(), std::move(json_derivative));
+                } else {
+                    SPDLOG_WARN("Schema for key {} is too large to submit",
+                        derivative.key());
+                }
             } else {
-                attributes_.emplace(derivative.key(), derivative);
+                if (derivative.is_signed()) {
+                    auto value =
+                        static_cast<double>(static_cast<int64_t>(derivative));
+                    metrics_attributes_.emplace(derivative.key(), value);
+                } else if (derivative.is_unsigned()) {
+                    auto value =
+                        static_cast<double>(static_cast<uint64_t>(derivative));
+                    metrics_attributes_.emplace(derivative.key(), value);
+                } else if (derivative.is_float()) {
+                    auto value = static_cast<double>(derivative);
+                    metrics_attributes_.emplace(derivative.key(), value);
+                } else if (derivative.is_string()) {
+                    meta_attributes_.emplace(derivative.key(), derivative);
+                } else {
+                    SPDLOG_WARN("Unsupported attribute type for key {}",
+                        derivative.key());
+                }
             }
         }
     }
@@ -696,21 +725,13 @@ void instance::listener::submit_metrics(
         }
     }
 
-    for (const auto &[key, value] : attributes_) {
-        std::string derivative = value;
-        if (value.length() > max_plain_schema_allowed &&
-            key.starts_with("_dd.appsec.s.")) {
-            auto encoded = compress(derivative);
-            if (encoded) {
-                derivative = base64_encode(encoded.value(), false);
-            }
-        }
+    for (const auto &[key, value] : meta_attributes_) {
+        msubmitter.submit_span_meta_copy_key(
+            std::string{key}, std::string(value));
+    }
 
-        if (derivative.length() <= max_schema_size) {
-            msubmitter.submit_span_meta_copy_key(key, std::move(derivative));
-        } else {
-            SPDLOG_WARN("Schema for key {} is too large to submit", key);
-        }
+    for (const auto &[key, value] : metrics_attributes_) {
+        msubmitter.submit_span_metric(key, value);
     }
 }
 

appsec/src/helper/subscriber/waf.hpp

@@ -71,7 +71,8 @@ class instance : public dds::subscriber {
         unsigned rasp_calls_{0};
         unsigned rasp_timeouts_{0};
         DDWAF_RET_CODE code_{DDWAF_OK};
-        std::map<std::string, std::string> attributes_;
+        std::map<std::string, std::string> meta_attributes_;
+        std::map<std::string, double> metrics_attributes_;
         telemetry::telemetry_tags base_tags_;
         bool rule_triggered_{};
         bool request_blocked_{};

appsec/tests/helper/engine_test.cpp

@@ -18,7 +18,7 @@
 using dds::remote_config::changeset;
 
 const std::string waf_rule =
-    R"({"version":"2.1","rules":[{"id":"1","name":"rule1","tags":{"type":"flow1","category":"category1"},"conditions":[{"operator":"match_regex","parameters":{"inputs":[{"address":"arg1","key_path":[]}],"regex":"^string.*"}},{"operator":"match_regex","parameters":{"inputs":[{"address":"arg2","key_path":[]}],"regex":".*"}}]},{"id":"2","name":"rule2","tags":{"type":"flow2","category":"category2"},"conditions":[{"operator":"match_regex","parameters":{"inputs":[{"address":"arg3","key_path":[]}],"regex":"^string.*"}}]}]})";
+    R"({"version": "2.1", "rules": [{"id": "1", "name": "rule1", "tags": {"type": "flow1", "category": "category1" }, "conditions": [{"operator": "match_regex", "parameters": {"inputs": [{"address": "arg1", "key_path": [] } ], "regex": "^string.*" } }, {"operator": "match_regex", "parameters": {"inputs": [{"address": "arg2", "key_path": [] } ], "regex": ".*" } } ] }, {"id": "2", "name": "rule2", "tags": {"type": "flow2", "category": "category2" }, "conditions": [{"operator": "match_regex", "parameters": {"inputs": [{"address": "arg3", "key_path": [] } ], "regex": "^string.*" } } ] } ], "rules_compat": [{"id": "ttr-000-001", "name": "Trace Tagging Rule: Attributes, Keep, No Event", "tags": {"type": "security_scanner", "category": "attack_attempt" }, "conditions": [{"operator": "match_regex", "parameters": {"inputs": [{"address": "arg4", "key_path": [] } ], "regex": "^string.*" } } ], "output": {"event": false, "keep": true, "attributes": {"_dd.appsec.trace.integer": {"value": 12345 }, "_dd.appsec.trace.string": {"value": "678" }, "_dd.appsec.trace.agent": {"address": "server.request.headers.no_cookies", "key_path": ["user-agent" ] } } }, "on_match": [] }, {"id": "ttr-000-002", "name": "Trace Tagging Rule: Attributes, No Keep, No Event", "tags": {"type": "security_scanner", "category": "attack_attempt" }, "conditions": [{"operator": "match_regex", "parameters": {"inputs": [{"address": "arg5", "key_path": [] } ], "regex": "^string.*" } } ], "output": {"event": false, "keep": false, "attributes": {"_dd.appsec.trace.integer": {"value": 12345 }, "_dd.appsec.trace.string": {"value": "678" }, "_dd.appsec.trace.agent": {"address": "server.request.headers.no_cookies", "key_path": ["user-agent" ] } } }, "on_match": [] } ] })";
 const std::string waf_rule_with_data =
     R"({"version":"2.1","rules":[{"id":"blk-001-001","name":"Block IP Addresses","tags":{"type":"block_ip","category":"security_response"},"conditions":[{"parameters":{"inputs":[{"address":"http.client_ip"}],"data":"blocked_ips"},"operator":"ip_match"}],"transformers":[],"on_match":["block"]}]})";
 
@@ -104,7 +104,7 @@ TEST(EngineTest, MultipleSubscriptors)
                                    std::string rasp) -> void {
             std::unordered_set<std::string_view> subs{"a", "b", "e", "f"};
             if (subs.find(data[0].parameterName) != subs.end()) {
-                event_.data.push_back("some event");
+                event_.triggers.push_back("some event");
                 event_.actions.push_back({dds::action_type::block, {}});
             }
         }));
@@ -115,7 +115,7 @@ TEST(EngineTest, MultipleSubscriptors)
                                    std::string rasp) -> void {
             std::unordered_set<std::string_view> subs{"c", "d", "e", "g"};
             if (subs.find(data[0].parameterName) != subs.end()) {
-                event_.data.push_back("some event");
+                event_.triggers.push_back("some event");
             }
         }));
 
@@ -382,8 +382,8 @@ TEST(EngineTest, WafSubscriptorBasic)
     Mock::VerifyAndClearExpectations(&msubmitter);
     EXPECT_TRUE(res);
     EXPECT_EQ(res->actions[0].type, dds::action_type::record);
-    EXPECT_EQ(res->events.size(), 1);
-    for (auto &match : res->events) {
+    EXPECT_EQ(res->triggers.size(), 1);
+    for (auto &match : res->triggers) {
         rapidjson::Document doc;
         doc.Parse(match);
         EXPECT_FALSE(doc.HasParseError());
@@ -562,7 +562,7 @@ TEST(EngineTest, WafSubscriptorUpdateRuleData)
         auto res = ctx.publish(std::move(p));
         EXPECT_TRUE(res);
         EXPECT_EQ(res->actions[0].type, dds::action_type::block);
-        EXPECT_EQ(res->events.size(), 1);
+        EXPECT_EQ(res->triggers.size(), 1);
     }
 
     {
@@ -672,7 +672,7 @@ TEST(EngineTest, WafSubscriptorUpdateRules)
         auto res = ctx.publish(std::move(p));
         EXPECT_TRUE(res);
         EXPECT_EQ(res->actions[0].type, dds::action_type::block);
-        EXPECT_EQ(res->events.size(), 1);
+        EXPECT_EQ(res->triggers.size(), 1);
     }
 }
 
@@ -791,6 +791,56 @@ TEST(EngineTest, WafSubscriptorUpdateRuleOverrideAndActions)
         EXPECT_TRUE(res);
         EXPECT_EQ(res->actions[0].type, dds::action_type::record);
     }
+
+    { // Test keep is true
+        auto ctx = e->get_context();
+
+        auto p = parameter::map();
+        p.add("arg4", parameter::string("string 4"sv));
+
+        auto res = ctx.publish(std::move(p));
+        EXPECT_TRUE(res);
+        EXPECT_EQ(res->actions[0].type, dds::action_type::record);
+        EXPECT_EQ(res->force_keep, true);
+    }
+}
+
+TEST(EngineTest, TestKeep)
+{
+    auto msubmitter = NiceMock<mock::tel_submitter>{};
+
+    auto e{engine::create()};
+    e->subscribe(waf::instance::from_string(waf_rule, msubmitter));
+
+    {
+        auto ctx = e->get_context();
+
+        auto p = parameter::map();
+        p.add("arg12", parameter::string("string 12"sv));
+
+        auto res = ctx.publish(std::move(p));
+        EXPECT_FALSE(res);
+    }
+    {
+        auto ctx = e->get_context();
+
+        auto p = parameter::map();
+        p.add("arg5", parameter::string("string 5"sv));
+
+        auto res = ctx.publish(std::move(p));
+        EXPECT_FALSE(res);
+    }
+    {
+        auto ctx = e->get_context();
+
+        auto p = parameter::map();
+        p.add("arg4", parameter::string("string 4"sv));
+
+        auto res = ctx.publish(std::move(p));
+        EXPECT_TRUE(res);
+        EXPECT_EQ(res->actions[0].type, dds::action_type::record);
+        EXPECT_EQ(res->force_keep, true);
+    }
 }
 
 TEST(EngineTest, WafSubscriptorExclusions)

appsec/tests/helper/remote_config/listeners/engine_listener_test.cpp

@@ -164,7 +164,7 @@ TEST(RemoteConfigEngineListener, EngineRuleUpdate)
         auto res = ctx.publish(std::move(p));
         ASSERT_TRUE(res);
         EXPECT_EQ(res->actions[0].type, dds::action_type::block);
-        EXPECT_EQ(res->events.size(), 1);
+        EXPECT_EQ(res->triggers.size(), 1);
     }
 }
 
@@ -201,7 +201,7 @@ TEST(RemoteConfigEngineListener, EngineRuleUpdateFallback)
         auto res = ctx.publish(std::move(p));
         EXPECT_TRUE(res);
         EXPECT_EQ(res->actions[0].type, dds::action_type::block);
-        EXPECT_EQ(res->events.size(), 1);
+        EXPECT_EQ(res->triggers.size(), 1);
     }
 
     remote_config::engine_listener listener(
@@ -586,7 +586,7 @@ TEST(RemoteConfigEngineListener, EngineRuleDataUpdate)
         auto res = ctx.publish(std::move(p));
         EXPECT_TRUE(res);
         EXPECT_EQ(res->actions[0].type, dds::action_type::block);
-        EXPECT_EQ(res->events.size(), 1);
+        EXPECT_EQ(res->triggers.size(), 1);
     }
 }