From 390fe4f46b7e418ddca5e18adebb75de9f15ea25 Mon Sep 17 00:00:00 2001 From: Anil Mahtani <929854+Anilm3@users.noreply.github.com> Date: Sun, 18 Aug 2024 21:13:31 +0100 Subject: [PATCH 01/11] [PoC] Global context and threshold rules --- cmake/objects.cmake | 3 + src/context.cpp | 14 +- src/context.hpp | 5 +- src/event.cpp | 4 +- src/event.hpp | 4 +- src/global_context.cpp | 34 +++++ src/global_context.hpp | 35 +++++ src/parser/global_rule_parser.cpp | 124 ++++++++++++++++++ src/parser/parser.hpp | 3 + src/rule.cpp | 74 +++++++++++ src/rule.hpp | 177 ++++++++++++++++++++----- src/ruleset.hpp | 3 + src/ruleset_builder.cpp | 20 +++ src/ruleset_builder.hpp | 5 + src/timed_counter.hpp | 211 ++++++++++++++++++++++++++++++ tests/context_test.cpp | 117 +++++++++++------ tests/timed_counter_test.cpp | 104 +++++++++++++++ tools/CMakeLists.txt | 2 +- tools/waf_streamer.cpp | 142 ++++++++++++++++++++ 19 files changed, 993 insertions(+), 88 deletions(-) create mode 100644 src/global_context.cpp create mode 100644 src/global_context.hpp create mode 100644 src/parser/global_rule_parser.cpp create mode 100644 src/rule.cpp create mode 100644 src/timed_counter.hpp create mode 100644 tests/timed_counter_test.cpp create mode 100644 tools/waf_streamer.cpp diff --git a/cmake/objects.cmake b/cmake/objects.cmake index 0c50f21ff..252df49b5 100644 --- a/cmake/objects.cmake +++ b/cmake/objects.cmake @@ -4,6 +4,7 @@ set(LIBDDWAF_SOURCE ${libddwaf_SOURCE_DIR}/src/parameter.cpp ${libddwaf_SOURCE_DIR}/src/interface.cpp ${libddwaf_SOURCE_DIR}/src/context.cpp + ${libddwaf_SOURCE_DIR}/src/global_context.cpp ${libddwaf_SOURCE_DIR}/src/context_allocator.cpp ${libddwaf_SOURCE_DIR}/src/event.cpp ${libddwaf_SOURCE_DIR}/src/object.cpp @@ -21,6 +22,7 @@ set(LIBDDWAF_SOURCE ${libddwaf_SOURCE_DIR}/src/platform.cpp ${libddwaf_SOURCE_DIR}/src/sha256.cpp ${libddwaf_SOURCE_DIR}/src/uuid.cpp + ${libddwaf_SOURCE_DIR}/src/rule.cpp ${libddwaf_SOURCE_DIR}/src/action_mapper.cpp ${libddwaf_SOURCE_DIR}/src/builder/processor_builder.cpp ${libddwaf_SOURCE_DIR}/src/tokenizer/sql_base.cpp @@ -45,6 +47,7 @@ set(LIBDDWAF_SOURCE ${libddwaf_SOURCE_DIR}/src/parser/rule_override_parser.cpp ${libddwaf_SOURCE_DIR}/src/parser/scanner_parser.cpp ${libddwaf_SOURCE_DIR}/src/parser/exclusion_parser.cpp + ${libddwaf_SOURCE_DIR}/src/parser/global_rule_parser.cpp ${libddwaf_SOURCE_DIR}/src/processor/extract_schema.cpp ${libddwaf_SOURCE_DIR}/src/processor/fingerprint.cpp ${libddwaf_SOURCE_DIR}/src/condition/lfi_detector.cpp diff --git a/src/context.cpp b/src/context.cpp index 7e12e4b0f..44b31889e 100644 --- a/src/context.cpp +++ b/src/context.cpp @@ -92,6 +92,10 @@ DDWAF_RET_CODE context::run(optional_ref persistent, try { eval_preprocessors(derived, deadline); + if (ruleset_->gctx) { + ruleset_->gctx->eval(events, store_, gctx_cache_, deadline); + } + // If no rule targets are available, there is no point in evaluating them const bool should_eval_rules = check_new_rule_targets(); const bool should_eval_filters = should_eval_rules || check_new_filter_targets(); @@ -103,7 +107,7 @@ DDWAF_RET_CODE context::run(optional_ref persistent, const auto &policy = eval_filters(deadline); if (should_eval_rules) { - events = eval_rules(policy, deadline); + eval_rules(events, policy, deadline); if (!events.empty()) { set_context_event_address(store_); } @@ -218,11 +222,9 @@ exclusion::context_policy &context::eval_filters(ddwaf::timer &deadline) return exclusion_policy_; } -std::vector context::eval_rules( - const exclusion::context_policy &policy, ddwaf::timer &deadline) +void context::eval_rules(std::vector &events, const exclusion::context_policy &policy, + ddwaf::timer &deadline) { - std::vector events; - auto eval_collection = [&](const auto &type, const auto &collection) { auto it = collection_cache_.find(type); if (it == collection_cache_.end()) { @@ -255,8 +257,6 @@ std::vector context::eval_rules( DDWAF_DEBUG("Evaluating base collection '{}'", type); eval_collection(type, collection); } - - return events; } } // namespace ddwaf diff --git a/src/context.hpp b/src/context.hpp index df6dcc2bf..7798e080c 100644 --- a/src/context.hpp +++ b/src/context.hpp @@ -50,7 +50,8 @@ class context { // This function below returns a reference to an internal object, // however using them this way helps with testing exclusion::context_policy &eval_filters(ddwaf::timer &deadline); - std::vector eval_rules(const exclusion::context_policy &policy, ddwaf::timer &deadline); + void eval_rules(std::vector &events, const exclusion::context_policy &policy, + ddwaf::timer &deadline); protected: bool is_first_run() const { return collection_cache_.empty(); } @@ -89,6 +90,8 @@ class context { // Cache of collections to avoid processing once a result has been obtained memory::unordered_map collection_cache_{}; + + global_context::cache_type gctx_cache_{}; }; class context_wrapper { diff --git a/src/event.cpp b/src/event.cpp index efd78ba8f..84433ef26 100644 --- a/src/event.cpp +++ b/src/event.cpp @@ -145,7 +145,7 @@ void add_action_to_tracker(action_tracker &actions, std::string_view id, action_ } } -void serialize_rule(const ddwaf::rule &rule, ddwaf_object &rule_map) +void serialize_rule(const ddwaf::base_rule &rule, ddwaf_object &rule_map) { ddwaf_object tmp; ddwaf_object tags_map; @@ -180,7 +180,7 @@ void serialize_empty_rule(ddwaf_object &rule_map) ddwaf_object_map_add(&rule_map, "tags", &tags_map); } -void serialize_and_consolidate_rule_actions(const ddwaf::rule &rule, ddwaf_object &rule_map, +void serialize_and_consolidate_rule_actions(const ddwaf::base_rule &rule, ddwaf_object &rule_map, std::string_view action_override, action_tracker &actions, ddwaf_object &stack_id) { const auto &rule_actions = rule.get_actions(); diff --git a/src/event.hpp b/src/event.hpp index 1d1d34445..a55384205 100644 --- a/src/event.hpp +++ b/src/event.hpp @@ -15,10 +15,10 @@ namespace ddwaf { -class rule; +class base_rule; struct event { - const ddwaf::rule *rule{nullptr}; + const ddwaf::base_rule *rule{nullptr}; std::vector matches; bool ephemeral{false}; std::string_view action_override; diff --git a/src/global_context.cpp b/src/global_context.cpp new file mode 100644 index 000000000..21d5879c7 --- /dev/null +++ b/src/global_context.cpp @@ -0,0 +1,34 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#include "global_context.hpp" +#include "rule.hpp" + +namespace ddwaf { +void global_context::eval(std::vector &events, const object_store &store, cache_type &cache, + ddwaf::timer &deadline) +{ + auto timepoint = monotonic_clock::now(); + + DDWAF_DEBUG("Evaluating global rules"); + for (const auto &rule : rules_) { + auto cache_it = cache.find(rule.get()); + if (cache_it == cache.end()) { + auto [new_it, res] = cache.emplace(rule.get(), base_threshold_rule::cache_type{}); + if (!res) { + continue; + } + cache_it = new_it; + } + DDWAF_DEBUG("Evaluating rule {}", rule->get_id()); + auto opt_evt = rule->eval(store, cache_it->second, timepoint, deadline); + if (opt_evt.has_value()) { + events.emplace_back(std::move(opt_evt.value())); + } + } +} + +} // namespace ddwaf diff --git a/src/global_context.hpp b/src/global_context.hpp new file mode 100644 index 000000000..099a4ebb6 --- /dev/null +++ b/src/global_context.hpp @@ -0,0 +1,35 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#pragma once + +#include + +#include "rule.hpp" + +namespace ddwaf { + +class global_context { +public: + using cache_type = std::unordered_map; + + explicit global_context(std::vector> rules) + : rules_(std::move(rules)) + {} + global_context(const global_context &) = delete; + global_context &operator=(const global_context &) = delete; + global_context(global_context &&) = default; + global_context &operator=(global_context &&) = delete; + ~global_context() = default; + + void eval(std::vector &events, const object_store &store, cache_type &lcache, + ddwaf::timer &deadline); + +protected: + std::vector> rules_; +}; + +} // namespace ddwaf diff --git a/src/parser/global_rule_parser.cpp b/src/parser/global_rule_parser.cpp new file mode 100644 index 000000000..e31c42199 --- /dev/null +++ b/src/parser/global_rule_parser.cpp @@ -0,0 +1,124 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#include "parser/common.hpp" +#include "parser/parser.hpp" +#include "parser/specification.hpp" + +namespace ddwaf::parser::v2 { + +namespace { +// NOLINTNEXTLINE(bugprone-easily-swappable-parameters) +std::unique_ptr parse_indexed_threshold_rule( + std::string id, parameter::map &rule, parameter::map &criteria_map, const object_limits &limits) +{ + auto conditions_array = at(rule, "conditions", {}); + + address_container addresses; + auto expr = parse_simplified_expression(conditions_array, addresses, limits); + std::unordered_map tags; + for (auto &[key, value] : at(rule, "tags")) { + try { + tags.emplace(key, std::string(value)); + } catch (const bad_cast &e) { + throw invalid_type(std::string(key), e); + } + } + + if (tags.find("type") == tags.end()) { + throw ddwaf::parsing_error("missing key 'type'"); + } + + indexed_threshold_rule::evaluation_criteria criteria; + criteria.threshold = at(criteria_map, "threshold"); + criteria.period = std::chrono::milliseconds(at(criteria_map, "period")); + criteria.name = at(criteria_map, "input"); + criteria.target = get_target_index(criteria.name); + + return std::make_unique(std::move(id), at(rule, "name"), + std::move(tags), std::move(expr), criteria, + at>(rule, "on_match", {}), at(rule, "enabled", true)); +} + +// NOLINTNEXTLINE(bugprone-easily-swappable-parameters) +std::unique_ptr parse_threshold_rule( + std::string id, parameter::map &rule, parameter::map &criteria_map, const object_limits &limits) +{ + auto conditions_array = at(rule, "conditions", {}); + + address_container addresses; + auto expr = parse_simplified_expression(conditions_array, addresses, limits); + std::unordered_map tags; + for (auto &[key, value] : at(rule, "tags")) { + try { + tags.emplace(key, std::string(value)); + } catch (const bad_cast &e) { + throw invalid_type(std::string(key), e); + } + } + + if (tags.find("type") == tags.end()) { + throw ddwaf::parsing_error("missing key 'type'"); + } + + threshold_rule::evaluation_criteria criteria; + criteria.threshold = at(criteria_map, "threshold"); + criteria.period = std::chrono::milliseconds(at(criteria_map, "period")); + + return std::make_unique(std::move(id), at(rule, "name"), + std::move(tags), std::move(expr), criteria, + at>(rule, "on_match", {}), at(rule, "enabled", true)); +} + +std::unique_ptr parse_global_rule( + std::string id, parameter::map &rule, const object_limits &limits) +{ + auto criteria = at(rule, "criteria"); + if (criteria.contains("input")) { + return parse_indexed_threshold_rule(std::move(id), rule, criteria, limits); + } + return parse_threshold_rule(std::move(id), rule, criteria, limits); +} +} // namespace + +std::shared_ptr parse_global_rules( + parameter::vector &rule_array, base_section_info &info, const object_limits &limits) +{ + std::vector> rules; + + std::unordered_set ids; + for (unsigned i = 0; i < rule_array.size(); ++i) { + const auto &rule_param = rule_array[i]; + auto rule_map = static_cast(rule_param); + std::string id; + try { + address_container addresses; + id = at(rule_map, "id"); + if (ids.find(id) != ids.end()) { + DDWAF_WARN("Duplicate global rule {}", id); + info.add_failed(id, "duplicate rule"); + continue; + } + + auto rule = parse_global_rule(id, rule_map, limits); + DDWAF_DEBUG("Parsed global rule {}", id); + info.add_loaded(id); + add_addresses_to_info(addresses, info); + + rules.emplace_back(std::move(rule)); + } catch (const std::exception &e) { + if (id.empty()) { + id = index_to_id(i); + } + DDWAF_WARN("Failed to parse rule '{}': {}", id, e.what()); + info.add_failed(id, e.what()); + } + } + + return std::make_shared(std::move(rules)); +} + +} // namespace ddwaf::parser::v2 diff --git a/src/parser/parser.hpp b/src/parser/parser.hpp index cf3bb64aa..55037415c 100644 --- a/src/parser/parser.hpp +++ b/src/parser/parser.hpp @@ -63,5 +63,8 @@ std::vector parse_transformers(const parameter::vector &root, da std::pair> parse_matcher( std::string_view name, const parameter::map ¶ms); +std::shared_ptr parse_global_rules( + parameter::vector &rule_array, base_section_info &info, const object_limits &limits); + } // namespace v2 } // namespace ddwaf::parser diff --git a/src/rule.cpp b/src/rule.cpp new file mode 100644 index 000000000..0abe43363 --- /dev/null +++ b/src/rule.cpp @@ -0,0 +1,74 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#include "rule.hpp" +#include "timed_counter.hpp" + +using namespace std::literals; + +namespace ddwaf { + +std::optional threshold_rule::eval(const object_store &store, cache_type &cache, + monotonic_clock::time_point now, ddwaf::timer &deadline) +{ + if (expression::get_result(cache)) { + // An event was already produced, so we skip the rule + return std::nullopt; + } + + auto res = expr_->eval(cache, store, {}, {}, deadline); + if (!res.outcome) { + return std::nullopt; + } + + auto ms = std::chrono::duration_cast(now.time_since_epoch()); + auto count = counter_.add_timepoint_and_count(ms); + if (count > criteria_.threshold) { + // Match should be generated differently + // Match should be generated differently + auto matches = expression::get_matches(cache); + matches.emplace_back(condition_match{{}, {}, "threshold", threshold_str_, false}); + return {ddwaf::event{this, std::move(matches), false}}; + } + + return std::nullopt; +} + +std::optional indexed_threshold_rule::eval(const object_store &store, cache_type &cache, + monotonic_clock::time_point now, ddwaf::timer &deadline) +{ + if (expression::get_result(cache)) { + // An event was already produced, so we skip the rule + return std::nullopt; + } + + auto [obj, attr] = store.get_target(criteria_.target); + if (obj == nullptr || obj->type != DDWAF_OBJ_STRING) { + return std::nullopt; + } + + auto res = expr_->eval(cache, store, {}, {}, deadline); + if (!res.outcome) { + return std::nullopt; + } + + auto ms = std::chrono::duration_cast(now.time_since_epoch()); + std::string_view key{obj->stringValue, static_cast(obj->nbEntries)}; + + auto count = counter_.add_timepoint_and_count(key, ms); + if (count > criteria_.threshold) { + // Match should be generated differently + auto matches = expression::get_matches(cache); + matches.emplace_back( + condition_match{{{"input"sv, object_to_string(*obj), criteria_.name, {}}}, {}, + "threshold", threshold_str_, false}); + return {ddwaf::event{this, std::move(matches), false}}; + } + + return std::nullopt; +} + +} // namespace ddwaf diff --git a/src/rule.hpp b/src/rule.hpp index a449e56f1..2999225ed 100644 --- a/src/rule.hpp +++ b/src/rule.hpp @@ -6,7 +6,6 @@ #pragma once -#include #include #include #include @@ -16,59 +15,36 @@ #include "event.hpp" #include "exclusion/common.hpp" #include "expression.hpp" -#include "iterator.hpp" #include "matcher/base.hpp" #include "object_store.hpp" +#include "timed_counter.hpp" namespace ddwaf { -class rule { +class base_rule { public: - enum class source_type : uint8_t { base = 1, user = 2 }; - - using cache_type = expression::cache_type; - - rule(std::string id, std::string name, std::unordered_map tags, + base_rule(std::string id, std::string name, std::unordered_map tags, std::shared_ptr expr, std::vector actions = {}, - bool enabled = true, source_type source = source_type::base) - : enabled_(enabled), source_(source), id_(std::move(id)), name_(std::move(name)), - tags_(std::move(tags)), expr_(std::move(expr)), actions_(std::move(actions)) + bool enabled = true) + : enabled_(enabled), id_(std::move(id)), name_(std::move(name)), tags_(std::move(tags)), + expr_(std::move(expr)), actions_(std::move(actions)) { if (!expr_) { throw std::invalid_argument("rule constructed with null expression"); } } - rule(const rule &) = delete; - rule &operator=(const rule &) = delete; - - rule(rule &&rhs) noexcept = default; - rule &operator=(rule &&rhs) = default; - - virtual ~rule() = default; - - virtual std::optional match(const object_store &store, cache_type &cache, - const exclusion::object_set_ref &objects_excluded, - const std::unordered_map> &dynamic_matchers, - ddwaf::timer &deadline) const - { - if (expression::get_result(cache)) { - // An event was already produced, so we skip the rule - return std::nullopt; - } + base_rule(const base_rule &) = delete; + base_rule &operator=(const base_rule &) = delete; - auto res = expr_->eval(cache, store, objects_excluded, dynamic_matchers, deadline); - if (!res.outcome) { - return std::nullopt; - } + base_rule(base_rule &&rhs) noexcept = default; + base_rule &operator=(base_rule &&rhs) noexcept = default; - return {ddwaf::event{this, expression::get_matches(cache), res.ephemeral, {}}}; - } + virtual ~base_rule() = default; [[nodiscard]] bool is_enabled() const { return enabled_; } void toggle(bool value) { enabled_ = value; } - source_type get_source() const { return source_; } const std::string &get_id() const { return id_; } const std::string &get_name() const { return name_; } @@ -109,7 +85,6 @@ class rule { protected: bool enabled_{true}; - source_type source_; std::string id_; std::string name_; std::unordered_map tags_; @@ -118,4 +93,134 @@ class rule { std::vector actions_; }; +class rule : public base_rule { +public: + enum class source_type : uint8_t { base = 1, user = 2 }; + + using cache_type = expression::cache_type; + + rule(std::string id, std::string name, std::unordered_map tags, + std::shared_ptr expr, std::vector actions = {}, + bool enabled = true, source_type source = source_type::base) + : base_rule(std::move(id), std::move(name), std::move(tags), std::move(expr), + std::move(actions), enabled), + source_(source) + {} + + rule(const rule &) = delete; + rule &operator=(const rule &) = delete; + + rule(rule &&rhs) noexcept = default; + rule &operator=(rule &&rhs) noexcept = default; + + ~rule() override = default; + + virtual std::optional match(const object_store &store, cache_type &cache, + const exclusion::object_set_ref &objects_excluded, + const std::unordered_map> &dynamic_matchers, + ddwaf::timer &deadline) const + { + if (expression::get_result(cache)) { + // An event was already produced, so we skip the rule + return std::nullopt; + } + + auto res = expr_->eval(cache, store, objects_excluded, dynamic_matchers, deadline); + if (!res.outcome) { + return std::nullopt; + } + + return {ddwaf::event{this, expression::get_matches(cache), res.ephemeral, {}}}; + } + + source_type get_source() const { return source_; } + +protected: + source_type source_; +}; + +class base_threshold_rule : public base_rule { +public: + using cache_type = expression::cache_type; + + base_threshold_rule(std::string id, std::string name, + std::unordered_map tags, std::shared_ptr expr, + std::vector actions = {}, bool enabled = true) + : base_rule(std::move(id), std::move(name), std::move(tags), std::move(expr), + std::move(actions), enabled) + {} + ~base_threshold_rule() override = default; + base_threshold_rule(const base_threshold_rule &) = delete; + base_threshold_rule &operator=(const base_threshold_rule &) = delete; + base_threshold_rule(base_threshold_rule &&rhs) noexcept = default; + base_threshold_rule &operator=(base_threshold_rule &&rhs) noexcept = default; + + virtual std::optional eval(const object_store &store, cache_type &cache, + monotonic_clock::time_point now, ddwaf::timer &deadline) = 0; +}; + +class threshold_rule : public base_threshold_rule { +public: + struct evaluation_criteria { + std::size_t threshold; + std::chrono::milliseconds period{}; + }; + + threshold_rule(std::string id, std::string name, + std::unordered_map tags, std::shared_ptr expr, + evaluation_criteria criteria, std::vector actions = {}, bool enabled = true) + : base_threshold_rule(std::move(id), std::move(name), std::move(tags), std::move(expr), + std::move(actions), enabled), + criteria_(criteria), counter_(criteria_.period, criteria_.threshold * 2), + threshold_str_(to_string(criteria_.threshold)) + {} + + ~threshold_rule() override = default; + threshold_rule(const threshold_rule &) = delete; + threshold_rule &operator=(const threshold_rule &) = delete; + threshold_rule(threshold_rule &&rhs) noexcept = delete; + threshold_rule &operator=(threshold_rule &&rhs) noexcept = delete; + + std::optional eval(const object_store &store, cache_type &cache, + monotonic_clock::time_point now, ddwaf::timer &deadline) override; + +protected: + evaluation_criteria criteria_; + timed_counter_ts_ms counter_; + std::string threshold_str_; +}; + +class indexed_threshold_rule : public base_threshold_rule { +public: + struct evaluation_criteria { + std::string name; + target_index target; + std::size_t threshold; + std::chrono::milliseconds period; + }; + + indexed_threshold_rule(std::string id, std::string name, + std::unordered_map tags, std::shared_ptr expr, + evaluation_criteria criteria, std::vector actions = {}, bool enabled = true) + : base_threshold_rule(std::move(id), std::move(name), std::move(tags), std::move(expr), + std::move(actions), enabled), + criteria_(std::move(criteria)), counter_(criteria_.period, 128, criteria_.threshold * 2), + threshold_str_(to_string(criteria_.threshold)) + {} + + ~indexed_threshold_rule() override = default; + indexed_threshold_rule(const indexed_threshold_rule &) = delete; + indexed_threshold_rule &operator=(const indexed_threshold_rule &) = delete; + indexed_threshold_rule(indexed_threshold_rule &&rhs) noexcept = delete; + indexed_threshold_rule &operator=(indexed_threshold_rule &&rhs) noexcept = delete; + + std::optional eval(const object_store &store, cache_type &cache, + monotonic_clock::time_point now, ddwaf::timer &deadline) override; + +protected: + evaluation_criteria criteria_; + indexed_timed_counter_ts_ms counter_; + std::string threshold_str_; +}; + } // namespace ddwaf diff --git a/src/ruleset.hpp b/src/ruleset.hpp index 19e9edd0e..cde05b9f3 100644 --- a/src/ruleset.hpp +++ b/src/ruleset.hpp @@ -14,6 +14,7 @@ #include "collection.hpp" #include "exclusion/input_filter.hpp" #include "exclusion/rule_filter.hpp" +#include "global_context.hpp" #include "obfuscator.hpp" #include "processor/base.hpp" #include "rule.hpp" @@ -194,6 +195,8 @@ struct ruleset { // These are lazily computed andthe underlying memory of each string is // owned by the action mapper. std::vector available_action_types; + + std::shared_ptr gctx; }; } // namespace ddwaf diff --git a/src/ruleset_builder.cpp b/src/ruleset_builder.cpp index 3001b1cf0..f02566100 100644 --- a/src/ruleset_builder.cpp +++ b/src/ruleset_builder.cpp @@ -198,6 +198,7 @@ std::shared_ptr ruleset_builder::build(parameter::map &root, base_rules rs->rule_matchers = rule_matchers_; rs->exclusion_matchers = exclusion_matchers_; rs->scanners = scanners_.items(); + rs->gctx = gctx_; rs->actions = actions_; rs->free_fn = free_fn_; rs->event_obfuscator = event_obfuscator_; @@ -429,6 +430,25 @@ ruleset_builder::change_state ruleset_builder::load(parameter::map &root, base_r } } + it = root.find("global_rules"); + if (it != root.end()) { + DDWAF_DEBUG("Parsing global rules"); + auto §ion = info.add_section("global_rules"); + try { + auto global_rules = static_cast(it->second); + if (!global_rules.empty()) { + gctx_ = parser::v2::parse_global_rules(global_rules, section, limits_); + } else { + DDWAF_DEBUG("Clearing all global rules"); + gctx_ = nullptr; + } + state = state | change_state::global_rules; + } catch (const std::exception &e) { + DDWAF_WARN("Failed to parse global rules: {}", e.what()); + section.set_error(e.what()); + } + } + return state; } diff --git a/src/ruleset_builder.hpp b/src/ruleset_builder.hpp index 453e752f5..370626b87 100644 --- a/src/ruleset_builder.hpp +++ b/src/ruleset_builder.hpp @@ -12,6 +12,7 @@ #include #include "builder/processor_builder.hpp" +#include "global_context.hpp" #include "indexer.hpp" #include "parameter.hpp" #include "parser/specification.hpp" @@ -54,6 +55,7 @@ class ruleset_builder { scanners = 64, actions = 128, exclusion_data = 256, + global_rules = 512, }; friend constexpr change_state operator|(change_state lhs, change_state rhs); @@ -112,6 +114,9 @@ class ruleset_builder { // Actions std::shared_ptr actions_; + + // Global Context + std::shared_ptr gctx_; }; } // namespace ddwaf diff --git a/src/timed_counter.hpp b/src/timed_counter.hpp new file mode 100644 index 000000000..25bcc84ba --- /dev/null +++ b/src/timed_counter.hpp @@ -0,0 +1,211 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace ddwaf { + +template +concept is_duration = std::is_same_v, T>; + +// Perhaps this should use an std::chrono::time_point instead +template + requires is_duration +class timed_counter { +public: + timed_counter() = default; + explicit timed_counter(T period, std::size_t max_window_size = 100) : period_(period) + { + time_points_.resize(max_window_size); + } + + std::size_t add_timepoint_and_count(T point) + { + // Discard old elements + update_count(point); + + // Check if the latest element is beyond the current one (concurrent writers) + auto index = decrement(right); + if (buckets > 0 && time_points_[index].point > point) { + ++time_points_[index].count; + return ++count; + } + + if (buckets < time_points_.size()) { + // Add a new element + time_points_[right].point = point; + time_points_[right].count = 1; + right = increment(right); + ++count; + ++buckets; + } else if (buckets == time_points_.size()) { + // Discard the oldest one + time_points_[right].point = point; + count -= (time_points_[right].count - 1); + time_points_[right].count = 1; + right = increment(right); + left = increment(left); + } + + return count; + } + + T last_timepoint() const + { + if (buckets == 0) { + [[unlikely]] return static_cast(0); + } + + auto index = decrement(right); + return time_points_[index].point; + } + + std::size_t update_count(T point) + { + // Discard old elements + auto window_begin = point - period_; + while (buckets > 0 && time_points_[left].point <= window_begin) { + count -= time_points_[left].count; + --buckets; + left = increment(left); + } + return count; + } + + void reset() { left = right = count = buckets = 0; } + +protected: + [[nodiscard]] std::size_t increment(std::size_t value) const + { + return (value + 1) % time_points_.size(); + } + + [[nodiscard]] std::size_t decrement(std::size_t value) const + { + return (value + time_points_.size() - 1) % time_points_.size(); + } + + struct time_bucket { + T point; + std::size_t count; + }; + + std::chrono::milliseconds period_{}; + std::vector time_points_; + std::size_t left{0}; + std::size_t right{0}; + std::size_t count{0}; + std::size_t buckets{0}; +}; + +template + requires is_duration +class timed_counter_ts : protected timed_counter { +public: + timed_counter_ts() = default; + explicit timed_counter_ts(T period, std::size_t max_window_size = 100) + : timed_counter(period, max_window_size) + {} + std::size_t add_timepoint_and_count(T point) + { + std::lock_guard lock(mtx_); + return timed_counter::add_timepoint_and_count(point); + } + + T last_timepoint() const + { + std::lock_guard lock(mtx_); + return timed_counter::last_timepoint(); + } + + std::size_t update_count(T point) + { + std::lock_guard lock(mtx_); + return timed_counter::update_count(point); + } + + void reset() + { + std::lock_guard lock(mtx_); + timed_counter::reset(); + } + +protected: + mutable std::mutex mtx_; +}; + +template + requires is_duration +class indexed_timed_counter_ts { +public: + indexed_timed_counter_ts() = default; + explicit indexed_timed_counter_ts( + // NOLINTNEXTLINE(bugprone-easily-swappable-parameters) + Duration period, std::size_t max_index_size = 32, std::size_t max_window_size = 100) + : period_(period), max_index_size_(max_index_size), max_window_size_(max_window_size) + {} + + template + requires std::is_constructible_v + std::size_t add_timepoint_and_count(T key, Duration point) + { + std::lock_guard lock(mtx_); + auto it = index_.find(key); + if (it == index_.end()) { + if (index_.size() == max_index_size_) { + remove_oldest_entry(point); + } + + auto [new_it, res] = + index_.emplace(Key{key}, timed_counter{period_, max_window_size_}); + if (!res) { + return 0; + } + + it = new_it; + } + + return it->second.add_timepoint_and_count(point); + } + +protected: + void remove_oldest_entry(Duration point) + { + using iterator_type = typename decltype(index_)::iterator; + + Duration max_delta{0}; + iterator_type oldest_it; + for (auto it = index_.begin(); it != index_.end(); ++it) { + auto window_last = it->second.last_timepoint(); + auto delta = point - window_last; + if (delta > max_delta) { + max_delta = delta; + oldest_it = it; + } + } + + index_.erase(oldest_it); + } + + Duration period_; + std::size_t max_index_size_{}; + std::size_t max_window_size_{}; + std::map, std::less<>> index_; + mutable std::mutex mtx_; +}; + +using timed_counter_ts_ms = timed_counter_ts; +using indexed_timed_counter_ts_ms = + indexed_timed_counter_ts; + +} // namespace ddwaf diff --git a/tests/context_test.cpp b/tests/context_test.cpp index a9bd9d7b8..e047d933b 100644 --- a/tests/context_test.cpp +++ b/tests/context_test.cpp @@ -229,7 +229,8 @@ TEST(TestContext, MatchTimeout) ddwaf_object_map_add(&root, "http.client_ip", ddwaf_object_string(&tmp, "192.168.0.1")); ctx.insert(root); - EXPECT_THROW(ctx.eval_rules({}, deadline), ddwaf::timeout_exception); + std::vector events; + EXPECT_THROW(ctx.eval_rules(events, {}, deadline), ddwaf::timeout_exception); } TEST(TestContext, NoMatch) @@ -256,7 +257,8 @@ TEST(TestContext, NoMatch) ddwaf_object_map_add(&root, "http.client_ip", ddwaf_object_string(&tmp, "192.168.0.2")); ctx.insert(root); - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 0); } @@ -284,7 +286,8 @@ TEST(TestContext, Match) ddwaf_object_map_add(&root, "http.client_ip", ddwaf_object_string(&tmp, "192.168.0.1")); ctx.insert(root); - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 1); } @@ -331,7 +334,8 @@ TEST(TestContext, MatchMultipleRulesInCollectionSingleRun) ddwaf_object_map_add(&root, "usr.id", ddwaf_object_string(&tmp, "admin")); ctx.insert(root); - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 1); auto &event = events[0]; @@ -397,7 +401,8 @@ TEST(TestContext, MatchMultipleRulesWithPrioritySingleRun) ctx.insert(root); ddwaf::timer deadline{2s}; - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 1); auto event = events[0]; @@ -417,7 +422,8 @@ TEST(TestContext, MatchMultipleRulesWithPrioritySingleRun) ctx.insert(root); ddwaf::timer deadline{2s}; - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 1); auto event = events[0]; @@ -470,7 +476,8 @@ TEST(TestContext, MatchMultipleRulesInCollectionDoubleRun) ddwaf_object_map_add(&root, "http.client_ip", ddwaf_object_string(&tmp, "192.168.0.1")); ctx.insert(root); - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 1); auto &event = events[0]; @@ -498,7 +505,8 @@ TEST(TestContext, MatchMultipleRulesInCollectionDoubleRun) ddwaf_object_map_add(&root, "usr.id", ddwaf_object_string(&tmp, "admin")); ctx.insert(root); - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 0); } } @@ -547,7 +555,8 @@ TEST(TestContext, MatchMultipleRulesWithPriorityDoubleRunPriorityLast) ddwaf_object_map_add(&root, "http.client_ip", ddwaf_object_string(&tmp, "192.168.0.1")); ctx.insert(root); - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 1); auto &event = events[0]; @@ -577,7 +586,8 @@ TEST(TestContext, MatchMultipleRulesWithPriorityDoubleRunPriorityLast) ddwaf_object_map_add(&root, "usr.id", ddwaf_object_string(&tmp, "admin")); ctx.insert(root); - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 1); auto &event = events[0]; @@ -644,7 +654,8 @@ TEST(TestContext, MatchMultipleRulesWithPriorityDoubleRunPriorityFirst) ddwaf_object_map_add(&root, "http.client_ip", ddwaf_object_string(&tmp, "192.168.0.1")); ctx.insert(root); - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 1); auto &event = events[0]; @@ -674,7 +685,8 @@ TEST(TestContext, MatchMultipleRulesWithPriorityDoubleRunPriorityFirst) ddwaf_object_map_add(&root, "usr.id", ddwaf_object_string(&tmp, "admin")); ctx.insert(root); - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 0); } } @@ -723,7 +735,8 @@ TEST(TestContext, MatchMultipleRulesWithPriorityUntilAllActionsMet) ddwaf_object_map_add(&root, "http.client_ip", ddwaf_object_string(&tmp, "192.168.0.1")); ctx.insert(root); - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 1); auto &event = events[0]; @@ -751,7 +764,8 @@ TEST(TestContext, MatchMultipleRulesWithPriorityUntilAllActionsMet) ddwaf_object_map_add(&root, "usr.id", ddwaf_object_string(&tmp, "admin")); ctx.insert(root); - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 1); auto &event = events[0]; @@ -817,7 +831,8 @@ TEST(TestContext, MatchMultipleCollectionsSingleRun) ddwaf_object_map_add(&root, "usr.id", ddwaf_object_string(&tmp, "admin")); ctx.insert(root); - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 2); } @@ -866,7 +881,8 @@ TEST(TestContext, MatchMultiplePriorityCollectionsSingleRun) ddwaf_object_map_add(&root, "usr.id", ddwaf_object_string(&tmp, "admin")); ctx.insert(root); - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 2); } @@ -913,7 +929,8 @@ TEST(TestContext, MatchMultipleCollectionsDoubleRun) ddwaf_object_map_add(&root, "usr.id", ddwaf_object_string(&tmp, "admin")); ctx.insert(root); - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 1); } @@ -924,7 +941,8 @@ TEST(TestContext, MatchMultipleCollectionsDoubleRun) ddwaf_object_map_add(&root, "http.client_ip", ddwaf_object_string(&tmp, "192.168.0.1")); ctx.insert(root); - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 1); } } @@ -974,7 +992,8 @@ TEST(TestContext, MatchMultiplePriorityCollectionsDoubleRun) ddwaf_object_map_add(&root, "usr.id", ddwaf_object_string(&tmp, "admin")); ctx.insert(root); - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 1); } @@ -985,7 +1004,8 @@ TEST(TestContext, MatchMultiplePriorityCollectionsDoubleRun) ddwaf_object_map_add(&root, "http.client_ip", ddwaf_object_string(&tmp, "192.168.0.1")); ctx.insert(root); - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 1); } } @@ -1136,7 +1156,8 @@ TEST(TestContext, RuleFilterWithCondition) EXPECT_EQ(rules_to_exclude.size(), 1); EXPECT_TRUE(rules_to_exclude.contains(rule.get())); - auto events = ctx.eval_rules(rules_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, rules_to_exclude, deadline); EXPECT_EQ(events.size(), 0); } @@ -1446,7 +1467,8 @@ TEST(TestContext, NoRuleFilterWithCondition) auto rules_to_exclude = ctx.eval_filters(deadline); EXPECT_TRUE(rules_to_exclude.empty()); - auto events = ctx.eval_rules(rules_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, rules_to_exclude, deadline); EXPECT_EQ(events.size(), 1); } @@ -1936,7 +1958,8 @@ TEST(TestContext, InputFilterExclude) auto objects_to_exclude = ctx.eval_filters(deadline); EXPECT_EQ(objects_to_exclude.size(), 1); - auto events = ctx.eval_rules(objects_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, objects_to_exclude, deadline); EXPECT_EQ(events.size(), 0); } @@ -2085,7 +2108,8 @@ TEST(TestContext, InputFilterExcludeRule) it->second.mode = filter_mode::none; EXPECT_TRUE(it->second.objects.empty()); - auto events = ctx.eval_rules(objects_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, objects_to_exclude, deadline); EXPECT_EQ(events.size(), 1); } @@ -2369,7 +2393,8 @@ TEST(TestContext, InputFilterWithCondition) auto objects_to_exclude = ctx.eval_filters(deadline); EXPECT_EQ(objects_to_exclude.size(), 0); - auto events = ctx.eval_rules(objects_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, objects_to_exclude, deadline); EXPECT_EQ(events.size(), 1); } @@ -2387,7 +2412,8 @@ TEST(TestContext, InputFilterWithCondition) auto objects_to_exclude = ctx.eval_filters(deadline); EXPECT_EQ(objects_to_exclude.size(), 0); - auto events = ctx.eval_rules(objects_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, objects_to_exclude, deadline); EXPECT_EQ(events.size(), 1); } @@ -2405,7 +2431,8 @@ TEST(TestContext, InputFilterWithCondition) auto objects_to_exclude = ctx.eval_filters(deadline); EXPECT_EQ(objects_to_exclude.size(), 1); - auto events = ctx.eval_rules(objects_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, objects_to_exclude, deadline); EXPECT_EQ(events.size(), 0); } } @@ -2534,7 +2561,8 @@ TEST(TestContext, InputFilterMultipleRules) EXPECT_EQ(policy.objects.size(), 1); } - auto events = ctx.eval_rules(objects_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, objects_to_exclude, deadline); EXPECT_EQ(events.size(), 0); } @@ -2556,7 +2584,8 @@ TEST(TestContext, InputFilterMultipleRules) EXPECT_EQ(policy.objects.size(), 2); } - auto events = ctx.eval_rules(objects_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, objects_to_exclude, deadline); EXPECT_EQ(events.size(), 0); } @@ -2578,7 +2607,8 @@ TEST(TestContext, InputFilterMultipleRules) EXPECT_EQ(policy.objects.size(), 2); } - auto events = ctx.eval_rules(objects_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, objects_to_exclude, deadline); EXPECT_EQ(events.size(), 0); } } @@ -2658,7 +2688,8 @@ TEST(TestContext, InputFilterMultipleRulesMultipleFilters) EXPECT_EQ(objects.size(), 1); } - auto events = ctx.eval_rules(objects_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, objects_to_exclude, deadline); EXPECT_EQ(events.size(), 0); } @@ -2681,7 +2712,8 @@ TEST(TestContext, InputFilterMultipleRulesMultipleFilters) EXPECT_EQ(objects.size(), 1); } - auto events = ctx.eval_rules(objects_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, objects_to_exclude, deadline); EXPECT_EQ(events.size(), 0); } @@ -2704,7 +2736,8 @@ TEST(TestContext, InputFilterMultipleRulesMultipleFilters) EXPECT_EQ(objects.size(), 1); } - auto events = ctx.eval_rules(objects_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, objects_to_exclude, deadline); EXPECT_EQ(events.size(), 0); } } @@ -2818,7 +2851,8 @@ TEST(TestContext, InputFilterMultipleRulesMultipleFiltersMultipleObjects) EXPECT_TRUE(objects.contains(&root.array[0])); } - auto events = ctx.eval_rules(objects_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, objects_to_exclude, deadline); EXPECT_EQ(events.size(), 0); } @@ -2840,7 +2874,8 @@ TEST(TestContext, InputFilterMultipleRulesMultipleFiltersMultipleObjects) EXPECT_TRUE(objects.contains(&root.array[0])); } - auto events = ctx.eval_rules(objects_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, objects_to_exclude, deadline); EXPECT_EQ(events.size(), 0); } @@ -2867,7 +2902,8 @@ TEST(TestContext, InputFilterMultipleRulesMultipleFiltersMultipleObjects) EXPECT_TRUE(objects.contains(&root.array[0])); } - auto events = ctx.eval_rules(objects_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, objects_to_exclude, deadline); EXPECT_EQ(events.size(), 0); } @@ -2890,7 +2926,8 @@ TEST(TestContext, InputFilterMultipleRulesMultipleFiltersMultipleObjects) EXPECT_TRUE(objects.contains(&root.array[0])); EXPECT_TRUE(objects.contains(&root.array[1])); } - auto events = ctx.eval_rules(objects_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, objects_to_exclude, deadline); EXPECT_EQ(events.size(), 0); } @@ -2918,7 +2955,8 @@ TEST(TestContext, InputFilterMultipleRulesMultipleFiltersMultipleObjects) EXPECT_TRUE(objects.contains(&root.array[0])); EXPECT_TRUE(objects.contains(&root.array[1])); } - auto events = ctx.eval_rules(objects_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, objects_to_exclude, deadline); EXPECT_EQ(events.size(), 0); } @@ -2948,7 +2986,8 @@ TEST(TestContext, InputFilterMultipleRulesMultipleFiltersMultipleObjects) EXPECT_TRUE(objects.contains(&root.array[1])); EXPECT_TRUE(objects.contains(&root.array[2])); } - auto events = ctx.eval_rules(objects_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, objects_to_exclude, deadline); EXPECT_EQ(events.size(), 0); } } diff --git a/tests/timed_counter_test.cpp b/tests/timed_counter_test.cpp new file mode 100644 index 000000000..df91c19c5 --- /dev/null +++ b/tests/timed_counter_test.cpp @@ -0,0 +1,104 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#include "test.hpp" +#include "test_utils.hpp" +#include "timed_counter.hpp" + +using namespace std::literals; +using namespace std::chrono_literals; + +namespace { + +TEST(TestTimedCounter, BasicMs) +{ + ddwaf::timed_counter_ts window{10ms, 5}; + + EXPECT_EQ(window.add_timepoint_and_count(1ms), 1); + EXPECT_EQ(window.add_timepoint_and_count(11ms), 1); + EXPECT_EQ(window.add_timepoint_and_count(12ms), 2); + EXPECT_EQ(window.add_timepoint_and_count(13ms), 3); + EXPECT_EQ(window.add_timepoint_and_count(14ms), 4); + EXPECT_EQ(window.add_timepoint_and_count(15ms), 5); + EXPECT_EQ(window.add_timepoint_and_count(16ms), 5); + + EXPECT_EQ(window.add_timepoint_and_count(17ms), 5); + EXPECT_EQ(window.add_timepoint_and_count(18ms), 5); + EXPECT_EQ(window.add_timepoint_and_count(19ms), 5); + EXPECT_EQ(window.add_timepoint_and_count(20ms), 5); + EXPECT_EQ(window.add_timepoint_and_count(21ms), 5); + EXPECT_EQ(window.add_timepoint_and_count(40ms), 1); +} + +TEST(TestTimedCounter, BasicS) +{ + ddwaf::timed_counter_ts window{10s, 5}; + + EXPECT_EQ(window.add_timepoint_and_count(1s), 1); + EXPECT_EQ(window.add_timepoint_and_count(11s), 1); + EXPECT_EQ(window.add_timepoint_and_count(12s), 2); + EXPECT_EQ(window.add_timepoint_and_count(13s), 3); + EXPECT_EQ(window.add_timepoint_and_count(14s), 4); + EXPECT_EQ(window.add_timepoint_and_count(15s), 5); + EXPECT_EQ(window.add_timepoint_and_count(16s), 5); + EXPECT_EQ(window.add_timepoint_and_count(17s), 5); + EXPECT_EQ(window.add_timepoint_and_count(18s), 5); + EXPECT_EQ(window.add_timepoint_and_count(19s), 5); + EXPECT_EQ(window.add_timepoint_and_count(20s), 5); + EXPECT_EQ(window.add_timepoint_and_count(21s), 5); + EXPECT_EQ(window.add_timepoint_and_count(40s), 1); +} + +TEST(TestIndexedTimedCounter, BasicString) +{ + ddwaf::indexed_timed_counter_ts window{10s, 5, 5}; + + EXPECT_EQ(window.add_timepoint_and_count("admin"sv, 1s), 1); + EXPECT_EQ(window.add_timepoint_and_count("user"sv, 10s), 1); + EXPECT_EQ(window.add_timepoint_and_count("docker"sv, 11s), 1); + EXPECT_EQ(window.add_timepoint_and_count("nobody"sv, 11s), 1); + EXPECT_EQ(window.add_timepoint_and_count("root"sv, 11s), 1); + // Admin should be removed, as it's the latest + EXPECT_EQ(window.add_timepoint_and_count("mail"sv, 11s), 1); + + // User will now be removed + EXPECT_EQ(window.add_timepoint_and_count("admin"sv, 11s), 1); + EXPECT_EQ(window.add_timepoint_and_count("admin"sv, 12s), 2); + EXPECT_EQ(window.add_timepoint_and_count("admin"sv, 13s), 3); + EXPECT_EQ(window.add_timepoint_and_count("admin"sv, 14s), 4); + EXPECT_EQ(window.add_timepoint_and_count("admin"sv, 15s), 5); + EXPECT_EQ(window.add_timepoint_and_count("admin"sv, 16s), 5); + + EXPECT_EQ(window.add_timepoint_and_count("docker"sv, 17s), 2); + EXPECT_EQ(window.add_timepoint_and_count("nobody"sv, 17s), 2); + EXPECT_EQ(window.add_timepoint_and_count("root"sv, 17s), 2); + EXPECT_EQ(window.add_timepoint_and_count("mail"sv, 17s), 2); + + EXPECT_EQ(window.add_timepoint_and_count("docker"sv, 18s), 3); + EXPECT_EQ(window.add_timepoint_and_count("nobody"sv, 18s), 3); + EXPECT_EQ(window.add_timepoint_and_count("root"sv, 18s), 3); + EXPECT_EQ(window.add_timepoint_and_count("mail"sv, 18s), 3); + + EXPECT_EQ(window.add_timepoint_and_count("docker"sv, 19s), 4); + EXPECT_EQ(window.add_timepoint_and_count("nobody"sv, 19s), 4); + EXPECT_EQ(window.add_timepoint_and_count("root"sv, 19s), 4); + EXPECT_EQ(window.add_timepoint_and_count("mail"sv, 19s), 4); + + EXPECT_EQ(window.add_timepoint_and_count("docker"sv, 20s), 5); + EXPECT_EQ(window.add_timepoint_and_count("nobody"sv, 20s), 5); + EXPECT_EQ(window.add_timepoint_and_count("root"sv, 20s), 5); + EXPECT_EQ(window.add_timepoint_and_count("mail"sv, 20s), 5); + EXPECT_EQ(window.add_timepoint_and_count("admin"sv, 20s), 5); + + EXPECT_EQ(window.add_timepoint_and_count("nobody"sv, 21s), 5); + EXPECT_EQ(window.add_timepoint_and_count("root"sv, 21s), 5); + EXPECT_EQ(window.add_timepoint_and_count("mail"sv, 21s), 5); + EXPECT_EQ(window.add_timepoint_and_count("admin"sv, 21s), 5); + // Docker will now be removed + EXPECT_EQ(window.add_timepoint_and_count("user"sv, 21s), 1); +} + +} // namespace diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 3379ca540..b1a5303a5 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -5,7 +5,7 @@ foreach(TOOL ${LIBDDWAF_TOOL_SOURCE}) get_filename_component(TOOL_NAME ${TOOL} NAME_WLE) add_executable(${TOOL_NAME} ${TOOL} ${LIBDDWAF_TOOL_COMMON_SOURCE}) - target_link_libraries(${TOOL_NAME} PRIVATE libddwaf_objects lib_yamlcpp lib_rapidjson) + target_link_libraries(${TOOL_NAME} PRIVATE libddwaf_objects lib_yamlcpp lib_rapidjson readline) target_include_directories(${TOOL_NAME} PRIVATE ${LIBDDWAF_PRIVATE_INCLUDES}) set_target_properties(${TOOL_NAME} PROPERTIES diff --git a/tools/waf_streamer.cpp b/tools/waf_streamer.cpp new file mode 100644 index 000000000..bf8b88010 --- /dev/null +++ b/tools/waf_streamer.cpp @@ -0,0 +1,142 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/utils.hpp" +#include "ddwaf.h" + +// NOLINTNEXTLINE +auto parse_args(int argc, char *argv[]) +{ + const std::map> arg_mapping{ + {"-r", "--ruleset"}, {"--ruleset", "--ruleset"}}; + + std::unordered_map args; + auto last_arg = args.end(); + for (int i = 1; i < argc; i++) { + std::string_view arg = argv[i]; + if (arg.starts_with('-')) { + if (auto long_arg = arg_mapping.find(arg); long_arg != arg_mapping.end()) { + arg = long_arg->second; + } else { + continue; // Unknown option + } + + auto [it, res] = args.emplace(arg, std::string{}); + last_arg = it; + } else if (last_arg != args.end()) { + last_arg->second = arg; + } + } + return args; +} + +int main(int argc, char *argv[]) +{ + //ddwaf_set_log_cb(log_cb, DDWAF_LOG_TRACE); + + auto args = parse_args(argc, argv); + + std::string ruleset = args["--ruleset"]; + if (ruleset.empty()) { + std::cout << "Usage: " << argv[0] << " --ruleset [..]\n"; + return EXIT_FAILURE; + } + + auto rule = YAML::Load(read_file(ruleset)).as(); + const ddwaf_config config{{0, 0, 0}, {nullptr, nullptr}, ddwaf_object_free}; + auto *handle = ddwaf_init(&rule, &config, nullptr); + ddwaf_object_free(&rule); + if (handle == nullptr) { + std::cout << "Failed to load " << ruleset << '\n'; + return EXIT_FAILURE; + } + + while (true) { + char *inpt = readline("Input: "); + add_history(inpt); + std::string json_str = inpt; + + + ddwaf_context context = ddwaf_context_init(handle); + if (context == nullptr) { + ddwaf_destroy(handle); + std::cout << "Failed to initialise context\n"; + return EXIT_FAILURE; + } + + auto input = YAML::Load(json_str).as(); + + ddwaf_result ret; + auto code = + ddwaf_run(context, &input, nullptr, &ret, std::numeric_limits::max()); + + if (code == DDWAF_MATCH) { + std::cout << "Evaluating " << json_str << " --> Match!\n"; + } else if (code == DDWAF_OK) { + std::cout << "Evaluating " << json_str << " --> No match!\n"; + } else { + std::cout << "Evaluating " << json_str << " --> Error!\n"; + } + + if (code == DDWAF_MATCH && ddwaf_object_size(&ret.events) > 0) { + std::stringstream ss; + YAML::Emitter out(ss); + out.SetIndent(2); + out.SetMapFormat(YAML::Block); + out.SetSeqFormat(YAML::Block); + out << object_to_yaml(ret.events); + + std::cout << "Events:\n" << ss.str() << "\n\n"; + } + + if (code == DDWAF_MATCH && ddwaf_object_size(&ret.actions) > 0) { + std::stringstream ss; + YAML::Emitter out(ss); + out.SetIndent(2); + out.SetMapFormat(YAML::Block); + out.SetSeqFormat(YAML::Block); + out << object_to_yaml(ret.actions); + + std::cout << "Actions:\n" << ss.str() << "\n\n"; + } + + if (ddwaf_object_size(&ret.derivatives) > 0) { + std::stringstream ss; + YAML::Emitter out(ss); + out.SetIndent(2); + out.SetMapFormat(YAML::Block); + out.SetSeqFormat(YAML::Block); + out << object_to_yaml(ret.derivatives); + + std::cout << "Derivatives:\n" << ss.str() << "\n\n"; + } + + ddwaf_result_free(&ret); + ddwaf_context_destroy(context); + } + + ddwaf_destroy(handle); + + return EXIT_SUCCESS; +} + From fbc1f40b454f18567496cfe1d4a1e2f8b0553943 Mon Sep 17 00:00:00 2001 From: Anil Mahtani <929854+Anilm3@users.noreply.github.com> Date: Mon, 19 Aug 2024 14:11:06 +0100 Subject: [PATCH 02/11] Reorganize rules, rename timed_counter to sliding_window_counter, move thread-safety to monitor --- cmake/objects.cmake | 2 +- src/collection.hpp | 2 +- src/context.hpp | 2 +- src/event.cpp | 2 +- src/exclusion/input_filter.hpp | 2 +- src/exclusion/rule_filter.hpp | 2 +- src/global_context.cpp | 2 +- src/global_context.hpp | 2 +- src/monitor.hpp | 33 +++ src/parser/global_rule_parser.cpp | 5 +- src/parser/parser.hpp | 2 +- src/parser/parser_v1.cpp | 2 +- src/parser/specification.hpp | 2 +- src/rule.hpp | 226 ------------------ src/rule/base.hpp | 116 +++++++++ src/rule/rule.hpp | 70 ++++++ src/{rule.cpp => rule/threshold_rule.cpp} | 11 +- src/rule/threshold_rule.hpp | 90 +++++++ src/ruleset.hpp | 2 +- src/ruleset_builder.hpp | 2 +- ...counter.hpp => sliding_window_counter.hpp} | 62 +---- tests/event_serializer_test.cpp | 2 +- tests/rule_test.cpp | 2 +- ...st.cpp => sliding_window_counter_test.cpp} | 8 +- 24 files changed, 348 insertions(+), 303 deletions(-) create mode 100644 src/monitor.hpp delete mode 100644 src/rule.hpp create mode 100644 src/rule/base.hpp create mode 100644 src/rule/rule.hpp rename src/{rule.cpp => rule/threshold_rule.cpp} (88%) create mode 100644 src/rule/threshold_rule.hpp rename src/{timed_counter.hpp => sliding_window_counter.hpp} (73%) rename tests/{timed_counter_test.cpp => sliding_window_counter_test.cpp} (94%) diff --git a/cmake/objects.cmake b/cmake/objects.cmake index 252df49b5..a8b67da41 100644 --- a/cmake/objects.cmake +++ b/cmake/objects.cmake @@ -22,7 +22,7 @@ set(LIBDDWAF_SOURCE ${libddwaf_SOURCE_DIR}/src/platform.cpp ${libddwaf_SOURCE_DIR}/src/sha256.cpp ${libddwaf_SOURCE_DIR}/src/uuid.cpp - ${libddwaf_SOURCE_DIR}/src/rule.cpp + ${libddwaf_SOURCE_DIR}/src/rule/threshold_rule.cpp ${libddwaf_SOURCE_DIR}/src/action_mapper.cpp ${libddwaf_SOURCE_DIR}/src/builder/processor_builder.cpp ${libddwaf_SOURCE_DIR}/src/tokenizer/sql_base.cpp diff --git a/src/collection.hpp b/src/collection.hpp index 6f4e883ce..c1e753c48 100644 --- a/src/collection.hpp +++ b/src/collection.hpp @@ -9,7 +9,7 @@ #include "context_allocator.hpp" #include "event.hpp" #include "exclusion/rule_filter.hpp" -#include "rule.hpp" +#include "rule/rule.hpp" namespace ddwaf { diff --git a/src/context.hpp b/src/context.hpp index 7798e080c..d30029baa 100644 --- a/src/context.hpp +++ b/src/context.hpp @@ -17,7 +17,7 @@ #include "exclusion/input_filter.hpp" #include "exclusion/rule_filter.hpp" #include "obfuscator.hpp" -#include "rule.hpp" +#include "rule/rule.hpp" #include "ruleset.hpp" #include "utils.hpp" diff --git a/src/event.cpp b/src/event.cpp index 84433ef26..2123bb4c5 100644 --- a/src/event.cpp +++ b/src/event.cpp @@ -9,7 +9,7 @@ #include "action_mapper.hpp" #include "ddwaf.h" #include "event.hpp" -#include "rule.hpp" +#include "rule/rule.hpp" #include "uuid.hpp" namespace ddwaf { diff --git a/src/exclusion/input_filter.hpp b/src/exclusion/input_filter.hpp index b4869fe9e..92c299313 100644 --- a/src/exclusion/input_filter.hpp +++ b/src/exclusion/input_filter.hpp @@ -13,7 +13,7 @@ #include "clock.hpp" #include "exclusion/object_filter.hpp" #include "object_store.hpp" -#include "rule.hpp" +#include "rule/rule.hpp" namespace ddwaf::exclusion { diff --git a/src/exclusion/rule_filter.hpp b/src/exclusion/rule_filter.hpp index db9be61b7..5e5e3ce03 100644 --- a/src/exclusion/rule_filter.hpp +++ b/src/exclusion/rule_filter.hpp @@ -13,7 +13,7 @@ #include "clock.hpp" #include "exclusion/common.hpp" #include "object_store.hpp" -#include "rule.hpp" +#include "rule/rule.hpp" namespace ddwaf::exclusion { diff --git a/src/global_context.cpp b/src/global_context.cpp index 21d5879c7..1801ca57e 100644 --- a/src/global_context.cpp +++ b/src/global_context.cpp @@ -5,7 +5,7 @@ // Copyright 2021 Datadog, Inc. #include "global_context.hpp" -#include "rule.hpp" +#include "rule/rule.hpp" namespace ddwaf { void global_context::eval(std::vector &events, const object_store &store, cache_type &cache, diff --git a/src/global_context.hpp b/src/global_context.hpp index 099a4ebb6..6f9898840 100644 --- a/src/global_context.hpp +++ b/src/global_context.hpp @@ -8,7 +8,7 @@ #include -#include "rule.hpp" +#include "rule/rule.hpp" namespace ddwaf { diff --git a/src/monitor.hpp b/src/monitor.hpp new file mode 100644 index 000000000..5faac9dc1 --- /dev/null +++ b/src/monitor.hpp @@ -0,0 +1,33 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#pragma once + +#include +namespace ddwaf { + +template class monitor { +public: + template explicit monitor(Args... args) : underlying_value_(args...) {} + + class locked_monitor { + public: + explicit locked_monitor(monitor &m) : m_(m), lock_(m.mtx_) {} + T *operator->() { return &m_.underlying_value_; } + + protected: + monitor &m_; + std::unique_lock lock_; + }; + + locked_monitor operator->() { return locked_monitor{*this}; } + +protected: + std::mutex mtx_{}; + T underlying_value_; +}; + +} // namespace ddwaf diff --git a/src/parser/global_rule_parser.cpp b/src/parser/global_rule_parser.cpp index e31c42199..b30300c50 100644 --- a/src/parser/global_rule_parser.cpp +++ b/src/parser/global_rule_parser.cpp @@ -7,12 +7,13 @@ #include "parser/common.hpp" #include "parser/parser.hpp" #include "parser/specification.hpp" +#include "rule/threshold_rule.hpp" namespace ddwaf::parser::v2 { namespace { -// NOLINTNEXTLINE(bugprone-easily-swappable-parameters) std::unique_ptr parse_indexed_threshold_rule( + // NOLINTNEXTLINE(bugprone-easily-swappable-parameters) std::string id, parameter::map &rule, parameter::map &criteria_map, const object_limits &limits) { auto conditions_array = at(rule, "conditions", {}); @@ -43,8 +44,8 @@ std::unique_ptr parse_indexed_threshold_rule( at>(rule, "on_match", {}), at(rule, "enabled", true)); } -// NOLINTNEXTLINE(bugprone-easily-swappable-parameters) std::unique_ptr parse_threshold_rule( + // NOLINTNEXTLINE(bugprone-easily-swappable-parameters) std::string id, parameter::map &rule, parameter::map &criteria_map, const object_limits &limits) { auto conditions_array = at(rule, "conditions", {}); diff --git a/src/parser/parser.hpp b/src/parser/parser.hpp index 55037415c..98386f22a 100644 --- a/src/parser/parser.hpp +++ b/src/parser/parser.hpp @@ -15,7 +15,7 @@ #include "parameter.hpp" #include "parser/common.hpp" #include "parser/specification.hpp" -#include "rule.hpp" +#include "rule/rule.hpp" #include "ruleset.hpp" #include "ruleset_info.hpp" diff --git a/src/parser/parser_v1.cpp b/src/parser/parser_v1.cpp index 8286e5761..792c368cc 100644 --- a/src/parser/parser_v1.cpp +++ b/src/parser/parser_v1.cpp @@ -19,7 +19,7 @@ #include "parameter.hpp" #include "parser/common.hpp" #include "parser/parser.hpp" -#include "rule.hpp" +#include "rule/rule.hpp" #include "ruleset.hpp" #include "ruleset_info.hpp" diff --git a/src/parser/specification.hpp b/src/parser/specification.hpp index 750ba2c1e..8c02b2121 100644 --- a/src/parser/specification.hpp +++ b/src/parser/specification.hpp @@ -14,7 +14,7 @@ #include "expression.hpp" #include "parameter.hpp" #include "processor/base.hpp" -#include "rule.hpp" +#include "rule/rule.hpp" #include "scanner.hpp" namespace ddwaf::parser { diff --git a/src/rule.hpp b/src/rule.hpp deleted file mode 100644 index 2999225ed..000000000 --- a/src/rule.hpp +++ /dev/null @@ -1,226 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are -// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. -// -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2021 Datadog, Inc. - -#pragma once - -#include -#include -#include -#include - -#include "clock.hpp" -#include "event.hpp" -#include "exclusion/common.hpp" -#include "expression.hpp" -#include "matcher/base.hpp" -#include "object_store.hpp" -#include "timed_counter.hpp" - -namespace ddwaf { - -class base_rule { -public: - base_rule(std::string id, std::string name, std::unordered_map tags, - std::shared_ptr expr, std::vector actions = {}, - bool enabled = true) - : enabled_(enabled), id_(std::move(id)), name_(std::move(name)), tags_(std::move(tags)), - expr_(std::move(expr)), actions_(std::move(actions)) - { - if (!expr_) { - throw std::invalid_argument("rule constructed with null expression"); - } - } - - base_rule(const base_rule &) = delete; - base_rule &operator=(const base_rule &) = delete; - - base_rule(base_rule &&rhs) noexcept = default; - base_rule &operator=(base_rule &&rhs) noexcept = default; - - virtual ~base_rule() = default; - - [[nodiscard]] bool is_enabled() const { return enabled_; } - void toggle(bool value) { enabled_ = value; } - - const std::string &get_id() const { return id_; } - const std::string &get_name() const { return name_; } - - std::string_view get_tag(const std::string &tag) const - { - auto it = tags_.find(tag); - return it == tags_.end() ? std::string_view() : it->second; - } - - std::string_view get_tag_or(const std::string &tag, std::string_view or_value) const - { - auto it = tags_.find(tag); - return it == tags_.end() ? or_value : it->second; - } - - const std::unordered_map &get_tags() const { return tags_; } - const std::unordered_map &get_ancillary_tags() const - { - return ancillary_tags_; - } - - void set_ancillary_tag(const std::string &key, const std::string &value) - { - // Ancillary tags aren't allowed to overlap with standard tags - if (!tags_.contains(key)) { - ancillary_tags_[key] = value; - } - } - - const std::vector &get_actions() const { return actions_; } - - void get_addresses(std::unordered_map &addresses) const - { - return expr_->get_addresses(addresses); - } - - void set_actions(std::vector new_actions) { actions_ = std::move(new_actions); } - -protected: - bool enabled_{true}; - std::string id_; - std::string name_; - std::unordered_map tags_; - std::unordered_map ancillary_tags_; - std::shared_ptr expr_; - std::vector actions_; -}; - -class rule : public base_rule { -public: - enum class source_type : uint8_t { base = 1, user = 2 }; - - using cache_type = expression::cache_type; - - rule(std::string id, std::string name, std::unordered_map tags, - std::shared_ptr expr, std::vector actions = {}, - bool enabled = true, source_type source = source_type::base) - : base_rule(std::move(id), std::move(name), std::move(tags), std::move(expr), - std::move(actions), enabled), - source_(source) - {} - - rule(const rule &) = delete; - rule &operator=(const rule &) = delete; - - rule(rule &&rhs) noexcept = default; - rule &operator=(rule &&rhs) noexcept = default; - - ~rule() override = default; - - virtual std::optional match(const object_store &store, cache_type &cache, - const exclusion::object_set_ref &objects_excluded, - const std::unordered_map> &dynamic_matchers, - ddwaf::timer &deadline) const - { - if (expression::get_result(cache)) { - // An event was already produced, so we skip the rule - return std::nullopt; - } - - auto res = expr_->eval(cache, store, objects_excluded, dynamic_matchers, deadline); - if (!res.outcome) { - return std::nullopt; - } - - return {ddwaf::event{this, expression::get_matches(cache), res.ephemeral, {}}}; - } - - source_type get_source() const { return source_; } - -protected: - source_type source_; -}; - -class base_threshold_rule : public base_rule { -public: - using cache_type = expression::cache_type; - - base_threshold_rule(std::string id, std::string name, - std::unordered_map tags, std::shared_ptr expr, - std::vector actions = {}, bool enabled = true) - : base_rule(std::move(id), std::move(name), std::move(tags), std::move(expr), - std::move(actions), enabled) - {} - ~base_threshold_rule() override = default; - base_threshold_rule(const base_threshold_rule &) = delete; - base_threshold_rule &operator=(const base_threshold_rule &) = delete; - base_threshold_rule(base_threshold_rule &&rhs) noexcept = default; - base_threshold_rule &operator=(base_threshold_rule &&rhs) noexcept = default; - - virtual std::optional eval(const object_store &store, cache_type &cache, - monotonic_clock::time_point now, ddwaf::timer &deadline) = 0; -}; - -class threshold_rule : public base_threshold_rule { -public: - struct evaluation_criteria { - std::size_t threshold; - std::chrono::milliseconds period{}; - }; - - threshold_rule(std::string id, std::string name, - std::unordered_map tags, std::shared_ptr expr, - evaluation_criteria criteria, std::vector actions = {}, bool enabled = true) - : base_threshold_rule(std::move(id), std::move(name), std::move(tags), std::move(expr), - std::move(actions), enabled), - criteria_(criteria), counter_(criteria_.period, criteria_.threshold * 2), - threshold_str_(to_string(criteria_.threshold)) - {} - - ~threshold_rule() override = default; - threshold_rule(const threshold_rule &) = delete; - threshold_rule &operator=(const threshold_rule &) = delete; - threshold_rule(threshold_rule &&rhs) noexcept = delete; - threshold_rule &operator=(threshold_rule &&rhs) noexcept = delete; - - std::optional eval(const object_store &store, cache_type &cache, - monotonic_clock::time_point now, ddwaf::timer &deadline) override; - -protected: - evaluation_criteria criteria_; - timed_counter_ts_ms counter_; - std::string threshold_str_; -}; - -class indexed_threshold_rule : public base_threshold_rule { -public: - struct evaluation_criteria { - std::string name; - target_index target; - std::size_t threshold; - std::chrono::milliseconds period; - }; - - indexed_threshold_rule(std::string id, std::string name, - std::unordered_map tags, std::shared_ptr expr, - evaluation_criteria criteria, std::vector actions = {}, bool enabled = true) - : base_threshold_rule(std::move(id), std::move(name), std::move(tags), std::move(expr), - std::move(actions), enabled), - criteria_(std::move(criteria)), counter_(criteria_.period, 128, criteria_.threshold * 2), - threshold_str_(to_string(criteria_.threshold)) - {} - - ~indexed_threshold_rule() override = default; - indexed_threshold_rule(const indexed_threshold_rule &) = delete; - indexed_threshold_rule &operator=(const indexed_threshold_rule &) = delete; - indexed_threshold_rule(indexed_threshold_rule &&rhs) noexcept = delete; - indexed_threshold_rule &operator=(indexed_threshold_rule &&rhs) noexcept = delete; - - std::optional eval(const object_store &store, cache_type &cache, - monotonic_clock::time_point now, ddwaf::timer &deadline) override; - -protected: - evaluation_criteria criteria_; - indexed_timed_counter_ts_ms counter_; - std::string threshold_str_; -}; - -} // namespace ddwaf diff --git a/src/rule/base.hpp b/src/rule/base.hpp new file mode 100644 index 000000000..9d9150156 --- /dev/null +++ b/src/rule/base.hpp @@ -0,0 +1,116 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#pragma once + +#include +#include +#include +#include + +#include "clock.hpp" +#include "event.hpp" +#include "exclusion/common.hpp" +#include "expression.hpp" +#include "matcher/base.hpp" +#include "object_store.hpp" +#include "sliding_window_counter.hpp" + +namespace ddwaf { + +class base_rule { +public: + base_rule(std::string id, std::string name, std::unordered_map tags, + std::shared_ptr expr, std::vector actions = {}, + bool enabled = true) + : enabled_(enabled), id_(std::move(id)), name_(std::move(name)), tags_(std::move(tags)), + expr_(std::move(expr)), actions_(std::move(actions)) + { + if (!expr_) { + throw std::invalid_argument("rule constructed with null expression"); + } + } + + base_rule(const base_rule &) = delete; + base_rule &operator=(const base_rule &) = delete; + + base_rule(base_rule &&rhs) noexcept = default; + base_rule &operator=(base_rule &&rhs) noexcept = default; + + virtual ~base_rule() = default; + + [[nodiscard]] bool is_enabled() const { return enabled_; } + void toggle(bool value) { enabled_ = value; } + + const std::string &get_id() const { return id_; } + const std::string &get_name() const { return name_; } + + std::string_view get_tag(const std::string &tag) const + { + auto it = tags_.find(tag); + return it == tags_.end() ? std::string_view() : it->second; + } + + std::string_view get_tag_or(const std::string &tag, std::string_view or_value) const + { + auto it = tags_.find(tag); + return it == tags_.end() ? or_value : it->second; + } + + const std::unordered_map &get_tags() const { return tags_; } + const std::unordered_map &get_ancillary_tags() const + { + return ancillary_tags_; + } + + void set_ancillary_tag(const std::string &key, const std::string &value) + { + // Ancillary tags aren't allowed to overlap with standard tags + if (!tags_.contains(key)) { + ancillary_tags_[key] = value; + } + } + + const std::vector &get_actions() const { return actions_; } + + void get_addresses(std::unordered_map &addresses) const + { + return expr_->get_addresses(addresses); + } + + void set_actions(std::vector new_actions) { actions_ = std::move(new_actions); } + +protected: + bool enabled_{true}; + std::string id_; + std::string name_; + std::unordered_map tags_; + std::unordered_map ancillary_tags_; + std::shared_ptr expr_; + std::vector actions_; +}; + +class base_threshold_rule : public base_rule { +public: + using cache_type = expression::cache_type; + + base_threshold_rule(std::string id, std::string name, + std::unordered_map tags, std::shared_ptr expr, + std::vector actions = {}, bool enabled = true) + : base_rule(std::move(id), std::move(name), std::move(tags), std::move(expr), + std::move(actions), enabled) + {} + ~base_threshold_rule() override = default; + base_threshold_rule(const base_threshold_rule &) = delete; + base_threshold_rule &operator=(const base_threshold_rule &) = delete; + base_threshold_rule(base_threshold_rule &&rhs) noexcept = default; + base_threshold_rule &operator=(base_threshold_rule &&rhs) noexcept = default; + + virtual std::optional eval(const object_store &store, cache_type &cache, + monotonic_clock::time_point now, ddwaf::timer &deadline) = 0; +}; + +} // namespace ddwaf diff --git a/src/rule/rule.hpp b/src/rule/rule.hpp new file mode 100644 index 000000000..d2337a16d --- /dev/null +++ b/src/rule/rule.hpp @@ -0,0 +1,70 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#pragma once + +#include +#include +#include +#include + +#include "clock.hpp" +#include "event.hpp" +#include "exclusion/common.hpp" +#include "expression.hpp" +#include "matcher/base.hpp" +#include "object_store.hpp" +#include "rule/base.hpp" + +namespace ddwaf { + +class rule : public base_rule { +public: + enum class source_type : uint8_t { base = 1, user = 2 }; + + using cache_type = expression::cache_type; + + rule(std::string id, std::string name, std::unordered_map tags, + std::shared_ptr expr, std::vector actions = {}, + bool enabled = true, source_type source = source_type::base) + : base_rule(std::move(id), std::move(name), std::move(tags), std::move(expr), + std::move(actions), enabled), + source_(source) + {} + + rule(const rule &) = delete; + rule &operator=(const rule &) = delete; + + rule(rule &&rhs) noexcept = default; + rule &operator=(rule &&rhs) noexcept = default; + + ~rule() override = default; + + virtual std::optional match(const object_store &store, cache_type &cache, + const exclusion::object_set_ref &objects_excluded, + const std::unordered_map> &dynamic_matchers, + ddwaf::timer &deadline) const + { + if (expression::get_result(cache)) { + // An event was already produced, so we skip the rule + return std::nullopt; + } + + auto res = expr_->eval(cache, store, objects_excluded, dynamic_matchers, deadline); + if (!res.outcome) { + return std::nullopt; + } + + return {ddwaf::event{this, expression::get_matches(cache), res.ephemeral, {}}}; + } + + source_type get_source() const { return source_; } + +protected: + source_type source_; +}; + +} // namespace ddwaf diff --git a/src/rule.cpp b/src/rule/threshold_rule.cpp similarity index 88% rename from src/rule.cpp rename to src/rule/threshold_rule.cpp index 0abe43363..a77fcf104 100644 --- a/src/rule.cpp +++ b/src/rule/threshold_rule.cpp @@ -4,8 +4,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2021 Datadog, Inc. -#include "rule.hpp" -#include "timed_counter.hpp" +#include "rule/threshold_rule.hpp" using namespace std::literals; @@ -25,13 +24,13 @@ std::optional threshold_rule::eval(const object_store &store, cache_type } auto ms = std::chrono::duration_cast(now.time_since_epoch()); - auto count = counter_.add_timepoint_and_count(ms); + auto count = counter_->add_timepoint_and_count(ms); if (count > criteria_.threshold) { // Match should be generated differently // Match should be generated differently auto matches = expression::get_matches(cache); matches.emplace_back(condition_match{{}, {}, "threshold", threshold_str_, false}); - return {ddwaf::event{this, std::move(matches), false}}; + return {ddwaf::event{this, std::move(matches), false, {}}}; } return std::nullopt; @@ -58,14 +57,14 @@ std::optional indexed_threshold_rule::eval(const object_store &store, cac auto ms = std::chrono::duration_cast(now.time_since_epoch()); std::string_view key{obj->stringValue, static_cast(obj->nbEntries)}; - auto count = counter_.add_timepoint_and_count(key, ms); + auto count = counter_->add_timepoint_and_count(key, ms); if (count > criteria_.threshold) { // Match should be generated differently auto matches = expression::get_matches(cache); matches.emplace_back( condition_match{{{"input"sv, object_to_string(*obj), criteria_.name, {}}}, {}, "threshold", threshold_str_, false}); - return {ddwaf::event{this, std::move(matches), false}}; + return {ddwaf::event{this, std::move(matches), false, {}}}; } return std::nullopt; diff --git a/src/rule/threshold_rule.hpp b/src/rule/threshold_rule.hpp new file mode 100644 index 000000000..3ac278c01 --- /dev/null +++ b/src/rule/threshold_rule.hpp @@ -0,0 +1,90 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#pragma once + +#include +#include +#include +#include + +#include "clock.hpp" +#include "event.hpp" +#include "exclusion/common.hpp" +#include "expression.hpp" +#include "matcher/base.hpp" +#include "monitor.hpp" +#include "object_store.hpp" +#include "rule/base.hpp" +#include "sliding_window_counter.hpp" + +namespace ddwaf { + +class threshold_rule : public base_threshold_rule { +public: + struct evaluation_criteria { + std::size_t threshold; + std::chrono::milliseconds period{}; + }; + + threshold_rule(std::string id, std::string name, + std::unordered_map tags, std::shared_ptr expr, + evaluation_criteria criteria, std::vector actions = {}, bool enabled = true) + : base_threshold_rule(std::move(id), std::move(name), std::move(tags), std::move(expr), + std::move(actions), enabled), + criteria_(criteria), counter_(criteria_.period, criteria_.threshold * 2), + threshold_str_(to_string(criteria_.threshold)) + {} + + ~threshold_rule() override = default; + threshold_rule(const threshold_rule &) = delete; + threshold_rule &operator=(const threshold_rule &) = delete; + threshold_rule(threshold_rule &&rhs) noexcept = delete; + threshold_rule &operator=(threshold_rule &&rhs) noexcept = delete; + + std::optional eval(const object_store &store, cache_type &cache, + monotonic_clock::time_point now, ddwaf::timer &deadline) override; + +protected: + evaluation_criteria criteria_; + monitor counter_; + std::string threshold_str_; +}; + +class indexed_threshold_rule : public base_threshold_rule { +public: + struct evaluation_criteria { + std::string name; + target_index target; + std::size_t threshold; + std::chrono::milliseconds period; + }; + + indexed_threshold_rule(std::string id, std::string name, + std::unordered_map tags, std::shared_ptr expr, + evaluation_criteria criteria, std::vector actions = {}, bool enabled = true) + : base_threshold_rule(std::move(id), std::move(name), std::move(tags), std::move(expr), + std::move(actions), enabled), + criteria_(std::move(criteria)), counter_(criteria_.period, 128, criteria_.threshold * 2), + threshold_str_(to_string(criteria_.threshold)) + {} + + ~indexed_threshold_rule() override = default; + indexed_threshold_rule(const indexed_threshold_rule &) = delete; + indexed_threshold_rule &operator=(const indexed_threshold_rule &) = delete; + indexed_threshold_rule(indexed_threshold_rule &&rhs) noexcept = delete; + indexed_threshold_rule &operator=(indexed_threshold_rule &&rhs) noexcept = delete; + + std::optional eval(const object_store &store, cache_type &cache, + monotonic_clock::time_point now, ddwaf::timer &deadline) override; + +protected: + evaluation_criteria criteria_; + monitor counter_; + std::string threshold_str_; +}; + +} // namespace ddwaf diff --git a/src/ruleset.hpp b/src/ruleset.hpp index cde05b9f3..57ad1f9cf 100644 --- a/src/ruleset.hpp +++ b/src/ruleset.hpp @@ -17,7 +17,7 @@ #include "global_context.hpp" #include "obfuscator.hpp" #include "processor/base.hpp" -#include "rule.hpp" +#include "rule/rule.hpp" #include "scanner.hpp" namespace ddwaf { diff --git a/src/ruleset_builder.hpp b/src/ruleset_builder.hpp index 370626b87..0769393f7 100644 --- a/src/ruleset_builder.hpp +++ b/src/ruleset_builder.hpp @@ -16,7 +16,7 @@ #include "indexer.hpp" #include "parameter.hpp" #include "parser/specification.hpp" -#include "rule.hpp" +#include "rule/rule.hpp" #include "ruleset.hpp" #include "ruleset_info.hpp" diff --git a/src/timed_counter.hpp b/src/sliding_window_counter.hpp similarity index 73% rename from src/timed_counter.hpp rename to src/sliding_window_counter.hpp index 25bcc84ba..7088e0deb 100644 --- a/src/timed_counter.hpp +++ b/src/sliding_window_counter.hpp @@ -21,10 +21,10 @@ concept is_duration = std::is_same_v instead template requires is_duration -class timed_counter { +class sliding_window_counter { public: - timed_counter() = default; - explicit timed_counter(T period, std::size_t max_window_size = 100) : period_(period) + sliding_window_counter() = default; + explicit sliding_window_counter(T period, std::size_t max_window_size = 100) : period_(period) { time_points_.resize(max_window_size); } @@ -108,48 +108,12 @@ class timed_counter { std::size_t buckets{0}; }; -template - requires is_duration -class timed_counter_ts : protected timed_counter { -public: - timed_counter_ts() = default; - explicit timed_counter_ts(T period, std::size_t max_window_size = 100) - : timed_counter(period, max_window_size) - {} - std::size_t add_timepoint_and_count(T point) - { - std::lock_guard lock(mtx_); - return timed_counter::add_timepoint_and_count(point); - } - - T last_timepoint() const - { - std::lock_guard lock(mtx_); - return timed_counter::last_timepoint(); - } - - std::size_t update_count(T point) - { - std::lock_guard lock(mtx_); - return timed_counter::update_count(point); - } - - void reset() - { - std::lock_guard lock(mtx_); - timed_counter::reset(); - } - -protected: - mutable std::mutex mtx_; -}; - template requires is_duration -class indexed_timed_counter_ts { +class indexed_sliding_window_counter { public: - indexed_timed_counter_ts() = default; - explicit indexed_timed_counter_ts( + indexed_sliding_window_counter() = default; + explicit indexed_sliding_window_counter( // NOLINTNEXTLINE(bugprone-easily-swappable-parameters) Duration period, std::size_t max_index_size = 32, std::size_t max_window_size = 100) : period_(period), max_index_size_(max_index_size), max_window_size_(max_window_size) @@ -159,15 +123,14 @@ class indexed_timed_counter_ts { requires std::is_constructible_v std::size_t add_timepoint_and_count(T key, Duration point) { - std::lock_guard lock(mtx_); auto it = index_.find(key); if (it == index_.end()) { if (index_.size() == max_index_size_) { remove_oldest_entry(point); } - auto [new_it, res] = - index_.emplace(Key{key}, timed_counter{period_, max_window_size_}); + auto [new_it, res] = index_.emplace( + Key{key}, sliding_window_counter{period_, max_window_size_}); if (!res) { return 0; } @@ -200,12 +163,11 @@ class indexed_timed_counter_ts { Duration period_; std::size_t max_index_size_{}; std::size_t max_window_size_{}; - std::map, std::less<>> index_; - mutable std::mutex mtx_; + std::map, std::less<>> index_; }; -using timed_counter_ts_ms = timed_counter_ts; -using indexed_timed_counter_ts_ms = - indexed_timed_counter_ts; +using sliding_window_counter_ms = sliding_window_counter; +using indexed_sliding_window_counter_ms = + indexed_sliding_window_counter; } // namespace ddwaf diff --git a/tests/event_serializer_test.cpp b/tests/event_serializer_test.cpp index 36d9227bf..285e7b041 100644 --- a/tests/event_serializer_test.cpp +++ b/tests/event_serializer_test.cpp @@ -7,7 +7,7 @@ #include "test_utils.hpp" #include "event.hpp" -#include "rule.hpp" +#include "rule/rule.hpp" #include "utils.hpp" using namespace ddwaf; diff --git a/tests/rule_test.cpp b/tests/rule_test.cpp index 33addcaa7..36f014860 100644 --- a/tests/rule_test.cpp +++ b/tests/rule_test.cpp @@ -8,7 +8,7 @@ #include "matcher/exact_match.hpp" #include "matcher/ip_match.hpp" #include "object_store.hpp" -#include "rule.hpp" +#include "rule/rule.hpp" #include "test_utils.hpp" using namespace ddwaf; diff --git a/tests/timed_counter_test.cpp b/tests/sliding_window_counter_test.cpp similarity index 94% rename from tests/timed_counter_test.cpp rename to tests/sliding_window_counter_test.cpp index df91c19c5..a868d7625 100644 --- a/tests/timed_counter_test.cpp +++ b/tests/sliding_window_counter_test.cpp @@ -4,9 +4,9 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2021 Datadog, Inc. +#include "sliding_window_counter.hpp" #include "test.hpp" #include "test_utils.hpp" -#include "timed_counter.hpp" using namespace std::literals; using namespace std::chrono_literals; @@ -15,7 +15,7 @@ namespace { TEST(TestTimedCounter, BasicMs) { - ddwaf::timed_counter_ts window{10ms, 5}; + ddwaf::sliding_window_counter window{10ms, 5}; EXPECT_EQ(window.add_timepoint_and_count(1ms), 1); EXPECT_EQ(window.add_timepoint_and_count(11ms), 1); @@ -35,7 +35,7 @@ TEST(TestTimedCounter, BasicMs) TEST(TestTimedCounter, BasicS) { - ddwaf::timed_counter_ts window{10s, 5}; + ddwaf::sliding_window_counter window{10s, 5}; EXPECT_EQ(window.add_timepoint_and_count(1s), 1); EXPECT_EQ(window.add_timepoint_and_count(11s), 1); @@ -54,7 +54,7 @@ TEST(TestTimedCounter, BasicS) TEST(TestIndexedTimedCounter, BasicString) { - ddwaf::indexed_timed_counter_ts window{10s, 5, 5}; + ddwaf::indexed_sliding_window_counter window{10s, 5, 5}; EXPECT_EQ(window.add_timepoint_and_count("admin"sv, 1s), 1); EXPECT_EQ(window.add_timepoint_and_count("user"sv, 10s), 1); From f46d8c45fb56c34629b101874733fddf88e0ca2a Mon Sep 17 00:00:00 2001 From: Anil Mahtani <929854+Anilm3@users.noreply.github.com> Date: Mon, 19 Aug 2024 14:39:26 +0100 Subject: [PATCH 03/11] Fix types --- src/parser/global_rule_parser.cpp | 4 ++-- src/rule/threshold_rule.hpp | 4 ++-- src/sliding_window_counter.hpp | 20 ++++++++++---------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/parser/global_rule_parser.cpp b/src/parser/global_rule_parser.cpp index b30300c50..9381dd298 100644 --- a/src/parser/global_rule_parser.cpp +++ b/src/parser/global_rule_parser.cpp @@ -34,7 +34,7 @@ std::unique_ptr parse_indexed_threshold_rule( } indexed_threshold_rule::evaluation_criteria criteria; - criteria.threshold = at(criteria_map, "threshold"); + criteria.threshold = at(criteria_map, "threshold"); criteria.period = std::chrono::milliseconds(at(criteria_map, "period")); criteria.name = at(criteria_map, "input"); criteria.target = get_target_index(criteria.name); @@ -66,7 +66,7 @@ std::unique_ptr parse_threshold_rule( } threshold_rule::evaluation_criteria criteria; - criteria.threshold = at(criteria_map, "threshold"); + criteria.threshold = at(criteria_map, "threshold"); criteria.period = std::chrono::milliseconds(at(criteria_map, "period")); return std::make_unique(std::move(id), at(rule, "name"), diff --git a/src/rule/threshold_rule.hpp b/src/rule/threshold_rule.hpp index 3ac278c01..61bb8ac93 100644 --- a/src/rule/threshold_rule.hpp +++ b/src/rule/threshold_rule.hpp @@ -26,7 +26,7 @@ namespace ddwaf { class threshold_rule : public base_threshold_rule { public: struct evaluation_criteria { - std::size_t threshold; + uint64_t threshold; std::chrono::milliseconds period{}; }; @@ -59,7 +59,7 @@ class indexed_threshold_rule : public base_threshold_rule { struct evaluation_criteria { std::string name; target_index target; - std::size_t threshold; + uint64_t threshold; std::chrono::milliseconds period; }; diff --git a/src/sliding_window_counter.hpp b/src/sliding_window_counter.hpp index 7088e0deb..371ab427b 100644 --- a/src/sliding_window_counter.hpp +++ b/src/sliding_window_counter.hpp @@ -29,7 +29,7 @@ class sliding_window_counter { time_points_.resize(max_window_size); } - std::size_t add_timepoint_and_count(T point) + uint64_t add_timepoint_and_count(T point) { // Discard old elements update_count(point); @@ -70,7 +70,7 @@ class sliding_window_counter { return time_points_[index].point; } - std::size_t update_count(T point) + uint64_t update_count(T point) { // Discard old elements auto window_begin = point - period_; @@ -85,27 +85,27 @@ class sliding_window_counter { void reset() { left = right = count = buckets = 0; } protected: - [[nodiscard]] std::size_t increment(std::size_t value) const + [[nodiscard]] uint64_t increment(uint64_t value) const { return (value + 1) % time_points_.size(); } - [[nodiscard]] std::size_t decrement(std::size_t value) const + [[nodiscard]] uint64_t decrement(uint64_t value) const { return (value + time_points_.size() - 1) % time_points_.size(); } struct time_bucket { T point; - std::size_t count; + uint64_t count; }; std::chrono::milliseconds period_{}; std::vector time_points_; - std::size_t left{0}; - std::size_t right{0}; - std::size_t count{0}; - std::size_t buckets{0}; + uint64_t left{0}; + uint64_t right{0}; + uint64_t count{0}; + uint64_t buckets{0}; }; template @@ -121,7 +121,7 @@ class indexed_sliding_window_counter { template requires std::is_constructible_v - std::size_t add_timepoint_and_count(T key, Duration point) + uint64_t add_timepoint_and_count(T key, Duration point) { auto it = index_.find(key); if (it == index_.end()) { From c0093ff60c57f7e79f69aed3eb456ba1c094367f Mon Sep 17 00:00:00 2001 From: Anil Mahtani <929854+Anilm3@users.noreply.github.com> Date: Mon, 19 Aug 2024 17:27:28 +0100 Subject: [PATCH 04/11] Filtered matcher for threshold rules --- src/parser/global_rule_parser.cpp | 37 ++++++++++++++++++++++--------- src/rule/threshold_rule.cpp | 27 ++++++++++++++++++---- src/rule/threshold_rule.hpp | 7 ++++-- src/sliding_window_counter.hpp | 2 ++ 4 files changed, 56 insertions(+), 17 deletions(-) diff --git a/src/parser/global_rule_parser.cpp b/src/parser/global_rule_parser.cpp index 9381dd298..d8ecff57e 100644 --- a/src/parser/global_rule_parser.cpp +++ b/src/parser/global_rule_parser.cpp @@ -13,8 +13,7 @@ namespace ddwaf::parser::v2 { namespace { std::unique_ptr parse_indexed_threshold_rule( - // NOLINTNEXTLINE(bugprone-easily-swappable-parameters) - std::string id, parameter::map &rule, parameter::map &criteria_map, const object_limits &limits) + std::string id, parameter::map &rule, const object_limits &limits) { auto conditions_array = at(rule, "conditions", {}); @@ -33,20 +32,36 @@ std::unique_ptr parse_indexed_threshold_rule( throw ddwaf::parsing_error("missing key 'type'"); } + auto criteria_map = at(rule, "criteria"); indexed_threshold_rule::evaluation_criteria criteria; criteria.threshold = at(criteria_map, "threshold"); criteria.period = std::chrono::milliseconds(at(criteria_map, "period")); - criteria.name = at(criteria_map, "input"); - criteria.target = get_target_index(criteria.name); + + auto filter_map = at(rule, "filter"); + + auto input_map = at(filter_map, "input"); + criteria.filter.name = at(input_map, "address"); + criteria.filter.target = get_target_index(criteria.filter.name); + + if (filter_map.contains("operator")) { + auto operator_name = at(filter_map, "operator"); + auto params = at(filter_map, "parameters"); + + auto [data_id, matcher] = parse_matcher(operator_name, params); + if (!data_id.empty()) { + throw ddwaf::parsing_error("unsupported: global rule filter with data ID"); + } + + criteria.filter.matcher = std::move(matcher); + } return std::make_unique(std::move(id), at(rule, "name"), - std::move(tags), std::move(expr), criteria, + std::move(tags), std::move(expr), std::move(criteria), at>(rule, "on_match", {}), at(rule, "enabled", true)); } std::unique_ptr parse_threshold_rule( - // NOLINTNEXTLINE(bugprone-easily-swappable-parameters) - std::string id, parameter::map &rule, parameter::map &criteria_map, const object_limits &limits) + std::string id, parameter::map &rule, const object_limits &limits) { auto conditions_array = at(rule, "conditions", {}); @@ -65,6 +80,7 @@ std::unique_ptr parse_threshold_rule( throw ddwaf::parsing_error("missing key 'type'"); } + auto criteria_map = at(rule, "criteria"); threshold_rule::evaluation_criteria criteria; criteria.threshold = at(criteria_map, "threshold"); criteria.period = std::chrono::milliseconds(at(criteria_map, "period")); @@ -77,11 +93,10 @@ std::unique_ptr parse_threshold_rule( std::unique_ptr parse_global_rule( std::string id, parameter::map &rule, const object_limits &limits) { - auto criteria = at(rule, "criteria"); - if (criteria.contains("input")) { - return parse_indexed_threshold_rule(std::move(id), rule, criteria, limits); + if (rule.contains("filter")) { + return parse_indexed_threshold_rule(std::move(id), rule, limits); } - return parse_threshold_rule(std::move(id), rule, criteria, limits); + return parse_threshold_rule(std::move(id), rule, limits); } } // namespace diff --git a/src/rule/threshold_rule.cpp b/src/rule/threshold_rule.cpp index a77fcf104..01a9f12de 100644 --- a/src/rule/threshold_rule.cpp +++ b/src/rule/threshold_rule.cpp @@ -5,6 +5,7 @@ // Copyright 2021 Datadog, Inc. #include "rule/threshold_rule.hpp" +#include using namespace std::literals; @@ -44,7 +45,7 @@ std::optional indexed_threshold_rule::eval(const object_store &store, cac return std::nullopt; } - auto [obj, attr] = store.get_target(criteria_.target); + auto [obj, attr] = store.get_target(criteria_.filter.target); if (obj == nullptr || obj->type != DDWAF_OBJ_STRING) { return std::nullopt; } @@ -55,14 +56,32 @@ std::optional indexed_threshold_rule::eval(const object_store &store, cac } auto ms = std::chrono::duration_cast(now.time_since_epoch()); - std::string_view key{obj->stringValue, static_cast(obj->nbEntries)}; - auto count = counter_->add_timepoint_and_count(key, ms); + std::string filtered_key; + if (criteria_.filter.matcher) { + // The value must be filtered + auto [res, highlight] = criteria_.filter.matcher->match(*obj); + if (!res) { + // If the matcher fails, there is nothing to filter on + return std::nullopt; + } + + filtered_key = std::move(highlight); + } + + uint64_t count = 0; + if (filtered_key.empty()) { + std::string_view key{obj->stringValue, static_cast(obj->nbEntries)}; + count = counter_->add_timepoint_and_count(key, ms); + } else { + count = counter_->add_timepoint_and_count(std::move(filtered_key), ms); + } + if (count > criteria_.threshold) { // Match should be generated differently auto matches = expression::get_matches(cache); matches.emplace_back( - condition_match{{{"input"sv, object_to_string(*obj), criteria_.name, {}}}, {}, + condition_match{{{"input"sv, object_to_string(*obj), criteria_.filter.name, {}}}, {}, "threshold", threshold_str_, false}); return {ddwaf::event{this, std::move(matches), false, {}}}; } diff --git a/src/rule/threshold_rule.hpp b/src/rule/threshold_rule.hpp index 61bb8ac93..3231085c3 100644 --- a/src/rule/threshold_rule.hpp +++ b/src/rule/threshold_rule.hpp @@ -57,10 +57,13 @@ class threshold_rule : public base_threshold_rule { class indexed_threshold_rule : public base_threshold_rule { public: struct evaluation_criteria { - std::string name; - target_index target; uint64_t threshold; std::chrono::milliseconds period; + struct { + std::string name; + target_index target; + std::unique_ptr matcher; + } filter; }; indexed_threshold_rule(std::string id, std::string name, diff --git a/src/sliding_window_counter.hpp b/src/sliding_window_counter.hpp index 371ab427b..ca7cb2aab 100644 --- a/src/sliding_window_counter.hpp +++ b/src/sliding_window_counter.hpp @@ -7,6 +7,7 @@ #pragma once #include +#include #include #include #include @@ -129,6 +130,7 @@ class indexed_sliding_window_counter { remove_oldest_entry(point); } + std::cout << key << '\n'; auto [new_it, res] = index_.emplace( Key{key}, sliding_window_counter{period_, max_window_size_}); if (!res) { From 3560323dc24fbf845e9389cf49f191c5389fdce4 Mon Sep 17 00:00:00 2001 From: Anil Mahtani <929854+Anilm3@users.noreply.github.com> Date: Mon, 19 Aug 2024 17:46:22 +0100 Subject: [PATCH 05/11] Add transformers to filtered threshold rule --- src/parser/global_rule_parser.cpp | 9 +++++++ src/parser/parser.hpp | 1 + src/parser/transformer_parser.cpp | 23 +++++++++++++++++ src/rule/threshold_rule.cpp | 41 ++++++++++++++++++++++++------- src/rule/threshold_rule.hpp | 1 + 5 files changed, 66 insertions(+), 9 deletions(-) diff --git a/src/parser/global_rule_parser.cpp b/src/parser/global_rule_parser.cpp index d8ecff57e..831d37e70 100644 --- a/src/parser/global_rule_parser.cpp +++ b/src/parser/global_rule_parser.cpp @@ -43,6 +43,15 @@ std::unique_ptr parse_indexed_threshold_rule( criteria.filter.name = at(input_map, "address"); criteria.filter.target = get_target_index(criteria.filter.name); + if (input_map.contains("transformers")) { + auto input_transformers = at(input_map, "transformers"); + if (input_transformers.size() > limits.max_transformers_per_address) { + throw ddwaf::parsing_error("number of transformers beyond allowed limit"); + } + + criteria.filter.transformers = parse_transformers(input_transformers); + } + if (filter_map.contains("operator")) { auto operator_name = at(filter_map, "operator"); auto params = at(filter_map, "parameters"); diff --git a/src/parser/parser.hpp b/src/parser/parser.hpp index 98386f22a..b93658299 100644 --- a/src/parser/parser.hpp +++ b/src/parser/parser.hpp @@ -59,6 +59,7 @@ std::shared_ptr parse_simplified_expression(const parameter::vector address_container &addresses, const object_limits &limits); std::vector parse_transformers(const parameter::vector &root, data_source &source); +std::vector parse_transformers(const parameter::vector &root); std::pair> parse_matcher( std::string_view name, const parameter::map ¶ms); diff --git a/src/parser/transformer_parser.cpp b/src/parser/transformer_parser.cpp index f8159fe5c..19a4b4d7b 100644 --- a/src/parser/transformer_parser.cpp +++ b/src/parser/transformer_parser.cpp @@ -39,4 +39,27 @@ std::vector parse_transformers(const parameter::vector &root, da return transformers; } +std::vector parse_transformers(const parameter::vector &root) +{ + if (root.empty()) { + return {}; + } + + std::vector transformers; + transformers.reserve(root.size()); + + for (const auto &transformer_param : root) { + auto transformer = static_cast(transformer_param); + auto id = transformer_from_string(transformer); + if (id.has_value()) { + transformers.emplace_back(id.value()); + } else if (transformer == "keys_only" || transformer == "values_only") { + throw ddwaf::parsing_error("source transformer not supported within this context"); + } else { + throw ddwaf::parsing_error("invalid transformer " + std::string(transformer)); + } + } + return transformers; +} + } // namespace ddwaf::parser::v2 diff --git a/src/rule/threshold_rule.cpp b/src/rule/threshold_rule.cpp index 01a9f12de..b8a9b6995 100644 --- a/src/rule/threshold_rule.cpp +++ b/src/rule/threshold_rule.cpp @@ -5,12 +5,42 @@ // Copyright 2021 Datadog, Inc. #include "rule/threshold_rule.hpp" -#include +#include "transformer/manager.hpp" using namespace std::literals; namespace ddwaf { +namespace { + +std::string filter_with_matcher(ddwaf_object &src, auto &filter) +{ + if (!filter.transformers.empty()) { + ddwaf_object dst; + ddwaf_object_invalid(&dst); + + auto transformed = transformer::manager::transform(src, dst, filter.transformers); + const scope_exit on_exit([&dst] { ddwaf_object_free(&dst); }); + if (transformed) { + auto [res, highlight] = filter.matcher->match(dst); + if (!res) { + return {}; + } + return highlight; + } + } + + // The value must be filtered + auto [res, highlight] = filter.matcher->match(src); + if (!res) { + // If the matcher fails, there is nothing to filter on + return {}; + } + return highlight; +} + +} // namespace + std::optional threshold_rule::eval(const object_store &store, cache_type &cache, monotonic_clock::time_point now, ddwaf::timer &deadline) { @@ -59,14 +89,7 @@ std::optional indexed_threshold_rule::eval(const object_store &store, cac std::string filtered_key; if (criteria_.filter.matcher) { - // The value must be filtered - auto [res, highlight] = criteria_.filter.matcher->match(*obj); - if (!res) { - // If the matcher fails, there is nothing to filter on - return std::nullopt; - } - - filtered_key = std::move(highlight); + filter_with_matcher(*obj, criteria_.filter); } uint64_t count = 0; diff --git a/src/rule/threshold_rule.hpp b/src/rule/threshold_rule.hpp index 3231085c3..d1d7e778a 100644 --- a/src/rule/threshold_rule.hpp +++ b/src/rule/threshold_rule.hpp @@ -62,6 +62,7 @@ class indexed_threshold_rule : public base_threshold_rule { struct { std::string name; target_index target; + std::vector transformers{}; std::unique_ptr matcher; } filter; }; From f3c0de6c395c30aa57e05ff175b9ea9c0343f589 Mon Sep 17 00:00:00 2001 From: Anil Mahtani <929854+Anilm3@users.noreply.github.com> Date: Wed, 21 Aug 2024 17:37:44 +0100 Subject: [PATCH 06/11] Lint fixes --- src/collection.cpp | 2 +- src/event.cpp | 2 +- src/exclusion/input_filter.cpp | 2 +- src/exclusion/rule_filter.cpp | 2 +- src/global_context.cpp | 8 +++++++- src/parser/global_rule_parser.cpp | 21 ++++++++++++++++++--- src/parser/rule_parser.cpp | 2 +- src/rule/threshold_rule.cpp | 28 +++++++++++++++++++++------- src/ruleset_builder.cpp | 2 +- 9 files changed, 52 insertions(+), 17 deletions(-) diff --git a/src/collection.cpp b/src/collection.cpp index 5cc3fc1e2..76d565e15 100644 --- a/src/collection.cpp +++ b/src/collection.cpp @@ -20,7 +20,7 @@ #include "log.hpp" #include "matcher/base.hpp" #include "object_store.hpp" -#include "rule.hpp" +#include "rule/rule.hpp" namespace ddwaf { diff --git a/src/event.cpp b/src/event.cpp index e1d2027d4..f8d0795ec 100644 --- a/src/event.cpp +++ b/src/event.cpp @@ -14,7 +14,7 @@ #include "ddwaf.h" #include "event.hpp" #include "obfuscator.hpp" -#include "rule/rule.hpp" +#include "rule/rule.hpp" // IWYU pragma: keep #include "uuid.hpp" namespace ddwaf { diff --git a/src/exclusion/input_filter.cpp b/src/exclusion/input_filter.cpp index dfcdc7330..26ce153dc 100644 --- a/src/exclusion/input_filter.cpp +++ b/src/exclusion/input_filter.cpp @@ -19,7 +19,7 @@ #include "log.hpp" #include "matcher/base.hpp" #include "object_store.hpp" -#include "rule.hpp" +#include "rule/rule.hpp" namespace ddwaf::exclusion { diff --git a/src/exclusion/rule_filter.cpp b/src/exclusion/rule_filter.cpp index b117a1a83..2325cdab5 100644 --- a/src/exclusion/rule_filter.cpp +++ b/src/exclusion/rule_filter.cpp @@ -18,7 +18,7 @@ #include "log.hpp" #include "matcher/base.hpp" #include "object_store.hpp" -#include "rule.hpp" +#include "rule/rule.hpp" namespace ddwaf::exclusion { diff --git a/src/global_context.cpp b/src/global_context.cpp index 1801ca57e..e061e90ed 100644 --- a/src/global_context.cpp +++ b/src/global_context.cpp @@ -3,9 +3,15 @@ // // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2021 Datadog, Inc. +#include +#include +#include "clock.hpp" +#include "event.hpp" #include "global_context.hpp" -#include "rule/rule.hpp" +#include "log.hpp" +#include "object_store.hpp" +#include "rule/base.hpp" namespace ddwaf { void global_context::eval(std::vector &events, const object_store &store, cache_type &cache, diff --git a/src/parser/global_rule_parser.cpp b/src/parser/global_rule_parser.cpp index 831d37e70..df43619aa 100644 --- a/src/parser/global_rule_parser.cpp +++ b/src/parser/global_rule_parser.cpp @@ -3,11 +3,26 @@ // // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2021 Datadog, Inc. - +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "exception.hpp" +#include "global_context.hpp" +#include "log.hpp" +#include "parameter.hpp" #include "parser/common.hpp" #include "parser/parser.hpp" -#include "parser/specification.hpp" +#include "rule/base.hpp" #include "rule/threshold_rule.hpp" +#include "utils.hpp" namespace ddwaf::parser::v2 { @@ -120,7 +135,7 @@ std::shared_ptr parse_global_rules( auto rule_map = static_cast(rule_param); std::string id; try { - address_container addresses; + const address_container addresses; id = at(rule_map, "id"); if (ids.find(id) != ids.end()) { DDWAF_WARN("Duplicate global rule {}", id); diff --git a/src/parser/rule_parser.cpp b/src/parser/rule_parser.cpp index 9d142c04c..8df423117 100644 --- a/src/parser/rule_parser.cpp +++ b/src/parser/rule_parser.cpp @@ -16,7 +16,7 @@ #include "parser/common.hpp" #include "parser/parser.hpp" #include "parser/specification.hpp" -#include "rule.hpp" +#include "rule/rule.hpp" #include "transformer/base.hpp" #include "utils.hpp" diff --git a/src/rule/threshold_rule.cpp b/src/rule/threshold_rule.cpp index b8a9b6995..f6f965e97 100644 --- a/src/rule/threshold_rule.cpp +++ b/src/rule/threshold_rule.cpp @@ -3,9 +3,23 @@ // // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2021 Datadog, Inc. - +#include +#include +#include +#include +#include +#include +#include + +#include "clock.hpp" +#include "condition/base.hpp" +#include "ddwaf.h" +#include "event.hpp" +#include "expression.hpp" +#include "object_store.hpp" #include "rule/threshold_rule.hpp" #include "transformer/manager.hpp" +#include "utils.hpp" using namespace std::literals; @@ -57,10 +71,10 @@ std::optional threshold_rule::eval(const object_store &store, cache_type auto ms = std::chrono::duration_cast(now.time_since_epoch()); auto count = counter_->add_timepoint_and_count(ms); if (count > criteria_.threshold) { - // Match should be generated differently // Match should be generated differently auto matches = expression::get_matches(cache); - matches.emplace_back(condition_match{{}, {}, "threshold", threshold_str_, false}); + condition_match match{{}, {}, "threshold", threshold_str_, false}; + matches.emplace_back(std::move(match)); return {ddwaf::event{this, std::move(matches), false, {}}}; } @@ -94,7 +108,7 @@ std::optional indexed_threshold_rule::eval(const object_store &store, cac uint64_t count = 0; if (filtered_key.empty()) { - std::string_view key{obj->stringValue, static_cast(obj->nbEntries)}; + const std::string_view key{obj->stringValue, static_cast(obj->nbEntries)}; count = counter_->add_timepoint_and_count(key, ms); } else { count = counter_->add_timepoint_and_count(std::move(filtered_key), ms); @@ -103,9 +117,9 @@ std::optional indexed_threshold_rule::eval(const object_store &store, cac if (count > criteria_.threshold) { // Match should be generated differently auto matches = expression::get_matches(cache); - matches.emplace_back( - condition_match{{{"input"sv, object_to_string(*obj), criteria_.filter.name, {}}}, {}, - "threshold", threshold_str_, false}); + condition_match match{{{"input"sv, object_to_string(*obj), criteria_.filter.name, {}}}, {}, + "threshold", threshold_str_, false}; + matches.emplace_back(std::move(match)); return {ddwaf::event{this, std::move(matches), false, {}}}; } diff --git a/src/ruleset_builder.cpp b/src/ruleset_builder.cpp index 527a10f7c..b40968126 100644 --- a/src/ruleset_builder.cpp +++ b/src/ruleset_builder.cpp @@ -22,7 +22,7 @@ #include "parser/common.hpp" #include "parser/parser.hpp" #include "parser/specification.hpp" -#include "rule.hpp" +#include "rule/rule.hpp" #include "ruleset.hpp" #include "ruleset_builder.hpp" #include "ruleset_info.hpp" From d85b34cc77511ef22cb4828558018c7bb624ea90 Mon Sep 17 00:00:00 2001 From: Anil Mahtani <929854+Anilm3@users.noreply.github.com> Date: Thu, 22 Aug 2024 13:07:54 +0100 Subject: [PATCH 07/11] Filter with keypath --- src/parser/global_rule_parser.cpp | 1 + src/rule/threshold_rule.cpp | 32 +++++++++++++++++++++++++++---- src/rule/threshold_rule.hpp | 1 + src/sliding_window_counter.hpp | 1 - 4 files changed, 30 insertions(+), 5 deletions(-) diff --git a/src/parser/global_rule_parser.cpp b/src/parser/global_rule_parser.cpp index df43619aa..b7b70cfc0 100644 --- a/src/parser/global_rule_parser.cpp +++ b/src/parser/global_rule_parser.cpp @@ -57,6 +57,7 @@ std::unique_ptr parse_indexed_threshold_rule( auto input_map = at(filter_map, "input"); criteria.filter.name = at(input_map, "address"); criteria.filter.target = get_target_index(criteria.filter.name); + criteria.filter.key_path = at>(input_map, "key_path", {}); if (input_map.contains("transformers")) { auto input_transformers = at(input_map, "transformers"); diff --git a/src/rule/threshold_rule.cpp b/src/rule/threshold_rule.cpp index f6f965e97..344263f2e 100644 --- a/src/rule/threshold_rule.cpp +++ b/src/rule/threshold_rule.cpp @@ -16,6 +16,7 @@ #include "ddwaf.h" #include "event.hpp" #include "expression.hpp" +#include "iterator.hpp" #include "object_store.hpp" #include "rule/threshold_rule.hpp" #include "transformer/manager.hpp" @@ -27,7 +28,7 @@ namespace ddwaf { namespace { -std::string filter_with_matcher(ddwaf_object &src, auto &filter) +std::string filter_with_matcher(const ddwaf_object &src, auto &filter) { if (!filter.transformers.empty()) { ddwaf_object dst; @@ -53,6 +54,29 @@ std::string filter_with_matcher(ddwaf_object &src, auto &filter) return highlight; } +const ddwaf_object *get_object(const object_store &store, const auto &filter) +{ + auto [obj, attr] = store.get_target(filter.target); + if (obj == nullptr) { + return nullptr; + } + + if (filter.key_path.empty()) { + if (obj->type != DDWAF_OBJ_STRING) { + return nullptr; + } + + return obj; + } + + object::value_iterator it{obj, filter.key_path, {}}; + if (!it || it.type() != DDWAF_OBJ_STRING) { + return nullptr; + } + + return *it; +} + } // namespace std::optional threshold_rule::eval(const object_store &store, cache_type &cache, @@ -89,8 +113,8 @@ std::optional indexed_threshold_rule::eval(const object_store &store, cac return std::nullopt; } - auto [obj, attr] = store.get_target(criteria_.filter.target); - if (obj == nullptr || obj->type != DDWAF_OBJ_STRING) { + const auto *obj = get_object(store, criteria_.filter); + if (obj == nullptr) { return std::nullopt; } @@ -103,7 +127,7 @@ std::optional indexed_threshold_rule::eval(const object_store &store, cac std::string filtered_key; if (criteria_.filter.matcher) { - filter_with_matcher(*obj, criteria_.filter); + filtered_key = filter_with_matcher(*obj, criteria_.filter); } uint64_t count = 0; diff --git a/src/rule/threshold_rule.hpp b/src/rule/threshold_rule.hpp index d1d7e778a..444292444 100644 --- a/src/rule/threshold_rule.hpp +++ b/src/rule/threshold_rule.hpp @@ -62,6 +62,7 @@ class indexed_threshold_rule : public base_threshold_rule { struct { std::string name; target_index target; + std::vector key_path; std::vector transformers{}; std::unique_ptr matcher; } filter; diff --git a/src/sliding_window_counter.hpp b/src/sliding_window_counter.hpp index ca7cb2aab..5cd18c915 100644 --- a/src/sliding_window_counter.hpp +++ b/src/sliding_window_counter.hpp @@ -130,7 +130,6 @@ class indexed_sliding_window_counter { remove_oldest_entry(point); } - std::cout << key << '\n'; auto [new_it, res] = index_.emplace( Key{key}, sliding_window_counter{period_, max_window_size_}); if (!res) { From d4609c1eb71ea9c10f9a2fa66b0909f739eaa1a2 Mon Sep 17 00:00:00 2001 From: Anil Mahtani <929854+Anilm3@users.noreply.github.com> Date: Thu, 22 Aug 2024 16:59:22 +0100 Subject: [PATCH 08/11] Enforce expiration once threshold is breached, refactor into lru_cache --- src/lru_cache.hpp | 85 ++++++++++++++++++++++++ src/parser/global_rule_parser.cpp | 2 + src/rule/threshold_rule.cpp | 20 +++--- src/rule/threshold_rule.hpp | 57 ++++++++++++++-- src/sliding_window_counter.hpp | 65 +----------------- src/traits.hpp | 4 ++ tests/sliding_window_counter_test.cpp | 96 +++++++++++++-------------- 7 files changed, 204 insertions(+), 125 deletions(-) create mode 100644 src/lru_cache.hpp diff --git a/src/lru_cache.hpp b/src/lru_cache.hpp new file mode 100644 index 000000000..7672e99d7 --- /dev/null +++ b/src/lru_cache.hpp @@ -0,0 +1,85 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "traits.hpp" + +namespace ddwaf { + +template + requires is_duration +class lru_cache { +public: + explicit lru_cache(ConstructorType constructor, std::size_t max_index_size = 32) + : constructor_(std::move(constructor)), max_index_size_(max_index_size) + {} + + + template + requires std::is_constructible_v + DataType& emplace_or_retrieve(CompatKeyType key, DurationType timepoint) + { + auto it = index_.find(key); + if (it == index_.end()) { + if (index_.size() == max_index_size_) { + remove_oldest_entry(timepoint); + } + + + auto [new_it, res] = index_.emplace(KeyType{key}, cache_entry{timepoint, constructor_()}); + if (!res) { + throw std::out_of_range("failed to add element to cache"); + } + return new_it->second.data; + } + + it->second.latest_timepoint = std::max(timepoint, it->second.latest_timepoint); + + return it->second.data; + } + +protected: + struct cache_entry { + DurationType latest_timepoint; + DataType data; + }; + + void remove_oldest_entry(DurationType timepoint) + { + using iterator_type = typename decltype(index_)::iterator; + + DurationType max_delta{0}; + iterator_type oldest_it; + for (auto it = index_.begin(); it != index_.end(); ++it) { + auto window_last = it->second.latest_timepoint; + auto delta = timepoint - window_last; + if (delta > max_delta) { + max_delta = delta; + oldest_it = it; + } + } + + index_.erase(oldest_it); + } + + ConstructorType constructor_; + std::size_t max_index_size_{}; + std::map> index_; +}; + +template +using lru_cache_ms = lru_cache; + +} // namespace ddwaf diff --git a/src/parser/global_rule_parser.cpp b/src/parser/global_rule_parser.cpp index b7b70cfc0..12cb50087 100644 --- a/src/parser/global_rule_parser.cpp +++ b/src/parser/global_rule_parser.cpp @@ -51,6 +51,7 @@ std::unique_ptr parse_indexed_threshold_rule( indexed_threshold_rule::evaluation_criteria criteria; criteria.threshold = at(criteria_map, "threshold"); criteria.period = std::chrono::milliseconds(at(criteria_map, "period")); + criteria.duration = std::chrono::milliseconds(at(criteria_map, "duration")); auto filter_map = at(rule, "filter"); @@ -109,6 +110,7 @@ std::unique_ptr parse_threshold_rule( threshold_rule::evaluation_criteria criteria; criteria.threshold = at(criteria_map, "threshold"); criteria.period = std::chrono::milliseconds(at(criteria_map, "period")); + criteria.duration = std::chrono::milliseconds(at(criteria_map, "duration")); return std::make_unique(std::move(id), at(rule, "name"), std::move(tags), std::move(expr), criteria, diff --git a/src/rule/threshold_rule.cpp b/src/rule/threshold_rule.cpp index 344263f2e..2e64e5c82 100644 --- a/src/rule/threshold_rule.cpp +++ b/src/rule/threshold_rule.cpp @@ -5,7 +5,7 @@ // Copyright 2021 Datadog, Inc. #include #include -#include +#include #include #include #include @@ -93,8 +93,7 @@ std::optional threshold_rule::eval(const object_store &store, cache_type } auto ms = std::chrono::duration_cast(now.time_since_epoch()); - auto count = counter_->add_timepoint_and_count(ms); - if (count > criteria_.threshold) { + if (counter_->add_timepoint_and_count(ms)) { // Match should be generated differently auto matches = expression::get_matches(cache); condition_match match{{}, {}, "threshold", threshold_str_, false}; @@ -128,17 +127,18 @@ std::optional indexed_threshold_rule::eval(const object_store &store, cac std::string filtered_key; if (criteria_.filter.matcher) { filtered_key = filter_with_matcher(*obj, criteria_.filter); + } else { + filtered_key = {obj->stringValue, static_cast(obj->nbEntries)}; } - uint64_t count = 0; - if (filtered_key.empty()) { - const std::string_view key{obj->stringValue, static_cast(obj->nbEntries)}; - count = counter_->add_timepoint_and_count(key, ms); - } else { - count = counter_->add_timepoint_and_count(std::move(filtered_key), ms); + bool match = false; + { + const std::unique_lock lock{mtx_}; + auto &counter = counter_cache_.emplace_or_retrieve(filtered_key, ms); + match = counter.add_timepoint_and_count(ms); } - if (count > criteria_.threshold) { + if (match) { // Match should be generated differently auto matches = expression::get_matches(cache); condition_match match{{{"input"sv, object_to_string(*obj), criteria_.filter.name, {}}}, {}, diff --git a/src/rule/threshold_rule.hpp b/src/rule/threshold_rule.hpp index 444292444..2de538578 100644 --- a/src/rule/threshold_rule.hpp +++ b/src/rule/threshold_rule.hpp @@ -15,6 +15,7 @@ #include "event.hpp" #include "exclusion/common.hpp" #include "expression.hpp" +#include "lru_cache.hpp" #include "matcher/base.hpp" #include "monitor.hpp" #include "object_store.hpp" @@ -23,11 +24,42 @@ namespace ddwaf { +class threshold_counter { +public: + threshold_counter(std::chrono::milliseconds period, uint64_t threshold, + std::chrono::milliseconds threshold_duration) + : threshold_(threshold), counter_{period, threshold * 2}, + threshold_duration_(threshold_duration) + {} + + bool add_timepoint_and_count(std::chrono::milliseconds ms) + { + if (expiration_ > ms) { + return true; + } + + auto count = counter_.add_timepoint_and_count(ms); + if (count > threshold_) { + expiration_ = ms + threshold_duration_; + return true; + } + + return false; + } + +protected: + uint64_t threshold_; + sliding_window_counter_ms counter_; + std::chrono::milliseconds threshold_duration_; + std::chrono::milliseconds expiration_{0}; +}; + class threshold_rule : public base_threshold_rule { public: struct evaluation_criteria { uint64_t threshold; std::chrono::milliseconds period{}; + std::chrono::milliseconds duration{}; }; threshold_rule(std::string id, std::string name, @@ -35,7 +67,8 @@ class threshold_rule : public base_threshold_rule { evaluation_criteria criteria, std::vector actions = {}, bool enabled = true) : base_threshold_rule(std::move(id), std::move(name), std::move(tags), std::move(expr), std::move(actions), enabled), - criteria_(criteria), counter_(criteria_.period, criteria_.threshold * 2), + criteria_(criteria), + counter_(criteria_.period, criteria_.threshold * 2, criteria_.duration), threshold_str_(to_string(criteria_.threshold)) {} @@ -50,7 +83,7 @@ class threshold_rule : public base_threshold_rule { protected: evaluation_criteria criteria_; - monitor counter_; + monitor counter_; std::string threshold_str_; }; @@ -59,6 +92,7 @@ class indexed_threshold_rule : public base_threshold_rule { struct evaluation_criteria { uint64_t threshold; std::chrono::milliseconds period; + std::chrono::milliseconds duration{}; struct { std::string name; target_index target; @@ -73,7 +107,10 @@ class indexed_threshold_rule : public base_threshold_rule { evaluation_criteria criteria, std::vector actions = {}, bool enabled = true) : base_threshold_rule(std::move(id), std::move(name), std::move(tags), std::move(expr), std::move(actions), enabled), - criteria_(std::move(criteria)), counter_(criteria_.period, 128, criteria_.threshold * 2), + criteria_(std::move(criteria)), + counter_cache_(threshold_counter_constructor{criteria_.period, criteria_.threshold, + criteria_.duration}, + 128), threshold_str_(to_string(criteria_.threshold)) {} @@ -87,9 +124,21 @@ class indexed_threshold_rule : public base_threshold_rule { monotonic_clock::time_point now, ddwaf::timer &deadline) override; protected: + struct threshold_counter_constructor { + std::chrono::milliseconds period; + uint64_t threshold; + std::chrono::milliseconds threshold_duration; + + threshold_counter operator()() const + { + return threshold_counter{period, threshold, threshold_duration}; + } + }; + evaluation_criteria criteria_; - monitor counter_; + lru_cache_ms counter_cache_; std::string threshold_str_; + std::mutex mtx_; }; } // namespace ddwaf diff --git a/src/sliding_window_counter.hpp b/src/sliding_window_counter.hpp index 5cd18c915..e6ee233ab 100644 --- a/src/sliding_window_counter.hpp +++ b/src/sliding_window_counter.hpp @@ -14,10 +14,9 @@ #include #include -namespace ddwaf { +#include "traits.hpp" -template -concept is_duration = std::is_same_v, T>; +namespace ddwaf { // Perhaps this should use an std::chrono::time_point instead template @@ -109,66 +108,6 @@ class sliding_window_counter { uint64_t buckets{0}; }; -template - requires is_duration -class indexed_sliding_window_counter { -public: - indexed_sliding_window_counter() = default; - explicit indexed_sliding_window_counter( - // NOLINTNEXTLINE(bugprone-easily-swappable-parameters) - Duration period, std::size_t max_index_size = 32, std::size_t max_window_size = 100) - : period_(period), max_index_size_(max_index_size), max_window_size_(max_window_size) - {} - - template - requires std::is_constructible_v - uint64_t add_timepoint_and_count(T key, Duration point) - { - auto it = index_.find(key); - if (it == index_.end()) { - if (index_.size() == max_index_size_) { - remove_oldest_entry(point); - } - - auto [new_it, res] = index_.emplace( - Key{key}, sliding_window_counter{period_, max_window_size_}); - if (!res) { - return 0; - } - - it = new_it; - } - - return it->second.add_timepoint_and_count(point); - } - -protected: - void remove_oldest_entry(Duration point) - { - using iterator_type = typename decltype(index_)::iterator; - - Duration max_delta{0}; - iterator_type oldest_it; - for (auto it = index_.begin(); it != index_.end(); ++it) { - auto window_last = it->second.last_timepoint(); - auto delta = point - window_last; - if (delta > max_delta) { - max_delta = delta; - oldest_it = it; - } - } - - index_.erase(oldest_it); - } - - Duration period_; - std::size_t max_index_size_{}; - std::size_t max_window_size_{}; - std::map, std::less<>> index_; -}; - using sliding_window_counter_ms = sliding_window_counter; -using indexed_sliding_window_counter_ms = - indexed_sliding_window_counter; } // namespace ddwaf diff --git a/src/traits.hpp b/src/traits.hpp index 8be8629f0..b7c97b6d5 100644 --- a/src/traits.hpp +++ b/src/traits.hpp @@ -6,6 +6,7 @@ #pragma once +#include #include // Generate a tuple containing a subset of the arguments @@ -49,3 +50,6 @@ function_traits make_traits(Result (Class::*)(Args...) const) template function_traits make_traits(Result (Class::*)(Args...)); + +template +concept is_duration = std::is_same_v, T>; diff --git a/tests/sliding_window_counter_test.cpp b/tests/sliding_window_counter_test.cpp index a868d7625..f5ae30499 100644 --- a/tests/sliding_window_counter_test.cpp +++ b/tests/sliding_window_counter_test.cpp @@ -52,53 +52,53 @@ TEST(TestTimedCounter, BasicS) EXPECT_EQ(window.add_timepoint_and_count(40s), 1); } -TEST(TestIndexedTimedCounter, BasicString) -{ - ddwaf::indexed_sliding_window_counter window{10s, 5, 5}; - - EXPECT_EQ(window.add_timepoint_and_count("admin"sv, 1s), 1); - EXPECT_EQ(window.add_timepoint_and_count("user"sv, 10s), 1); - EXPECT_EQ(window.add_timepoint_and_count("docker"sv, 11s), 1); - EXPECT_EQ(window.add_timepoint_and_count("nobody"sv, 11s), 1); - EXPECT_EQ(window.add_timepoint_and_count("root"sv, 11s), 1); - // Admin should be removed, as it's the latest - EXPECT_EQ(window.add_timepoint_and_count("mail"sv, 11s), 1); - - // User will now be removed - EXPECT_EQ(window.add_timepoint_and_count("admin"sv, 11s), 1); - EXPECT_EQ(window.add_timepoint_and_count("admin"sv, 12s), 2); - EXPECT_EQ(window.add_timepoint_and_count("admin"sv, 13s), 3); - EXPECT_EQ(window.add_timepoint_and_count("admin"sv, 14s), 4); - EXPECT_EQ(window.add_timepoint_and_count("admin"sv, 15s), 5); - EXPECT_EQ(window.add_timepoint_and_count("admin"sv, 16s), 5); - - EXPECT_EQ(window.add_timepoint_and_count("docker"sv, 17s), 2); - EXPECT_EQ(window.add_timepoint_and_count("nobody"sv, 17s), 2); - EXPECT_EQ(window.add_timepoint_and_count("root"sv, 17s), 2); - EXPECT_EQ(window.add_timepoint_and_count("mail"sv, 17s), 2); - - EXPECT_EQ(window.add_timepoint_and_count("docker"sv, 18s), 3); - EXPECT_EQ(window.add_timepoint_and_count("nobody"sv, 18s), 3); - EXPECT_EQ(window.add_timepoint_and_count("root"sv, 18s), 3); - EXPECT_EQ(window.add_timepoint_and_count("mail"sv, 18s), 3); - - EXPECT_EQ(window.add_timepoint_and_count("docker"sv, 19s), 4); - EXPECT_EQ(window.add_timepoint_and_count("nobody"sv, 19s), 4); - EXPECT_EQ(window.add_timepoint_and_count("root"sv, 19s), 4); - EXPECT_EQ(window.add_timepoint_and_count("mail"sv, 19s), 4); - - EXPECT_EQ(window.add_timepoint_and_count("docker"sv, 20s), 5); - EXPECT_EQ(window.add_timepoint_and_count("nobody"sv, 20s), 5); - EXPECT_EQ(window.add_timepoint_and_count("root"sv, 20s), 5); - EXPECT_EQ(window.add_timepoint_and_count("mail"sv, 20s), 5); - EXPECT_EQ(window.add_timepoint_and_count("admin"sv, 20s), 5); - - EXPECT_EQ(window.add_timepoint_and_count("nobody"sv, 21s), 5); - EXPECT_EQ(window.add_timepoint_and_count("root"sv, 21s), 5); - EXPECT_EQ(window.add_timepoint_and_count("mail"sv, 21s), 5); - EXPECT_EQ(window.add_timepoint_and_count("admin"sv, 21s), 5); - // Docker will now be removed - EXPECT_EQ(window.add_timepoint_and_count("user"sv, 21s), 1); -} +/*TEST(TestIndexedTimedCounter, BasicString)*/ +/*{*/ +/*ddwaf::indexed_sliding_window_counter window{10s, 5, 5};*/ + +/*EXPECT_EQ(window.add_timepoint_and_count("admin"sv, 1s), 1);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("user"sv, 10s), 1);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("docker"sv, 11s), 1);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("nobody"sv, 11s), 1);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("root"sv, 11s), 1);*/ +/*// Admin should be removed, as it's the latest*/ +/*EXPECT_EQ(window.add_timepoint_and_count("mail"sv, 11s), 1);*/ + +/*// User will now be removed*/ +/*EXPECT_EQ(window.add_timepoint_and_count("admin"sv, 11s), 1);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("admin"sv, 12s), 2);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("admin"sv, 13s), 3);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("admin"sv, 14s), 4);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("admin"sv, 15s), 5);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("admin"sv, 16s), 5);*/ + +/*EXPECT_EQ(window.add_timepoint_and_count("docker"sv, 17s), 2);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("nobody"sv, 17s), 2);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("root"sv, 17s), 2);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("mail"sv, 17s), 2);*/ + +/*EXPECT_EQ(window.add_timepoint_and_count("docker"sv, 18s), 3);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("nobody"sv, 18s), 3);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("root"sv, 18s), 3);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("mail"sv, 18s), 3);*/ + +/*EXPECT_EQ(window.add_timepoint_and_count("docker"sv, 19s), 4);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("nobody"sv, 19s), 4);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("root"sv, 19s), 4);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("mail"sv, 19s), 4);*/ + +/*EXPECT_EQ(window.add_timepoint_and_count("docker"sv, 20s), 5);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("nobody"sv, 20s), 5);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("root"sv, 20s), 5);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("mail"sv, 20s), 5);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("admin"sv, 20s), 5);*/ + +/*EXPECT_EQ(window.add_timepoint_and_count("nobody"sv, 21s), 5);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("root"sv, 21s), 5);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("mail"sv, 21s), 5);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("admin"sv, 21s), 5);*/ +/*// Docker will now be removed*/ +/*EXPECT_EQ(window.add_timepoint_and_count("user"sv, 21s), 1);*/ +/*}*/ } // namespace From 9674750f828111283ba9422496874e0ebc2831c5 Mon Sep 17 00:00:00 2001 From: Anil Mahtani <929854+Anilm3@users.noreply.github.com> Date: Fri, 13 Sep 2024 09:10:22 +0100 Subject: [PATCH 09/11] Add key path --- src/rule/threshold_rule.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rule/threshold_rule.cpp b/src/rule/threshold_rule.cpp index 2e64e5c82..5939013ac 100644 --- a/src/rule/threshold_rule.cpp +++ b/src/rule/threshold_rule.cpp @@ -141,7 +141,7 @@ std::optional indexed_threshold_rule::eval(const object_store &store, cac if (match) { // Match should be generated differently auto matches = expression::get_matches(cache); - condition_match match{{{"input"sv, object_to_string(*obj), criteria_.filter.name, {}}}, {}, + condition_match match{{{"input"sv, object_to_string(*obj), criteria_.filter.name, criteria_.filter.key_path}}, {}, "threshold", threshold_str_, false}; matches.emplace_back(std::move(match)); return {ddwaf::event{this, std::move(matches), false, {}}}; From 984f4fcbf381db5425552529943f157b9943d7f1 Mon Sep 17 00:00:00 2001 From: Anil Mahtani <929854+Anilm3@users.noreply.github.com> Date: Tue, 15 Oct 2024 21:53:40 +0100 Subject: [PATCH 10/11] Fix after merge --- src/lru_cache.hpp | 9 ++++----- src/parser/global_rule_parser.cpp | 3 ++- src/rule/threshold_rule.cpp | 5 +++-- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/lru_cache.hpp b/src/lru_cache.hpp index 7672e99d7..189ba4aa8 100644 --- a/src/lru_cache.hpp +++ b/src/lru_cache.hpp @@ -26,10 +26,9 @@ class lru_cache { : constructor_(std::move(constructor)), max_index_size_(max_index_size) {} - template requires std::is_constructible_v - DataType& emplace_or_retrieve(CompatKeyType key, DurationType timepoint) + DataType &emplace_or_retrieve(CompatKeyType key, DurationType timepoint) { auto it = index_.find(key); if (it == index_.end()) { @@ -37,15 +36,15 @@ class lru_cache { remove_oldest_entry(timepoint); } - - auto [new_it, res] = index_.emplace(KeyType{key}, cache_entry{timepoint, constructor_()}); + auto [new_it, res] = + index_.emplace(KeyType{key}, cache_entry{timepoint, constructor_()}); if (!res) { throw std::out_of_range("failed to add element to cache"); } return new_it->second.data; } - it->second.latest_timepoint = std::max(timepoint, it->second.latest_timepoint); + it->second.latest_timepoint = std::max(timepoint, it->second.latest_timepoint); return it->second.data; } diff --git a/src/parser/global_rule_parser.cpp b/src/parser/global_rule_parser.cpp index 12cb50087..0d90ac65b 100644 --- a/src/parser/global_rule_parser.cpp +++ b/src/parser/global_rule_parser.cpp @@ -19,6 +19,7 @@ #include "log.hpp" #include "parameter.hpp" #include "parser/common.hpp" +#include "parser/matcher_parser.hpp" #include "parser/parser.hpp" #include "rule/base.hpp" #include "rule/threshold_rule.hpp" @@ -73,7 +74,7 @@ std::unique_ptr parse_indexed_threshold_rule( auto operator_name = at(filter_map, "operator"); auto params = at(filter_map, "parameters"); - auto [data_id, matcher] = parse_matcher(operator_name, params); + auto [data_id, matcher] = parse_any_matcher(operator_name, params); if (!data_id.empty()) { throw ddwaf::parsing_error("unsupported: global rule filter with data ID"); } diff --git a/src/rule/threshold_rule.cpp b/src/rule/threshold_rule.cpp index 5939013ac..1f77f3453 100644 --- a/src/rule/threshold_rule.cpp +++ b/src/rule/threshold_rule.cpp @@ -141,8 +141,9 @@ std::optional indexed_threshold_rule::eval(const object_store &store, cac if (match) { // Match should be generated differently auto matches = expression::get_matches(cache); - condition_match match{{{"input"sv, object_to_string(*obj), criteria_.filter.name, criteria_.filter.key_path}}, {}, - "threshold", threshold_str_, false}; + condition_match match{ + {{"input"sv, object_to_string(*obj), criteria_.filter.name, criteria_.filter.key_path}}, + {}, "threshold", threshold_str_, false}; matches.emplace_back(std::move(match)); return {ddwaf::event{this, std::move(matches), false, {}}}; } From d595b3eb5139c98d6bd9dbfb41bcf9f8c94d12f0 Mon Sep 17 00:00:00 2001 From: Anil Mahtani <929854+Anilm3@users.noreply.github.com> Date: Tue, 15 Oct 2024 21:57:25 +0100 Subject: [PATCH 11/11] Install libreadline on tools job --- .github/workflows/test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3151451f3..e281352a2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -264,6 +264,9 @@ jobs: - name: Create directories run: mkdir Debug + - name: Install dependencies + run: sudo apt update ; sudo apt install -y libreadline-dev + - name: CMake env: CC: gcc-12