diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3151451f3..e281352a2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -264,6 +264,9 @@ jobs: - name: Create directories run: mkdir Debug + - name: Install dependencies + run: sudo apt update ; sudo apt install -y libreadline-dev + - name: CMake env: CC: gcc-12 diff --git a/cmake/objects.cmake b/cmake/objects.cmake index efc8011de..6edd83157 100644 --- a/cmake/objects.cmake +++ b/cmake/objects.cmake @@ -4,6 +4,7 @@ set(LIBDDWAF_SOURCE ${libddwaf_SOURCE_DIR}/src/parameter.cpp ${libddwaf_SOURCE_DIR}/src/interface.cpp ${libddwaf_SOURCE_DIR}/src/context.cpp + ${libddwaf_SOURCE_DIR}/src/global_context.cpp ${libddwaf_SOURCE_DIR}/src/context_allocator.cpp ${libddwaf_SOURCE_DIR}/src/event.cpp ${libddwaf_SOURCE_DIR}/src/object.cpp @@ -21,6 +22,7 @@ set(LIBDDWAF_SOURCE ${libddwaf_SOURCE_DIR}/src/platform.cpp ${libddwaf_SOURCE_DIR}/src/sha256.cpp ${libddwaf_SOURCE_DIR}/src/uuid.cpp + ${libddwaf_SOURCE_DIR}/src/rule/threshold_rule.cpp ${libddwaf_SOURCE_DIR}/src/action_mapper.cpp ${libddwaf_SOURCE_DIR}/src/builder/processor_builder.cpp ${libddwaf_SOURCE_DIR}/src/tokenizer/sql_base.cpp @@ -45,6 +47,7 @@ set(LIBDDWAF_SOURCE ${libddwaf_SOURCE_DIR}/src/parser/rule_override_parser.cpp ${libddwaf_SOURCE_DIR}/src/parser/scanner_parser.cpp ${libddwaf_SOURCE_DIR}/src/parser/exclusion_parser.cpp + ${libddwaf_SOURCE_DIR}/src/parser/global_rule_parser.cpp ${libddwaf_SOURCE_DIR}/src/processor/extract_schema.cpp ${libddwaf_SOURCE_DIR}/src/processor/fingerprint.cpp ${libddwaf_SOURCE_DIR}/src/condition/exists.cpp diff --git a/src/collection.cpp b/src/collection.cpp index 5cc3fc1e2..76d565e15 100644 --- a/src/collection.cpp +++ b/src/collection.cpp @@ -20,7 +20,7 @@ #include "log.hpp" #include "matcher/base.hpp" #include "object_store.hpp" -#include "rule.hpp" +#include "rule/rule.hpp" namespace ddwaf { diff --git a/src/collection.hpp b/src/collection.hpp index 6f4e883ce..c1e753c48 100644 --- a/src/collection.hpp +++ b/src/collection.hpp @@ -9,7 +9,7 @@ #include "context_allocator.hpp" #include "event.hpp" #include "exclusion/rule_filter.hpp" -#include "rule.hpp" +#include "rule/rule.hpp" namespace ddwaf { diff --git a/src/context.cpp b/src/context.cpp index f66c57bde..a981231d2 100644 --- a/src/context.cpp +++ b/src/context.cpp @@ -103,6 +103,10 @@ DDWAF_RET_CODE context::run(optional_ref persistent, try { eval_preprocessors(derived, deadline); + if (ruleset_->gctx) { + ruleset_->gctx->eval(events, store_, gctx_cache_, deadline); + } + // If no rule targets are available, there is no point in evaluating them const bool should_eval_rules = check_new_rule_targets(); const bool should_eval_filters = should_eval_rules || check_new_filter_targets(); @@ -114,7 +118,7 @@ DDWAF_RET_CODE context::run(optional_ref persistent, const auto &policy = eval_filters(deadline); if (should_eval_rules) { - events = eval_rules(policy, deadline); + eval_rules(events, policy, deadline); if (!events.empty()) { set_context_event_address(store_); } @@ -230,11 +234,9 @@ exclusion::context_policy &context::eval_filters(ddwaf::timer &deadline) return exclusion_policy_; } -std::vector context::eval_rules( - const exclusion::context_policy &policy, ddwaf::timer &deadline) +void context::eval_rules(std::vector &events, const exclusion::context_policy &policy, + ddwaf::timer &deadline) { - std::vector events; - auto eval_collection = [&](const auto &type, const auto &collection) { auto it = collection_cache_.find(type); if (it == collection_cache_.end()) { @@ -267,8 +269,6 @@ std::vector context::eval_rules( DDWAF_DEBUG("Evaluating base collection '{}'", type); eval_collection(type, collection); } - - return events; } } // namespace ddwaf diff --git a/src/context.hpp b/src/context.hpp index df6dcc2bf..d30029baa 100644 --- a/src/context.hpp +++ b/src/context.hpp @@ -17,7 +17,7 @@ #include "exclusion/input_filter.hpp" #include "exclusion/rule_filter.hpp" #include "obfuscator.hpp" -#include "rule.hpp" +#include "rule/rule.hpp" #include "ruleset.hpp" #include "utils.hpp" @@ -50,7 +50,8 @@ class context { // This function below returns a reference to an internal object, // however using them this way helps with testing exclusion::context_policy &eval_filters(ddwaf::timer &deadline); - std::vector eval_rules(const exclusion::context_policy &policy, ddwaf::timer &deadline); + void eval_rules(std::vector &events, const exclusion::context_policy &policy, + ddwaf::timer &deadline); protected: bool is_first_run() const { return collection_cache_.empty(); } @@ -89,6 +90,8 @@ class context { // Cache of collections to avoid processing once a result has been obtained memory::unordered_map collection_cache_{}; + + global_context::cache_type gctx_cache_{}; }; class context_wrapper { diff --git a/src/event.cpp b/src/event.cpp index 2d6c64342..f8d0795ec 100644 --- a/src/event.cpp +++ b/src/event.cpp @@ -14,7 +14,7 @@ #include "ddwaf.h" #include "event.hpp" #include "obfuscator.hpp" -#include "rule.hpp" +#include "rule/rule.hpp" // IWYU pragma: keep #include "uuid.hpp" namespace ddwaf { @@ -151,7 +151,7 @@ void add_action_to_tracker(action_tracker &actions, std::string_view id, action_ } } -void serialize_rule(const ddwaf::rule &rule, ddwaf_object &rule_map) +void serialize_rule(const ddwaf::base_rule &rule, ddwaf_object &rule_map) { ddwaf_object tmp; ddwaf_object tags_map; @@ -186,7 +186,7 @@ void serialize_empty_rule(ddwaf_object &rule_map) ddwaf_object_map_add(&rule_map, "tags", &tags_map); } -void serialize_and_consolidate_rule_actions(const ddwaf::rule &rule, ddwaf_object &rule_map, +void serialize_and_consolidate_rule_actions(const ddwaf::base_rule &rule, ddwaf_object &rule_map, std::string_view action_override, action_tracker &actions, ddwaf_object &stack_id) { const auto &rule_actions = rule.get_actions(); diff --git a/src/event.hpp b/src/event.hpp index 1d1d34445..a55384205 100644 --- a/src/event.hpp +++ b/src/event.hpp @@ -15,10 +15,10 @@ namespace ddwaf { -class rule; +class base_rule; struct event { - const ddwaf::rule *rule{nullptr}; + const ddwaf::base_rule *rule{nullptr}; std::vector matches; bool ephemeral{false}; std::string_view action_override; diff --git a/src/exclusion/input_filter.cpp b/src/exclusion/input_filter.cpp index dfcdc7330..26ce153dc 100644 --- a/src/exclusion/input_filter.cpp +++ b/src/exclusion/input_filter.cpp @@ -19,7 +19,7 @@ #include "log.hpp" #include "matcher/base.hpp" #include "object_store.hpp" -#include "rule.hpp" +#include "rule/rule.hpp" namespace ddwaf::exclusion { diff --git a/src/exclusion/input_filter.hpp b/src/exclusion/input_filter.hpp index b4869fe9e..92c299313 100644 --- a/src/exclusion/input_filter.hpp +++ b/src/exclusion/input_filter.hpp @@ -13,7 +13,7 @@ #include "clock.hpp" #include "exclusion/object_filter.hpp" #include "object_store.hpp" -#include "rule.hpp" +#include "rule/rule.hpp" namespace ddwaf::exclusion { diff --git a/src/exclusion/rule_filter.cpp b/src/exclusion/rule_filter.cpp index b117a1a83..2325cdab5 100644 --- a/src/exclusion/rule_filter.cpp +++ b/src/exclusion/rule_filter.cpp @@ -18,7 +18,7 @@ #include "log.hpp" #include "matcher/base.hpp" #include "object_store.hpp" -#include "rule.hpp" +#include "rule/rule.hpp" namespace ddwaf::exclusion { diff --git a/src/exclusion/rule_filter.hpp b/src/exclusion/rule_filter.hpp index db9be61b7..5e5e3ce03 100644 --- a/src/exclusion/rule_filter.hpp +++ b/src/exclusion/rule_filter.hpp @@ -13,7 +13,7 @@ #include "clock.hpp" #include "exclusion/common.hpp" #include "object_store.hpp" -#include "rule.hpp" +#include "rule/rule.hpp" namespace ddwaf::exclusion { diff --git a/src/global_context.cpp b/src/global_context.cpp new file mode 100644 index 000000000..e061e90ed --- /dev/null +++ b/src/global_context.cpp @@ -0,0 +1,40 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. +#include +#include + +#include "clock.hpp" +#include "event.hpp" +#include "global_context.hpp" +#include "log.hpp" +#include "object_store.hpp" +#include "rule/base.hpp" + +namespace ddwaf { +void global_context::eval(std::vector &events, const object_store &store, cache_type &cache, + ddwaf::timer &deadline) +{ + auto timepoint = monotonic_clock::now(); + + DDWAF_DEBUG("Evaluating global rules"); + for (const auto &rule : rules_) { + auto cache_it = cache.find(rule.get()); + if (cache_it == cache.end()) { + auto [new_it, res] = cache.emplace(rule.get(), base_threshold_rule::cache_type{}); + if (!res) { + continue; + } + cache_it = new_it; + } + DDWAF_DEBUG("Evaluating rule {}", rule->get_id()); + auto opt_evt = rule->eval(store, cache_it->second, timepoint, deadline); + if (opt_evt.has_value()) { + events.emplace_back(std::move(opt_evt.value())); + } + } +} + +} // namespace ddwaf diff --git a/src/global_context.hpp b/src/global_context.hpp new file mode 100644 index 000000000..6f9898840 --- /dev/null +++ b/src/global_context.hpp @@ -0,0 +1,35 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#pragma once + +#include + +#include "rule/rule.hpp" + +namespace ddwaf { + +class global_context { +public: + using cache_type = std::unordered_map; + + explicit global_context(std::vector> rules) + : rules_(std::move(rules)) + {} + global_context(const global_context &) = delete; + global_context &operator=(const global_context &) = delete; + global_context(global_context &&) = default; + global_context &operator=(global_context &&) = delete; + ~global_context() = default; + + void eval(std::vector &events, const object_store &store, cache_type &lcache, + ddwaf::timer &deadline); + +protected: + std::vector> rules_; +}; + +} // namespace ddwaf diff --git a/src/lru_cache.hpp b/src/lru_cache.hpp new file mode 100644 index 000000000..189ba4aa8 --- /dev/null +++ b/src/lru_cache.hpp @@ -0,0 +1,84 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "traits.hpp" + +namespace ddwaf { + +template + requires is_duration +class lru_cache { +public: + explicit lru_cache(ConstructorType constructor, std::size_t max_index_size = 32) + : constructor_(std::move(constructor)), max_index_size_(max_index_size) + {} + + template + requires std::is_constructible_v + DataType &emplace_or_retrieve(CompatKeyType key, DurationType timepoint) + { + auto it = index_.find(key); + if (it == index_.end()) { + if (index_.size() == max_index_size_) { + remove_oldest_entry(timepoint); + } + + auto [new_it, res] = + index_.emplace(KeyType{key}, cache_entry{timepoint, constructor_()}); + if (!res) { + throw std::out_of_range("failed to add element to cache"); + } + return new_it->second.data; + } + + it->second.latest_timepoint = std::max(timepoint, it->second.latest_timepoint); + + return it->second.data; + } + +protected: + struct cache_entry { + DurationType latest_timepoint; + DataType data; + }; + + void remove_oldest_entry(DurationType timepoint) + { + using iterator_type = typename decltype(index_)::iterator; + + DurationType max_delta{0}; + iterator_type oldest_it; + for (auto it = index_.begin(); it != index_.end(); ++it) { + auto window_last = it->second.latest_timepoint; + auto delta = timepoint - window_last; + if (delta > max_delta) { + max_delta = delta; + oldest_it = it; + } + } + + index_.erase(oldest_it); + } + + ConstructorType constructor_; + std::size_t max_index_size_{}; + std::map> index_; +}; + +template +using lru_cache_ms = lru_cache; + +} // namespace ddwaf diff --git a/src/monitor.hpp b/src/monitor.hpp new file mode 100644 index 000000000..5faac9dc1 --- /dev/null +++ b/src/monitor.hpp @@ -0,0 +1,33 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#pragma once + +#include +namespace ddwaf { + +template class monitor { +public: + template explicit monitor(Args... args) : underlying_value_(args...) {} + + class locked_monitor { + public: + explicit locked_monitor(monitor &m) : m_(m), lock_(m.mtx_) {} + T *operator->() { return &m_.underlying_value_; } + + protected: + monitor &m_; + std::unique_lock lock_; + }; + + locked_monitor operator->() { return locked_monitor{*this}; } + +protected: + std::mutex mtx_{}; + T underlying_value_; +}; + +} // namespace ddwaf diff --git a/src/parser/global_rule_parser.cpp b/src/parser/global_rule_parser.cpp new file mode 100644 index 000000000..0d90ac65b --- /dev/null +++ b/src/parser/global_rule_parser.cpp @@ -0,0 +1,168 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "exception.hpp" +#include "global_context.hpp" +#include "log.hpp" +#include "parameter.hpp" +#include "parser/common.hpp" +#include "parser/matcher_parser.hpp" +#include "parser/parser.hpp" +#include "rule/base.hpp" +#include "rule/threshold_rule.hpp" +#include "utils.hpp" + +namespace ddwaf::parser::v2 { + +namespace { +std::unique_ptr parse_indexed_threshold_rule( + std::string id, parameter::map &rule, const object_limits &limits) +{ + auto conditions_array = at(rule, "conditions", {}); + + address_container addresses; + auto expr = parse_simplified_expression(conditions_array, addresses, limits); + std::unordered_map tags; + for (auto &[key, value] : at(rule, "tags")) { + try { + tags.emplace(key, std::string(value)); + } catch (const bad_cast &e) { + throw invalid_type(std::string(key), e); + } + } + + if (tags.find("type") == tags.end()) { + throw ddwaf::parsing_error("missing key 'type'"); + } + + auto criteria_map = at(rule, "criteria"); + indexed_threshold_rule::evaluation_criteria criteria; + criteria.threshold = at(criteria_map, "threshold"); + criteria.period = std::chrono::milliseconds(at(criteria_map, "period")); + criteria.duration = std::chrono::milliseconds(at(criteria_map, "duration")); + + auto filter_map = at(rule, "filter"); + + auto input_map = at(filter_map, "input"); + criteria.filter.name = at(input_map, "address"); + criteria.filter.target = get_target_index(criteria.filter.name); + criteria.filter.key_path = at>(input_map, "key_path", {}); + + if (input_map.contains("transformers")) { + auto input_transformers = at(input_map, "transformers"); + if (input_transformers.size() > limits.max_transformers_per_address) { + throw ddwaf::parsing_error("number of transformers beyond allowed limit"); + } + + criteria.filter.transformers = parse_transformers(input_transformers); + } + + if (filter_map.contains("operator")) { + auto operator_name = at(filter_map, "operator"); + auto params = at(filter_map, "parameters"); + + auto [data_id, matcher] = parse_any_matcher(operator_name, params); + if (!data_id.empty()) { + throw ddwaf::parsing_error("unsupported: global rule filter with data ID"); + } + + criteria.filter.matcher = std::move(matcher); + } + + return std::make_unique(std::move(id), at(rule, "name"), + std::move(tags), std::move(expr), std::move(criteria), + at>(rule, "on_match", {}), at(rule, "enabled", true)); +} + +std::unique_ptr parse_threshold_rule( + std::string id, parameter::map &rule, const object_limits &limits) +{ + auto conditions_array = at(rule, "conditions", {}); + + address_container addresses; + auto expr = parse_simplified_expression(conditions_array, addresses, limits); + std::unordered_map tags; + for (auto &[key, value] : at(rule, "tags")) { + try { + tags.emplace(key, std::string(value)); + } catch (const bad_cast &e) { + throw invalid_type(std::string(key), e); + } + } + + if (tags.find("type") == tags.end()) { + throw ddwaf::parsing_error("missing key 'type'"); + } + + auto criteria_map = at(rule, "criteria"); + threshold_rule::evaluation_criteria criteria; + criteria.threshold = at(criteria_map, "threshold"); + criteria.period = std::chrono::milliseconds(at(criteria_map, "period")); + criteria.duration = std::chrono::milliseconds(at(criteria_map, "duration")); + + return std::make_unique(std::move(id), at(rule, "name"), + std::move(tags), std::move(expr), criteria, + at>(rule, "on_match", {}), at(rule, "enabled", true)); +} + +std::unique_ptr parse_global_rule( + std::string id, parameter::map &rule, const object_limits &limits) +{ + if (rule.contains("filter")) { + return parse_indexed_threshold_rule(std::move(id), rule, limits); + } + return parse_threshold_rule(std::move(id), rule, limits); +} +} // namespace + +std::shared_ptr parse_global_rules( + parameter::vector &rule_array, base_section_info &info, const object_limits &limits) +{ + std::vector> rules; + + std::unordered_set ids; + for (unsigned i = 0; i < rule_array.size(); ++i) { + const auto &rule_param = rule_array[i]; + auto rule_map = static_cast(rule_param); + std::string id; + try { + const address_container addresses; + id = at(rule_map, "id"); + if (ids.find(id) != ids.end()) { + DDWAF_WARN("Duplicate global rule {}", id); + info.add_failed(id, "duplicate rule"); + continue; + } + + auto rule = parse_global_rule(id, rule_map, limits); + DDWAF_DEBUG("Parsed global rule {}", id); + info.add_loaded(id); + add_addresses_to_info(addresses, info); + + rules.emplace_back(std::move(rule)); + } catch (const std::exception &e) { + if (id.empty()) { + id = index_to_id(i); + } + DDWAF_WARN("Failed to parse rule '{}': {}", id, e.what()); + info.add_failed(id, e.what()); + } + } + + return std::make_shared(std::move(rules)); +} + +} // namespace ddwaf::parser::v2 diff --git a/src/parser/parser.hpp b/src/parser/parser.hpp index 3e3bd5a08..d81733d6a 100644 --- a/src/parser/parser.hpp +++ b/src/parser/parser.hpp @@ -15,7 +15,7 @@ #include "parameter.hpp" #include "parser/common.hpp" #include "parser/specification.hpp" -#include "rule.hpp" +#include "rule/rule.hpp" #include "ruleset.hpp" #include "ruleset_info.hpp" @@ -59,9 +59,13 @@ std::shared_ptr parse_simplified_expression(const parameter::vector address_container &addresses, const object_limits &limits); std::vector parse_transformers(const parameter::vector &root, data_source &source); +std::vector parse_transformers(const parameter::vector &root); // std::pair> parse_matcher( // std::string_view name, const parameter::map ¶ms); +std::shared_ptr parse_global_rules( + parameter::vector &rule_array, base_section_info &info, const object_limits &limits); + } // namespace v2 } // namespace ddwaf::parser diff --git a/src/parser/parser_v1.cpp b/src/parser/parser_v1.cpp index edb141d32..b1be6d4bc 100644 --- a/src/parser/parser_v1.cpp +++ b/src/parser/parser_v1.cpp @@ -29,7 +29,7 @@ #include "parameter.hpp" #include "parser/common.hpp" #include "parser/parser.hpp" -#include "rule.hpp" +#include "rule/rule.hpp" #include "ruleset.hpp" #include "ruleset_info.hpp" #include "transformer/base.hpp" diff --git a/src/parser/rule_parser.cpp b/src/parser/rule_parser.cpp index 3d2eadbe7..e3f42778e 100644 --- a/src/parser/rule_parser.cpp +++ b/src/parser/rule_parser.cpp @@ -16,7 +16,7 @@ #include "parser/common.hpp" #include "parser/parser.hpp" #include "parser/specification.hpp" -#include "rule.hpp" +#include "rule/rule.hpp" #include "semver.hpp" #include "transformer/base.hpp" #include "utils.hpp" diff --git a/src/parser/specification.hpp b/src/parser/specification.hpp index 750ba2c1e..8c02b2121 100644 --- a/src/parser/specification.hpp +++ b/src/parser/specification.hpp @@ -14,7 +14,7 @@ #include "expression.hpp" #include "parameter.hpp" #include "processor/base.hpp" -#include "rule.hpp" +#include "rule/rule.hpp" #include "scanner.hpp" namespace ddwaf::parser { diff --git a/src/parser/transformer_parser.cpp b/src/parser/transformer_parser.cpp index 5d240f941..8882e5da4 100644 --- a/src/parser/transformer_parser.cpp +++ b/src/parser/transformer_parser.cpp @@ -45,4 +45,27 @@ std::vector parse_transformers(const parameter::vector &root, da return transformers; } +std::vector parse_transformers(const parameter::vector &root) +{ + if (root.empty()) { + return {}; + } + + std::vector transformers; + transformers.reserve(root.size()); + + for (const auto &transformer_param : root) { + auto transformer = static_cast(transformer_param); + auto id = transformer_from_string(transformer); + if (id.has_value()) { + transformers.emplace_back(id.value()); + } else if (transformer == "keys_only" || transformer == "values_only") { + throw ddwaf::parsing_error("source transformer not supported within this context"); + } else { + throw ddwaf::parsing_error("invalid transformer " + std::string(transformer)); + } + } + return transformers; +} + } // namespace ddwaf::parser::v2 diff --git a/src/rule.hpp b/src/rule/base.hpp similarity index 62% rename from src/rule.hpp rename to src/rule/base.hpp index a449e56f1..9d9150156 100644 --- a/src/rule.hpp +++ b/src/rule/base.hpp @@ -6,7 +6,6 @@ #pragma once -#include #include #include #include @@ -16,59 +15,36 @@ #include "event.hpp" #include "exclusion/common.hpp" #include "expression.hpp" -#include "iterator.hpp" #include "matcher/base.hpp" #include "object_store.hpp" +#include "sliding_window_counter.hpp" namespace ddwaf { -class rule { +class base_rule { public: - enum class source_type : uint8_t { base = 1, user = 2 }; - - using cache_type = expression::cache_type; - - rule(std::string id, std::string name, std::unordered_map tags, + base_rule(std::string id, std::string name, std::unordered_map tags, std::shared_ptr expr, std::vector actions = {}, - bool enabled = true, source_type source = source_type::base) - : enabled_(enabled), source_(source), id_(std::move(id)), name_(std::move(name)), - tags_(std::move(tags)), expr_(std::move(expr)), actions_(std::move(actions)) + bool enabled = true) + : enabled_(enabled), id_(std::move(id)), name_(std::move(name)), tags_(std::move(tags)), + expr_(std::move(expr)), actions_(std::move(actions)) { if (!expr_) { throw std::invalid_argument("rule constructed with null expression"); } } - rule(const rule &) = delete; - rule &operator=(const rule &) = delete; - - rule(rule &&rhs) noexcept = default; - rule &operator=(rule &&rhs) = default; + base_rule(const base_rule &) = delete; + base_rule &operator=(const base_rule &) = delete; - virtual ~rule() = default; + base_rule(base_rule &&rhs) noexcept = default; + base_rule &operator=(base_rule &&rhs) noexcept = default; - virtual std::optional match(const object_store &store, cache_type &cache, - const exclusion::object_set_ref &objects_excluded, - const std::unordered_map> &dynamic_matchers, - ddwaf::timer &deadline) const - { - if (expression::get_result(cache)) { - // An event was already produced, so we skip the rule - return std::nullopt; - } - - auto res = expr_->eval(cache, store, objects_excluded, dynamic_matchers, deadline); - if (!res.outcome) { - return std::nullopt; - } - - return {ddwaf::event{this, expression::get_matches(cache), res.ephemeral, {}}}; - } + virtual ~base_rule() = default; [[nodiscard]] bool is_enabled() const { return enabled_; } void toggle(bool value) { enabled_ = value; } - source_type get_source() const { return source_; } const std::string &get_id() const { return id_; } const std::string &get_name() const { return name_; } @@ -109,7 +85,6 @@ class rule { protected: bool enabled_{true}; - source_type source_; std::string id_; std::string name_; std::unordered_map tags_; @@ -118,4 +93,24 @@ class rule { std::vector actions_; }; +class base_threshold_rule : public base_rule { +public: + using cache_type = expression::cache_type; + + base_threshold_rule(std::string id, std::string name, + std::unordered_map tags, std::shared_ptr expr, + std::vector actions = {}, bool enabled = true) + : base_rule(std::move(id), std::move(name), std::move(tags), std::move(expr), + std::move(actions), enabled) + {} + ~base_threshold_rule() override = default; + base_threshold_rule(const base_threshold_rule &) = delete; + base_threshold_rule &operator=(const base_threshold_rule &) = delete; + base_threshold_rule(base_threshold_rule &&rhs) noexcept = default; + base_threshold_rule &operator=(base_threshold_rule &&rhs) noexcept = default; + + virtual std::optional eval(const object_store &store, cache_type &cache, + monotonic_clock::time_point now, ddwaf::timer &deadline) = 0; +}; + } // namespace ddwaf diff --git a/src/rule/rule.hpp b/src/rule/rule.hpp new file mode 100644 index 000000000..d2337a16d --- /dev/null +++ b/src/rule/rule.hpp @@ -0,0 +1,70 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#pragma once + +#include +#include +#include +#include + +#include "clock.hpp" +#include "event.hpp" +#include "exclusion/common.hpp" +#include "expression.hpp" +#include "matcher/base.hpp" +#include "object_store.hpp" +#include "rule/base.hpp" + +namespace ddwaf { + +class rule : public base_rule { +public: + enum class source_type : uint8_t { base = 1, user = 2 }; + + using cache_type = expression::cache_type; + + rule(std::string id, std::string name, std::unordered_map tags, + std::shared_ptr expr, std::vector actions = {}, + bool enabled = true, source_type source = source_type::base) + : base_rule(std::move(id), std::move(name), std::move(tags), std::move(expr), + std::move(actions), enabled), + source_(source) + {} + + rule(const rule &) = delete; + rule &operator=(const rule &) = delete; + + rule(rule &&rhs) noexcept = default; + rule &operator=(rule &&rhs) noexcept = default; + + ~rule() override = default; + + virtual std::optional match(const object_store &store, cache_type &cache, + const exclusion::object_set_ref &objects_excluded, + const std::unordered_map> &dynamic_matchers, + ddwaf::timer &deadline) const + { + if (expression::get_result(cache)) { + // An event was already produced, so we skip the rule + return std::nullopt; + } + + auto res = expr_->eval(cache, store, objects_excluded, dynamic_matchers, deadline); + if (!res.outcome) { + return std::nullopt; + } + + return {ddwaf::event{this, expression::get_matches(cache), res.ephemeral, {}}}; + } + + source_type get_source() const { return source_; } + +protected: + source_type source_; +}; + +} // namespace ddwaf diff --git a/src/rule/threshold_rule.cpp b/src/rule/threshold_rule.cpp new file mode 100644 index 000000000..1f77f3453 --- /dev/null +++ b/src/rule/threshold_rule.cpp @@ -0,0 +1,154 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. +#include +#include +#include +#include +#include +#include +#include + +#include "clock.hpp" +#include "condition/base.hpp" +#include "ddwaf.h" +#include "event.hpp" +#include "expression.hpp" +#include "iterator.hpp" +#include "object_store.hpp" +#include "rule/threshold_rule.hpp" +#include "transformer/manager.hpp" +#include "utils.hpp" + +using namespace std::literals; + +namespace ddwaf { + +namespace { + +std::string filter_with_matcher(const ddwaf_object &src, auto &filter) +{ + if (!filter.transformers.empty()) { + ddwaf_object dst; + ddwaf_object_invalid(&dst); + + auto transformed = transformer::manager::transform(src, dst, filter.transformers); + const scope_exit on_exit([&dst] { ddwaf_object_free(&dst); }); + if (transformed) { + auto [res, highlight] = filter.matcher->match(dst); + if (!res) { + return {}; + } + return highlight; + } + } + + // The value must be filtered + auto [res, highlight] = filter.matcher->match(src); + if (!res) { + // If the matcher fails, there is nothing to filter on + return {}; + } + return highlight; +} + +const ddwaf_object *get_object(const object_store &store, const auto &filter) +{ + auto [obj, attr] = store.get_target(filter.target); + if (obj == nullptr) { + return nullptr; + } + + if (filter.key_path.empty()) { + if (obj->type != DDWAF_OBJ_STRING) { + return nullptr; + } + + return obj; + } + + object::value_iterator it{obj, filter.key_path, {}}; + if (!it || it.type() != DDWAF_OBJ_STRING) { + return nullptr; + } + + return *it; +} + +} // namespace + +std::optional threshold_rule::eval(const object_store &store, cache_type &cache, + monotonic_clock::time_point now, ddwaf::timer &deadline) +{ + if (expression::get_result(cache)) { + // An event was already produced, so we skip the rule + return std::nullopt; + } + + auto res = expr_->eval(cache, store, {}, {}, deadline); + if (!res.outcome) { + return std::nullopt; + } + + auto ms = std::chrono::duration_cast(now.time_since_epoch()); + if (counter_->add_timepoint_and_count(ms)) { + // Match should be generated differently + auto matches = expression::get_matches(cache); + condition_match match{{}, {}, "threshold", threshold_str_, false}; + matches.emplace_back(std::move(match)); + return {ddwaf::event{this, std::move(matches), false, {}}}; + } + + return std::nullopt; +} + +std::optional indexed_threshold_rule::eval(const object_store &store, cache_type &cache, + monotonic_clock::time_point now, ddwaf::timer &deadline) +{ + if (expression::get_result(cache)) { + // An event was already produced, so we skip the rule + return std::nullopt; + } + + const auto *obj = get_object(store, criteria_.filter); + if (obj == nullptr) { + return std::nullopt; + } + + auto res = expr_->eval(cache, store, {}, {}, deadline); + if (!res.outcome) { + return std::nullopt; + } + + auto ms = std::chrono::duration_cast(now.time_since_epoch()); + + std::string filtered_key; + if (criteria_.filter.matcher) { + filtered_key = filter_with_matcher(*obj, criteria_.filter); + } else { + filtered_key = {obj->stringValue, static_cast(obj->nbEntries)}; + } + + bool match = false; + { + const std::unique_lock lock{mtx_}; + auto &counter = counter_cache_.emplace_or_retrieve(filtered_key, ms); + match = counter.add_timepoint_and_count(ms); + } + + if (match) { + // Match should be generated differently + auto matches = expression::get_matches(cache); + condition_match match{ + {{"input"sv, object_to_string(*obj), criteria_.filter.name, criteria_.filter.key_path}}, + {}, "threshold", threshold_str_, false}; + matches.emplace_back(std::move(match)); + return {ddwaf::event{this, std::move(matches), false, {}}}; + } + + return std::nullopt; +} + +} // namespace ddwaf diff --git a/src/rule/threshold_rule.hpp b/src/rule/threshold_rule.hpp new file mode 100644 index 000000000..2de538578 --- /dev/null +++ b/src/rule/threshold_rule.hpp @@ -0,0 +1,144 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#pragma once + +#include +#include +#include +#include + +#include "clock.hpp" +#include "event.hpp" +#include "exclusion/common.hpp" +#include "expression.hpp" +#include "lru_cache.hpp" +#include "matcher/base.hpp" +#include "monitor.hpp" +#include "object_store.hpp" +#include "rule/base.hpp" +#include "sliding_window_counter.hpp" + +namespace ddwaf { + +class threshold_counter { +public: + threshold_counter(std::chrono::milliseconds period, uint64_t threshold, + std::chrono::milliseconds threshold_duration) + : threshold_(threshold), counter_{period, threshold * 2}, + threshold_duration_(threshold_duration) + {} + + bool add_timepoint_and_count(std::chrono::milliseconds ms) + { + if (expiration_ > ms) { + return true; + } + + auto count = counter_.add_timepoint_and_count(ms); + if (count > threshold_) { + expiration_ = ms + threshold_duration_; + return true; + } + + return false; + } + +protected: + uint64_t threshold_; + sliding_window_counter_ms counter_; + std::chrono::milliseconds threshold_duration_; + std::chrono::milliseconds expiration_{0}; +}; + +class threshold_rule : public base_threshold_rule { +public: + struct evaluation_criteria { + uint64_t threshold; + std::chrono::milliseconds period{}; + std::chrono::milliseconds duration{}; + }; + + threshold_rule(std::string id, std::string name, + std::unordered_map tags, std::shared_ptr expr, + evaluation_criteria criteria, std::vector actions = {}, bool enabled = true) + : base_threshold_rule(std::move(id), std::move(name), std::move(tags), std::move(expr), + std::move(actions), enabled), + criteria_(criteria), + counter_(criteria_.period, criteria_.threshold * 2, criteria_.duration), + threshold_str_(to_string(criteria_.threshold)) + {} + + ~threshold_rule() override = default; + threshold_rule(const threshold_rule &) = delete; + threshold_rule &operator=(const threshold_rule &) = delete; + threshold_rule(threshold_rule &&rhs) noexcept = delete; + threshold_rule &operator=(threshold_rule &&rhs) noexcept = delete; + + std::optional eval(const object_store &store, cache_type &cache, + monotonic_clock::time_point now, ddwaf::timer &deadline) override; + +protected: + evaluation_criteria criteria_; + monitor counter_; + std::string threshold_str_; +}; + +class indexed_threshold_rule : public base_threshold_rule { +public: + struct evaluation_criteria { + uint64_t threshold; + std::chrono::milliseconds period; + std::chrono::milliseconds duration{}; + struct { + std::string name; + target_index target; + std::vector key_path; + std::vector transformers{}; + std::unique_ptr matcher; + } filter; + }; + + indexed_threshold_rule(std::string id, std::string name, + std::unordered_map tags, std::shared_ptr expr, + evaluation_criteria criteria, std::vector actions = {}, bool enabled = true) + : base_threshold_rule(std::move(id), std::move(name), std::move(tags), std::move(expr), + std::move(actions), enabled), + criteria_(std::move(criteria)), + counter_cache_(threshold_counter_constructor{criteria_.period, criteria_.threshold, + criteria_.duration}, + 128), + threshold_str_(to_string(criteria_.threshold)) + {} + + ~indexed_threshold_rule() override = default; + indexed_threshold_rule(const indexed_threshold_rule &) = delete; + indexed_threshold_rule &operator=(const indexed_threshold_rule &) = delete; + indexed_threshold_rule(indexed_threshold_rule &&rhs) noexcept = delete; + indexed_threshold_rule &operator=(indexed_threshold_rule &&rhs) noexcept = delete; + + std::optional eval(const object_store &store, cache_type &cache, + monotonic_clock::time_point now, ddwaf::timer &deadline) override; + +protected: + struct threshold_counter_constructor { + std::chrono::milliseconds period; + uint64_t threshold; + std::chrono::milliseconds threshold_duration; + + threshold_counter operator()() const + { + return threshold_counter{period, threshold, threshold_duration}; + } + }; + + evaluation_criteria criteria_; + lru_cache_ms counter_cache_; + std::string threshold_str_; + std::mutex mtx_; +}; + +} // namespace ddwaf diff --git a/src/ruleset.hpp b/src/ruleset.hpp index 19e9edd0e..57ad1f9cf 100644 --- a/src/ruleset.hpp +++ b/src/ruleset.hpp @@ -14,9 +14,10 @@ #include "collection.hpp" #include "exclusion/input_filter.hpp" #include "exclusion/rule_filter.hpp" +#include "global_context.hpp" #include "obfuscator.hpp" #include "processor/base.hpp" -#include "rule.hpp" +#include "rule/rule.hpp" #include "scanner.hpp" namespace ddwaf { @@ -194,6 +195,8 @@ struct ruleset { // These are lazily computed andthe underlying memory of each string is // owned by the action mapper. std::vector available_action_types; + + std::shared_ptr gctx; }; } // namespace ddwaf diff --git a/src/ruleset_builder.cpp b/src/ruleset_builder.cpp index 4ebe3e1a8..b40968126 100644 --- a/src/ruleset_builder.cpp +++ b/src/ruleset_builder.cpp @@ -22,7 +22,7 @@ #include "parser/common.hpp" #include "parser/parser.hpp" #include "parser/specification.hpp" -#include "rule.hpp" +#include "rule/rule.hpp" #include "ruleset.hpp" #include "ruleset_builder.hpp" #include "ruleset_info.hpp" @@ -212,6 +212,7 @@ std::shared_ptr ruleset_builder::build(parameter::map &root, base_rules rs->rule_matchers = rule_matchers_; rs->exclusion_matchers = exclusion_matchers_; rs->scanners = scanners_.items(); + rs->gctx = gctx_; rs->actions = actions_; rs->free_fn = free_fn_; rs->event_obfuscator = event_obfuscator_; @@ -443,6 +444,25 @@ ruleset_builder::change_state ruleset_builder::load(parameter::map &root, base_r } } + it = root.find("global_rules"); + if (it != root.end()) { + DDWAF_DEBUG("Parsing global rules"); + auto §ion = info.add_section("global_rules"); + try { + auto global_rules = static_cast(it->second); + if (!global_rules.empty()) { + gctx_ = parser::v2::parse_global_rules(global_rules, section, limits_); + } else { + DDWAF_DEBUG("Clearing all global rules"); + gctx_ = nullptr; + } + state = state | change_state::global_rules; + } catch (const std::exception &e) { + DDWAF_WARN("Failed to parse global rules: {}", e.what()); + section.set_error(e.what()); + } + } + return state; } diff --git a/src/ruleset_builder.hpp b/src/ruleset_builder.hpp index 453e752f5..0769393f7 100644 --- a/src/ruleset_builder.hpp +++ b/src/ruleset_builder.hpp @@ -12,10 +12,11 @@ #include #include "builder/processor_builder.hpp" +#include "global_context.hpp" #include "indexer.hpp" #include "parameter.hpp" #include "parser/specification.hpp" -#include "rule.hpp" +#include "rule/rule.hpp" #include "ruleset.hpp" #include "ruleset_info.hpp" @@ -54,6 +55,7 @@ class ruleset_builder { scanners = 64, actions = 128, exclusion_data = 256, + global_rules = 512, }; friend constexpr change_state operator|(change_state lhs, change_state rhs); @@ -112,6 +114,9 @@ class ruleset_builder { // Actions std::shared_ptr actions_; + + // Global Context + std::shared_ptr gctx_; }; } // namespace ddwaf diff --git a/src/sliding_window_counter.hpp b/src/sliding_window_counter.hpp new file mode 100644 index 000000000..e6ee233ab --- /dev/null +++ b/src/sliding_window_counter.hpp @@ -0,0 +1,113 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "traits.hpp" + +namespace ddwaf { + +// Perhaps this should use an std::chrono::time_point instead +template + requires is_duration +class sliding_window_counter { +public: + sliding_window_counter() = default; + explicit sliding_window_counter(T period, std::size_t max_window_size = 100) : period_(period) + { + time_points_.resize(max_window_size); + } + + uint64_t add_timepoint_and_count(T point) + { + // Discard old elements + update_count(point); + + // Check if the latest element is beyond the current one (concurrent writers) + auto index = decrement(right); + if (buckets > 0 && time_points_[index].point > point) { + ++time_points_[index].count; + return ++count; + } + + if (buckets < time_points_.size()) { + // Add a new element + time_points_[right].point = point; + time_points_[right].count = 1; + right = increment(right); + ++count; + ++buckets; + } else if (buckets == time_points_.size()) { + // Discard the oldest one + time_points_[right].point = point; + count -= (time_points_[right].count - 1); + time_points_[right].count = 1; + right = increment(right); + left = increment(left); + } + + return count; + } + + T last_timepoint() const + { + if (buckets == 0) { + [[unlikely]] return static_cast(0); + } + + auto index = decrement(right); + return time_points_[index].point; + } + + uint64_t update_count(T point) + { + // Discard old elements + auto window_begin = point - period_; + while (buckets > 0 && time_points_[left].point <= window_begin) { + count -= time_points_[left].count; + --buckets; + left = increment(left); + } + return count; + } + + void reset() { left = right = count = buckets = 0; } + +protected: + [[nodiscard]] uint64_t increment(uint64_t value) const + { + return (value + 1) % time_points_.size(); + } + + [[nodiscard]] uint64_t decrement(uint64_t value) const + { + return (value + time_points_.size() - 1) % time_points_.size(); + } + + struct time_bucket { + T point; + uint64_t count; + }; + + std::chrono::milliseconds period_{}; + std::vector time_points_; + uint64_t left{0}; + uint64_t right{0}; + uint64_t count{0}; + uint64_t buckets{0}; +}; + +using sliding_window_counter_ms = sliding_window_counter; + +} // namespace ddwaf diff --git a/src/traits.hpp b/src/traits.hpp index fa89a9e3e..107db7785 100644 --- a/src/traits.hpp +++ b/src/traits.hpp @@ -6,6 +6,7 @@ #pragma once +#include #include #include #include @@ -52,6 +53,9 @@ function_traits make_traits(Result (Class::*)(Args...) const) template function_traits make_traits(Result (Class::*)(Args...)); +template +concept is_duration = std::is_same_v, T>; + // https://stackoverflow.com/questions/43992510/enable-if-to-check-if-value-type-of-iterator-is-a-pair template struct is_pair : std::false_type {}; template struct is_pair> : std::true_type {}; diff --git a/tests/context_test.cpp b/tests/context_test.cpp index a9bd9d7b8..e047d933b 100644 --- a/tests/context_test.cpp +++ b/tests/context_test.cpp @@ -229,7 +229,8 @@ TEST(TestContext, MatchTimeout) ddwaf_object_map_add(&root, "http.client_ip", ddwaf_object_string(&tmp, "192.168.0.1")); ctx.insert(root); - EXPECT_THROW(ctx.eval_rules({}, deadline), ddwaf::timeout_exception); + std::vector events; + EXPECT_THROW(ctx.eval_rules(events, {}, deadline), ddwaf::timeout_exception); } TEST(TestContext, NoMatch) @@ -256,7 +257,8 @@ TEST(TestContext, NoMatch) ddwaf_object_map_add(&root, "http.client_ip", ddwaf_object_string(&tmp, "192.168.0.2")); ctx.insert(root); - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 0); } @@ -284,7 +286,8 @@ TEST(TestContext, Match) ddwaf_object_map_add(&root, "http.client_ip", ddwaf_object_string(&tmp, "192.168.0.1")); ctx.insert(root); - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 1); } @@ -331,7 +334,8 @@ TEST(TestContext, MatchMultipleRulesInCollectionSingleRun) ddwaf_object_map_add(&root, "usr.id", ddwaf_object_string(&tmp, "admin")); ctx.insert(root); - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 1); auto &event = events[0]; @@ -397,7 +401,8 @@ TEST(TestContext, MatchMultipleRulesWithPrioritySingleRun) ctx.insert(root); ddwaf::timer deadline{2s}; - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 1); auto event = events[0]; @@ -417,7 +422,8 @@ TEST(TestContext, MatchMultipleRulesWithPrioritySingleRun) ctx.insert(root); ddwaf::timer deadline{2s}; - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 1); auto event = events[0]; @@ -470,7 +476,8 @@ TEST(TestContext, MatchMultipleRulesInCollectionDoubleRun) ddwaf_object_map_add(&root, "http.client_ip", ddwaf_object_string(&tmp, "192.168.0.1")); ctx.insert(root); - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 1); auto &event = events[0]; @@ -498,7 +505,8 @@ TEST(TestContext, MatchMultipleRulesInCollectionDoubleRun) ddwaf_object_map_add(&root, "usr.id", ddwaf_object_string(&tmp, "admin")); ctx.insert(root); - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 0); } } @@ -547,7 +555,8 @@ TEST(TestContext, MatchMultipleRulesWithPriorityDoubleRunPriorityLast) ddwaf_object_map_add(&root, "http.client_ip", ddwaf_object_string(&tmp, "192.168.0.1")); ctx.insert(root); - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 1); auto &event = events[0]; @@ -577,7 +586,8 @@ TEST(TestContext, MatchMultipleRulesWithPriorityDoubleRunPriorityLast) ddwaf_object_map_add(&root, "usr.id", ddwaf_object_string(&tmp, "admin")); ctx.insert(root); - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 1); auto &event = events[0]; @@ -644,7 +654,8 @@ TEST(TestContext, MatchMultipleRulesWithPriorityDoubleRunPriorityFirst) ddwaf_object_map_add(&root, "http.client_ip", ddwaf_object_string(&tmp, "192.168.0.1")); ctx.insert(root); - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 1); auto &event = events[0]; @@ -674,7 +685,8 @@ TEST(TestContext, MatchMultipleRulesWithPriorityDoubleRunPriorityFirst) ddwaf_object_map_add(&root, "usr.id", ddwaf_object_string(&tmp, "admin")); ctx.insert(root); - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 0); } } @@ -723,7 +735,8 @@ TEST(TestContext, MatchMultipleRulesWithPriorityUntilAllActionsMet) ddwaf_object_map_add(&root, "http.client_ip", ddwaf_object_string(&tmp, "192.168.0.1")); ctx.insert(root); - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 1); auto &event = events[0]; @@ -751,7 +764,8 @@ TEST(TestContext, MatchMultipleRulesWithPriorityUntilAllActionsMet) ddwaf_object_map_add(&root, "usr.id", ddwaf_object_string(&tmp, "admin")); ctx.insert(root); - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 1); auto &event = events[0]; @@ -817,7 +831,8 @@ TEST(TestContext, MatchMultipleCollectionsSingleRun) ddwaf_object_map_add(&root, "usr.id", ddwaf_object_string(&tmp, "admin")); ctx.insert(root); - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 2); } @@ -866,7 +881,8 @@ TEST(TestContext, MatchMultiplePriorityCollectionsSingleRun) ddwaf_object_map_add(&root, "usr.id", ddwaf_object_string(&tmp, "admin")); ctx.insert(root); - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 2); } @@ -913,7 +929,8 @@ TEST(TestContext, MatchMultipleCollectionsDoubleRun) ddwaf_object_map_add(&root, "usr.id", ddwaf_object_string(&tmp, "admin")); ctx.insert(root); - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 1); } @@ -924,7 +941,8 @@ TEST(TestContext, MatchMultipleCollectionsDoubleRun) ddwaf_object_map_add(&root, "http.client_ip", ddwaf_object_string(&tmp, "192.168.0.1")); ctx.insert(root); - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 1); } } @@ -974,7 +992,8 @@ TEST(TestContext, MatchMultiplePriorityCollectionsDoubleRun) ddwaf_object_map_add(&root, "usr.id", ddwaf_object_string(&tmp, "admin")); ctx.insert(root); - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 1); } @@ -985,7 +1004,8 @@ TEST(TestContext, MatchMultiplePriorityCollectionsDoubleRun) ddwaf_object_map_add(&root, "http.client_ip", ddwaf_object_string(&tmp, "192.168.0.1")); ctx.insert(root); - auto events = ctx.eval_rules({}, deadline); + std::vector events; + ctx.eval_rules(events, {}, deadline); EXPECT_EQ(events.size(), 1); } } @@ -1136,7 +1156,8 @@ TEST(TestContext, RuleFilterWithCondition) EXPECT_EQ(rules_to_exclude.size(), 1); EXPECT_TRUE(rules_to_exclude.contains(rule.get())); - auto events = ctx.eval_rules(rules_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, rules_to_exclude, deadline); EXPECT_EQ(events.size(), 0); } @@ -1446,7 +1467,8 @@ TEST(TestContext, NoRuleFilterWithCondition) auto rules_to_exclude = ctx.eval_filters(deadline); EXPECT_TRUE(rules_to_exclude.empty()); - auto events = ctx.eval_rules(rules_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, rules_to_exclude, deadline); EXPECT_EQ(events.size(), 1); } @@ -1936,7 +1958,8 @@ TEST(TestContext, InputFilterExclude) auto objects_to_exclude = ctx.eval_filters(deadline); EXPECT_EQ(objects_to_exclude.size(), 1); - auto events = ctx.eval_rules(objects_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, objects_to_exclude, deadline); EXPECT_EQ(events.size(), 0); } @@ -2085,7 +2108,8 @@ TEST(TestContext, InputFilterExcludeRule) it->second.mode = filter_mode::none; EXPECT_TRUE(it->second.objects.empty()); - auto events = ctx.eval_rules(objects_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, objects_to_exclude, deadline); EXPECT_EQ(events.size(), 1); } @@ -2369,7 +2393,8 @@ TEST(TestContext, InputFilterWithCondition) auto objects_to_exclude = ctx.eval_filters(deadline); EXPECT_EQ(objects_to_exclude.size(), 0); - auto events = ctx.eval_rules(objects_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, objects_to_exclude, deadline); EXPECT_EQ(events.size(), 1); } @@ -2387,7 +2412,8 @@ TEST(TestContext, InputFilterWithCondition) auto objects_to_exclude = ctx.eval_filters(deadline); EXPECT_EQ(objects_to_exclude.size(), 0); - auto events = ctx.eval_rules(objects_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, objects_to_exclude, deadline); EXPECT_EQ(events.size(), 1); } @@ -2405,7 +2431,8 @@ TEST(TestContext, InputFilterWithCondition) auto objects_to_exclude = ctx.eval_filters(deadline); EXPECT_EQ(objects_to_exclude.size(), 1); - auto events = ctx.eval_rules(objects_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, objects_to_exclude, deadline); EXPECT_EQ(events.size(), 0); } } @@ -2534,7 +2561,8 @@ TEST(TestContext, InputFilterMultipleRules) EXPECT_EQ(policy.objects.size(), 1); } - auto events = ctx.eval_rules(objects_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, objects_to_exclude, deadline); EXPECT_EQ(events.size(), 0); } @@ -2556,7 +2584,8 @@ TEST(TestContext, InputFilterMultipleRules) EXPECT_EQ(policy.objects.size(), 2); } - auto events = ctx.eval_rules(objects_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, objects_to_exclude, deadline); EXPECT_EQ(events.size(), 0); } @@ -2578,7 +2607,8 @@ TEST(TestContext, InputFilterMultipleRules) EXPECT_EQ(policy.objects.size(), 2); } - auto events = ctx.eval_rules(objects_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, objects_to_exclude, deadline); EXPECT_EQ(events.size(), 0); } } @@ -2658,7 +2688,8 @@ TEST(TestContext, InputFilterMultipleRulesMultipleFilters) EXPECT_EQ(objects.size(), 1); } - auto events = ctx.eval_rules(objects_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, objects_to_exclude, deadline); EXPECT_EQ(events.size(), 0); } @@ -2681,7 +2712,8 @@ TEST(TestContext, InputFilterMultipleRulesMultipleFilters) EXPECT_EQ(objects.size(), 1); } - auto events = ctx.eval_rules(objects_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, objects_to_exclude, deadline); EXPECT_EQ(events.size(), 0); } @@ -2704,7 +2736,8 @@ TEST(TestContext, InputFilterMultipleRulesMultipleFilters) EXPECT_EQ(objects.size(), 1); } - auto events = ctx.eval_rules(objects_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, objects_to_exclude, deadline); EXPECT_EQ(events.size(), 0); } } @@ -2818,7 +2851,8 @@ TEST(TestContext, InputFilterMultipleRulesMultipleFiltersMultipleObjects) EXPECT_TRUE(objects.contains(&root.array[0])); } - auto events = ctx.eval_rules(objects_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, objects_to_exclude, deadline); EXPECT_EQ(events.size(), 0); } @@ -2840,7 +2874,8 @@ TEST(TestContext, InputFilterMultipleRulesMultipleFiltersMultipleObjects) EXPECT_TRUE(objects.contains(&root.array[0])); } - auto events = ctx.eval_rules(objects_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, objects_to_exclude, deadline); EXPECT_EQ(events.size(), 0); } @@ -2867,7 +2902,8 @@ TEST(TestContext, InputFilterMultipleRulesMultipleFiltersMultipleObjects) EXPECT_TRUE(objects.contains(&root.array[0])); } - auto events = ctx.eval_rules(objects_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, objects_to_exclude, deadline); EXPECT_EQ(events.size(), 0); } @@ -2890,7 +2926,8 @@ TEST(TestContext, InputFilterMultipleRulesMultipleFiltersMultipleObjects) EXPECT_TRUE(objects.contains(&root.array[0])); EXPECT_TRUE(objects.contains(&root.array[1])); } - auto events = ctx.eval_rules(objects_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, objects_to_exclude, deadline); EXPECT_EQ(events.size(), 0); } @@ -2918,7 +2955,8 @@ TEST(TestContext, InputFilterMultipleRulesMultipleFiltersMultipleObjects) EXPECT_TRUE(objects.contains(&root.array[0])); EXPECT_TRUE(objects.contains(&root.array[1])); } - auto events = ctx.eval_rules(objects_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, objects_to_exclude, deadline); EXPECT_EQ(events.size(), 0); } @@ -2948,7 +2986,8 @@ TEST(TestContext, InputFilterMultipleRulesMultipleFiltersMultipleObjects) EXPECT_TRUE(objects.contains(&root.array[1])); EXPECT_TRUE(objects.contains(&root.array[2])); } - auto events = ctx.eval_rules(objects_to_exclude, deadline); + std::vector events; + ctx.eval_rules(events, objects_to_exclude, deadline); EXPECT_EQ(events.size(), 0); } } diff --git a/tests/event_serializer_test.cpp b/tests/event_serializer_test.cpp index 36d9227bf..285e7b041 100644 --- a/tests/event_serializer_test.cpp +++ b/tests/event_serializer_test.cpp @@ -7,7 +7,7 @@ #include "test_utils.hpp" #include "event.hpp" -#include "rule.hpp" +#include "rule/rule.hpp" #include "utils.hpp" using namespace ddwaf; diff --git a/tests/rule_test.cpp b/tests/rule_test.cpp index 33addcaa7..36f014860 100644 --- a/tests/rule_test.cpp +++ b/tests/rule_test.cpp @@ -8,7 +8,7 @@ #include "matcher/exact_match.hpp" #include "matcher/ip_match.hpp" #include "object_store.hpp" -#include "rule.hpp" +#include "rule/rule.hpp" #include "test_utils.hpp" using namespace ddwaf; diff --git a/tests/sliding_window_counter_test.cpp b/tests/sliding_window_counter_test.cpp new file mode 100644 index 000000000..f5ae30499 --- /dev/null +++ b/tests/sliding_window_counter_test.cpp @@ -0,0 +1,104 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#include "sliding_window_counter.hpp" +#include "test.hpp" +#include "test_utils.hpp" + +using namespace std::literals; +using namespace std::chrono_literals; + +namespace { + +TEST(TestTimedCounter, BasicMs) +{ + ddwaf::sliding_window_counter window{10ms, 5}; + + EXPECT_EQ(window.add_timepoint_and_count(1ms), 1); + EXPECT_EQ(window.add_timepoint_and_count(11ms), 1); + EXPECT_EQ(window.add_timepoint_and_count(12ms), 2); + EXPECT_EQ(window.add_timepoint_and_count(13ms), 3); + EXPECT_EQ(window.add_timepoint_and_count(14ms), 4); + EXPECT_EQ(window.add_timepoint_and_count(15ms), 5); + EXPECT_EQ(window.add_timepoint_and_count(16ms), 5); + + EXPECT_EQ(window.add_timepoint_and_count(17ms), 5); + EXPECT_EQ(window.add_timepoint_and_count(18ms), 5); + EXPECT_EQ(window.add_timepoint_and_count(19ms), 5); + EXPECT_EQ(window.add_timepoint_and_count(20ms), 5); + EXPECT_EQ(window.add_timepoint_and_count(21ms), 5); + EXPECT_EQ(window.add_timepoint_and_count(40ms), 1); +} + +TEST(TestTimedCounter, BasicS) +{ + ddwaf::sliding_window_counter window{10s, 5}; + + EXPECT_EQ(window.add_timepoint_and_count(1s), 1); + EXPECT_EQ(window.add_timepoint_and_count(11s), 1); + EXPECT_EQ(window.add_timepoint_and_count(12s), 2); + EXPECT_EQ(window.add_timepoint_and_count(13s), 3); + EXPECT_EQ(window.add_timepoint_and_count(14s), 4); + EXPECT_EQ(window.add_timepoint_and_count(15s), 5); + EXPECT_EQ(window.add_timepoint_and_count(16s), 5); + EXPECT_EQ(window.add_timepoint_and_count(17s), 5); + EXPECT_EQ(window.add_timepoint_and_count(18s), 5); + EXPECT_EQ(window.add_timepoint_and_count(19s), 5); + EXPECT_EQ(window.add_timepoint_and_count(20s), 5); + EXPECT_EQ(window.add_timepoint_and_count(21s), 5); + EXPECT_EQ(window.add_timepoint_and_count(40s), 1); +} + +/*TEST(TestIndexedTimedCounter, BasicString)*/ +/*{*/ +/*ddwaf::indexed_sliding_window_counter window{10s, 5, 5};*/ + +/*EXPECT_EQ(window.add_timepoint_and_count("admin"sv, 1s), 1);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("user"sv, 10s), 1);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("docker"sv, 11s), 1);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("nobody"sv, 11s), 1);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("root"sv, 11s), 1);*/ +/*// Admin should be removed, as it's the latest*/ +/*EXPECT_EQ(window.add_timepoint_and_count("mail"sv, 11s), 1);*/ + +/*// User will now be removed*/ +/*EXPECT_EQ(window.add_timepoint_and_count("admin"sv, 11s), 1);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("admin"sv, 12s), 2);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("admin"sv, 13s), 3);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("admin"sv, 14s), 4);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("admin"sv, 15s), 5);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("admin"sv, 16s), 5);*/ + +/*EXPECT_EQ(window.add_timepoint_and_count("docker"sv, 17s), 2);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("nobody"sv, 17s), 2);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("root"sv, 17s), 2);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("mail"sv, 17s), 2);*/ + +/*EXPECT_EQ(window.add_timepoint_and_count("docker"sv, 18s), 3);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("nobody"sv, 18s), 3);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("root"sv, 18s), 3);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("mail"sv, 18s), 3);*/ + +/*EXPECT_EQ(window.add_timepoint_and_count("docker"sv, 19s), 4);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("nobody"sv, 19s), 4);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("root"sv, 19s), 4);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("mail"sv, 19s), 4);*/ + +/*EXPECT_EQ(window.add_timepoint_and_count("docker"sv, 20s), 5);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("nobody"sv, 20s), 5);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("root"sv, 20s), 5);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("mail"sv, 20s), 5);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("admin"sv, 20s), 5);*/ + +/*EXPECT_EQ(window.add_timepoint_and_count("nobody"sv, 21s), 5);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("root"sv, 21s), 5);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("mail"sv, 21s), 5);*/ +/*EXPECT_EQ(window.add_timepoint_and_count("admin"sv, 21s), 5);*/ +/*// Docker will now be removed*/ +/*EXPECT_EQ(window.add_timepoint_and_count("user"sv, 21s), 1);*/ +/*}*/ + +} // namespace diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 3379ca540..b1a5303a5 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -5,7 +5,7 @@ foreach(TOOL ${LIBDDWAF_TOOL_SOURCE}) get_filename_component(TOOL_NAME ${TOOL} NAME_WLE) add_executable(${TOOL_NAME} ${TOOL} ${LIBDDWAF_TOOL_COMMON_SOURCE}) - target_link_libraries(${TOOL_NAME} PRIVATE libddwaf_objects lib_yamlcpp lib_rapidjson) + target_link_libraries(${TOOL_NAME} PRIVATE libddwaf_objects lib_yamlcpp lib_rapidjson readline) target_include_directories(${TOOL_NAME} PRIVATE ${LIBDDWAF_PRIVATE_INCLUDES}) set_target_properties(${TOOL_NAME} PROPERTIES diff --git a/tools/waf_streamer.cpp b/tools/waf_streamer.cpp new file mode 100644 index 000000000..bf8b88010 --- /dev/null +++ b/tools/waf_streamer.cpp @@ -0,0 +1,142 @@ +// Unless explicitly stated otherwise all files in this repository are +// dual-licensed under the Apache-2.0 License or BSD-3-Clause License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2021 Datadog, Inc. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/utils.hpp" +#include "ddwaf.h" + +// NOLINTNEXTLINE +auto parse_args(int argc, char *argv[]) +{ + const std::map> arg_mapping{ + {"-r", "--ruleset"}, {"--ruleset", "--ruleset"}}; + + std::unordered_map args; + auto last_arg = args.end(); + for (int i = 1; i < argc; i++) { + std::string_view arg = argv[i]; + if (arg.starts_with('-')) { + if (auto long_arg = arg_mapping.find(arg); long_arg != arg_mapping.end()) { + arg = long_arg->second; + } else { + continue; // Unknown option + } + + auto [it, res] = args.emplace(arg, std::string{}); + last_arg = it; + } else if (last_arg != args.end()) { + last_arg->second = arg; + } + } + return args; +} + +int main(int argc, char *argv[]) +{ + //ddwaf_set_log_cb(log_cb, DDWAF_LOG_TRACE); + + auto args = parse_args(argc, argv); + + std::string ruleset = args["--ruleset"]; + if (ruleset.empty()) { + std::cout << "Usage: " << argv[0] << " --ruleset [..]\n"; + return EXIT_FAILURE; + } + + auto rule = YAML::Load(read_file(ruleset)).as(); + const ddwaf_config config{{0, 0, 0}, {nullptr, nullptr}, ddwaf_object_free}; + auto *handle = ddwaf_init(&rule, &config, nullptr); + ddwaf_object_free(&rule); + if (handle == nullptr) { + std::cout << "Failed to load " << ruleset << '\n'; + return EXIT_FAILURE; + } + + while (true) { + char *inpt = readline("Input: "); + add_history(inpt); + std::string json_str = inpt; + + + ddwaf_context context = ddwaf_context_init(handle); + if (context == nullptr) { + ddwaf_destroy(handle); + std::cout << "Failed to initialise context\n"; + return EXIT_FAILURE; + } + + auto input = YAML::Load(json_str).as(); + + ddwaf_result ret; + auto code = + ddwaf_run(context, &input, nullptr, &ret, std::numeric_limits::max()); + + if (code == DDWAF_MATCH) { + std::cout << "Evaluating " << json_str << " --> Match!\n"; + } else if (code == DDWAF_OK) { + std::cout << "Evaluating " << json_str << " --> No match!\n"; + } else { + std::cout << "Evaluating " << json_str << " --> Error!\n"; + } + + if (code == DDWAF_MATCH && ddwaf_object_size(&ret.events) > 0) { + std::stringstream ss; + YAML::Emitter out(ss); + out.SetIndent(2); + out.SetMapFormat(YAML::Block); + out.SetSeqFormat(YAML::Block); + out << object_to_yaml(ret.events); + + std::cout << "Events:\n" << ss.str() << "\n\n"; + } + + if (code == DDWAF_MATCH && ddwaf_object_size(&ret.actions) > 0) { + std::stringstream ss; + YAML::Emitter out(ss); + out.SetIndent(2); + out.SetMapFormat(YAML::Block); + out.SetSeqFormat(YAML::Block); + out << object_to_yaml(ret.actions); + + std::cout << "Actions:\n" << ss.str() << "\n\n"; + } + + if (ddwaf_object_size(&ret.derivatives) > 0) { + std::stringstream ss; + YAML::Emitter out(ss); + out.SetIndent(2); + out.SetMapFormat(YAML::Block); + out.SetSeqFormat(YAML::Block); + out << object_to_yaml(ret.derivatives); + + std::cout << "Derivatives:\n" << ss.str() << "\n\n"; + } + + ddwaf_result_free(&ret); + ddwaf_context_destroy(context); + } + + ddwaf_destroy(handle); + + return EXIT_SUCCESS; +} +