diff --git a/modules/deduplicator/src/deduplicator.hpp b/modules/deduplicator/src/deduplicator.hpp index 56a27f35..34055494 100644 --- a/modules/deduplicator/src/deduplicator.hpp +++ b/modules/deduplicator/src/deduplicator.hpp @@ -46,7 +46,7 @@ class Deduplicator { std::function, std::function>; - static inline const int DEFAULT_HASHMAP_TIMEOUT = 5000; ///< Default timeout - 5s + static inline const uint64_t DEFAULT_HASHMAP_TIMEOUT = 5000; ///< Default timeout - 5s /** * @brief Deduplicator constructor diff --git a/modules/deduplicator/src/main.cpp b/modules/deduplicator/src/main.cpp index 984c4afa..a38ac3ba 100644 --- a/modules/deduplicator/src/main.cpp +++ b/modules/deduplicator/src/main.cpp @@ -112,16 +112,16 @@ int main(int argc, char** argv) try { program.add_argument("-s", "--size") .required() - .help("Size of the hash map. Default value is 2^20 for 1'048'576 records.") + .help("Exponent N for the hash map size (2^N entries). Default: 20 (~1 048 576).") .default_value(Deduplicator::Deduplicator::DeduplicatorHashMap:: TimeoutHashMapParameters::DEFAULT_HASHMAP_EXPONENT) - .scan<'i', int>(); + .scan<'u', uint32_t>(); program.add_argument("-t", "--timeout") .required() .help( - "Count of millisecond to consider flows as duplicates. Default value is 5000(5s).") + "Count of millisecond to consider flows as duplicates. Default value is 5000 (5s).") .default_value(Deduplicator::Deduplicator::DEFAULT_HASHMAP_TIMEOUT) - .scan<'i', int>(); + .scan<'u', uint64_t>(); program.add_argument("-m", "--appfs-mountpoint") .required() .help("path where the appFs directory will be mounted") @@ -161,7 +161,7 @@ int main(int argc, char** argv) std::cerr << "Table size must be at least 8.\n"; return EXIT_FAILURE; } - const auto timeout = program.get("--timeout"); + const auto timeout = program.get("--timeout"); if (timeout <= 0) { std::cerr << "Timeout must be higher than zero.\n"; return EXIT_FAILURE; @@ -185,6 +185,7 @@ int main(int argc, char** argv) biInterface.setRequieredFormat( "uint16 SRC_PORT, uint16 DST_PORT, ipaddr DST_IP,ipaddr SRC_IP, uint64 LINK_BIT_FIELD, " "uint8 PROTOCOL, time TIME_LAST"); + deduplicator.updateUnirecIds(); processUnirecRecords(biInterface, deduplicator); } catch (std::exception& ex) { diff --git a/modules/deduplicator/src/timeoutHashMap.hpp b/modules/deduplicator/src/timeoutHashMap.hpp index 5b695366..12ed8382 100644 --- a/modules/deduplicator/src/timeoutHashMap.hpp +++ b/modules/deduplicator/src/timeoutHashMap.hpp @@ -212,7 +212,7 @@ class TimeoutHashMap { /** * @brief Default size for the hash map. */ - static inline const int DEFAULT_HASHMAP_EXPONENT = 20; // 1'048'576 records + static inline const uint32_t DEFAULT_HASHMAP_EXPONENT = 20; // 1'048'576 records uint32_t bucketCountExponent; ///< Total amount of records in table uint64_t timeout; ///< Time interval to consider flow unique diff --git a/modules/deduplicator/tests/test.sh b/modules/deduplicator/tests/test.sh new file mode 100755 index 00000000..1a4f8fb7 --- /dev/null +++ b/modules/deduplicator/tests/test.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +function exit_with_error { + pkill logger + pkill logreplay + pkill deduplicator + exit 1 +} + +function process_started { + pid=$1 + if ! ps -p $pid > /dev/null + then + echo "Failed to start process" + exit_with_error + fi +} + +data_path="$(dirname "$0")/testsData/" +deduplicator=$1 + +set -e +trap 'echo "Command \"$BASH_COMMAND\" failed!"; exit_with_error' ERR +for input_file in $data_path/inputs/*; do + index=$(echo "$input_file" | grep -o '[0-9]\+') + echo "Running test $index" + + res_file="/tmp/res" + logger -i "u:deduplicator" -w $res_file & + logger_pid=$! + sleep 0.1 + + process_started $logger_pid + + $deduplicator \ + -i "u:din,u:deduplicator" & + + detector_pid=$! + sleep 0.1 + process_started $detector_pid + + logreplay -i "u:din" -f "$data_path/inputs/input$index.csv" 2>/dev/null & + sleep 0.1 + process_started $! + + wait $logger_pid + wait $detector_pid + + if [ -f "$res_file" ]; then + if ! cmp -s "$data_path/results/res$index.csv" "$res_file"; then + echo "Files results/res$index.csv and $res_file are not equal" + exit_with_error + fi + else + echo "File $res_file not found" + exit_with_error + fi +done + +echo "All tests passed" +exit 0 diff --git a/modules/deduplicator/tests/testsData/inputs/input1.csv b/modules/deduplicator/tests/testsData/inputs/input1.csv new file mode 100644 index 00000000..b455482d --- /dev/null +++ b/modules/deduplicator/tests/testsData/inputs/input1.csv @@ -0,0 +1,9 @@ +ipaddr SRC_IP, ipaddr DST_IP, uint16 SRC_PORT, uint16 DST_PORT, uint8 PROTOCOL, uint64 LINK_BIT_FIELD, time TIME_LAST +154.175.219.8,155.175.219.8,123,123,6,10,2020-01-01T00:00:01Z +54.175.29.123,54.175.219.24,123,80,6,10,2020-01-01T00:00:02Z +54.175.21.66,54.175.219.77,443,22,6,10,2020-01-01T00:00:03Z +54.175.3.17,54.175.219.99,8080,53,17,10,2020-01-01T00:00:04Z +154.175.219.8,155.175.219.8,123,123,6,10,2020-01-01T00:00:01Z +154.175.219.8,155.175.219.8,123,123,6,11,2020-01-01T00:00:01Z +154.175.219.8,155.175.219.8,123,123,6,10,2020-01-01T00:00:01Z +154.175.219.8,155.175.219.8,123,123,6,11,2020-01-01T00:00:01Z diff --git a/modules/deduplicator/tests/testsData/results/res1.csv b/modules/deduplicator/tests/testsData/results/res1.csv new file mode 100644 index 00000000..1f34da1e --- /dev/null +++ b/modules/deduplicator/tests/testsData/results/res1.csv @@ -0,0 +1,6 @@ +155.175.219.8,154.175.219.8,10,2020-01-01T00:00:01.000000,123,123,6 +54.175.219.24,54.175.29.123,10,2020-01-01T00:00:02.000000,80,123,6 +54.175.219.77,54.175.21.66,10,2020-01-01T00:00:03.000000,22,443,6 +54.175.219.99,54.175.3.17,10,2020-01-01T00:00:04.000000,53,8080,17 +155.175.219.8,154.175.219.8,10,2020-01-01T00:00:01.000000,123,123,6 +155.175.219.8,154.175.219.8,10,2020-01-01T00:00:01.000000,123,123,6