From 108c84c140e7ae1c6df69a6a5a58c140bebadebc Mon Sep 17 00:00:00 2001 From: Michalsus Date: Thu, 25 Jul 2024 17:35:30 +0200 Subject: [PATCH 1/2] flow_age_stats: Added categorization based on FLOW_END_REASON MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Václav Bartoš --- flow_age_stats/README.md | 15 +- flow_age_stats/flow_age_stats.c | 454 ++++++++++++++++++++++++-------- flow_age_stats/graphs.sh | 20 ++ flow_age_stats/plot.gp | 40 ++- 4 files changed, 399 insertions(+), 130 deletions(-) create mode 100644 flow_age_stats/graphs.sh diff --git a/flow_age_stats/README.md b/flow_age_stats/README.md index e599ba85..1baedc99 100644 --- a/flow_age_stats/README.md +++ b/flow_age_stats/README.md @@ -1,9 +1,15 @@ +--- # Flow Age Stats module - README ## Description -This module is used for making statistics about the age of incoming flow data. The statistics produced are minimal, maximal and average values of the differences between the time a flow was received and its TIME_FIRST and TIME_LAST timestamps. +This module is used for making statistics about the age of incoming flow data. The statistics produced are minimal, maximal and average values of the differences between the time a flow was received and its TIME_FIRST and TIME_LAST timestamps. + +Additionally the module can output two text files (0_time_first.txt, 0_time_last.txt) that each have a table of three columns. First is the max age of the flow (the end of bin range). Second is the percentage of flows that are in that age group. Third is the flow count. By default, the bins are 0-1s, 1s-10s, 10s-20s, ... 590s-600s, >600s. + +If -e is specified, the statistis are computed separately by reason why the flow is exported (i.e. the value of the FLOW_END_REASON field). In this case, the files are named "_time_{first/last}.txt", where is the value of the FLOW_END_REASON field (0-5). + +Reference for FLOW_END_RESON values: https://www.iana.org/assignments/ipfix/ipfix.xhtml#ipfix-flow-end-reason -Additionally, the module can output histograms of flow age distribution. These are written as two text files (time_first.txt, time_last.txt) that each have a table of three columns. First is the max age of the flow (the end of bin range). Second is the percentage of flows that are in that age group. Third is the flow count. By default, the bins are 0-1s, 1s-10s, 10s-20s, ... 590s-600s, >600s. ## Interfaces - Input: One UniRec interface @@ -11,10 +17,11 @@ Additionally, the module can output histograms of flow age distribution. These a - Output: None ## Parameters -- '-t' If specified, the module writes a file where the tables will be outputted. (Caution - the module will overwrite files labeled time_first.txt, time_last.txt) +- '-t' If specified the module writes a file where the tables will be outputed. (Caution - the module will overwrite files labeled *_time_first.txt, *_time_last.txt) +- '-e' 'If specified the module creates statistics categorized based on FLOW_END_REASON.' ## Graphs -This module also comes with a script that makes use of GNUplot to make graphs from the data that is outputted into files. You can see how the graph looks like below. +This module also comes with a script (graphs.sh) that makes use of GNUplot to make graphs from the data that is outputed into files. You can see how the graph looks like below. ![ExampleGraph](example.png) diff --git a/flow_age_stats/flow_age_stats.c b/flow_age_stats/flow_age_stats.c index bf710cdf..de460624 100644 --- a/flow_age_stats/flow_age_stats.c +++ b/flow_age_stats/flow_age_stats.c @@ -47,6 +47,7 @@ #include #include +#include #include #include #include @@ -56,10 +57,12 @@ #include /** - * Linked list structure for storing histogram of flows ages + * @brief Linked list structure for storing flows + * + * This structure is a linked list of bins, that are used to categorize flows based on their age. */ typedef struct bins_t { - uint64_t max_age; //maximal duration of the bin TO DO + uint64_t max_age; size_t count_first; size_t count_last; struct bins_t *next; @@ -67,7 +70,9 @@ typedef struct bins_t { /** - * Structure for storing statistics about flow ages + * @brief Structure for storing statistics about flow ages + * + * This structure is used to storing general statistics about flow ages encountered during runtime of the program. */ typedef struct stats_t { uint64_t max; @@ -76,52 +81,64 @@ typedef struct stats_t { } stat; /** - * Definition of fields used in unirec templates (for both input and output interfaces) + * @brief Structure for categorization by FLOW_END_REASON + * + * This structure is used for separating flows into categories based on the reason they ended. + * It makes use of the structs defined above to store general statistcs about them and their ages. + */ +typedef struct category_t { + bin* bins; + stat* first; + stat* last; + uint8_t reason; + int count; + struct category_t* next; +} category; + +/** + * @brief Definition of fields used in unirec templates (for both input and output interfaces) */ UR_FIELDS ( time TIME_FIRST, time TIME_LAST, + uint8 FLOW_END_REASON, ) trap_module_info_t *module_info = NULL; /** - * Definition of basic module information - module name, module description, number of input and output interfaces + * @brief Definition of basic module information + * + * The module information include: module name, module description, number of input and output interfaces */ #define MODULE_BASIC_INFO(BASIC) \ BASIC("Flow Age Stats module", \ "This module finds min, max and avg of ages of flow data from input.\n" \ - "It can also make histograms of flow ages and output them into a file when -t is specified.\n", 1, 0) + "The second function is making percentual histograms of flow ages and outputs them into a file when -t is specified.\n" \ + "The third function is making percentual histograms of flow ages and the reasons why the flow ended into files when -e is specified.\n" , 1, 0) /** - * Definition of module parameter + * @brief Definition of module parameter */ #define MODULE_PARAMS(PARAM)\ - PARAM('t', "table", "store data about the flows in files", no_argument, "none") + PARAM('t', "table", "Store statistics (histograms) in files", no_argument, "none")\ + PARAM('e', "end reason", "Make separate statistics for different values of FLOW_END_REASON field", no_argument, "none") +//declaration of functions +bin* createNode(uint64_t max, uint64_t count); +void categorizeIntoCats(category* curr, uint64_t first_diff, uint64_t last_diff, uint8_t end_reason); +category* createCategory(category* next, uint8_t reason); +void destroyCategory(category* current); +void printCategories(category* head, int flow_count); +void outputInFiles(category* head, int flow_count); -/** - * Function for creating the bins -*/ -bin* createNode(uint64_t max, uint64_t count){ - bin* new_node = (bin*)malloc(sizeof(bin)); - if (new_node == NULL) { - fprintf(stderr, "Error: Memory allocation failed\n"); - return NULL; - } - new_node->max_age = max; - new_node->count_first = count; - new_node->count_last = count; - new_node->next = NULL; - return new_node; -} static int stop = 0; /** - * Function to handle SIGTERM and SIGINT signals (used to stop the module) + * @brief Function to handle SIGTERM and SIGINT signals (used to stop the module) */ TRAP_DEFAULT_SIGNAL_HANDLER(stop = 1) @@ -149,8 +166,8 @@ int main(int argc, char **argv) */ TRAP_REGISTER_DEFAULT_SIGNAL_HANDLER(); - FILE* out = NULL; int file = NULL; + int endReas = 1; /** * Handling of arguments */ @@ -159,6 +176,9 @@ int main(int argc, char **argv) case 't': file = 1; break; + case 'e': + endReas = 5; + break; default: fprintf(stderr, "Invalid arguments.\n"); FREE_MODULE_INFO_STRUCT(MODULE_BASIC_INFO, MODULE_PARAMS); @@ -173,20 +193,29 @@ int main(int argc, char **argv) fprintf(stderr, "Error: Input template could not be created.\n"); return -1; } + if(endReas == 5){ + in_tmplt = ur_define_fields_and_update_template("uint8 FLOW_END_REASON, time TIME_FIRST, time TIME_LAST", in_tmplt); + } - //initialization of the structs for statistics like max, min, avg - stat first = {0, UINT64_MAX, 0}; - - stat last = {0, UINT64_MAX, 0}; + - //initialization of age bins - bin *head = createNode(1, 0); - bin *current = head; - for (uint64_t i = 10; i <= 600; i+=10) { - current->next = createNode(i, 0); - current = current->next; - } - current->next = createNode(0, 0); + category* head = createCategory(NULL, 0); + if(head == NULL){ + destroyCategory(head); + } + category* curr = head; + //initialization of categories + if (endReas == 5) { + head->reason = 1; + for (int i = 2; i <= 5; ++i){ + curr->next = createCategory(NULL, i); + if(curr->next == NULL){ + goto failure;//jump to cleanup + } + curr = curr->next; + } + curr->next = createCategory(NULL, 0); + } //initialization of time time_t rawTime; @@ -253,80 +282,298 @@ int main(int argc, char **argv) flow_count++; //categorization into bins - bin* curr = head; - int first_inc = 0;// to make sure it only increments once - int last_inc = 0; - //loop for putting the flows into correct bins - while (curr != NULL){ - if (first_inc == 0){ - if(curr->max_age >= (first_diff/1000)){ - curr->count_first++; - first_inc++; + if(endReas == 1){ + categorizeIntoCats(head, first_diff, last_diff, 0); + } + else{ + uint8_t reas = ur_get(in_tmplt, in_rec, F_FLOW_END_REASON); + categorizeIntoCats(head, first_diff, last_diff, reas); + } + + free(received); + } + + time_t end_time; + time(&end_time); + double runtime = difftime(end_time, start_time);//calculating runtimes + + printf("\nRuntime: %0.2lfs\n", runtime); + printf("Number of flows processed: %zu\n \n", flow_count); + printCategories(head, flow_count); + + + //should be outputed to file if specified + if (file == 1){ + outputInFiles(head, flow_count); + } + + /* **** Cleanup **** */ + failure: + + // clean categories + while (head != NULL){ + category* next = head->next; + destroyCategory(head); + head = next; + } + + // Do all necessary cleanup in libtrap before exiting + TRAP_DEFAULT_FINALIZATION(); + + // Release allocated memory for module_info structure + FREE_MODULE_INFO_STRUCT(MODULE_BASIC_INFO, MODULE_PARAMS) + + // Free unirec template + ur_free_template(in_tmplt); + ur_finalize(); + + return 0; +} + +/** + * @brief Function for creating the bins + * + * This function creates a Node in the bin list. + * + * @param max maximal age of the FLOW in this bin + * @param count counts inside the Node are initialized to this value +*/ +bin* createNode(uint64_t max, uint64_t count){ + bin* new_node = (bin*)malloc(sizeof(bin)); + if (new_node == NULL) { + fprintf(stderr, "Error: Memory allocation failed\n"); + return NULL; + } + new_node->max_age = max; + new_node->count_first = count; + new_node->count_last = count; + new_node->next = NULL; + return new_node; +} + +/** + * @brief Function to create category_t + * + * This function creates a dynamically allocated category for statistics about flow age stats, + * and based on the argument -e for FLOW_END_REASON. + * + * @param next next category in list + * @param reason if -e is specified there are 5 different (default is 1) + */ +category* createCategory(category* next, uint8_t reason){ + category* newCat = (category*)malloc(sizeof(category)); + if (newCat == NULL) { + fprintf(stderr, "Error: Memory allocation failed\n"); + return NULL; + } + //creating bins + newCat->bins = createNode(1, 0); + bin *current = newCat->bins; + for (uint64_t i = 10; i <= 600; i+=10) { + current->next = createNode(i, 0); + if (current->next == NULL){ + return NULL; + } + current = current->next; + } + current->next = createNode(0, 0); + + //creating stat structs + newCat->first = (stat*)malloc(sizeof(stat)); + if (newCat->first == NULL) { + fprintf(stderr, "Error: Memory allocation failed\n"); + return NULL; + } + newCat->first->avg = 0; + newCat->first->min = UINT64_MAX; + newCat->first->max = 0; + + newCat->last = (stat*)malloc(sizeof(stat)); + if (newCat->last == NULL) { + fprintf(stderr, "Error: Memory allocation failed\n"); + return NULL; + } + newCat->last->avg = 0; + newCat->last->min = UINT64_MAX; + newCat->last->max = 0; + + newCat->next = next; + newCat->reason = reason; + newCat->count = 0; + return newCat; +} + +/** + * @brief Function for destroying categories + * + * This function frees the dynamically allocated categories. It also checks if the allocations failed + * beforehand in createCategory(). + * + * @param current category to be destroyed + */ +void destroyCategory(category* current){ + if(current == NULL){ + return; + } + //free bins + bin* curr = current->bins; + while (curr != NULL){ + bin* next = curr->next; + free(curr); + curr = next; + } + //free stat structs + if(current->first == NULL){ + return; + } + free(current->first); + if(current->last == NULL){ + return; + } + free(current->last); + free(current); +} + +/** + * @brief Function for categorization + * + * This function goes through the list of categories and updates the statistics based on FLOW_END_REASON (if -e is specified) + * or by the default (which is 1). + * + * @param curr head of the category list + * @param first_diff difference of TIME_FIRST and current time (in ms) + * @param last_diff difference of TIME_LAST and current time (in ms) + * @param end_reason FLOW_END_REASON + * + */ +void categorizeIntoCats(category* curr, uint64_t first_diff, uint64_t last_diff, uint8_t end_reason){ + while (curr != NULL){//loop for categorization + if(curr->reason != end_reason){ + curr = curr->next; + continue; + } + curr->count++; + bin* tmp = curr->bins; + bool first_inc = true; + bool last_inc = true; + while (tmp != NULL){ + if (first_inc){ + if(tmp->max_age >= (first_diff/1000)){ + tmp->count_first++; + first_inc = false; } } - if (last_inc == 0){ - if (curr->max_age >= (last_diff/1000)){ - curr->count_last++; - last_inc++; + if (last_inc){ + if (tmp->max_age >= (last_diff/1000)){ + tmp->count_last++; + last_inc = false; } } - if(last_inc == 1 && first_inc == 1){ + if((!last_inc) && (!first_inc)){ break; } - if(curr->next == NULL){ - if (first_inc == 0){ - curr->count_first++; + if(tmp->next == NULL){ + if (first_inc){ + tmp->count_first++; } - if(last_inc == 0){ - curr->count_last++; + if(last_inc){ + tmp->count_last++; } break; } - curr = curr->next; + tmp = tmp->next; } - - first.avg += first_diff; - last.avg += last_diff; + curr->first->avg += first_diff; + curr->last->avg += last_diff; //setting new max or min if needed for first - if(first.max < first_diff){ - first.max = first_diff; + if(curr->first->max < first_diff){ + curr->first->max = first_diff; } - else if (first.min > first_diff){ - first.min = first_diff; + else if (curr->first->min > first_diff){ + curr->first->min = first_diff; } //setting new max or min if needed for last - if(last.max < last_diff){ - last.max = last_diff; - } - else if (last.min > last_diff){ - last.min = last_diff; + if(curr->last->max < last_diff){ + curr->last->max = last_diff; } - free(received); + else if (curr->last->min > last_diff){ + curr->last->min = last_diff; + } + break; } +} - time_t end_time; - time(&end_time); - double runtime = difftime(end_time, start_time);//calculating runtimes +/** + * @brief Function to print out basic statistics + * + * Function prints out basic statistics of the flow age data. If -e is specified it prints out 5 separate + * statistics based on which FLOW_END_REASON was encountered. + * + * @param head head of the list of categories + * @param flow_count count of flows that were received by module + */ +void printCategories(category* head, int flow_count){ + int count = 0; + while(head != NULL){ + count++; + switch(head->reason){ + case 1: + printf("Stats for FLOW_END_REASON = 1 (idle timeout):\nNumber of flows:%d\nPercentage of the flows with this reason: %0.2lf %%\n", head->count, ((double)head->count/flow_count) * 100); + break; + case 2: + printf("Stats for FLOW_END_REASON = 2 (active timeout):\nNumber of flows:%d\nPercentage of the flows with this reason: %0.2lf %%\n", head->count, ((double)head->count/flow_count) * 100); + break; + case 3: + printf("Stats for FLOW_END_REASON = 3 (end of flow detected):\nNumber of flows:%d\nPercentage of the flows with this reason: %0.2lf %%\n", head->count, ((double)head->count/flow_count) * 100); + break; + case 4: + printf("Stats for FLOW_END_REASON = 4 (forced end):\nNumber of flows:%d\nPercentage of the flows with this reason: %0.2lf %%\n", head->count, ((double)head->count/flow_count) * 100); + break; + case 5: + printf("Stats for FLOW_END_REASON = 5 (lack of resources)\nNumber of flows:%d\nPercentage of the flows with this reason: %0.2lf %%\n", head->count, ((double)head->count/flow_count) * 100); + break; + default: + if(count == 1){ + printf("Stats for all flows encountered:\nNumber of flows:%d\n", head->count); + } + else{ + printf("Stats for other values of FLOW_END_REASON:\nNumber of flows:%d\nPercentage of the flows with this reason: %0.2lf %%\n", head->count, ((double)head->count/flow_count) * 100); + } + break; + } + printf("\tMinimal age of time_first: %0.2lf s\n", (double)head->first->min/1000);//from milliseconds to seconds + printf("\tMaximal age of time_first: %0.2lf s\n", (double)head->first->max/1000); + printf("\tAverage age of time_first: %0.2lf s\n", (double)(head->first->avg/flow_count)/1000); + printf("\tMinimal age of time_last: %0.2lf s\n", (double)head->last->min/1000); + printf("\tMaximal age of time_last: %0.2lf s\n", (double)head->last->max/1000); + printf("\tAverage age of time_last: %0.2lf s\n\n", (double)(head->last->avg/flow_count)/1000); + head = head->next; + } +} - printf("\nRuntime: %0.2lfs\n", runtime); - printf("Number of flows processed: %zu\n \n", flow_count); - printf("Minimal age of time_first: %0.2lf s\n", (double)first.min/1000);//from milliseconds to seconds - printf("Maximal age of time_first: %0.2lf s\n", (double)first.max/1000); - printf("Average age of time_first: %0.2lf s\n", (double)(first.avg/flow_count)/1000); - printf("Minimal age of time_last: %0.2lf s\n", (double)last.min/1000); - printf("Maximal age of time_last: %0.2lf s\n", (double)last.max/1000); - printf("Average age of time_last: %0.2lf s\n", (double)(last.avg/flow_count)/1000); +/** + * @brief Function for outputting into files + * + * This function outputs the tables into files. If the -e is specified tables for each FLOW_END_REASON are created. + * + * @param head head of the category list + * @param flow_count count of flows encountered + */ +void outputInFiles(category* head, int flow_count){ + char first_file[] = "0_time_first.txt"; + char last_file[] = "0_time_last.txt"; + FILE* out = NULL; - //should be outputed to file if specified - if(file == 1){ - out = fopen("time_first.txt", "w"); + while (head != NULL){ + first_file[0] = '0' + head->reason; + out = fopen(first_file, "w"); if (out == NULL){ - fprintf(stderr, "Error: Could not open file 'time_first.txt'.\n"); - goto skip_output; + fprintf(stderr, "Error: Could not open file '%s'.\n", first_file); + return; } - current = head; + bin* current = head->bins; while(current != NULL){ if (current->next == NULL){ // last bin - print label as "+" instead of "0" fprintf(out, "%" PRIu64 "+\t%0.2lf%%\t%zu\n", current->max_age, ((double)(current->count_first * 100)/flow_count), current->count_first); @@ -341,12 +588,13 @@ int main(int argc, char **argv) } fclose(out); - out = fopen("time_last.txt", "w"); + last_file[0] = '0' + head->reason; + out = fopen(last_file, "w"); if (out == NULL){ - fprintf(stderr, "Error: Could not open file 'time_last.txt'.\n"); - goto skip_output; + fprintf(stderr, "Error: Could not open file '%s'.\n", last_file); + return; } - current = head; + current = head->bins; while(current != NULL){ if (current->next == NULL){ // last bin - print label as "+" instead of "0" fprintf(out, "%" PRIu64 "+\t%0.2lf%%\t%zu\n", current->max_age, ((double)(current->count_last * 100)/flow_count), current->count_last); @@ -360,27 +608,7 @@ int main(int argc, char **argv) current = current->next; } fclose(out); + head = head->next; } - - /* **** Cleanup **** */ - skip_output: - //cleanup of bins - current = head; - while(current != NULL){ - bin* next = current->next; - free(current); - current = next; - } - - // Do all necessary cleanup in libtrap before exiting - TRAP_DEFAULT_FINALIZATION(); - - // Release allocated memory for module_info structure - FREE_MODULE_INFO_STRUCT(MODULE_BASIC_INFO, MODULE_PARAMS) - - // Free unirec template - ur_free_template(in_tmplt); - ur_finalize(); - - return 0; } + diff --git a/flow_age_stats/graphs.sh b/flow_age_stats/graphs.sh new file mode 100644 index 00000000..1d1648ae --- /dev/null +++ b/flow_age_stats/graphs.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +echo "Press 1 if you want graphs for TIME_FIRST and TIME_LAST. Press 2 if you want graphs for each FLOW_END_REASON:" +read choice + +if [ "$choice" = "1" ]; then + echo "Making graphs for all the flows encountered..." + gnuplot -c plot.gp "0_time_first.txt" "0_time_last.txt" "" +elif [ "$choice" = "2" ]; then + echo "Making graphs for each FLOW_END_REASON..." + gnuplot -c plot.gp "0_time_first.txt" "0_time_last.txt" "0_no_FLOW_END_REASON" + gnuplot -c plot.gp "1_time_first.txt" "1_time_last.txt" "1_idle_timeout" + gnuplot -c plot.gp "2_time_first.txt" "2_time_last.txt" "2_active_timeout" + gnuplot -c plot.gp "3_time_first.txt" "3_time_last.txt" "3_end_of_flow_detected" + gnuplot -c plot.gp "4_time_first.txt" "4_time_last.txt" "4_forced_end" + gnuplot -c plot.gp "5_time_first.txt" "5_time_last.txt" "5_lack_of_resources" +else + echo "Invalid input. Please run the script again and enter either 1 or 2." + exit 1 +fi \ No newline at end of file diff --git a/flow_age_stats/plot.gp b/flow_age_stats/plot.gp index 1ba77365..4dc88278 100644 --- a/flow_age_stats/plot.gp +++ b/flow_age_stats/plot.gp @@ -1,9 +1,22 @@ -# Set the output terminal +# Check if we have the right number of arguments +if (ARGC != 3) { + print "Error: Two data files and a title suffix are required." + print "Usage: gnuplot -c plot.gp time_first.txt time_last.txt title_suffix" + exit +} + +# Store the file names and title suffix in variables +time_first_file = ARG1 +time_last_file = ARG2 +title_suffix = ARG3 +title_suffix_no_underscores = system("echo ".ARG3." | sed 's,_, ,g'") + +# Set the output terminal for the first graph set terminal png enhanced font "Arial,12" -set output "time_first.png" +set output sprintf("time_first_%s.png", title_suffix) # Set the title and axis labels -set title "TIME FIRST" +set title sprintf("TIME FIRST %s", title_suffix_no_underscores) set xlabel "Age (s)" set ylabel "Percentage (%)" set y2label "Number of Flows" @@ -17,20 +30,21 @@ set y2range [0:*] set ytics nomirror set y2tics nomirror set grid -set xtics 10, 50 # Set x-axis tick marks at every 10th value, with minor ticks every 50th value +set xtics 10, 50 # Set the style for solid bars set style fill solid 1.0 -# Plot the data -plot "time_first.txt" using 1:3 with boxes lc rgb "#4daf4a" title "Flow Counts" axes x1y2, \ - "time_first.txt" using 1:2 with lines lc rgb "#e41a1c" title "Percentage" axes x1y1 +# Plot the data for the first graph +plot time_first_file using 1:3 with boxes lc rgb "#4daf4a" title "Flow Counts" axes x1y2, \ + time_first_file using 1:2 with lines lc rgb "#e41a1c" title "Percentage" axes x1y1 +# Set the output terminal for the second graph set terminal png enhanced font "Arial,12" -set output "time_last.png" +set output sprintf("time_last_%s.png", title_suffix) # Set the title and axis labels -set title "TIME LAST" +set title sprintf("TIME LAST %s", title_suffix_no_underscores) set xlabel "Age (s)" set ylabel "Percentage (%)" set y2label "Number of Flows" @@ -44,11 +58,11 @@ set y2range [0:*] set ytics nomirror set y2tics nomirror set grid -set xtics 10, 50 # Set x-axis tick marks at every 10th value, with minor ticks every 50th value +set xtics 10, 50 # Set the style for solid bars set style fill solid 1.0 -# Plot the data -plot "time_last.txt" using 1:3 with boxes lc rgb "#4daf4a" title "Flow Counts" axes x1y2, \ - "time_last.txt" using 1:2 with lines lc rgb "#e41a1c" title "Percentage" axes x1y1 \ No newline at end of file +# Plot the data for the second graph +plot time_last_file using 1:3 with boxes lc rgb "#4daf4a" title "Flow Counts" axes x1y2, \ + time_last_file using 1:2 with lines lc rgb "#e41a1c" title "Percentage" axes x1y1 \ No newline at end of file From c8624f5790e299e12a6103ef9bc32d680cd34a16 Mon Sep 17 00:00:00 2001 From: Michalsus Date: Sat, 31 Aug 2024 10:49:11 +0200 Subject: [PATCH 2/2] flow_age_stats: Fixed coding style --- flow_age_stats/flow_age_stats.c | 323 ++++++++++++++++---------------- 1 file changed, 162 insertions(+), 161 deletions(-) diff --git a/flow_age_stats/flow_age_stats.c b/flow_age_stats/flow_age_stats.c index de460624..1a8472eb 100644 --- a/flow_age_stats/flow_age_stats.c +++ b/flow_age_stats/flow_age_stats.c @@ -49,54 +49,48 @@ #include #include #include +#include +#include +#include #include #include -#include -#include #include "fields.h" -#include /** - * @brief Linked list structure for storing flows - * * This structure is a linked list of bins, that are used to categorize flows based on their age. */ typedef struct bins_t { - uint64_t max_age; - size_t count_first; - size_t count_last; - struct bins_t *next; + uint64_t max_age; ///< Maximal age in a bin + size_t count_first; ///< Counter of TIME_FIRST for this age + size_t count_last; ///< Counter of TIME_LAST for this age + struct bins_t *next; ///< Pointer to next bin in line } bin; - /** - * @brief Structure for storing statistics about flow ages - * - * This structure is used to storing general statistics about flow ages encountered during runtime of the program. + * This structure is used to store general statistics about + * flow ages encountered during runtime of the program. */ typedef struct stats_t { - uint64_t max; - uint64_t min; - uint64_t avg; + uint64_t max; ///< Maximal age encountered + uint64_t min; ///< Minimal age encountered + uint64_t avg; ///< Average age encountered } stat; /** - * @brief Structure for categorization by FLOW_END_REASON - * * This structure is used for separating flows into categories based on the reason they ended. * It makes use of the structs defined above to store general statistcs about them and their ages. */ typedef struct category_t { - bin* bins; - stat* first; - stat* last; - uint8_t reason; - int count; - struct category_t* next; + bin *bins; ///< Pointer to first bin + stat *first; ///< Pointer to stat struct + stat *last; ///< Pointer to stat struct + uint8_t reason; ///< Number representing which FLOW_END_REASON this category is for + int count; ///< Number of flows encountered for this reason + struct category_t *next; ///< Pointer to next category } category; /** - * @brief Definition of fields used in unirec templates (for both input and output interfaces) + * Definition of fields used in unirec templates (for both input and output interfaces) */ UR_FIELDS ( time TIME_FIRST, @@ -106,9 +100,8 @@ UR_FIELDS ( trap_module_info_t *module_info = NULL; - /** - * @brief Definition of basic module information + * Definition of basic module information * * The module information include: module name, module description, number of input and output interfaces */ @@ -118,27 +111,24 @@ trap_module_info_t *module_info = NULL; "The second function is making percentual histograms of flow ages and outputs them into a file when -t is specified.\n" \ "The third function is making percentual histograms of flow ages and the reasons why the flow ended into files when -e is specified.\n" , 1, 0) - /** - * @brief Definition of module parameter + * Definition of module parameters */ #define MODULE_PARAMS(PARAM)\ PARAM('t', "table", "Store statistics (histograms) in files", no_argument, "none")\ PARAM('e', "end reason", "Make separate statistics for different values of FLOW_END_REASON field", no_argument, "none") -//declaration of functions -bin* createNode(uint64_t max, uint64_t count); -void categorizeIntoCats(category* curr, uint64_t first_diff, uint64_t last_diff, uint8_t end_reason); -category* createCategory(category* next, uint8_t reason); -void destroyCategory(category* current); -void printCategories(category* head, int flow_count); -void outputInFiles(category* head, int flow_count); - +bin *create_node(uint64_t max, uint64_t count); +void categorize_into_cats(category *curr, uint64_t first_diff, uint64_t last_diff, uint8_t end_reason); +category *create_category(category *next, uint8_t reason); +void destroy_category(category *current); +void print_categories(category *head, int flow_count); +void output_in_files(category *head, int flow_count); static int stop = 0; /** - * @brief Function to handle SIGTERM and SIGINT signals (used to stop the module) + * Function to handle SIGTERM and SIGINT signals (used to stop the module) */ TRAP_DEFAULT_SIGNAL_HANDLER(stop = 1) @@ -167,7 +157,7 @@ int main(int argc, char **argv) TRAP_REGISTER_DEFAULT_SIGNAL_HANDLER(); int file = NULL; - int endReas = 1; + int end_reason = 1; /** * Handling of arguments */ @@ -176,9 +166,11 @@ int main(int argc, char **argv) case 't': file = 1; break; + case 'e': - endReas = 5; + end_reason = 5; break; + default: fprintf(stderr, "Invalid arguments.\n"); FREE_MODULE_INFO_STRUCT(MODULE_BASIC_INFO, MODULE_PARAMS); @@ -189,44 +181,47 @@ int main(int argc, char **argv) /* **** Create UniRec templates **** */ ur_template_t *in_tmplt = ur_create_input_template(0, "TIME_FIRST,TIME_LAST", NULL); - if (in_tmplt == NULL){ + if (in_tmplt == NULL) { fprintf(stderr, "Error: Input template could not be created.\n"); return -1; } - if(endReas == 5){ + if (end_reason == 5) { in_tmplt = ur_define_fields_and_update_template("uint8 FLOW_END_REASON, time TIME_FIRST, time TIME_LAST", in_tmplt); } - - - category* head = createCategory(NULL, 0); - if(head == NULL){ - destroyCategory(head); + category *head = create_category(NULL, 0); + if (head == NULL) { + destroy_category(head); } - category* curr = head; - //initialization of categories - if (endReas == 5) { + category *curr = head; + /** + * initialization of categories + */ + if (end_reason == 5) { head->reason = 1; - for (int i = 2; i <= 5; ++i){ - curr->next = createCategory(NULL, i); - if(curr->next == NULL){ - goto failure;//jump to cleanup + for (int i = 2; i <= 5; ++i) { + curr->next = create_category(NULL, i); + if (curr->next == NULL) { + goto failure; } curr = curr->next; } - curr->next = createCategory(NULL, 0); + curr->next = create_category(NULL, 0); } - //initialization of time + /** + * initialization of time + */ time_t rawTime; - /* **** Main processing loop **** */ size_t flow_count = 0; time_t start_time; time(&start_time); - // Read data from input, process them and output them into file if specified + /** + * Read data from input, process them and output them into file if specified + */ while (!stop) { const void *in_rec; uint16_t in_rec_size; @@ -250,7 +245,6 @@ int main(int argc, char **argv) } // PROCESS THE DATA - // TODO: there is probably a faster method to get current time in ur_time_t than by conversion from string time(&rawTime); struct tm* utc_timeinfo; #ifdef _WIN32 @@ -262,32 +256,34 @@ int main(int argc, char **argv) strftime(time_received, 20, "%Y-%m-%dT%H:%M:%S", utc_timeinfo); ur_time_t* received = malloc(sizeof(ur_time_t)); - if(received == NULL){ + if (received == NULL) { fprintf(stderr, "Error: Malloc for ur_time_t failed.\n"); break; } uint8_t check = ur_time_from_string(received, time_received); - if(check == 1){ + if (check == 1) { fprintf(stderr, "Error: could not convert string to ur_time_t\n"); break; } ur_time_t time_first = ur_get(in_tmplt, in_rec, F_TIME_FIRST); ur_time_t time_last = ur_get(in_tmplt, in_rec, F_TIME_LAST); - //time difference between time at which the flow was received vs the time in the record itself + /** + * time difference between time at which the flow was received vs the time in the record itself + */ uint64_t first_diff = ur_timediff(*received, time_first); uint64_t last_diff = ur_timediff(*received, time_last); - //time will be in milliseconds flow_count++; - //categorization into bins - if(endReas == 1){ - categorizeIntoCats(head, first_diff, last_diff, 0); - } - else{ + /** + * categorization into bins + */ + if (end_reason == 1) { + categorize_into_cats(head, first_diff, last_diff, 0); + } else { uint8_t reas = ur_get(in_tmplt, in_rec, F_FLOW_END_REASON); - categorizeIntoCats(head, first_diff, last_diff, reas); + categorize_into_cats(head, first_diff, last_diff, reas); } free(received); @@ -295,25 +291,24 @@ int main(int argc, char **argv) time_t end_time; time(&end_time); - double runtime = difftime(end_time, start_time);//calculating runtimes + double runtime = difftime(end_time, start_time); printf("\nRuntime: %0.2lfs\n", runtime); printf("Number of flows processed: %zu\n \n", flow_count); - printCategories(head, flow_count); - + print_categories(head, flow_count); //should be outputed to file if specified - if (file == 1){ - outputInFiles(head, flow_count); + if (file == 1) { + output_in_files(head, flow_count); } /* **** Cleanup **** */ failure: // clean categories - while (head != NULL){ - category* next = head->next; - destroyCategory(head); + while (head != NULL) { + category *next = head->next; + destroy_category(head); head = next; } @@ -331,15 +326,15 @@ int main(int argc, char **argv) } /** - * @brief Function for creating the bins - * * This function creates a Node in the bin list. * - * @param max maximal age of the FLOW in this bin - * @param count counts inside the Node are initialized to this value + * \param[in] max maximal age of the FLOW in this bin + * \param[in] count counts inside the Node are initialized to this value + * \returns bin* on success, otherwise NULL */ -bin* createNode(uint64_t max, uint64_t count){ - bin* new_node = (bin*)malloc(sizeof(bin)); +bin *create_node(uint64_t max, uint64_t count) +{ + bin *new_node = (bin *)malloc(sizeof(bin)); if (new_node == NULL) { fprintf(stderr, "Error: Memory allocation failed\n"); return NULL; @@ -352,34 +347,34 @@ bin* createNode(uint64_t max, uint64_t count){ } /** - * @brief Function to create category_t - * * This function creates a dynamically allocated category for statistics about flow age stats, * and based on the argument -e for FLOW_END_REASON. * - * @param next next category in list - * @param reason if -e is specified there are 5 different (default is 1) + * \param[in] next next category in list + * \param[in] reason if -e is specified there are 5 different (default is 1) + * \returns category* on success, otherwise NULL */ -category* createCategory(category* next, uint8_t reason){ - category* newCat = (category*)malloc(sizeof(category)); +category *create_category(category *next, uint8_t reason) +{ + category *newCat = (category *)malloc(sizeof(category)); if (newCat == NULL) { fprintf(stderr, "Error: Memory allocation failed\n"); return NULL; } //creating bins - newCat->bins = createNode(1, 0); + newCat->bins = create_node(1, 0); bin *current = newCat->bins; for (uint64_t i = 10; i <= 600; i+=10) { - current->next = createNode(i, 0); - if (current->next == NULL){ + current->next = create_node(i, 0); + if (current->next == NULL) { return NULL; } current = current->next; } - current->next = createNode(0, 0); + current->next = create_node(0, 0); //creating stat structs - newCat->first = (stat*)malloc(sizeof(stat)); + newCat->first = (stat *)malloc(sizeof(stat)); if (newCat->first == NULL) { fprintf(stderr, "Error: Memory allocation failed\n"); return NULL; @@ -388,7 +383,7 @@ category* createCategory(category* next, uint8_t reason){ newCat->first->min = UINT64_MAX; newCat->first->max = 0; - newCat->last = (stat*)malloc(sizeof(stat)); + newCat->last = (stat *)malloc(sizeof(stat)); if (newCat->last == NULL) { fprintf(stderr, "Error: Memory allocation failed\n"); return NULL; @@ -404,30 +399,29 @@ category* createCategory(category* next, uint8_t reason){ } /** - * @brief Function for destroying categories - * - * This function frees the dynamically allocated categories. It also checks if the allocations failed - * beforehand in createCategory(). + * This function frees the dynamically allocated categories. It also checks if the allocations failed + * beforehand in create_category(). * - * @param current category to be destroyed + * \param[in] current category to be destroyed */ -void destroyCategory(category* current){ - if(current == NULL){ +void destroy_category(category *current) +{ + if(current == NULL) { return; } //free bins - bin* curr = current->bins; - while (curr != NULL){ - bin* next = curr->next; + bin *curr = current->bins; + while (curr != NULL) { + bin *next = curr->next; free(curr); curr = next; } //free stat structs - if(current->first == NULL){ + if (current->first == NULL) { return; } free(current->first); - if(current->last == NULL){ + if (current->last == NULL) { return; } free(current->last); @@ -435,48 +429,47 @@ void destroyCategory(category* current){ } /** - * @brief Function for categorization - * * This function goes through the list of categories and updates the statistics based on FLOW_END_REASON (if -e is specified) * or by the default (which is 1). * - * @param curr head of the category list - * @param first_diff difference of TIME_FIRST and current time (in ms) - * @param last_diff difference of TIME_LAST and current time (in ms) - * @param end_reason FLOW_END_REASON - * + * \param[in] curr head of the category list + * \param[in] first_diff difference of TIME_FIRST and current time (in ms) + * \param[in] last_diff difference of TIME_LAST and current time (in ms) + * \param[in] end_reason FLOW_END_REASON */ -void categorizeIntoCats(category* curr, uint64_t first_diff, uint64_t last_diff, uint8_t end_reason){ - while (curr != NULL){//loop for categorization - if(curr->reason != end_reason){ +void categorize_into_cats(category *curr, uint64_t first_diff, uint64_t last_diff, uint8_t end_reason) +{ + while (curr != NULL) {//loop for categorization + if (curr->reason != end_reason) { curr = curr->next; continue; } curr->count++; - bin* tmp = curr->bins; - bool first_inc = true; + bin *tmp = curr->bins; + bool first_inc = true; //so that it only increments once bool last_inc = true; - while (tmp != NULL){ - if (first_inc){ - if(tmp->max_age >= (first_diff/1000)){ + //go through bins + while (tmp != NULL) { + if (first_inc) { + if (tmp->max_age >= (first_diff/1000)) { tmp->count_first++; first_inc = false; } } - if (last_inc){ - if (tmp->max_age >= (last_diff/1000)){ + if (last_inc) { + if (tmp->max_age >= (last_diff/1000)) { tmp->count_last++; last_inc = false; } } - if((!last_inc) && (!first_inc)){ + if ((!last_inc) && (!first_inc)) { break; } - if(tmp->next == NULL){ - if (first_inc){ + if (tmp->next == NULL) { + if (first_inc) { tmp->count_first++; } - if(last_inc){ + if (last_inc) { tmp->count_last++; } break; @@ -487,18 +480,18 @@ void categorizeIntoCats(category* curr, uint64_t first_diff, uint64_t last_diff, curr->last->avg += last_diff; //setting new max or min if needed for first - if(curr->first->max < first_diff){ + if (curr->first->max < first_diff) { curr->first->max = first_diff; } - else if (curr->first->min > first_diff){ + else if (curr->first->min > first_diff) { curr->first->min = first_diff; } //setting new max or min if needed for last - if(curr->last->max < last_diff){ + if (curr->last->max < last_diff) { curr->last->max = last_diff; } - else if (curr->last->min > last_diff){ + else if (curr->last->min > last_diff) { curr->last->min = last_diff; } break; @@ -506,44 +499,54 @@ void categorizeIntoCats(category* curr, uint64_t first_diff, uint64_t last_diff, } /** - * @brief Function to print out basic statistics - * - * Function prints out basic statistics of the flow age data. If -e is specified it prints out 5 separate + * Function prints out basic statistics of the flow age data. If -e is specified it prints out 5 separate * statistics based on which FLOW_END_REASON was encountered. * - * @param head head of the list of categories - * @param flow_count count of flows that were received by module + * \param[in] head head of the list of categories + * \param[in] flow_count count of flows that were received by module */ -void printCategories(category* head, int flow_count){ +void print_categories(category *head, int flow_count) +{ int count = 0; - while(head != NULL){ + while (head != NULL) { count++; - switch(head->reason){ + switch(head->reason) { case 1: - printf("Stats for FLOW_END_REASON = 1 (idle timeout):\nNumber of flows:%d\nPercentage of the flows with this reason: %0.2lf %%\n", head->count, ((double)head->count/flow_count) * 100); + printf("Stats for FLOW_END_REASON = 1 (idle timeout):\nNumber of flows:%d\n", head->count); + printf("Percentage of the flows with this reason: %0.2lf %%\n", ((double)head->count/flow_count) * 100); break; + case 2: - printf("Stats for FLOW_END_REASON = 2 (active timeout):\nNumber of flows:%d\nPercentage of the flows with this reason: %0.2lf %%\n", head->count, ((double)head->count/flow_count) * 100); + printf("Stats for FLOW_END_REASON = 2 (active timeout):\nNumber of flows:%d\n", head->count); + printf("Percentage of the flows with this reason: %0.2lf %%\n", ((double)head->count/flow_count) * 100); break; + case 3: - printf("Stats for FLOW_END_REASON = 3 (end of flow detected):\nNumber of flows:%d\nPercentage of the flows with this reason: %0.2lf %%\n", head->count, ((double)head->count/flow_count) * 100); + printf("Stats for FLOW_END_REASON = 3 (end of flow detected):\nNumber of flows:%d\n", head->count); + printf("Percentage of the flows with this reason: %0.2lf %%\n", ((double)head->count/flow_count) * 100); break; + case 4: - printf("Stats for FLOW_END_REASON = 4 (forced end):\nNumber of flows:%d\nPercentage of the flows with this reason: %0.2lf %%\n", head->count, ((double)head->count/flow_count) * 100); + printf("Stats for FLOW_END_REASON = 4 (forced end):\nNumber of flows:%d\n", head->count); + printf("Percentage of the flows with this reason: %0.2lf %%\n", ((double)head->count/flow_count) * 100); break; + case 5: - printf("Stats for FLOW_END_REASON = 5 (lack of resources)\nNumber of flows:%d\nPercentage of the flows with this reason: %0.2lf %%\n", head->count, ((double)head->count/flow_count) * 100); + printf("Stats for FLOW_END_REASON = 5 (lack of resources)\nNumber of flows:%d\n", head->count); + printf("Percentage of the flows with this reason: %0.2lf %%\n", ((double)head->count/flow_count) * 100); break; + default: if(count == 1){ printf("Stats for all flows encountered:\nNumber of flows:%d\n", head->count); } else{ - printf("Stats for other values of FLOW_END_REASON:\nNumber of flows:%d\nPercentage of the flows with this reason: %0.2lf %%\n", head->count, ((double)head->count/flow_count) * 100); + printf("Stats for other values of FLOW_END_REASON:\nNumber of flows:%d\n", head->count); + printf("Percentage of the flows with this reason: %0.2lf %%\n", ((double)head->count/flow_count) * 100); } break; } - printf("\tMinimal age of time_first: %0.2lf s\n", (double)head->first->min/1000);//from milliseconds to seconds + printf("\tMinimal age of time_first: %0.2lf s\n", (double)head->first->min/1000); printf("\tMaximal age of time_first: %0.2lf s\n", (double)head->first->max/1000); printf("\tAverage age of time_first: %0.2lf s\n", (double)(head->first->avg/flow_count)/1000); printf("\tMinimal age of time_last: %0.2lf s\n", (double)head->last->min/1000); @@ -554,27 +557,26 @@ void printCategories(category* head, int flow_count){ } /** - * @brief Function for outputting into files - * - * This function outputs the tables into files. If the -e is specified tables for each FLOW_END_REASON are created. + * This function outputs the tables into files. If the -e is specified tables for each FLOW_END_REASON are created. * - * @param head head of the category list - * @param flow_count count of flows encountered + * \param[in] head head of the category list + * \param[in] flow_count count of flows encountered */ -void outputInFiles(category* head, int flow_count){ +void output_in_files(category *head, int flow_count) +{ char first_file[] = "0_time_first.txt"; char last_file[] = "0_time_last.txt"; - FILE* out = NULL; + FILE *out = NULL; - while (head != NULL){ + while (head != NULL) { first_file[0] = '0' + head->reason; out = fopen(first_file, "w"); - if (out == NULL){ + if (out == NULL) { fprintf(stderr, "Error: Could not open file '%s'.\n", first_file); return; } bin* current = head->bins; - while(current != NULL){ + while (current != NULL) { if (current->next == NULL){ // last bin - print label as "+" instead of "0" fprintf(out, "%" PRIu64 "+\t%0.2lf%%\t%zu\n", current->max_age, ((double)(current->count_first * 100)/flow_count), current->count_first); break; @@ -590,12 +592,12 @@ void outputInFiles(category* head, int flow_count){ last_file[0] = '0' + head->reason; out = fopen(last_file, "w"); - if (out == NULL){ + if (out == NULL) { fprintf(stderr, "Error: Could not open file '%s'.\n", last_file); return; } current = head->bins; - while(current != NULL){ + while (current != NULL) { if (current->next == NULL){ // last bin - print label as "+" instead of "0" fprintf(out, "%" PRIu64 "+\t%0.2lf%%\t%zu\n", current->max_age, ((double)(current->count_last * 100)/flow_count), current->count_last); break; @@ -610,5 +612,4 @@ void outputInFiles(category* head, int flow_count){ fclose(out); head = head->next; } -} - +} \ No newline at end of file