+ {% if page.heading %}
+
+ {{ page.heading }}
+ {% elsif page.title and page.layout != 'archive' %} +{{ page.title }}
+ {% endif %} + {% if page.description %} +{{ page.description }}
+ {% endif %} + {{ content }} +
+
+
+
./configure.sh -h to see all the available options. Nevertheless, for standard (max 1Gbps) network monitoroing without any specialized tools, you should use following configuration."
+ code:
+ - "./configure.sh --with-pcap --with-quic --with-unwind"
+ -
+ description: "Then just make the ipfixprobe and install it. You might need root privileges for installation."
+ code:
+ - "make -j 2"
+ - "sudo make install"
+
+ -
+ description: "Optional NEMEA plugin. Ipfixprobe can export data directly to NEMEA framework. If you want to use this feature, you need to install NEMEA dependencies and enable this feature in autotools script."
+ code:
+ - "dnf install libtrap-devel unirec-devel"
+ - "./configure.sh --with-pcap --with-quic --with-unwind --with-nemea"
+ - "make -j 2"
+ - sudo make install
+---
diff --git a/docs/_get_options/openwrt.md b/docs/_get_options/openwrt.md
new file mode 100644
index 000000000..66b92bf33
--- /dev/null
+++ b/docs/_get_options/openwrt.md
@@ -0,0 +1,23 @@
+---
+title: Installation on Turris (OpenWrt routers)
+description: CESNET feed is officially supported by CZ.NIC, so installation on Turris devices is easy! Contrary for other OpenWrt devices, it is most likely necessary to compile a package; see our NEMEA-OpenWrt feed for more details or contact us. Installation on Turris can be done via SSH, which is described bellow, or using LUCI intuitive interface.
+
+instructions:
+ -
+ description: "Update repository metadata"
+ code:
+ - opkg update
+
+ -
+ description: "Install ipfixprobe"
+ code:
+ - opkg install ipfixprobe
+
+ -
+ description: "Optionally for LUCI configuration page, install luci-app-ipfixprobe"
+ code:
+ - opkg install luci-app-ipfixprobe
+
+
+---
+
diff --git a/docs/_how/Input plugin.md b/docs/_how/Input plugin.md
new file mode 100644
index 000000000..9a68b2dc0
--- /dev/null
+++ b/docs/_how/Input plugin.md
@@ -0,0 +1,132 @@
+---
+title: Input plugin
+description: Input plugin defines source of incoming packets. Use -i to specify input plugin.
+
+options:
+ -
+ title: "Pcap reader"
+ description: "Input plugin for reading packets from a pcap file or a network interface"
+ parameters:
+ -
+ name: "f or file"
+ description: "Defines path to pcap file."
+ -
+ name: "i or ifc"
+ description: "Defines interface name."
+ -
+ name: "F or filter"
+ description: "Defines filter string."
+ -
+ name: "s or snaplen"
+ description: "Defines snapshot length in bytes (live capture only)."
+ -
+ name: "l or list"
+ description: "Print list of available interfaces."
+ runs:
+ -
+ explanation: "Read the pcap file specified by PATH value. Possible PATH value 'pcaps/bstats.pcap' "
+ code: "./ipfixprobe -i 'pcap;file=PATH;' -s 'cache'"
+ -
+ explanation: "Read packets from interface specified by IFC value. Possible IFC value 'eth0'"
+ code: "./ipfixprobe -i 'pcap;i=IFC;' -s 'cache'"
+ -
+ title: "DPDK"
+ description: "Input plugin for reading packets using DPDK interface"
+ parameters:
+ -
+ name: "b or bsize"
+ description: "Size of the MBUF packet buffer. Default: 64."
+ -
+ name: "p or port"
+ description: "DPDK port to be used as an input interface."
+ -
+ name: "m or mem"
+ description: "Size of the memory pool for received packets. Default: 16384."
+ -
+ name: "q or queue"
+ description: "Number of RX queues. Default: 1."
+ -
+ name: "e or eal"
+ description: "DPDK eal."
+ -
+ name: "M or mtu"
+ description: "Input interface MTU. Default: 1518."
+ runs:
+ -
+ explanation: "Read packets using DPDK input interface and 1 DPDK queue, enable plugins for basic statistics, http and tls, output to IPFIX on a local machine
+ DPDK EAL parameters are passed in `e, eal` parameters
+ DPDK plugin configuration has to be specified in the first input interface.
+ The following `dpdk` interfaces are given without parameters; their configuration is inherited from the first one.
+ Example for the queue of 3 DPDK input plugins (q=3): "
+ code: "./ipfixprobe -i 'dpdk;p=0;q=3;e=-c 0x1 -a <[domain:]bus:devid.func>' -i dpdk -i dpdk -p http -p bstats -p tls -o 'ipfix;h=127.0.0.1'"
+ -
+ explanation: "Same example for the multiport read from ports 0 and 1, note comma separated ports:"
+ code: "./ipfixprobe -i 'dpdk;p=0,1;q=3;e=-c 0x1 -a <[domain:]bus:devid.func>' -i dpdk -i dpdk -p http -p bstats -p tls -o 'ipfix;h=127.0.0.1'"
+ -
+ title: "DPDK-ring"
+ description: "DPDK ring input interface for ipfixprobe (secondary DPDK app)."
+ parameters:
+ -
+ name: "b or bsize"
+ description: "Size of the MBUF packet buffer. Default: 64."
+ -
+ name: "r or ring"
+ description: "Name of the ring to read packets from. Need to be specified explicitly thus no default provided."
+ -
+ name: "e or eal"
+ description: "DPDK eal."
+ runs:
+ -
+ explanation: "Read packets using DPDK input interface as secondary process with shared memory (DPDK rings) - in this case, 4 DPDK rings are used"
+ code: "./ipfixprobe -i 'dpdk-ring;r=rx_ipfixprobe_0;e= --proc-type=secondary' -i 'dpdk-ring;r=rx_ipfixprobe_1' -i 'dpdk-ring;r=rx_ipfixprobe_2' -i 'dpdk-ring;r=rx_ipfixprobe_3' -o 'text'"
+ -
+ title: "Raw"
+ description: "Input plugin for reading packets from raw interface"
+ parameters:
+ -
+ name: "i or ifc"
+ description: "Defines network interface name."
+ -
+ name: "b or blocks"
+ description: "Defines number of packet blocks."
+ -
+ name: "f or fanout"
+ description: "Enables packet fanout."
+ -
+ name: "p or pkts"
+ description: "Defines number of packets in block."
+ -
+ name: "l or list"
+ description: "Print list of available interfaces."
+ runs:
+ -
+ explanation: "Read packets from interface specified by IFC value. Possible IFC value 'eth0'"
+ code: "./ipfixprobe -i 'raw;ifc=IFC;' -s 'cache'"
+ -
+ title: "Benchmark"
+ description: "Input plugin for various benchmarking purposes."
+ parameters:
+ -
+ name: "m or mode"
+ description: "Defines benchmark mode: 1f (1x N-packet flow) or nf (Nx 1-packet flow)."
+ -
+ name: "S or seed"
+ description: "Defines string seed for random generator."
+ -
+ name: "d or duration"
+ description: "Defines duration in seconds."
+ -
+ name: "p or count"
+ description: "Defines packet count."
+ -
+ name: "s or size"
+ description: "Defines packet size."
+ -
+ name: "I or id"
+ description: "Defines link identifier number."
+ runs:
+ -
+ explanation: "Read packets from interface specified with DPDK ports 0 and 1"
+ code: "`./ipfixprobe -i 'dpdk;p=0,1;' -s 'cache'"
+
+---
\ No newline at end of file
diff --git a/docs/_how/output plugin.md b/docs/_how/output plugin.md
new file mode 100644
index 000000000..349c290c4
--- /dev/null
+++ b/docs/_how/output plugin.md
@@ -0,0 +1,88 @@
+---
+title: Output plugin
+description: Output plugin defines how flows are expoted. Use -o to specify output plugin.
+
+options:
+-
+ title: "Text"
+ description: "Provides human readable output to the terminal or file."
+ parameters:
+ -
+ name: "f or file"
+ description: "Defines path to savefile to write output in instead of stdout."
+ -
+ name: "m or mac"
+ description: "Boolean flag. Mac addresses are hidden if set."
+
+ runs:
+ -
+ explanation: "Print expoted flows to the terminal without mac adresses "
+ code: "./ipfixprobe -o 'text;mac'-i 'pcap;file=...;' -s 'cache'"
+ -
+ explanation: "Print expoted flows to the FILE"
+ code: "./ipfixprobe -o 'text;f=FILE'-i 'pcap;file=...;' -s 'cache'"
+-
+ title: "IPFIX"
+ description: "Exports data in the IPFIX format"
+ parameters:
+ -
+ name: "h or host"
+ description: "Defines ip address of remote collector."
+ -
+ name: "p or port "
+ description: "Defines collector port to send data to."
+ -
+ name: "m or mtu"
+ description: "Defines maximum size of ipfix packet payload sent."
+ -
+ name: "u or udp"
+ description: "Boolean flag. UDP is used if set."
+ -
+ name: "n or non-blocking-tcp"
+ description: "Boolean flag. Non-blocking-tcp socket is used if set."
+ -
+ name: "I or id"
+ description: "Defines exporter id."
+ -
+ name: "t or template"
+ description: "Defines template refresh rate in seconds."
+ runs:
+ -
+ explanation: "Send exported data to the localhost using UDP as an exporter 3."
+ code: "./ipfixprobe -o 'ipfix;h=127.0.0.1,u,I=3'-i 'pcap;file=...;' -s 'cache'"
+ -
+ explanation: "Send exported data to the localhost:4739 using non-blocking tcp as an exporter 3 with maximal transfer unit set to 2000."
+ code: "./ipfixprobe -o 'ipfix;h=127.0.0.1,p=4739,n,mtu=2000'-i 'pcap;file=...;' -s 'cache'"
+-
+ title: "UNIREC"
+ description: "Exports data in the UNIREC format"
+ parameters:
+ -
+ name: "i or ifc"
+ description: "Defines unirec interface to use."
+ -
+ name: "p or plugins"
+ description: "Defines plugin-interface mapping. Plugins can be grouped like '(p1,p2,p3),p4,(p5,p6)."
+ -
+ name: "o or odid"
+ description: "Boolean flag.If set exports ODID field."
+ -
+ name: "e or eof"
+ description: "Boolean flag.If set sends eof messag on exit."
+ -
+ name: "I or id"
+ description: "Defines exporter id."
+ -
+ name: "h or help"
+ description: "Prints libtrap help."
+ runs:
+ -
+ explanation: "Send exported data to the Unix socket 'ipfixprobe'"
+ code: "./ipfixprobe -o 'unirec;i=u:ipfixprobe'-i 'pcap;file=...;' -s 'cache'"
+ -
+ explanation: "Same as previous, but should be used with small pcap files to avoid not sending data"
+ code: "./ipfixprobe -o 'unirec;i=u:ipfixprobe:timeout=WAIT:buffer=off'-i 'pcap;file=...;' -s 'cache'"
+ -
+ explanation: "Save exported data to the data.trapcap"
+ code: "./ipfixprobe -o 'unirec;i=f:data.trapcap'-i 'pcap;file=...;' -s 'cache'"
+---
\ No newline at end of file
diff --git a/docs/_how/storage plugin.md b/docs/_how/storage plugin.md
new file mode 100644
index 000000000..1d182936b
--- /dev/null
+++ b/docs/_how/storage plugin.md
@@ -0,0 +1,32 @@
+---
+title: Storage plugin
+description: Storage plugin defines how flows are internally stored. Use -s to specify storage plugin.
+
+options:
+-
+ title: "Cache"
+ description: "Currently only available plugin. Hash table is used to keep flows. Hash table is divided into rows. Each row is managed as LRU. "
+ parameters:
+ -
+ name: "s or size"
+ description: "Defines count of flows that are kept in the cache at once. Cache size is 2s."
+ -
+ name: "l or line"
+ description: "Defines length of the cache line. Line length is 2l."
+ -
+ name: "a or active"
+ description: "Defines active timeout. When there is a flow, that is active for more than -a seconds, its exported."
+ -
+ name: "i or inactive"
+ description: "Defines inactive timeout. When there is a flow, that is inactive for more than -i seconds, its exported."
+ -
+ name: "S or split "
+ description: "Boolean flag. Defines if the bidirectional flow between two nodes is splitted into 2 separate unidirectional flows."
+ -
+ name: "fe/frag-enable, fs/frag-size, ft/frag-timeout"
+ description: "Used to enable completing fragmented packets into one packet. Framentation cache size is fs and timeout to consider fragments belong to same packet is ft."
+ runs:
+ -
+ explanation: "Store flows using 'cache' "
+ code: "./ipfixprobe -s 'cache' -i 'pcap;file=PATH;'"
+---
\ No newline at end of file
diff --git a/docs/_includes/list-posts.html b/docs/_includes/list-posts.html
new file mode 100644
index 000000000..0dd91fa9b
--- /dev/null
+++ b/docs/_includes/list-posts.html
@@ -0,0 +1,10 @@
+{% for post in include.posts %}
+ + {% for category in include.post.categories %} + + {{ category | capitalize }} + + {% endfor %} + {% if include.post.date %} + {{ include.post.date | date: "%B %d, %Y" }} + {% endif %} +
diff --git a/docs/_includes/relative-src.html b/docs/_includes/relative-src.html new file mode 100644 index 000000000..fb32308f5 --- /dev/null +++ b/docs/_includes/relative-src.html @@ -0,0 +1 @@ +{% assign prefix = include.src | slice: 0, 2 %}{% assign protocol = include.src | slice: 0, 4 %}{% unless protocol == 'http' or prefix == "//" %}{{ site.baseurl }}{% endunless %}{{ include.src }} diff --git a/docs/_includes/social-icon.html b/docs/_includes/social-icon.html new file mode 100644 index 000000000..c4d125ba9 --- /dev/null +++ b/docs/_includes/social-icon.html @@ -0,0 +1,20 @@ +{% case include.icon %} + {% when "Facebook" %} + + {% when "Instagram" %} + + {% when "LinkedIn" %} + + {% when "Pinterest" %} + + {% when "Tumblr" %} + + {% when "Twitter" %} + + {% when "YouTube" %} + + {% when "RSS" %} + + {% when "GitHub" %} +
+{% endcase %}
diff --git a/docs/_layouts/archive.html b/docs/_layouts/archive.html
new file mode 100644
index 000000000..4e8073ce3
--- /dev/null
+++ b/docs/_layouts/archive.html
@@ -0,0 +1,23 @@
+---
+layout: page
+---
+
+{{ blog.description }}
+ {% endif %} + + + +
{{ page.description }}
+ {% endif %} + {{ content }} +The architecture of the ipfixprobe can be described by the following diagram:
+
+ The ipfixprobe contains script that creates template for new process plugin. To use it follow these steps:
+
Run the script:
+ +cd process
+./create_plugin.sh
+ + +To create the process plugin follow these steps:
+Add plugin_name.hpp and plugin_name.cpp files to ipfixprobe_process_src variable in Makefile.am.
Implement process plugin event functions. Don't forget to remove unused events to keep default implementation.
Set PLUGIN_NAME_UNIREC_TEMPLATE and IPFIX_PLUGIN_NAME_TEMPLATE macros to export Unirec or IPFIX data respectively.
Define Unirec and IPFIX fields to export Unirec or IPFIX respectively.
Implement fill_ipfix and fill_unirec.
Update README.md.
pre_create Is called before the creation of new flow from the initial packet.
+post_create is called after the flow is created, taket newly created flow and initial packet.
+pre_update is called when incoming packet belongs to the existing flow, before the data from the packet are added to the flow.
+post_update is called after the data of the packet are added to the flow.
+pre_export is called right before the flow is exported.
+You can also create own input plugin.
+ +To create the input plugin follow these steps:
+Process plugins can export data. Export format of each plugin is described in this section
+{{export_table.description}}
+ {% assign tableId = export_table.title | append: "_table" %} +| Output field | +Type | +IPFIX Enterprise number/ID | +Description | +
|---|---|---|---|
| {{row.name}} | +{{row.type}} | +{{row.ipfix}} | +{{row.value}} | +
{{ option.description }}
+{{ instruction.description }}
+ + {% for line in instruction.code %} +{{line}} The ipfixprobe can be set up to be used as a daemon to continuously process incoming packets from the boot up:
+Create your instance.conf configuration, and save it in /etc/ipfixprobe/
+
+ All settings are explained in the example file
+
+
To start the systemd service, use:
+ +sudo systemctl start ipfixprobe@instance.service
+ + + "instance" serves as Your identifier of ipfixprobe instance, it must be equal in the name of the configuration file and in the systemctl command. + +To enable the service at system startup, run:
+ +sudo systemctl enable ipfixprobe@instance.service
+ + +The simplest way to use ipfixprobe is to process PCAP file using PCAP plugin (need to be ./configured with --with-pcap)
+./ipfixprobe -s cache -i "pcap;file=pcaps/http.pcap" -o "text;m"
+ +{{ plugin.description | raw }}
+The ipfixprobe consists of one input, zero or one output, one storage and zero or more process plugins.
+ {% for plugin in site.how %} +{{ plugin.description | raw }}
+{{ option.description }}
+ {% if plugin.options %} + + Command line parameters used by {{ option.title }} plugin:{{ run.explanation }}
+{{ run.code }}
+ + {% endfor %} + {% endfor %} + {% endfor %} + +The ipfixprobe can be used to convert given PCAP file to the CSV containing flows from that file in the Unirec format.
+ + Requirements:The script builds the image automatically, but be sure that Dockerfile is in the same directory.
+ To build the manually image, navigate to the directory containing the Dockerfile and run:
docker build -t docker_ipfixprobe .
+ +bash ./ipfixprobe_wrapper.sh <process_script.sh> <input_file.pcap> <output_file.csv>
+ +To process a file ../pcaps/mixed.pcap using a processing script process_script.sh and output the results to output.csv, use the following wrapper script:
+ +bash ./ipfixprobe_wrapper.sh ./process_script.sh ../pcaps/mixed.pcap ./output.csv
+ + +Create and save the configuration file to the /etc/config/ipfixprobe. Example of configuration file:
NEMEA ipfixprobe + # Copyright (C) 2022-2023 CESNET + + # Available options for profiles, 'list' options can be used repeatedly: + # list interfaces - list of NIC, e.g., eth0, enp0s1, ... + # list plugins - list of plugin names, see 'ipfixprobe -h process' for help + # ipfix_host - address of IPFIX collector + # ipfix_port - port of IPFIX collector, default: 4739 + # ipfix_udp - 1 to export to IPFIX collector via UDP, 0 via TCP + # cache_size - size of flow cache as exponent of 2, default: 1024 + # cache_line - size of flow cache line as exponent of 2, default: 4 + # active_timeout - active timeout in seconds, default: 300 + # inactive_timeout - inactive timeout in seconds, default: 30 + # link - unsigned integer as identification of link/router + # dir - unsigned integer as identification of direction/NIC + # split_biflow - 1 to split biflow to uniflow, default: 0 to use biflow + # ipfix_mtu - size of max transmission unit (MTU), default: 1452 + # + # respawn - enable respawn of crashed process + # respawn_threshold - timeout in seconds for restarting a service after it closes + # respawn_timeout - max time in seconds to wait for a process respawn to complete + # respawn_retry - max number of attempts to respawn before giving up, 0 means newer stop trying to respawn + # core - size of coredump, '0' - not generate, 'unlimited' - unlimited size + # + # enabled - 1 to enable start daemon instance for that profile, NOTE: if profile is directly specified for start script + # (example: '/etc/init.d/ipfixprobe start wan profileX profileY lan'), this option is ignored + + config profile 'lan' + option enabled '0' + list interfaces 'br-lan' + list plugins 'basicplus' + list plugins 'dns' + list plugins 'http' + list plugins 'pstats' + list plugins 'ovpn' + list plugins 'wg' + list plugins 'dnssd;txt' + list plugins 'ssdp' + list plugins 'tls' + list plugins 'quic' + option ipfix_host '127.0.0.1' + option ipfix_port '4739' + option ipfix_udp '1' + option link '1' + option dir '1'
+ +Prepare the init script:
+ +wget https://raw.githubusercontent.com/CESNET/Nemea-OpenWRT/master/net/ipfixprobe/files/init.d/ipfixprobe
+sudo mv ipfixprobe /etc/init.d/ipfixprobe
+ +To run the script manually use:
+ +/etc/init.d/ipfixprobe start
+ +To start the service at system startup:
+ +/etc/init.d/ipfixprobe enable
+ +Monitor and analyze your network traffic for free!
+ +The ipfixprobe exporter can be deployed to small SOHO routers with OpenWRT operating systems. In case of Turris routers, ipfixprobe is available in its standard repositories.
+The ipfixprobe supports multiple inputs. From the high-speed DPDK or NDP inputs produced by specialized 100 Gbps network monitoring cards, + to slower interfaces such as libpcap that is supported in any operating system.
+The ipfixprobe support multiple output plugins. From the basic ones, where flow data are dumped to standard output in form of text to the standard IPFIX format, that can be used together with IPFIX collectors such as ipfixcol2, which enables multiple additional format outputs. Just check out the documentation to learn how to use ipfixprobe and ipfixcol2 together with systems like Apache Kafka or ELK.
+