Skip to content

Field classifier dev #71

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jaroslavpesek merged 3 commits into from
Oct 29, 2025
Merged

Field classifier dev #71

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1e04a90
fieldClassifier: optimising module
Tomas-Vrana Oct 27, 2025
4f3e5c1
fieldClassifier: bugfix: fixing ipclassifier module
Tomas-Vrana Oct 29, 2025
636df2a
Merge branch 'staging' into field_classifier_dev
jaroslavpesek Oct 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
329 changes: 204 additions & 125 deletions modules/fieldClassifier/src/fieldClassifier.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,48 @@ void FieldClassifier::getUnirecIdsForDirection(const std::string& prefix, IDMap&
idMap.at(fieldName) = idField;
}
}

void FieldClassifier::getUnirecIdsForInputFields(std::string& templateStr)
{
m_inputFieldIds.clear();
m_inputFieldTypes.clear();
// split template to vector by commas
std::vector<std::string> templateFields;
size_t pos = 0;
std::string strCopy = templateStr;
if (templateStr.back() != ',') {
strCopy += ','; // Add a trailing comma to ensure the last field is processed
}
while ((pos = strCopy.find(',')) != std::string::npos) {
// Extract the field
std::string field = strCopy.substr(0, pos);
strCopy.erase(0, pos + 1);
if (!field.empty()) { // ignore empty fields
templateFields.push_back(field);
}
}
for (const auto& field : templateFields) {
// split field by space to get field name
size_t spacePos = field.find(' ');
if (spacePos == std::string::npos) {
throw std::runtime_error(
"FieldClassifier: Invalid Unirec field format in template: " + field);
}
std::string fieldName = field.substr(spacePos + 1);
std::string fieldType = field.substr(0, spacePos);

auto fieldId = static_cast<ur_field_id_t>(ur_get_id_by_name(fieldName.c_str()));
if (fieldId == UR_E_INVALID_NAME) {
throw std::runtime_error("FieldClassifier: Invalid Unirec field name: " + fieldName);
}
m_inputFieldIds.push_back(fieldId);
if (m_UnirecTypeMap.find(fieldType) == m_UnirecTypeMap.end()) {
throw std::runtime_error("FieldClassifier: Invalid Unirec field type: " + fieldType);
}
m_inputFieldTypes.push_back(m_UnirecTypeMap.at(fieldType));
}
}

void FieldClassifier::getUnirecIds()
{
// get Unirec field IDs for source and destination IP fields
Expand Down Expand Up @@ -285,138 +327,175 @@ void FieldClassifier::loadIP(
}
void FieldClassifier::fillInputFieldsToOutput(
std::optional<Nemea::UnirecRecordView>& input,
std::optional<Nemea::UnirecRecord>& output,
std::string& templateStr)
std::optional<Nemea::UnirecRecord>& output)

{
// split template to vector by commas
std::vector<std::string> templateFields;
size_t pos = 0;
std::string strCopy = templateStr;
if (templateStr.back() != ',') {
strCopy += ','; // Add a trailing comma to ensure the last field is processed
}
while ((pos = strCopy.find(',')) != std::string::npos) {
// Extract the field
std::string field = strCopy.substr(0, pos);
strCopy.erase(0, pos + 1);
if (!field.empty()) { // ignore empty fields
templateFields.push_back(field);
unsigned long size = m_inputFieldIds.size();
for (unsigned long i = 0; i < size; i++) {
auto fieldId = m_inputFieldIds[i];
auto fieldTypeIt = m_inputFieldTypes[i];

switch (fieldTypeIt) {
case DataType::STRING: {
auto value = input->getFieldAsType<std::string>(fieldId);
output->setFieldFromType<std::string>(value, fieldId);
break;
}
}
// for each field, get data from input and set to output
for (const auto& field : templateFields) {
// split field by space to get field name
size_t spacePos = field.find(' ');
if (spacePos == std::string::npos) {
throw std::runtime_error(
"FieldClassifier: Invalid Unirec field format in template: " + field);
case DataType::UINT8: {
auto value = input->getFieldAsType<uint8_t>(fieldId);
output->setFieldFromType<uint8_t>(value, fieldId);
break;
}
std::string fieldName = field.substr(spacePos + 1);
std::string fieldType = field.substr(0, spacePos);

auto fieldId = static_cast<ur_field_id_t>(ur_get_id_by_name(fieldName.c_str()));
if (fieldId == UR_E_INVALID_NAME) {
throw std::runtime_error("FieldClassifier: Invalid Unirec field name: " + fieldName);
case DataType::UINT16: {
auto value = input->getFieldAsType<uint16_t>(fieldId);
output->setFieldFromType<uint16_t>(value, fieldId);
break;
}
try {
if (fieldType == "string") {
auto value = input->getFieldAsType<std::string>(fieldId);
output->setFieldFromType<std::string>(value, fieldId);
} else if (fieldType == "uint8") {
auto value = input->getFieldAsType<uint8_t>(fieldId);
output->setFieldFromType<uint8_t>(value, fieldId);
} else if (fieldType == "uint16") {
auto value = input->getFieldAsType<uint16_t>(fieldId);
output->setFieldFromType<uint16_t>(value, fieldId);
} else if (fieldType == "uint32") {
auto value = input->getFieldAsType<uint32_t>(fieldId);
output->setFieldFromType<uint32_t>(value, fieldId);
} else if (fieldType == "uint64") {
auto value = input->getFieldAsType<uint64_t>(fieldId);
output->setFieldFromType<uint64_t>(value, fieldId);
} else if (fieldType == "ipaddr") {
auto value = input->getFieldAsType<Nemea::IpAddress>(fieldId);
output->setFieldFromType<Nemea::IpAddress>(value, fieldId);
} else if (fieldType == "int8") {
auto value = input->getFieldAsType<int8_t>(fieldId);
output->setFieldFromType<int8_t>(value, fieldId);
} else if (fieldType == "int16") {
auto value = input->getFieldAsType<int16_t>(fieldId);
output->setFieldFromType<int16_t>(value, fieldId);
} else if (fieldType == "int32") {
auto value = input->getFieldAsType<int32_t>(fieldId);
output->setFieldFromType<int32_t>(value, fieldId);
} else if (fieldType == "int64") {
auto value = input->getFieldAsType<int64_t>(fieldId);
output->setFieldFromType<int64_t>(value, fieldId);
} else if (fieldType == "char") {
auto value = input->getFieldAsType<char>(fieldId);
output->setFieldFromType<char>(value, fieldId);
} else if (fieldType == "float") {
auto value = input->getFieldAsType<float>(fieldId);
output->setFieldFromType<float>(value, fieldId);
} else if (fieldType == "double") {
auto value = input->getFieldAsType<double>(fieldId);
output->setFieldFromType<double>(value, fieldId);
} else if (fieldType == "macaddr") {
auto value = input->getFieldAsType<Nemea::MacAddress>(fieldId);
output->setFieldFromType<Nemea::MacAddress>(value, fieldId);
} else if (fieldType == "time") {
auto value = input->getFieldAsType<Nemea::UrTime>(fieldId);
output->setFieldFromType<Nemea::UrTime>(value, fieldId);
//} else if (fieldType == "bytes") {
// Nemea::UnirecArray<std::byte> const arr
// = input->getFieldAsUnirecArray<std::byte>(fieldId);
// output->setFieldFromUnirecArray<std::byte>(arr, fieldId);
} else if (fieldType == "int8*") {
Nemea::UnirecArray<int8_t> const arr
= input->getFieldAsUnirecArray<int8_t>(fieldId);
output->setFieldFromUnirecArray<int8_t>(arr, fieldId);
} else if (fieldType == "int16*") {
Nemea::UnirecArray<int16_t> const arr
= input->getFieldAsUnirecArray<int16_t>(fieldId);
output->setFieldFromUnirecArray<int16_t>(arr, fieldId);
} else if (fieldType == "int32*") {
Nemea::UnirecArray<int32_t> const arr
= input->getFieldAsUnirecArray<int32_t>(fieldId);
output->setFieldFromUnirecArray<int32_t>(arr, fieldId);
} else if (fieldType == "int64*") {
Nemea::UnirecArray<int64_t> const arr
= input->getFieldAsUnirecArray<int64_t>(fieldId);
output->setFieldFromUnirecArray<int64_t>(arr, fieldId);
} else if (fieldType == "uint8*") {
Nemea::UnirecArray<uint8_t> const arr
= input->getFieldAsUnirecArray<uint8_t>(fieldId);
output->setFieldFromUnirecArray<uint8_t>(arr, fieldId);
} else if (fieldType == "uint16*") {
Nemea::UnirecArray<uint16_t> const arr
= input->getFieldAsUnirecArray<uint16_t>(fieldId);
output->setFieldFromUnirecArray<uint16_t>(arr, fieldId);
} else if (fieldType == "uint32*") {
Nemea::UnirecArray<uint32_t> const arr
= input->getFieldAsUnirecArray<uint32_t>(fieldId);
output->setFieldFromUnirecArray<uint32_t>(arr, fieldId);
} else if (fieldType == "uint64*") {
Nemea::UnirecArray<uint64_t> const arr
= input->getFieldAsUnirecArray<uint64_t>(fieldId);
output->setFieldFromUnirecArray<uint64_t>(arr, fieldId);
} else if (fieldType == "float*") {
Nemea::UnirecArray<float> const arr = input->getFieldAsUnirecArray<float>(fieldId);
output->setFieldFromUnirecArray<float>(arr, fieldId);
} else if (fieldType == "double*") {
Nemea::UnirecArray<double> const arr
= input->getFieldAsUnirecArray<double>(fieldId);
output->setFieldFromUnirecArray<double>(arr, fieldId);
} else {
throw std::runtime_error(
"FieldClassifier: Unsupported Unirec field type in template: " + fieldType);
}
case DataType::UINT32: {
auto value = input->getFieldAsType<uint32_t>(fieldId);
output->setFieldFromType<uint32_t>(value, fieldId);
break;
}
case DataType::UINT64: {
auto value = input->getFieldAsType<uint64_t>(fieldId);
output->setFieldFromType<uint64_t>(value, fieldId);
break;
}
case DataType::IPADDR: {
auto value = input->getFieldAsType<Nemea::IpAddress>(fieldId);
output->setFieldFromType<Nemea::IpAddress>(value, fieldId);
break;
}
case DataType::INT8: {
auto value = input->getFieldAsType<int8_t>(fieldId);
output->setFieldFromType<int8_t>(value, fieldId);
break;
}
case DataType::INT16: {
auto value = input->getFieldAsType<int16_t>(fieldId);
output->setFieldFromType<int16_t>(value, fieldId);
break;
}
case DataType::INT32: {
auto value = input->getFieldAsType<int32_t>(fieldId);
output->setFieldFromType<int32_t>(value, fieldId);
break;
}
case DataType::INT64: {
auto value = input->getFieldAsType<int64_t>(fieldId);
output->setFieldFromType<int64_t>(value, fieldId);
break;
}
case DataType::CHAR: {
auto value = input->getFieldAsType<char>(fieldId);
output->setFieldFromType<char>(value, fieldId);
break;
}
case DataType::FLOAT: {
auto value = input->getFieldAsType<float>(fieldId);
output->setFieldFromType<float>(value, fieldId);
break;
}
case DataType::DOUBLE: {
auto value = input->getFieldAsType<double>(fieldId);
output->setFieldFromType<double>(value, fieldId);
break;
}
case DataType::MACADDR: {
auto value = input->getFieldAsType<Nemea::MacAddress>(fieldId);
output->setFieldFromType<Nemea::MacAddress>(value, fieldId);
break;
}
case DataType::TIME: {
auto value = input->getFieldAsType<Nemea::UrTime>(fieldId);
output->setFieldFromType<Nemea::UrTime>(value, fieldId);
break;
}
case DataType::BYTES: {
// Nemea::UnirecArray<std::byte> const arr
// = input->getFieldAsUnirecArray<std::byte>(fieldId);
// output->setFieldFromUnirecArray<std::byte>(arr, fieldId);

} catch (const std::exception& ex) {
// std::byte value = input->getFieldAsType<std::byte>(fieldId);
// output->setFieldFromType<std::byte>(value, fieldId);

break;
}
case DataType::A_INT8: {
Nemea::UnirecArray<int8_t> const arr = input->getFieldAsUnirecArray<int8_t>(fieldId);
output->setFieldFromUnirecArray<int8_t>(arr, fieldId);
break;
}
case DataType::A_INT16: {
Nemea::UnirecArray<int16_t> const arr = input->getFieldAsUnirecArray<int16_t>(fieldId);
output->setFieldFromUnirecArray<int16_t>(arr, fieldId);
break;
}
case DataType::A_INT32: {
Nemea::UnirecArray<int32_t> const arr = input->getFieldAsUnirecArray<int32_t>(fieldId);
output->setFieldFromUnirecArray<int32_t>(arr, fieldId);
break;
}
case DataType::A_INT64: {
Nemea::UnirecArray<int64_t> const arr = input->getFieldAsUnirecArray<int64_t>(fieldId);
output->setFieldFromUnirecArray<int64_t>(arr, fieldId);
break;
}
case DataType::A_UINT8: {
Nemea::UnirecArray<uint8_t> const arr = input->getFieldAsUnirecArray<uint8_t>(fieldId);
output->setFieldFromUnirecArray<uint8_t>(arr, fieldId);
break;
}
case DataType::A_UINT16: {
Nemea::UnirecArray<uint16_t> const arr
= input->getFieldAsUnirecArray<uint16_t>(fieldId);
output->setFieldFromUnirecArray<uint16_t>(arr, fieldId);
break;
}
case DataType::A_UINT32: {
Nemea::UnirecArray<uint32_t> const arr
= input->getFieldAsUnirecArray<uint32_t>(fieldId);
output->setFieldFromUnirecArray<uint32_t>(arr, fieldId);
break;
}
case DataType::A_UINT64: {
Nemea::UnirecArray<uint64_t> const arr
= input->getFieldAsUnirecArray<uint64_t>(fieldId);
output->setFieldFromUnirecArray<uint64_t>(arr, fieldId);
break;
}
case DataType::A_FLOAT: {
Nemea::UnirecArray<float> const arr = input->getFieldAsUnirecArray<float>(fieldId);
output->setFieldFromUnirecArray<float>(arr, fieldId);
break;
}
case DataType::A_DOUBLE: {
Nemea::UnirecArray<double> const arr = input->getFieldAsUnirecArray<double>(fieldId);
output->setFieldFromUnirecArray<double>(arr, fieldId);
break;
}
case DataType::A_IP: {
Nemea::UnirecArray<Nemea::IpAddress> const arr
= input->getFieldAsUnirecArray<Nemea::IpAddress>(fieldId);
output->setFieldFromUnirecArray<Nemea::IpAddress>(arr, fieldId);
break;
}
case DataType::A_MAC: {
Nemea::UnirecArray<Nemea::MacAddress> const arr
= input->getFieldAsUnirecArray<Nemea::MacAddress>(fieldId);
output->setFieldFromUnirecArray<Nemea::MacAddress>(arr, fieldId);
break;
}
case DataType::A_TIME: {
Nemea::UnirecArray<Nemea::UrTime> const arr
= input->getFieldAsUnirecArray<Nemea::UrTime>(fieldId);
output->setFieldFromUnirecArray<Nemea::UrTime>(arr, fieldId);
break;
}
default:
throw std::runtime_error(
std::string("FieldClassifier: Unable to copy field '") + field
+ "' from input to output Unirec record: " + ex.what());
"FieldClassifier: Unsupported Unirec field type for field ID: "
+ std::to_string(fieldId));
}
}
}
Expand Down
27 changes: 25 additions & 2 deletions modules/fieldClassifier/src/fieldClassifier.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,10 +76,12 @@ class FieldClassifier {
*/
void handleParams(int argc, char** argv, argparse::ArgumentParser& parser);

void getUnirecIdsForInputFields(std::string& templateStr);

void fillInputFieldsToOutput(
std::optional<Nemea::UnirecRecordView>& input,
std::optional<Nemea::UnirecRecord>& output,
std::string& templateStr);
std::optional<Nemea::UnirecRecord>& output);
ur_template_t* templateStringAll;

private:
std::vector<class Plugin*> m_plugins = g_PLUGINS;
Expand Down Expand Up @@ -107,6 +109,9 @@ class FieldClassifier {

std::string m_templateStr;

std::vector<ur_field_id_t> m_inputFieldIds;
std::vector<DataType> m_inputFieldTypes;

std::string getIPtoString(const Nemea::IpAddress& ipAddr);

void loadIP(
Expand Down Expand Up @@ -146,5 +151,23 @@ class FieldClassifier {

void getRequiredFields(const std::string& requiredFields);
// ###################

std::unordered_map<std::string, DataType> m_UnirecTypeMap = {
{"string", DataType::STRING}, {"int8", DataType::INT8},
{"int16", DataType::INT16}, {"int32", DataType::INT32},
{"int64", DataType::INT64}, {"uint8", DataType::UINT8},
{"uint16", DataType::UINT16}, {"uint32", DataType::UINT32},
{"uint64", DataType::UINT64}, {"char", DataType::CHAR},
{"float", DataType::FLOAT}, {"double", DataType::DOUBLE},
{"ipaddr", DataType::IPADDR}, {"macaddr", DataType::MACADDR},
{"bytes", DataType::BYTES}, {"int8*", DataType::A_INT8},
{"int16*", DataType::A_INT16}, {"int32*", DataType::A_INT32},
{"int64*", DataType::A_INT64}, {"uint8*", DataType::A_UINT8},
{"uint16*", DataType::A_UINT16}, {"uint32*", DataType::A_UINT32},
{"uint64*", DataType::A_UINT64}, {"float*", DataType::A_FLOAT},
{"double*", DataType::A_DOUBLE}, {"ipaddr*", DataType::A_IP},
{"time*", DataType::A_TIME}, {"time", DataType::TIME},
{"macaddr*", DataType::A_MAC},
};
};
} // namespace NFieldClassifier
Loading
Loading