From 432b94b7a8ddaef207e45a29ee52b5e039881e5c Mon Sep 17 00:00:00 2001 From: Adam Debreceni Date: Thu, 14 Jan 2021 13:26:10 +0100 Subject: [PATCH 01/15] MINIFICPP-1448 - Provide JSON output format (flavors) --- .github/workflows/ci.yml | 20 ++ .../ConsumeWindowsEventLog.cpp | 67 +++- .../ConsumeWindowsEventLog.h | 26 +- .../tests/ConsumeWindowsEventLogTests.cpp | 290 ++++++++++++++++-- .../windows-event-log/wel/JSONConverter.h | 161 ++++++++++ .../windows-event-log/wel/MetadataWalker.cpp | 5 +- .../test/resources/wel/unit-test-provider.man | 36 +++ 7 files changed, 552 insertions(+), 53 deletions(-) create mode 100644 extensions/windows-event-log/wel/JSONConverter.h create mode 100644 libminifi/test/resources/wel/unit-test-provider.man diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1e00d42e59..2fc202a4cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,6 +46,16 @@ jobs: uses: actions/checkout@v2 - name: Setup PATH uses: microsoft/setup-msbuild@v1.0.2 + - id: create-wel-provider + run: | + cd libminifi\test\resources\wel + PATH %PATH%;C:\Program Files (x86)\Windows Kits\10\bin\10.0.17763.0\x86 + PATH %PATH%;C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\MSBuild\15.0\Bin\Roslyn + mc -css Namespace unit-test-provider.man + rc unit-test-provider.rc + csc /target:library /unsafe /win32res:unit-test-provider.res unit-test-provider.cs + wevtutil im unit-test-provider.man + shell: cmd - id: build run: win_build_vs.bat build /CI /S /A shell: cmd @@ -58,6 +68,16 @@ jobs: uses: actions/checkout@v2 - name: Setup PATH uses: microsoft/setup-msbuild@v1.0.2 + - id: create-wel-provider + run: | + cd libminifi\test\resources\wel + PATH %PATH%;C:\Program Files (x86)\Windows Kits\10\bin\10.0.19041.0\x64 + PATH %PATH%;C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\Roslyn + mc -css Namespace unit-test-provider.man + rc unit-test-provider.rc + csc /target:library /unsafe /win32res:unit-test-provider.res unit-test-provider.cs + wevtutil im unit-test-provider.man + shell: cmd - id: build run: win_build_vs.bat build /2019 /64 /CI shell: cmd diff --git a/extensions/windows-event-log/ConsumeWindowsEventLog.cpp b/extensions/windows-event-log/ConsumeWindowsEventLog.cpp index 22442f9b33..aa16bad925 100644 --- a/extensions/windows-event-log/ConsumeWindowsEventLog.cpp +++ b/extensions/windows-event-log/ConsumeWindowsEventLog.cpp @@ -34,6 +34,7 @@ #include "wel/MetadataWalker.h" #include "wel/XMLString.h" #include "wel/UnicodeConversion.h" +#include "wel/JSONConverter.h" #include "io/BufferStream.h" #include "core/ProcessContext.h" @@ -134,8 +135,8 @@ core::Property ConsumeWindowsEventLog::OutputFormat( core::PropertyBuilder::createProperty("Output Format")-> isRequired(true)-> withDefaultValue(Both)-> - withAllowableValues({XML, Plaintext, Both})-> - withDescription("Set the output format type. In case \'Both\' is selected the processor generates two flow files for every event captured")-> + withAllowableValues({XML, Plaintext, Both, JSONSimple, JSONFlattened, JSONRaw})-> + withDescription("Set the output format type. In case \'Both\' is selected the processor generates two flow files for every event captured in format XML and Plaintext")-> build()); core::Property ConsumeWindowsEventLog::BatchCommitSize( @@ -162,7 +163,8 @@ core::Property ConsumeWindowsEventLog::ProcessOldEvents( core::Relationship ConsumeWindowsEventLog::Success("success", "Relationship for successfully consumed events."); ConsumeWindowsEventLog::ConsumeWindowsEventLog(const std::string& name, utils::Identifier uuid) - : core::Processor(name, uuid), logger_(logging::LoggerFactory::getLogger()), apply_identifier_function_(false), batch_commit_size_(0U) { + : core::Processor(name, uuid), + logger_(logging::LoggerFactory::getLogger()) { char buff[MAX_COMPUTERNAME_LENGTH + 1]; DWORD size = sizeof(buff); if (GetComputerName(buff, &size)) { @@ -170,9 +172,6 @@ ConsumeWindowsEventLog::ConsumeWindowsEventLog(const std::string& name, utils::I } else { LogWindowsError(); } - - writeXML_ = false; - writePlainText_ = false; } void ConsumeWindowsEventLog::notifyStop() { @@ -252,11 +251,27 @@ void ConsumeWindowsEventLog::onSchedule(const std::shared_ptrgetProperty(OutputFormat.getName(), mode); - writeXML_ = (mode == Both || mode == XML); - - writePlainText_ = (mode == Both || mode == Plaintext); + writeXML_ = false; + writePlainText_ = false; + jsonFormat_ = JSONFormat::None; + if (mode == XML) { + writeXML_ = true; + } else if (mode == Plaintext) { + writePlainText_ = true; + } else if (mode == Both) { + writeXML_ = true; + writePlainText_ = true; + } else if (mode == JSONRaw) { + jsonFormat_ = JSONFormat::Raw; + } else if (mode == JSONSimple) { + jsonFormat_ = JSONFormat::Simple; + } else if (mode == JSONFlattened) { + jsonFormat_ = JSONFormat::Flattened; + } else { + // throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Unrecognized output format: " + mode); + } - if (writeXML_ && !hMsobjsDll_) { + if ((writeXML_ || jsonFormat_ != JSONFormat::None) && !hMsobjsDll_) { char systemDir[MAX_PATH]; if (GetSystemDirectory(systemDir, sizeof(systemDir))) { hMsobjsDll_ = LoadLibrary((systemDir + std::string("\\msobjs.dll")).c_str()); @@ -590,14 +605,17 @@ bool ConsumeWindowsEventLog::createEventRender(EVT_HANDLE hEvent, EventRender& e logger_->log_trace("Finish writing in plain text"); } - if (writeXML_) { - logger_->log_trace("Writing event in XML"); + if (writeXML_ || jsonFormat_ != JSONFormat::None) { substituteXMLPercentageItems(doc); logger_->log_trace("Finish substituting %% in XML"); if (resolve_as_attributes_) { eventRender.matched_fields_ = walker.getFieldValues(); } + } + + if (writeXML_) { + logger_->log_trace("Writing event in XML"); wel::XmlString writer; doc.print(writer, "", pugi::format_raw); // no indentation or formatting @@ -607,6 +625,19 @@ bool ConsumeWindowsEventLog::createEventRender(EVT_HANDLE hEvent, EventRender& e logger_->log_trace("Finish writing in XML"); } + if (jsonFormat_ != JSONFormat::None) { + logger_->log_trace("Writing event in JSON"); + + if (jsonFormat_ == JSONFormat::Raw) { + eventRender.json_ = wel::jsonToString(wel::toRawJSON(doc)); + } else { + eventRender.json_ = wel::jsonToString(wel::toJSON(doc, jsonFormat_ == JSONFormat::Flattened)); + } + + + logger_->log_trace("Finish writing in JSON"); + } + return true; } @@ -692,6 +723,18 @@ void ConsumeWindowsEventLog::putEventRenderFlowFileToSession(const EventRender& session.getProvenanceReporter()->receive(flowFile, provenanceUri_, getUUIDStr(), "Consume windows event logs", 0); session.transfer(flowFile, Success); } + + if (jsonFormat_ != JSONFormat::None) { + auto flowFile = session.create(); + logger_->log_trace("Writing rendered JSON to a flow file"); + + session.write(flowFile, &WriteCallback(eventRender.json_)); + session.putAttribute(flowFile, core::SpecialFlowAttribute::MIME_TYPE, "application/json"); + session.putAttribute(flowFile, "Timezone name", timezone_name_); + session.putAttribute(flowFile, "Timezone offset", timezone_offset_); + session.getProvenanceReporter()->receive(flowFile, provenanceUri_, getUUIDStr(), "Consume windows event logs", 0); + session.transfer(flowFile, Success); + } } void ConsumeWindowsEventLog::LogWindowsError(std::string error) const { diff --git a/extensions/windows-event-log/ConsumeWindowsEventLog.h b/extensions/windows-event-log/ConsumeWindowsEventLog.h index db4f13b7a7..a7c3a0ee5a 100644 --- a/extensions/windows-event-log/ConsumeWindowsEventLog.h +++ b/extensions/windows-event-log/ConsumeWindowsEventLog.h @@ -46,6 +46,7 @@ struct EventRender { std::map matched_fields_; std::string text_; std::string rendered_text_; + std::string json_; }; class Bookmark; @@ -107,9 +108,12 @@ class ConsumeWindowsEventLog : public core::Processor { bool createEventRender(EVT_HANDLE eventHandle, EventRender& eventRender); void substituteXMLPercentageItems(pugi::xml_document& doc); - static constexpr const char * const XML = "XML"; - static constexpr const char * const Both = "Both"; - static constexpr const char * const Plaintext = "Plaintext"; + static constexpr const char* XML = "XML"; + static constexpr const char* Both = "Both"; + static constexpr const char* Plaintext = "Plaintext"; + static constexpr const char* JSONSimple = "JSON::Simple"; + static constexpr const char* JSONFlattened = "JSON::Flattened"; + static constexpr const char* JSONRaw = "JSON::Raw"; private: struct TimeDiff { @@ -133,17 +137,23 @@ class ConsumeWindowsEventLog : public core::Processor { std::wstring wstrQuery_; std::string regex_; bool resolve_as_attributes_; - bool apply_identifier_function_; + bool apply_identifier_function_{false}; std::string provenanceUri_; std::string computerName_; uint64_t maxBufferSize_{}; DWORD lastActivityTimestamp_{}; std::mutex cache_mutex_; std::map providers_; - uint64_t batch_commit_size_; - - bool writeXML_; - bool writePlainText_; + uint64_t batch_commit_size_{}; + + bool writeXML_{false}; + bool writePlainText_{false}; + enum class JSONFormat { + None, + Simple, + Flattened, + Raw + } jsonFormat_{JSONFormat::None}; std::unique_ptr bookmark_; std::mutex on_trigger_mutex_; std::unordered_map xmlPercentageItemsResolutions_; diff --git a/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp b/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp index 02aae26507..ba9a26d56c 100644 --- a/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp +++ b/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp @@ -15,14 +15,21 @@ * limitations under the License. */ +#include + #include "ConsumeWindowsEventLog.h" #include "core/ConfigurableComponent.h" #include "processors/LogAttribute.h" +#include "processors/PutFile.h" #include "TestBase.h" +#include "utils/TestUtils.h" +#include "utils/file/FileUtils.h" +#include "rapidjson/document.h" using ConsumeWindowsEventLog = org::apache::nifi::minifi::processors::ConsumeWindowsEventLog; using LogAttribute = org::apache::nifi::minifi::processors::LogAttribute; +using PutFile = org::apache::nifi::minifi::processors::PutFile; using ConfigurableComponent = org::apache::nifi::minifi::core::ConfigurableComponent; using IdGenerator = org::apache::nifi::minifi::utils::IdGenerator; @@ -41,6 +48,29 @@ void reportEvent(const std::string& channel, const char* message, WORD log_level ReportEventA(event_source, log_level, 0, CWEL_TESTS_OPCODE, nullptr, 1, 0, &message, nullptr); } +struct CustomEvent { + std::string first; + std::string second; + std::string third; + int binary_length; + int binary_data; +}; + +const std::string custom_provider_name = "minifi_unit_test_provider"; +const std::string custom_channel = custom_provider_name + "/Log"; + +bool dispatchCustomEvent(const CustomEvent& event) { + std::string command = "powershell \"New-WinEvent -ProviderName " + custom_provider_name + + " -Id 10000 -Payload @('" + event.first + "','" + event.second + "', '" + event.third + + "', " + std::to_string(event.binary_length) + ", " + std::to_string(event.binary_data) + ")\""; + auto result = std::system(command.c_str()); + return result == 0; +} + +bool supportsCustomEvents() { + return dispatchCustomEvent({"First", "Second", "Third", 2, 5}); +} + } // namespace TEST_CASE("ConsumeWindowsEventLog constructor works", "[create]") { @@ -300,53 +330,249 @@ TEST_CASE("ConsumeWindowsEventLog output format can be set", "[create][output_fo outputFormatSetterTestHelper("InvalidValue", 0); } +namespace { + +class OutputFormatTestController : public TestController { + public: + OutputFormatTestController(std::string channel, std::string query, std::string output_format) + : channel_(std::move(channel)), + query_(std::move(query)), + output_format_(std::move(output_format)) {} + + std::string run() { + LogTestController::getInstance().setDebug(); + LogTestController::getInstance().setDebug(); + std::shared_ptr test_plan = createPlan(); + + auto cwel_processor = test_plan->addProcessor("ConsumeWindowsEventLog", "cwel"); + test_plan->setProperty(cwel_processor, ConsumeWindowsEventLog::Channel.getName(), channel_); + test_plan->setProperty(cwel_processor, ConsumeWindowsEventLog::Query.getName(), query_); + test_plan->setProperty(cwel_processor, ConsumeWindowsEventLog::OutputFormat.getName(), output_format_); + + auto dir = utils::createTempDir(this); + + auto put_file = test_plan->addProcessor("PutFile", "putFile", Success, true); + test_plan->setProperty(put_file, PutFile::Directory.getName(), dir); + + { + dispatchBookmarkEvent(); + + runSession(test_plan); + } + + test_plan->reset(); + LogTestController::getInstance().resetStream(LogTestController::getInstance().log_output); + + + { + dispatchCollectedEvent(); + + runSession(test_plan); + + auto files = utils::file::list_dir_all(dir, LogTestController::getInstance().getLogger(), false); + REQUIRE(files.size() == 1); + + std::ifstream file{utils::file::concat_path(files[0].first, files[0].second)}; + return {std::istreambuf_iterator{file}, {}}; + } + } + + protected: + virtual void dispatchBookmarkEvent() = 0; + virtual void dispatchCollectedEvent() = 0; + + std::string channel_; + std::string query_; + std::string output_format_; +}; + +} // namespace + // NOTE(fgerlits): I don't know how to unit test this, as my manually published events all result in an empty string if OutputFormat is Plaintext // but it does seem to work, based on manual tests reading system logs // TEST_CASE("ConsumeWindowsEventLog prints events in plain text correctly", "[onTrigger]") TEST_CASE("ConsumeWindowsEventLog prints events in XML correctly", "[onTrigger]") { - TestController test_controller; - LogTestController::getInstance().setDebug(); - LogTestController::getInstance().setDebug(); - std::shared_ptr test_plan = test_controller.createPlan(); + class XMLFormat : public OutputFormatTestController { + public: + XMLFormat() : OutputFormatTestController(APPLICATION_CHANNEL, QUERY, "XML") {} - auto cwel_processor = test_plan->addProcessor("ConsumeWindowsEventLog", "cwel"); - test_plan->setProperty(cwel_processor, ConsumeWindowsEventLog::Channel.getName(), APPLICATION_CHANNEL); - test_plan->setProperty(cwel_processor, ConsumeWindowsEventLog::Query.getName(), QUERY); - test_plan->setProperty(cwel_processor, ConsumeWindowsEventLog::OutputFormat.getName(), "XML"); + protected: + void dispatchBookmarkEvent() override { + reportEvent(APPLICATION_CHANNEL, "Event zero: this is in the past"); + } + void dispatchCollectedEvent() override { + reportEvent(APPLICATION_CHANNEL, "Event one"); + } + } test_controller; + + std::string event = test_controller.run(); + + REQUIRE(event.find(R"()") != std::string::npos); + REQUIRE(event.find(R"(14985)") != std::string::npos); + REQUIRE(event.find(R"(4)") != std::string::npos); + REQUIRE(event.find(R"(0)") != std::string::npos); + REQUIRE(event.find(R"(0x80000000000000)") != std::string::npos); + // the ID of the event goes here (a number) + REQUIRE(event.find(R"()") != std::string::npos); + REQUIRE(event.find(R"(Application)") != std::string::npos); + // the computer name goes here + REQUIRE(event.find(R"(Event one)") != std::string::npos); +} - auto logger_processor = test_plan->addProcessor("LogAttribute", "logger", Success, true); - test_plan->setProperty(logger_processor, LogAttribute::FlowFilesToLog.getName(), "0"); - test_plan->setProperty(logger_processor, LogAttribute::LogPayload.getName(), "true"); - test_plan->setProperty(logger_processor, LogAttribute::MaxPayloadLineLength.getName(), "1024"); +namespace { +// carries out a loose match on objects, i.e. it doesn't matter if the +// actual object has extra fields than expected +void matchJSON(const rapidjson::Value& json, const rapidjson::Value& expected) { + if (expected.IsObject()) { + REQUIRE(json.IsObject()); + for (const auto& expected_member : expected.GetObject()) { + REQUIRE(json.HasMember(expected_member.name)); + matchJSON(json[expected_member.name], expected_member.value); + } + } else if (expected.IsArray()) { + REQUIRE(json.IsArray()); + REQUIRE(json.Size() == expected.Size()); + for (size_t idx{0}; idx < expected.Size(); ++idx) { + matchJSON(json[idx], expected[idx]); + } + } else { + REQUIRE(json == expected); + } +} - { - reportEvent(APPLICATION_CHANNEL, "Event zero: this is in the past"); +void verifyJSON(const std::string& json_str, const std::string& expected_str) { + rapidjson::Document json, expected; + REQUIRE(!json.Parse(json_str.c_str()).HasParseError()); + REQUIRE(!expected.Parse(expected_str.c_str()).HasParseError()); - test_controller.runSession(test_plan); - } + matchJSON(json, expected); +} - test_plan->reset(); - LogTestController::getInstance().resetStream(LogTestController::getInstance().log_output); +class JSONOutputController : public OutputFormatTestController { + public: + JSONOutputController(std::string format) : OutputFormatTestController(APPLICATION_CHANNEL, "*", std::move(format)) {} - { + protected: + void dispatchBookmarkEvent() override { + reportEvent(APPLICATION_CHANNEL, "Event zero: this is in the past"); + } + void dispatchCollectedEvent() override { reportEvent(APPLICATION_CHANNEL, "Event one"); + } +}; - test_controller.runSession(test_plan); +} // namespace + +TEST_CASE("ConsumeWindowsEventLog prints events in JSON::Simple correctly", "[onTrigger]") { + std::string event = JSONOutputController{"JSON::Simple"}.run(); + verifyJSON(event, R"json( + { + "System": { + "Provider": { + "Name": "Application" + }, + "Channel": "Application" + }, + "EventData": [{ + "Type": "Data", + "Content": "Event one", + "Name": "" + }] + } + )json"); +} + +TEST_CASE("ConsumeWindowsEventLog prints events in JSON::Raw correctly", "[onTrigger]") { + std::string event = JSONOutputController{"JSON::Raw"}.run(); + verifyJSON(event, R"json( + [ + { + "name": "Event", + "children": [ + {"name": "System"}, + { + "name": "EventData", + "children": [{ + "name": "Data", + "text": "Event one" + }] + } + ] + } + ] + )json"); +} + +class CustomProviderController : public OutputFormatTestController { + public: + CustomProviderController(std::string format) : OutputFormatTestController(custom_channel, "*", std::move(format)) {} - REQUIRE(LogTestController::getInstance().contains(R"()")); - REQUIRE(LogTestController::getInstance().contains(R"(14985)")); - REQUIRE(LogTestController::getInstance().contains(R"(4)")); - REQUIRE(LogTestController::getInstance().contains(R"(0)")); - REQUIRE(LogTestController::getInstance().contains(R"(0x80000000000000)")); - // the ID of the event goes here (a number) - REQUIRE(LogTestController::getInstance().contains(R"()")); - REQUIRE(LogTestController::getInstance().contains(R"(Application)")); - // the computer name goes here - REQUIRE(LogTestController::getInstance().contains(R"(Event one)")); + protected: + void dispatchBookmarkEvent() override { + REQUIRE(dispatchCustomEvent({"Bookmark", "Second", "Third", 3, 12})); + std::this_thread::sleep_for(std::chrono::seconds{1}); + } + void dispatchCollectedEvent() override { + REQUIRE(dispatchCustomEvent({"Actual event", "Second", "Third", 1, 9})); + std::this_thread::sleep_for(std::chrono::seconds{1}); } +}; + +const std::string event_data_json = R"( + [{ + "Type": "Data", + "Content": "Actual event", + "Name": "param1" + }, { + "Type": "Data", + "Content": "Second", + "Name": "param2" + }, { + "Type": "Data", + "Content": "Third", + "Name": "Channel" + }, { + "Type": "Binary", + "Content": "09", + "Name": "" + }] +)"; + +TEST_CASE("ConsumeWindowsEventLog prints events in JSON::Simple correctly custom provider", "[onTrigger]") { + if (!supportsCustomEvents()) { + return; + } + std::string event = CustomProviderController{"JSON::Simple"}.run(); + verifyJSON(event, R"( + { + "System": { + "Provider": { + "Name": ")" + custom_provider_name + R"(" + }, + "Channel": ")" + custom_channel + R"(" + }, + "EventData": )" + event_data_json + R"( + } + )"); +} + +TEST_CASE("ConsumeWindowsEventLog prints events in JSON::Flattened correctly custom provider", "[onTrigger]") { + if (!supportsCustomEvents()) { + return; + } + std::string event = CustomProviderController{"JSON::Flattened"}.run(); + verifyJSON(event, R"( + { + "Name": ")" + custom_provider_name + R"(", + "Channel": ")" + custom_channel /* Channel is not overwritten by data named "Channel" */ + R"(", + "EventData": )" + event_data_json /* EventData is not discarded */ + R"(, + "param1": "Actual event", + "param2": "Second" + } + )"); } namespace { diff --git a/extensions/windows-event-log/wel/JSONConverter.h b/extensions/windows-event-log/wel/JSONConverter.h new file mode 100644 index 0000000000..465656aeb6 --- /dev/null +++ b/extensions/windows-event-log/wel/JSONConverter.h @@ -0,0 +1,161 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#undef RAPIDJSON_ASSERT +#define RAPIDJSON_ASSERT(x) if (!(x)) throw std::logic_error("rapidjson exception"); // NOLINT + +#include +#include +#include + +#include + +#include "rapidjson/document.h" +#include "rapidjson/writer.h" +#include "rapidjson/stringbuffer.h" +#include "rapidjson/prettywriter.h" + +#include "gsl/gsl-lite.hpp"; + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace wel { + +rapidjson::Value xmlElementToJSON(const pugi::xml_node& node, rapidjson::Document& doc) { + gsl_Expects(node.type() == pugi::xml_node_type::node_element); + rapidjson::Value object(rapidjson::kObjectType); + object.AddMember("name", rapidjson::StringRef(node.name()), doc.GetAllocator()); + auto& attributes = object.AddMember("attributes", rapidjson::kObjectType, doc.GetAllocator())["attributes"]; + for (const auto& attr : node.attributes()) { + attributes.AddMember(rapidjson::StringRef(attr.name()), rapidjson::StringRef(attr.value()), doc.GetAllocator()); + } + auto& children = object.AddMember("children", rapidjson::kArrayType, doc.GetAllocator())["children"]; + for (const auto& child : node.children()) { + if (child.type() == pugi::xml_node_type::node_element) { + children.PushBack(xmlElementToJSON(child, doc), doc.GetAllocator()); + } + } + object.AddMember("text", rapidjson::StringRef(node.text().get()), doc.GetAllocator()); + return object; +} + +rapidjson::Value xmlDocumentToJSON(const pugi::xml_node& node, rapidjson::Document& doc) { + gsl_Expects(node.type() == pugi::xml_node_type::node_document); + rapidjson::Value children(rapidjson::kArrayType); + for (const auto& child : node.children()) { + if (child.type() == pugi::xml_node_type::node_element) { + children.PushBack(xmlElementToJSON(child, doc), doc.GetAllocator()); + } + } + return children; +} + +rapidjson::Document toRawJSON(const pugi::xml_node& root) { + rapidjson::Document doc; + if (root.type() == pugi::xml_node_type::node_document) { + static_cast(doc) = xmlDocumentToJSON(root, doc); + } + return doc; +} + +rapidjson::Document toJSON(const pugi::xml_node& root, bool flatten) { + rapidjson::Document doc{rapidjson::kObjectType}; + + auto event_xml = root.child("Event"); + + { + auto system_xml = event_xml.child("System"); + auto& system = flatten ? doc : doc.AddMember("System", rapidjson::kObjectType, doc.GetAllocator())["System"]; + + { + auto provider_xml = system_xml.child("Provider"); + auto& provider = flatten ? doc : system.AddMember("Provider", rapidjson::kObjectType, doc.GetAllocator())["Provider"]; + provider.AddMember("Name", rapidjson::StringRef(provider_xml.attribute("Name").value()), doc.GetAllocator()); + provider.AddMember("Guid", rapidjson::StringRef(provider_xml.attribute("Guid").value()), doc.GetAllocator()); + } + + system.AddMember("EventID", rapidjson::StringRef(system_xml.child("EventID").text().get()), doc.GetAllocator()); + system.AddMember("Version", rapidjson::StringRef(system_xml.child("Version").text().get()), doc.GetAllocator()); + system.AddMember("Level", rapidjson::StringRef(system_xml.child("Level").text().get()), doc.GetAllocator()); + system.AddMember("Task", rapidjson::StringRef(system_xml.child("Task").text().get()), doc.GetAllocator()); + system.AddMember("Opcode", rapidjson::StringRef(system_xml.child("Opcode").text().get()), doc.GetAllocator()); + system.AddMember("Keywords", rapidjson::StringRef(system_xml.child("Keywords").text().get()), doc.GetAllocator()); + + { + auto timeCreated_xml = system_xml.child("TimeCreated"); + auto& timeCreated = flatten ? doc : system.AddMember("TimeCreated", rapidjson::kObjectType, doc.GetAllocator())["TimeCreated"]; + timeCreated.AddMember("SystemTime", rapidjson::StringRef(timeCreated_xml.attribute("SystemTime").value()), doc.GetAllocator()); + } + + system.AddMember("EventRecordID", rapidjson::StringRef(system_xml.child("EventRecordID").text().get()), doc.GetAllocator()); + + { + auto correlation_xml = system_xml.child("Correlation"); + auto& correlation = flatten ? doc : system.AddMember("Correlation", rapidjson::kObjectType, doc.GetAllocator())["Correlation"]; + correlation.AddMember("ActivityID", rapidjson::StringRef(correlation_xml.attribute("ActivityID").value()), doc.GetAllocator()); + } + + { + auto execution_xml = system_xml.child("Execution"); + auto& execution = flatten ? doc : system.AddMember("Execution", rapidjson::kObjectType, doc.GetAllocator())["Execution"]; + execution.AddMember("ProcessID", rapidjson::StringRef(execution_xml.attribute("ProcessID").value()), doc.GetAllocator()); + execution.AddMember("ThreadID", rapidjson::StringRef(execution_xml.attribute("ThreadID").value()), doc.GetAllocator()); + } + + system.AddMember("Channel", rapidjson::StringRef(system_xml.child("Channel").text().get()), doc.GetAllocator()); + system.AddMember("Computer", rapidjson::StringRef(system_xml.child("Computer").text().get()), doc.GetAllocator()); + } + + { + auto eventData_xml = event_xml.child("EventData"); + // create EventData subarray even if flatten requested + doc.AddMember("EventData", rapidjson::kArrayType, doc.GetAllocator()); + for (const auto& data : eventData_xml.children()) { + auto name_attr = data.attribute("Name"); + rapidjson::Value item(rapidjson::kObjectType); + item.AddMember("Name", rapidjson::StringRef(name_attr.value()), doc.GetAllocator()); + item.AddMember("Content", rapidjson::StringRef(data.text().get()), doc.GetAllocator()); + item.AddMember("Type", rapidjson::StringRef(data.name()), doc.GetAllocator()); + // we need to query EventData because a reference to it wouldn't be stable, as we + // possibly add members to its parent which could result in reallocation + doc["EventData"].PushBack(item, doc.GetAllocator()); + // check collision + if (flatten && !name_attr.empty() && !doc.HasMember(name_attr.value())) { + doc.AddMember(rapidjson::StringRef(name_attr.value()), rapidjson::StringRef(data.text().get()), doc.GetAllocator()); + } + } + } + + return doc; +} + +std::string jsonToString(rapidjson::Document& doc) { + rapidjson::StringBuffer buffer; + rapidjson::PrettyWriter writer(buffer); + doc.Accept(writer); + return buffer.GetString(); +} + +} // namespace wel +} // namespace minifi +} // namespace nifi +} // namespace apache +} // namespace org \ No newline at end of file diff --git a/extensions/windows-event-log/wel/MetadataWalker.cpp b/extensions/windows-event-log/wel/MetadataWalker.cpp index 2c8e682e3f..0f4f46fa7f 100644 --- a/extensions/windows-event-log/wel/MetadataWalker.cpp +++ b/extensions/windows-event-log/wel/MetadataWalker.cpp @@ -79,7 +79,10 @@ bool MetadataWalker::for_each(pugi::xml_node &node) { metadata_["EventID"] = node.text().get(); } else { - static std::map formatFlagMap = { {"Channel", EvtFormatMessageChannel}, {"Keywords", EvtFormatMessageKeyword}, {"Level", EvtFormatMessageLevel}, {"Opcode", EvtFormatMessageOpcode}, {"Task",EvtFormatMessageTask} }; + static std::map formatFlagMap = { + {"Channel", EvtFormatMessageChannel}, {"Keywords", EvtFormatMessageKeyword}, {"Level", EvtFormatMessageLevel}, + {"Opcode", EvtFormatMessageOpcode}, {"Task",EvtFormatMessageTask} + }; auto it = formatFlagMap.find(node_name); if (it != formatFlagMap.end()) { std::function updateFunc = [&](const std::string &input) -> std::string { diff --git a/libminifi/test/resources/wel/unit-test-provider.man b/libminifi/test/resources/wel/unit-test-provider.man new file mode 100644 index 0000000000..c04cdacf2c --- /dev/null +++ b/libminifi/test/resources/wel/unit-test-provider.man @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + From e4d8c1fce7be82251aac47c383efe6f04ed5d7ad Mon Sep 17 00:00:00 2001 From: Adam Debreceni Date: Thu, 14 Jan 2021 14:51:33 +0100 Subject: [PATCH 02/15] MINIFICPP-1448 - Restructure CWEL output format --- .../ConsumeWindowsEventLog.cpp | 105 ++++++++++++------ .../ConsumeWindowsEventLog.h | 40 ++++--- 2 files changed, 97 insertions(+), 48 deletions(-) diff --git a/extensions/windows-event-log/ConsumeWindowsEventLog.cpp b/extensions/windows-event-log/ConsumeWindowsEventLog.cpp index aa16bad925..285cd21f44 100644 --- a/extensions/windows-event-log/ConsumeWindowsEventLog.cpp +++ b/extensions/windows-event-log/ConsumeWindowsEventLog.cpp @@ -251,27 +251,27 @@ void ConsumeWindowsEventLog::onSchedule(const std::shared_ptrgetProperty(OutputFormat.getName(), mode); - writeXML_ = false; - writePlainText_ = false; - jsonFormat_ = JSONFormat::None; + output_.reset(); if (mode == XML) { - writeXML_ = true; + output_.xml = true; } else if (mode == Plaintext) { - writePlainText_ = true; + output_.plaintext = true; } else if (mode == Both) { - writeXML_ = true; - writePlainText_ = true; + output_.xml = true; + output_.plaintext = true; } else if (mode == JSONRaw) { - jsonFormat_ = JSONFormat::Raw; + output_.json.raw = true; } else if (mode == JSONSimple) { - jsonFormat_ = JSONFormat::Simple; + output_.json.simple = true; } else if (mode == JSONFlattened) { - jsonFormat_ = JSONFormat::Flattened; + output_.json.flattened = true; } else { + // in the future this might be considered an error, but for now due to backwards + // compatibility we just fall through and execute the processor outputing nothing // throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Unrecognized output format: " + mode); } - if ((writeXML_ || jsonFormat_ != JSONFormat::None) && !hMsobjsDll_) { + if ((output_.xml || output_.json) && !hMsobjsDll_) { char systemDir[MAX_PATH]; if (GetSystemDirectory(systemDir, sizeof(systemDir))) { hMsobjsDll_ = LoadLibrary((systemDir + std::string("\\msobjs.dll")).c_str()); @@ -579,7 +579,7 @@ bool ConsumeWindowsEventLog::createEventRender(EVT_HANDLE hEvent, EventRender& e logger_->log_debug("Finish doc traversing, performing writing..."); - if (writePlainText_) { + if (output_.plaintext) { logger_->log_trace("Writing event in plain text"); auto handler = getEventLogHandler(providerName); @@ -598,44 +598,55 @@ bool ConsumeWindowsEventLog::createEventRender(EVT_HANDLE hEvent, EventRender& e // set the delimiter log_header.setDelimiter(header_delimiter_); // render the header. - eventRender.rendered_text_ = log_header.getEventHeader([&walker](wel::METADATA metadata) { return walker.getMetadata(metadata); }); - eventRender.rendered_text_ += "Message" + header_delimiter_ + " "; - eventRender.rendered_text_ += message; + eventRender.plaintext = log_header.getEventHeader([&walker](wel::METADATA metadata) { return walker.getMetadata(metadata); }); + eventRender.plaintext += "Message" + header_delimiter_ + " "; + eventRender.plaintext += message; } logger_->log_trace("Finish writing in plain text"); } - if (writeXML_ || jsonFormat_ != JSONFormat::None) { + if (output_.xml || output_.json) { substituteXMLPercentageItems(doc); logger_->log_trace("Finish substituting %% in XML"); if (resolve_as_attributes_) { - eventRender.matched_fields_ = walker.getFieldValues(); + eventRender.matched_fields = walker.getFieldValues(); } } - if (writeXML_) { + if (output_.xml) { logger_->log_trace("Writing event in XML"); wel::XmlString writer; doc.print(writer, "", pugi::format_raw); // no indentation or formatting xml = writer.xml_; - eventRender.text_ = std::move(xml); + eventRender.xml = std::move(xml); logger_->log_trace("Finish writing in XML"); } - if (jsonFormat_ != JSONFormat::None) { - logger_->log_trace("Writing event in JSON"); + if (output_.json.raw) { + logger_->log_trace("Writing event in raw JSON"); - if (jsonFormat_ == JSONFormat::Raw) { - eventRender.json_ = wel::jsonToString(wel::toRawJSON(doc)); - } else { - eventRender.json_ = wel::jsonToString(wel::toJSON(doc, jsonFormat_ == JSONFormat::Flattened)); - } + eventRender.json.raw = wel::jsonToString(wel::toRawJSON(doc)); + + logger_->log_trace("Finish writing in raw JSON"); + } + + if (output_.json.simple) { + logger_->log_trace("Writing event in simple JSON"); + + eventRender.json.simple = wel::jsonToString(wel::toJSON(doc, false)); + + logger_->log_trace("Finish writing in simple JSON"); + } + + if (output_.json.flattened) { + logger_->log_trace("Writing event in flattened JSON"); + eventRender.json.flattened = wel::jsonToString(wel::toJSON(doc, true)); - logger_->log_trace("Finish writing in JSON"); + logger_->log_trace("Finish writing in flattened JSON"); } return true; @@ -689,15 +700,15 @@ void ConsumeWindowsEventLog::putEventRenderFlowFileToSession(const EventRender& const std::string& str_; }; - if (writeXML_) { + if (output_.xml) { auto flowFile = session.create(); logger_->log_trace("Writing rendered XML to a flow file"); { - WriteCallback wc{ eventRender.text_ }; + WriteCallback wc{ eventRender.xml }; session.write(flowFile, &wc); } - for (const auto &fieldMapping : eventRender.matched_fields_) { + for (const auto &fieldMapping : eventRender.matched_fields) { if (!fieldMapping.second.empty()) { session.putAttribute(flowFile, fieldMapping.first, fieldMapping.second); } @@ -709,12 +720,12 @@ void ConsumeWindowsEventLog::putEventRenderFlowFileToSession(const EventRender& session.transfer(flowFile, Success); } - if (writePlainText_) { + if (output_.plaintext) { auto flowFile = session.create(); logger_->log_trace("Writing rendered plain text to a flow file"); { - WriteCallback wc{ eventRender.rendered_text_ }; + WriteCallback wc{ eventRender.plaintext }; session.write(flowFile, &wc); } session.putAttribute(flowFile, core::SpecialFlowAttribute::MIME_TYPE, "text/plain"); @@ -724,11 +735,35 @@ void ConsumeWindowsEventLog::putEventRenderFlowFileToSession(const EventRender& session.transfer(flowFile, Success); } - if (jsonFormat_ != JSONFormat::None) { + if (output_.json.raw) { + auto flowFile = session.create(); + logger_->log_trace("Writing rendered raw JSON to a flow file"); + + session.write(flowFile, &WriteCallback(eventRender.json.raw)); + session.putAttribute(flowFile, core::SpecialFlowAttribute::MIME_TYPE, "application/json"); + session.putAttribute(flowFile, "Timezone name", timezone_name_); + session.putAttribute(flowFile, "Timezone offset", timezone_offset_); + session.getProvenanceReporter()->receive(flowFile, provenanceUri_, getUUIDStr(), "Consume windows event logs", 0); + session.transfer(flowFile, Success); + } + + if (output_.json.simple) { + auto flowFile = session.create(); + logger_->log_trace("Writing rendered simple JSON to a flow file"); + + session.write(flowFile, &WriteCallback(eventRender.json.simple)); + session.putAttribute(flowFile, core::SpecialFlowAttribute::MIME_TYPE, "application/json"); + session.putAttribute(flowFile, "Timezone name", timezone_name_); + session.putAttribute(flowFile, "Timezone offset", timezone_offset_); + session.getProvenanceReporter()->receive(flowFile, provenanceUri_, getUUIDStr(), "Consume windows event logs", 0); + session.transfer(flowFile, Success); + } + + if (output_.json.flattened) { auto flowFile = session.create(); - logger_->log_trace("Writing rendered JSON to a flow file"); + logger_->log_trace("Writing rendered flattened JSON to a flow file"); - session.write(flowFile, &WriteCallback(eventRender.json_)); + session.write(flowFile, &WriteCallback(eventRender.json.flattened)); session.putAttribute(flowFile, core::SpecialFlowAttribute::MIME_TYPE, "application/json"); session.putAttribute(flowFile, "Timezone name", timezone_name_); session.putAttribute(flowFile, "Timezone offset", timezone_offset_); diff --git a/extensions/windows-event-log/ConsumeWindowsEventLog.h b/extensions/windows-event-log/ConsumeWindowsEventLog.h index a7c3a0ee5a..4705909261 100644 --- a/extensions/windows-event-log/ConsumeWindowsEventLog.h +++ b/extensions/windows-event-log/ConsumeWindowsEventLog.h @@ -43,10 +43,14 @@ namespace minifi { namespace processors { struct EventRender { - std::map matched_fields_; - std::string text_; - std::string rendered_text_; - std::string json_; + std::map matched_fields; + std::string xml; + std::string plaintext; + struct { + std::string raw; + std::string simple; + std::string flattened; + } json; }; class Bookmark; @@ -111,9 +115,9 @@ class ConsumeWindowsEventLog : public core::Processor { static constexpr const char* XML = "XML"; static constexpr const char* Both = "Both"; static constexpr const char* Plaintext = "Plaintext"; + static constexpr const char* JSONRaw = "JSON::Raw"; static constexpr const char* JSONSimple = "JSON::Simple"; static constexpr const char* JSONFlattened = "JSON::Flattened"; - static constexpr const char* JSONRaw = "JSON::Raw"; private: struct TimeDiff { @@ -146,14 +150,24 @@ class ConsumeWindowsEventLog : public core::Processor { std::map providers_; uint64_t batch_commit_size_{}; - bool writeXML_{false}; - bool writePlainText_{false}; - enum class JSONFormat { - None, - Simple, - Flattened, - Raw - } jsonFormat_{JSONFormat::None}; + struct OutputFormat { + void reset() { + *this = OutputFormat{}; + } + + bool xml{false}; + bool plaintext{false}; + struct { + explicit operator bool() const noexcept { + return raw || simple || flattened; + } + + bool raw{false}; + bool simple{false}; + bool flattened{false}; + } json; + } output_; + std::unique_ptr bookmark_; std::mutex on_trigger_mutex_; std::unordered_map xmlPercentageItemsResolutions_; From a9edf1fec1b3e0ad423711140f0c1199da29b58e Mon Sep 17 00:00:00 2001 From: Adam Debreceni Date: Thu, 14 Jan 2021 15:52:40 +0100 Subject: [PATCH 03/15] MINIFICPP-1448 - Build fix --- extensions/windows-event-log/wel/WindowsEventLog.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/extensions/windows-event-log/wel/WindowsEventLog.cpp b/extensions/windows-event-log/wel/WindowsEventLog.cpp index 8d1334bc2f..80fb3c177d 100644 --- a/extensions/windows-event-log/wel/WindowsEventLog.cpp +++ b/extensions/windows-event-log/wel/WindowsEventLog.cpp @@ -148,6 +148,11 @@ std::string WindowsEventLogMetadataImpl::getEventData(EVT_FORMAT_MESSAGE_FLAGS f EvtFormatMessage(metadata_ptr_, event_ptr_, 0, 0, NULL, flags, num_chars_in_buffer, buffer.get(), &num_chars_used); } } + + if (num_chars_used == 0) { + return event_data; + } + if (EvtFormatMessageKeyword == flags) { buffer.get()[num_chars_used - 1] = L'\0'; } From 0824c7536fce18caaec8113a9e4282222aa5d569 Mon Sep 17 00:00:00 2001 From: Adam Debreceni Date: Fri, 15 Jan 2021 08:38:32 +0100 Subject: [PATCH 04/15] MINIFICPP-1448 - Restructure and add comments --- .../ConsumeWindowsEventLog.cpp | 6 +- .../wel/{JSONConverter.h => JSONUtils.cpp} | 19 +++-- extensions/windows-event-log/wel/JSONUtils.h | 70 +++++++++++++++++++ 3 files changed, 85 insertions(+), 10 deletions(-) rename extensions/windows-event-log/wel/{JSONConverter.h => JSONUtils.cpp} (93%) create mode 100644 extensions/windows-event-log/wel/JSONUtils.h diff --git a/extensions/windows-event-log/ConsumeWindowsEventLog.cpp b/extensions/windows-event-log/ConsumeWindowsEventLog.cpp index 285cd21f44..e85dd4b3a5 100644 --- a/extensions/windows-event-log/ConsumeWindowsEventLog.cpp +++ b/extensions/windows-event-log/ConsumeWindowsEventLog.cpp @@ -34,7 +34,7 @@ #include "wel/MetadataWalker.h" #include "wel/XMLString.h" #include "wel/UnicodeConversion.h" -#include "wel/JSONConverter.h" +#include "wel/JSONUtils.h" #include "io/BufferStream.h" #include "core/ProcessContext.h" @@ -636,7 +636,7 @@ bool ConsumeWindowsEventLog::createEventRender(EVT_HANDLE hEvent, EventRender& e if (output_.json.simple) { logger_->log_trace("Writing event in simple JSON"); - eventRender.json.simple = wel::jsonToString(wel::toJSON(doc, false)); + eventRender.json.simple = wel::jsonToString(wel::toSimpleJSON(doc)); logger_->log_trace("Finish writing in simple JSON"); } @@ -644,7 +644,7 @@ bool ConsumeWindowsEventLog::createEventRender(EVT_HANDLE hEvent, EventRender& e if (output_.json.flattened) { logger_->log_trace("Writing event in flattened JSON"); - eventRender.json.flattened = wel::jsonToString(wel::toJSON(doc, true)); + eventRender.json.flattened = wel::jsonToString(wel::toFlattenedJSON(doc)); logger_->log_trace("Finish writing in flattened JSON"); } diff --git a/extensions/windows-event-log/wel/JSONConverter.h b/extensions/windows-event-log/wel/JSONUtils.cpp similarity index 93% rename from extensions/windows-event-log/wel/JSONConverter.h rename to extensions/windows-event-log/wel/JSONUtils.cpp index 465656aeb6..de3d2e6cea 100644 --- a/extensions/windows-event-log/wel/JSONConverter.h +++ b/extensions/windows-event-log/wel/JSONUtils.cpp @@ -15,10 +15,7 @@ * limitations under the License. */ -#pragma once - -#undef RAPIDJSON_ASSERT -#define RAPIDJSON_ASSERT(x) if (!(x)) throw std::logic_error("rapidjson exception"); // NOLINT +#include "JSONUtils.h" #include #include @@ -39,7 +36,7 @@ namespace nifi { namespace minifi { namespace wel { -rapidjson::Value xmlElementToJSON(const pugi::xml_node& node, rapidjson::Document& doc) { +static rapidjson::Value xmlElementToJSON(const pugi::xml_node& node, rapidjson::Document& doc) { gsl_Expects(node.type() == pugi::xml_node_type::node_element); rapidjson::Value object(rapidjson::kObjectType); object.AddMember("name", rapidjson::StringRef(node.name()), doc.GetAllocator()); @@ -57,7 +54,7 @@ rapidjson::Value xmlElementToJSON(const pugi::xml_node& node, rapidjson::Documen return object; } -rapidjson::Value xmlDocumentToJSON(const pugi::xml_node& node, rapidjson::Document& doc) { +static rapidjson::Value xmlDocumentToJSON(const pugi::xml_node& node, rapidjson::Document& doc) { gsl_Expects(node.type() == pugi::xml_node_type::node_document); rapidjson::Value children(rapidjson::kArrayType); for (const auto& child : node.children()) { @@ -76,7 +73,7 @@ rapidjson::Document toRawJSON(const pugi::xml_node& root) { return doc; } -rapidjson::Document toJSON(const pugi::xml_node& root, bool flatten) { +static rapidjson::Document toJSONImpl(const pugi::xml_node& root, bool flatten) { rapidjson::Document doc{rapidjson::kObjectType}; auto event_xml = root.child("Event"); @@ -147,6 +144,14 @@ rapidjson::Document toJSON(const pugi::xml_node& root, bool flatten) { return doc; } +rapidjson::Document toSimpleJSON(const pugi::xml_node& root) { + return toJSONImpl(root, false); +} + +rapidjson::Document toFlattenedJSON(const pugi::xml_node& root) { + return toJSONImpl(root, true); +} + std::string jsonToString(rapidjson::Document& doc) { rapidjson::StringBuffer buffer; rapidjson::PrettyWriter writer(buffer); diff --git a/extensions/windows-event-log/wel/JSONUtils.h b/extensions/windows-event-log/wel/JSONUtils.h new file mode 100644 index 0000000000..78681f8811 --- /dev/null +++ b/extensions/windows-event-log/wel/JSONUtils.h @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#undef RAPIDJSON_ASSERT +#define RAPIDJSON_ASSERT(x) if (!(x)) throw std::logic_error("rapidjson exception"); // NOLINT + +#include + +#include "rapidjson/document.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace wel { + +/** + * Converts each xml element node to a json object of + * the form {name: String, attributes: Object, children: Array, text: String} + * Aims to preserve most of the input xml structure. + */ +rapidjson::Document toRawJSON(const pugi::xml_node& root); + +/** + * Retains some hierarchical structure of the original xml event, + * e.g. {System: {Provider: {Name: String, Guid: String}}} + */ +rapidjson::Document toSimpleJSON(const pugi::xml_node& root); + +/** + * Flattens most of the structure, i.e. removes intermediate + * objects and lifts innermost string-valued keys to the root. + * e.g. {System: {Provider: {Name: String}}} => {Name: String} + * + * Moreover it also flattens each named data element where the + * name does not conflict with already existing members + * (e.g. a data with name "Guid" won't be flattened as it would + * overwrite the existing "Guid" field). + * + * e.g. {EventData: [{Name: "Test", Content: "X"}]} => {Test: "X"} + * + * In order to mitigate data loss, it preserves the EventData + * array in its entirety as well. + * (otherwise a "Guid" data would be lost) + */ +rapidjson::Document toFlattenedJSON(const pugi::xml_node& root); + +std::string jsonToString(rapidjson::Document& doc); + +} // namespace wel +} // namespace minifi +} // namespace nifi +} // namespace apache +} // namespace org \ No newline at end of file From 2c1ca5958d7c70e176f723891ac9b86c1797abbf Mon Sep 17 00:00:00 2001 From: Adam Debreceni Date: Fri, 15 Jan 2021 09:14:11 +0100 Subject: [PATCH 05/15] MINIFICPP-1448 - Build fix --- extensions/windows-event-log/wel/JSONUtils.cpp | 2 +- extensions/windows-event-log/wel/JSONUtils.h | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/extensions/windows-event-log/wel/JSONUtils.cpp b/extensions/windows-event-log/wel/JSONUtils.cpp index de3d2e6cea..1a10573efd 100644 --- a/extensions/windows-event-log/wel/JSONUtils.cpp +++ b/extensions/windows-event-log/wel/JSONUtils.cpp @@ -163,4 +163,4 @@ std::string jsonToString(rapidjson::Document& doc) { } // namespace minifi } // namespace nifi } // namespace apache -} // namespace org \ No newline at end of file +} // namespace org diff --git a/extensions/windows-event-log/wel/JSONUtils.h b/extensions/windows-event-log/wel/JSONUtils.h index 78681f8811..b9850f4aa5 100644 --- a/extensions/windows-event-log/wel/JSONUtils.h +++ b/extensions/windows-event-log/wel/JSONUtils.h @@ -22,6 +22,7 @@ #include +#include // for std::logic_error #include "rapidjson/document.h" namespace org { @@ -67,4 +68,4 @@ std::string jsonToString(rapidjson::Document& doc); } // namespace minifi } // namespace nifi } // namespace apache -} // namespace org \ No newline at end of file +} // namespace org From 4e8ecd859258f446de9eb128bd3eea8e19cf9cf6 Mon Sep 17 00:00:00 2001 From: Adam Debreceni Date: Mon, 18 Jan 2021 09:41:33 +0100 Subject: [PATCH 06/15] MINIFICPP-1448 - Review changes --- .../ConsumeWindowsEventLog.cpp | 2 +- .../ConsumeWindowsEventLog.h | 4 --- .../tests/ConsumeWindowsEventLogTests.cpp | 35 +++++++++++++------ .../windows-event-log/wel/JSONUtils.cpp | 26 ++++++++------ extensions/windows-event-log/wel/JSONUtils.h | 5 ++- 5 files changed, 45 insertions(+), 27 deletions(-) diff --git a/extensions/windows-event-log/ConsumeWindowsEventLog.cpp b/extensions/windows-event-log/ConsumeWindowsEventLog.cpp index e85dd4b3a5..6598ff23ce 100644 --- a/extensions/windows-event-log/ConsumeWindowsEventLog.cpp +++ b/extensions/windows-event-log/ConsumeWindowsEventLog.cpp @@ -251,7 +251,7 @@ void ConsumeWindowsEventLog::onSchedule(const std::shared_ptrgetProperty(OutputFormat.getName(), mode); - output_.reset(); + output_ = {}; if (mode == XML) { output_.xml = true; } else if (mode == Plaintext) { diff --git a/extensions/windows-event-log/ConsumeWindowsEventLog.h b/extensions/windows-event-log/ConsumeWindowsEventLog.h index 4705909261..87d0f37c4d 100644 --- a/extensions/windows-event-log/ConsumeWindowsEventLog.h +++ b/extensions/windows-event-log/ConsumeWindowsEventLog.h @@ -151,10 +151,6 @@ class ConsumeWindowsEventLog : public core::Processor { uint64_t batch_commit_size_{}; struct OutputFormat { - void reset() { - *this = OutputFormat{}; - } - bool xml{false}; bool plaintext{false}; struct { diff --git a/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp b/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp index ba9a26d56c..3fd3eb4d44 100644 --- a/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp +++ b/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp @@ -56,11 +56,11 @@ struct CustomEvent { int binary_data; }; -const std::string custom_provider_name = "minifi_unit_test_provider"; -const std::string custom_channel = custom_provider_name + "/Log"; +const std::string CUSTOM_PROVIDER_NAME = "minifi_unit_test_provider"; +const std::string CUSTOM_CHANNEL = CUSTOM_PROVIDER_NAME + "/Log"; bool dispatchCustomEvent(const CustomEvent& event) { - std::string command = "powershell \"New-WinEvent -ProviderName " + custom_provider_name + std::string command = "powershell \"New-WinEvent -ProviderName " + CUSTOM_PROVIDER_NAME + " -Id 10000 -Payload @('" + event.first + "','" + event.second + "', '" + event.third + "', " + std::to_string(event.binary_length) + ", " + std::to_string(event.binary_data) + ")\""; auto result = std::system(command.c_str()); @@ -445,8 +445,8 @@ void matchJSON(const rapidjson::Value& json, const rapidjson::Value& expected) { void verifyJSON(const std::string& json_str, const std::string& expected_str) { rapidjson::Document json, expected; - REQUIRE(!json.Parse(json_str.c_str()).HasParseError()); - REQUIRE(!expected.Parse(expected_str.c_str()).HasParseError()); + REQUIRE_FALSE(json.Parse(json_str.c_str()).HasParseError()); + REQUIRE_FALSE(expected.Parse(expected_str.c_str()).HasParseError()); matchJSON(json, expected); } @@ -485,6 +485,21 @@ TEST_CASE("ConsumeWindowsEventLog prints events in JSON::Simple correctly", "[on )json"); } +TEST_CASE("ConsumeWindowsEventLog prints events in JSON::Flattened correctly", "[onTrigger]") { + std::string event = JSONOutputController{"JSON::Flattened"}.run(); + verifyJSON(event, R"json( + { + "Name": "Application", + "Channel": "Application", + "EventData": [{ + "Type": "Data", + "Content": "Event one", + "Name": "" + }] + } + )json"); +} + TEST_CASE("ConsumeWindowsEventLog prints events in JSON::Raw correctly", "[onTrigger]") { std::string event = JSONOutputController{"JSON::Raw"}.run(); verifyJSON(event, R"json( @@ -508,7 +523,7 @@ TEST_CASE("ConsumeWindowsEventLog prints events in JSON::Raw correctly", "[onTri class CustomProviderController : public OutputFormatTestController { public: - CustomProviderController(std::string format) : OutputFormatTestController(custom_channel, "*", std::move(format)) {} + CustomProviderController(std::string format) : OutputFormatTestController(CUSTOM_CHANNEL, "*", std::move(format)) {} protected: void dispatchBookmarkEvent() override { @@ -550,9 +565,9 @@ TEST_CASE("ConsumeWindowsEventLog prints events in JSON::Simple correctly custom { "System": { "Provider": { - "Name": ")" + custom_provider_name + R"(" + "Name": ")" + CUSTOM_PROVIDER_NAME + R"(" }, - "Channel": ")" + custom_channel + R"(" + "Channel": ")" + CUSTOM_CHANNEL + R"(" }, "EventData": )" + event_data_json + R"( } @@ -566,8 +581,8 @@ TEST_CASE("ConsumeWindowsEventLog prints events in JSON::Flattened correctly cus std::string event = CustomProviderController{"JSON::Flattened"}.run(); verifyJSON(event, R"( { - "Name": ")" + custom_provider_name + R"(", - "Channel": ")" + custom_channel /* Channel is not overwritten by data named "Channel" */ + R"(", + "Name": ")" + CUSTOM_PROVIDER_NAME + R"(", + "Channel": ")" + CUSTOM_CHANNEL /* Channel is not overwritten by data named "Channel" */ + R"(", "EventData": )" + event_data_json /* EventData is not discarded */ + R"(, "param1": "Actual event", "param2": "Second" diff --git a/extensions/windows-event-log/wel/JSONUtils.cpp b/extensions/windows-event-log/wel/JSONUtils.cpp index 1a10573efd..4e59de752e 100644 --- a/extensions/windows-event-log/wel/JSONUtils.cpp +++ b/extensions/windows-event-log/wel/JSONUtils.cpp @@ -36,7 +36,9 @@ namespace nifi { namespace minifi { namespace wel { -static rapidjson::Value xmlElementToJSON(const pugi::xml_node& node, rapidjson::Document& doc) { +namespace { + +rapidjson::Value xmlElementToJSON(const pugi::xml_node& node, rapidjson::Document& doc) { gsl_Expects(node.type() == pugi::xml_node_type::node_element); rapidjson::Value object(rapidjson::kObjectType); object.AddMember("name", rapidjson::StringRef(node.name()), doc.GetAllocator()); @@ -54,7 +56,7 @@ static rapidjson::Value xmlElementToJSON(const pugi::xml_node& node, rapidjson:: return object; } -static rapidjson::Value xmlDocumentToJSON(const pugi::xml_node& node, rapidjson::Document& doc) { +rapidjson::Value xmlDocumentToJSON(const pugi::xml_node& node, rapidjson::Document& doc) { gsl_Expects(node.type() == pugi::xml_node_type::node_document); rapidjson::Value children(rapidjson::kArrayType); for (const auto& child : node.children()) { @@ -65,15 +67,7 @@ static rapidjson::Value xmlDocumentToJSON(const pugi::xml_node& node, rapidjson: return children; } -rapidjson::Document toRawJSON(const pugi::xml_node& root) { - rapidjson::Document doc; - if (root.type() == pugi::xml_node_type::node_document) { - static_cast(doc) = xmlDocumentToJSON(root, doc); - } - return doc; -} - -static rapidjson::Document toJSONImpl(const pugi::xml_node& root, bool flatten) { +rapidjson::Document toJSONImpl(const pugi::xml_node& root, bool flatten) { rapidjson::Document doc{rapidjson::kObjectType}; auto event_xml = root.child("Event"); @@ -144,6 +138,16 @@ static rapidjson::Document toJSONImpl(const pugi::xml_node& root, bool flatten) return doc; } +} // namespace + +rapidjson::Document toRawJSON(const pugi::xml_node& root) { + rapidjson::Document doc; + if (root.type() == pugi::xml_node_type::node_document) { + static_cast(doc) = xmlDocumentToJSON(root, doc); + } + return doc; +} + rapidjson::Document toSimpleJSON(const pugi::xml_node& root) { return toJSONImpl(root, false); } diff --git a/extensions/windows-event-log/wel/JSONUtils.h b/extensions/windows-event-log/wel/JSONUtils.h index b9850f4aa5..326f2688d5 100644 --- a/extensions/windows-event-log/wel/JSONUtils.h +++ b/extensions/windows-event-log/wel/JSONUtils.h @@ -40,7 +40,10 @@ rapidjson::Document toRawJSON(const pugi::xml_node& root); /** * Retains some hierarchical structure of the original xml event, - * e.g. {System: {Provider: {Name: String, Guid: String}}} + * e.g. transforms + * + * into + * {System: {Provider: {Name: "Banana", Guid: "{5}"}}} */ rapidjson::Document toSimpleJSON(const pugi::xml_node& root); From 7b99751fdb30e4a0eb96dc7e97c6627a9ced9114 Mon Sep 17 00:00:00 2001 From: Adam Debreceni Date: Mon, 18 Jan 2021 10:44:15 +0100 Subject: [PATCH 07/15] MINIFICPP-1448 - Use API to dispatch custom events --- .github/workflows/ci.yml | 4 +- .../tests/ConsumeWindowsEventLogTests.cpp | 61 +- .../custom-provider}/unit-test-provider.man | 4 +- .../tests/unit-test-provider.h | 705 ++++++++++++++++++ 4 files changed, 750 insertions(+), 24 deletions(-) rename {libminifi/test/resources/wel => extensions/windows-event-log/tests/custom-provider}/unit-test-provider.man (91%) create mode 100644 extensions/windows-event-log/tests/unit-test-provider.h diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2fc202a4cb..9bc04ab81b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,7 +48,7 @@ jobs: uses: microsoft/setup-msbuild@v1.0.2 - id: create-wel-provider run: | - cd libminifi\test\resources\wel + cd extensions\windows-event-log\tests\custom-provider PATH %PATH%;C:\Program Files (x86)\Windows Kits\10\bin\10.0.17763.0\x86 PATH %PATH%;C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\MSBuild\15.0\Bin\Roslyn mc -css Namespace unit-test-provider.man @@ -70,7 +70,7 @@ jobs: uses: microsoft/setup-msbuild@v1.0.2 - id: create-wel-provider run: | - cd libminifi\test\resources\wel + cd extensions\windows-event-log\tests\custom-provider PATH %PATH%;C:\Program Files (x86)\Windows Kits\10\bin\10.0.19041.0\x64 PATH %PATH%;C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\Roslyn mc -css Namespace unit-test-provider.man diff --git a/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp b/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp index 3fd3eb4d44..7c1d724517 100644 --- a/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp +++ b/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp @@ -15,8 +15,6 @@ * limitations under the License. */ -#include - #include "ConsumeWindowsEventLog.h" #include "core/ConfigurableComponent.h" @@ -27,6 +25,10 @@ #include "utils/file/FileUtils.h" #include "rapidjson/document.h" +// generated from the manifest file "custom-provider/unit-test-provider.man" +// using the command "mc -um unit-test-provider.man" +#include "unit-test-provider.h" + using ConsumeWindowsEventLog = org::apache::nifi::minifi::processors::ConsumeWindowsEventLog; using LogAttribute = org::apache::nifi::minifi::processors::LogAttribute; using PutFile = org::apache::nifi::minifi::processors::PutFile; @@ -48,27 +50,40 @@ void reportEvent(const std::string& channel, const char* message, WORD log_level ReportEventA(event_source, log_level, 0, CWEL_TESTS_OPCODE, nullptr, 1, 0, &message, nullptr); } -struct CustomEvent { - std::string first; - std::string second; - std::string third; +struct CustomEventData { + std::wstring first; + std::wstring second; + std::wstring third; int binary_length; - int binary_data; + const unsigned char* binary_data; }; const std::string CUSTOM_PROVIDER_NAME = "minifi_unit_test_provider"; const std::string CUSTOM_CHANNEL = CUSTOM_PROVIDER_NAME + "/Log"; -bool dispatchCustomEvent(const CustomEvent& event) { - std::string command = "powershell \"New-WinEvent -ProviderName " + CUSTOM_PROVIDER_NAME - + " -Id 10000 -Payload @('" + event.first + "','" + event.second + "', '" + event.third - + "', " + std::to_string(event.binary_length) + ", " + std::to_string(event.binary_data) + ")\""; - auto result = std::system(command.c_str()); - return result == 0; +bool initializeCustomProvider() { + static auto provider_initialized = [] { + return EventRegisterminifi_unit_test_provider(); + }(); + + return provider_initialized == ERROR_SUCCESS; +} + +bool dispatchCustomEvent(const CustomEventData& event) { + REQUIRE(initializeCustomProvider()); + + auto result = EventWriteCustomEvent( + event.first.c_str(), + event.second.c_str(), + event.third.c_str(), + event.binary_length, + event.binary_data + ); + return result == ERROR_SUCCESS; } bool supportsCustomEvents() { - return dispatchCustomEvent({"First", "Second", "Third", 2, 5}); + return initializeCustomProvider(); } } // namespace @@ -527,16 +542,22 @@ class CustomProviderController : public OutputFormatTestController { protected: void dispatchBookmarkEvent() override { - REQUIRE(dispatchCustomEvent({"Bookmark", "Second", "Third", 3, 12})); + auto binary = reinterpret_cast("\x0c\x10"); + REQUIRE(dispatchCustomEvent({L"Bookmark", L"Second", L"Third", 2, binary})); + // even though we are using the API, we still have to wait for the event to appear + // for CWEL processor std::this_thread::sleep_for(std::chrono::seconds{1}); } void dispatchCollectedEvent() override { - REQUIRE(dispatchCustomEvent({"Actual event", "Second", "Third", 1, 9})); + auto binary = reinterpret_cast("\x09\x01"); + REQUIRE(dispatchCustomEvent({L"Actual event", L"Second", L"Third", 2, binary})); + // even though we are using the API, we still have to wait for the event to appear + // for CWEL processor std::this_thread::sleep_for(std::chrono::seconds{1}); } }; -const std::string event_data_json = R"( +const std::string EVENT_DATA_JSON = R"( [{ "Type": "Data", "Content": "Actual event", @@ -551,7 +572,7 @@ const std::string event_data_json = R"( "Name": "Channel" }, { "Type": "Binary", - "Content": "09", + "Content": "0901", "Name": "" }] )"; @@ -569,7 +590,7 @@ TEST_CASE("ConsumeWindowsEventLog prints events in JSON::Simple correctly custom }, "Channel": ")" + CUSTOM_CHANNEL + R"(" }, - "EventData": )" + event_data_json + R"( + "EventData": )" + EVENT_DATA_JSON + R"( } )"); } @@ -583,7 +604,7 @@ TEST_CASE("ConsumeWindowsEventLog prints events in JSON::Flattened correctly cus { "Name": ")" + CUSTOM_PROVIDER_NAME + R"(", "Channel": ")" + CUSTOM_CHANNEL /* Channel is not overwritten by data named "Channel" */ + R"(", - "EventData": )" + event_data_json /* EventData is not discarded */ + R"(, + "EventData": )" + EVENT_DATA_JSON /* EventData is not discarded */ + R"(, "param1": "Actual event", "param2": "Second" } diff --git a/libminifi/test/resources/wel/unit-test-provider.man b/extensions/windows-event-log/tests/custom-provider/unit-test-provider.man similarity index 91% rename from libminifi/test/resources/wel/unit-test-provider.man rename to extensions/windows-event-log/tests/custom-provider/unit-test-provider.man index c04cdacf2c..a107c83604 100644 --- a/libminifi/test/resources/wel/unit-test-provider.man +++ b/extensions/windows-event-log/tests/custom-provider/unit-test-provider.man @@ -10,8 +10,8 @@ + resourceFileName="D:\a\nifi-minifi-cpp\nifi-minifi-cpp\extensions\windows-event-log\tests\custom-provider\unit-test-provider.dll" + messageFileName="D:\a\nifi-minifi-cpp\nifi-minifi-cpp\extensions\windows-event-log\tests\custom-provider\unit-test-provider.dll"> diff --git a/extensions/windows-event-log/tests/unit-test-provider.h b/extensions/windows-event-log/tests/unit-test-provider.h new file mode 100644 index 0000000000..3c520b2278 --- /dev/null +++ b/extensions/windows-event-log/tests/unit-test-provider.h @@ -0,0 +1,705 @@ +//**********************************************************************` +//* This is an include file generated by Message Compiler. *` +//* *` +//* Copyright (c) Microsoft Corporation. All Rights Reserved. *` +//**********************************************************************` +#pragma once + +//***************************************************************************** +// +// Notes on the ETW event code generated by MC: +// +// - Structures and arrays of structures are treated as an opaque binary blob. +// The caller is responsible for packing the data for the structure into a +// single region of memory, with no padding between values. The macro will +// have an extra parameter for the length of the blob. +// - Arrays of nul-terminated strings must be packed by the caller into a +// single binary blob containing the correct number of strings, with a nul +// after each string. The size of the blob is specified in characters, and +// includes the final nul. +// - If a SID is provided, its length will be determined by calling +// GetLengthSid. +// - Arrays of SID are treated as a single binary blob. The caller is +// responsible for packing the SID values into a single region of memory with +// no padding. +// - The length attribute on the data element in the manifest is significant +// for values with intype win:UnicodeString, win:AnsiString, or win:Binary. +// The length attribute must be specified for win:Binary, and is optional for +// win:UnicodeString and win:AnsiString (if no length is given, the strings +// are assumed to be nul-terminated). For win:UnicodeString, the length is +// measured in characters, not bytes. +// - For an array of win:UnicodeString, win:AnsiString, or win:Binary, the +// length attribute applies to every value in the array, so every value in +// the array must have the same length. The values in the array are provided +// to the macro via a single pointer -- the caller is responsible for packing +// all of the values into a single region of memory with no padding between +// values. +// - Values of type win:CountedUnicodeString, win:CountedAnsiString, and +// win:CountedBinary can be generated and collected on Vista or later. +// However, they may not decode properly without the Windows 10 2018 Fall +// Update. +// - Arrays of type win:CountedUnicodeString, win:CountedAnsiString, and +// win:CountedBinary must be packed by the caller into a single region of +// memory. The format for each item is a UINT16 byte-count followed by that +// many bytes of data. When providing the array to the generated macro, you +// must provide the total size of the packed array data, including the UINT16 +// sizes for each item. In the case of win:CountedUnicodeString, the data +// size is specified in WCHAR (16-bit) units. In the case of +// win:CountedAnsiString and win:CountedBinary, the data size is specified in +// bytes. +// +//***************************************************************************** + +#include +#include +#include + +#if !defined(ETW_INLINE) +#define ETW_INLINE DECLSPEC_NOINLINE __inline +#endif + +#if defined(__cplusplus) +extern "C" { +#endif + +// +// MCGEN_DISABLE_PROVIDER_CODE_GENERATION macro: +// Define this macro to have the compiler skip the generated functions in this +// header. +// +#ifndef MCGEN_DISABLE_PROVIDER_CODE_GENERATION + +// +// MCGEN_USE_KERNEL_MODE_APIS macro: +// Controls whether the generated code uses kernel-mode or user-mode APIs. +// - Set to 0 to use Windows user-mode APIs such as EventRegister. +// - Set to 1 to use Windows kernel-mode APIs such as EtwRegister. +// Default is based on whether the _ETW_KM_ macro is defined (i.e. by wdm.h). +// Note that the APIs can also be overridden directly, e.g. by setting the +// MCGEN_EVENTWRITETRANSFER or MCGEN_EVENTREGISTER macros. +// +#ifndef MCGEN_USE_KERNEL_MODE_APIS + #ifdef _ETW_KM_ + #define MCGEN_USE_KERNEL_MODE_APIS 1 + #else + #define MCGEN_USE_KERNEL_MODE_APIS 0 + #endif +#endif // MCGEN_USE_KERNEL_MODE_APIS + +// +// MCGEN_HAVE_EVENTSETINFORMATION macro: +// Controls how McGenEventSetInformation uses the EventSetInformation API. +// - Set to 0 to disable the use of EventSetInformation +// (McGenEventSetInformation will always return an error). +// - Set to 1 to directly invoke MCGEN_EVENTSETINFORMATION. +// - Set to 2 to to locate EventSetInformation at runtime via GetProcAddress +// (user-mode) or MmGetSystemRoutineAddress (kernel-mode). +// Default is determined as follows: +// - If MCGEN_EVENTSETINFORMATION has been customized, set to 1 +// (i.e. use MCGEN_EVENTSETINFORMATION). +// - Else if the target OS version has EventSetInformation, set to 1 +// (i.e. use MCGEN_EVENTSETINFORMATION). +// - Else set to 2 (i.e. try to dynamically locate EventSetInformation). +// Note that an McGenEventSetInformation function will only be generated if one +// or more provider in a manifest has provider traits. +// +#ifndef MCGEN_HAVE_EVENTSETINFORMATION + #ifdef MCGEN_EVENTSETINFORMATION // if MCGEN_EVENTSETINFORMATION has been customized, + #define MCGEN_HAVE_EVENTSETINFORMATION 1 // directly invoke MCGEN_EVENTSETINFORMATION(...). + #elif MCGEN_USE_KERNEL_MODE_APIS // else if using kernel-mode APIs, + #if NTDDI_VERSION >= 0x06040000 // if target OS is Windows 10 or later, + #define MCGEN_HAVE_EVENTSETINFORMATION 1 // directly invoke MCGEN_EVENTSETINFORMATION(...). + #else // else + #define MCGEN_HAVE_EVENTSETINFORMATION 2 // find "EtwSetInformation" via MmGetSystemRoutineAddress. + #endif // else (using user-mode APIs) + #else // if target OS and SDK is Windows 8 or later, + #if WINVER >= 0x0602 && defined(EVENT_FILTER_TYPE_SCHEMATIZED) + #define MCGEN_HAVE_EVENTSETINFORMATION 1 // directly invoke MCGEN_EVENTSETINFORMATION(...). + #else // else + #define MCGEN_HAVE_EVENTSETINFORMATION 2 // find "EventSetInformation" via GetModuleHandleExW/GetProcAddress. + #endif + #endif +#endif // MCGEN_HAVE_EVENTSETINFORMATION + +// +// MCGEN_EVENTWRITETRANSFER macro: +// Override to use a custom API. +// +#ifndef MCGEN_EVENTWRITETRANSFER + #if MCGEN_USE_KERNEL_MODE_APIS + #define MCGEN_EVENTWRITETRANSFER EtwWriteTransfer + #else + #define MCGEN_EVENTWRITETRANSFER EventWriteTransfer + #endif +#endif // MCGEN_EVENTWRITETRANSFER + +// +// MCGEN_EVENTREGISTER macro: +// Override to use a custom API. +// +#ifndef MCGEN_EVENTREGISTER + #if MCGEN_USE_KERNEL_MODE_APIS + #define MCGEN_EVENTREGISTER EtwRegister + #else + #define MCGEN_EVENTREGISTER EventRegister + #endif +#endif // MCGEN_EVENTREGISTER + +// +// MCGEN_EVENTSETINFORMATION macro: +// Override to use a custom API. +// (McGenEventSetInformation also affected by MCGEN_HAVE_EVENTSETINFORMATION.) +// +#ifndef MCGEN_EVENTSETINFORMATION + #if MCGEN_USE_KERNEL_MODE_APIS + #define MCGEN_EVENTSETINFORMATION EtwSetInformation + #else + #define MCGEN_EVENTSETINFORMATION EventSetInformation + #endif +#endif // MCGEN_EVENTSETINFORMATION + +// +// MCGEN_EVENTUNREGISTER macro: +// Override to use a custom API. +// +#ifndef MCGEN_EVENTUNREGISTER + #if MCGEN_USE_KERNEL_MODE_APIS + #define MCGEN_EVENTUNREGISTER EtwUnregister + #else + #define MCGEN_EVENTUNREGISTER EventUnregister + #endif +#endif // MCGEN_EVENTUNREGISTER + +// +// MCGEN_PENABLECALLBACK macro: +// Override to use a custom function pointer type. +// (Should match the type used by MCGEN_EVENTREGISTER.) +// +#ifndef MCGEN_PENABLECALLBACK + #if MCGEN_USE_KERNEL_MODE_APIS + #define MCGEN_PENABLECALLBACK PETWENABLECALLBACK + #else + #define MCGEN_PENABLECALLBACK PENABLECALLBACK + #endif +#endif // MCGEN_PENABLECALLBACK + +// +// MCGEN_GETLENGTHSID macro: +// Override to use a custom API. +// +#ifndef MCGEN_GETLENGTHSID + #if MCGEN_USE_KERNEL_MODE_APIS + #define MCGEN_GETLENGTHSID(p) RtlLengthSid((PSID)(p)) + #else + #define MCGEN_GETLENGTHSID(p) GetLengthSid((PSID)(p)) + #endif +#endif // MCGEN_GETLENGTHSID + +// +// MCGEN_EVENT_ENABLED macro: +// Controls how the EventWrite[EventName] macros determine whether an event is +// enabled. The default behavior is for EventWrite[EventName] to use the +// EventEnabled[EventName] macros. +// +#ifndef MCGEN_EVENT_ENABLED +#define MCGEN_EVENT_ENABLED(EventName) EventEnabled##EventName() +#endif + +// +// MCGEN_EVENT_BIT_SET macro: +// Implements testing a bit in an array of ULONG, optimized for CPU type. +// +#ifndef MCGEN_EVENT_BIT_SET +# if defined(_M_IX86) || defined(_M_X64) +# define MCGEN_EVENT_BIT_SET(EnableBits, BitPosition) ((((const unsigned char*)EnableBits)[BitPosition >> 3] & (1u << (BitPosition & 7))) != 0) +# else +# define MCGEN_EVENT_BIT_SET(EnableBits, BitPosition) ((EnableBits[BitPosition >> 5] & (1u << (BitPosition & 31))) != 0) +# endif +#endif // MCGEN_EVENT_BIT_SET + +// +// MCGEN_ENABLE_CHECK macro: +// Determines whether the specified event would be considered as enabled +// based on the state of the specified context. Slightly faster than calling +// McGenEventEnabled directly. +// +#ifndef MCGEN_ENABLE_CHECK +#define MCGEN_ENABLE_CHECK(Context, Descriptor) (Context.IsEnabled && McGenEventEnabled(&Context, &Descriptor)) +#endif + +#if !defined(MCGEN_TRACE_CONTEXT_DEF) +#define MCGEN_TRACE_CONTEXT_DEF +typedef struct _MCGEN_TRACE_CONTEXT +{ + TRACEHANDLE RegistrationHandle; + TRACEHANDLE Logger; // Used as pointer to provider traits. + ULONGLONG MatchAnyKeyword; + ULONGLONG MatchAllKeyword; + ULONG Flags; + ULONG IsEnabled; + UCHAR Level; + UCHAR Reserve; + USHORT EnableBitsCount; + PULONG EnableBitMask; + const ULONGLONG* EnableKeyWords; + const UCHAR* EnableLevel; +} MCGEN_TRACE_CONTEXT, *PMCGEN_TRACE_CONTEXT; +#endif // MCGEN_TRACE_CONTEXT_DEF + +#if !defined(MCGEN_LEVEL_KEYWORD_ENABLED_DEF) +#define MCGEN_LEVEL_KEYWORD_ENABLED_DEF +// +// Determines whether an event with a given Level and Keyword would be +// considered as enabled based on the state of the specified context. +// Note that you may want to use MCGEN_ENABLE_CHECK instead of calling this +// function directly. +// +FORCEINLINE +BOOLEAN +McGenLevelKeywordEnabled( + _In_ PMCGEN_TRACE_CONTEXT EnableInfo, + _In_ UCHAR Level, + _In_ ULONGLONG Keyword + ) +{ + // + // Check if the event Level is lower than the level at which + // the channel is enabled. + // If the event Level is 0 or the channel is enabled at level 0, + // all levels are enabled. + // + + if ((Level <= EnableInfo->Level) || // This also covers the case of Level == 0. + (EnableInfo->Level == 0)) { + + // + // Check if Keyword is enabled + // + + if ((Keyword == (ULONGLONG)0) || + ((Keyword & EnableInfo->MatchAnyKeyword) && + ((Keyword & EnableInfo->MatchAllKeyword) == EnableInfo->MatchAllKeyword))) { + return TRUE; + } + } + + return FALSE; +} +#endif // MCGEN_LEVEL_KEYWORD_ENABLED_DEF + +#if !defined(MCGEN_EVENT_ENABLED_DEF) +#define MCGEN_EVENT_ENABLED_DEF +// +// Determines whether the specified event would be considered as enabled based +// on the state of the specified context. Note that you may want to use +// MCGEN_ENABLE_CHECK instead of calling this function directly. +// +FORCEINLINE +BOOLEAN +McGenEventEnabled( + _In_ PMCGEN_TRACE_CONTEXT EnableInfo, + _In_ PCEVENT_DESCRIPTOR EventDescriptor + ) +{ + return McGenLevelKeywordEnabled(EnableInfo, EventDescriptor->Level, EventDescriptor->Keyword); +} +#endif // MCGEN_EVENT_ENABLED_DEF + +#if !defined(MCGEN_CONTROL_CALLBACK) +#define MCGEN_CONTROL_CALLBACK + +DECLSPEC_NOINLINE __inline +VOID +__stdcall +McGenControlCallbackV2( + _In_ LPCGUID SourceId, + _In_ ULONG ControlCode, + _In_ UCHAR Level, + _In_ ULONGLONG MatchAnyKeyword, + _In_ ULONGLONG MatchAllKeyword, + _In_opt_ PEVENT_FILTER_DESCRIPTOR FilterData, + _Inout_opt_ PVOID CallbackContext + ) +/*++ + +Routine Description: + + This is the notification callback for Windows Vista and later. + +Arguments: + + SourceId - The GUID that identifies the session that enabled the provider. + + ControlCode - The parameter indicates whether the provider + is being enabled or disabled. + + Level - The level at which the event is enabled. + + MatchAnyKeyword - The bitmask of keywords that the provider uses to + determine the category of events that it writes. + + MatchAllKeyword - This bitmask additionally restricts the category + of events that the provider writes. + + FilterData - The provider-defined data. + + CallbackContext - The context of the callback that is defined when the provider + called EtwRegister to register itself. + +Remarks: + + ETW calls this function to notify provider of enable/disable + +--*/ +{ + PMCGEN_TRACE_CONTEXT Ctx = (PMCGEN_TRACE_CONTEXT)CallbackContext; + ULONG Ix; +#ifndef MCGEN_PRIVATE_ENABLE_CALLBACK_V2 + UNREFERENCED_PARAMETER(SourceId); + UNREFERENCED_PARAMETER(FilterData); +#endif + + if (Ctx == NULL) { + return; + } + + switch (ControlCode) { + + case EVENT_CONTROL_CODE_ENABLE_PROVIDER: + Ctx->Level = Level; + Ctx->MatchAnyKeyword = MatchAnyKeyword; + Ctx->MatchAllKeyword = MatchAllKeyword; + Ctx->IsEnabled = EVENT_CONTROL_CODE_ENABLE_PROVIDER; + + for (Ix = 0; Ix < Ctx->EnableBitsCount; Ix += 1) { + if (McGenLevelKeywordEnabled(Ctx, Ctx->EnableLevel[Ix], Ctx->EnableKeyWords[Ix]) != FALSE) { + Ctx->EnableBitMask[Ix >> 5] |= (1 << (Ix % 32)); + } else { + Ctx->EnableBitMask[Ix >> 5] &= ~(1 << (Ix % 32)); + } + } + break; + + case EVENT_CONTROL_CODE_DISABLE_PROVIDER: + Ctx->IsEnabled = EVENT_CONTROL_CODE_DISABLE_PROVIDER; + Ctx->Level = 0; + Ctx->MatchAnyKeyword = 0; + Ctx->MatchAllKeyword = 0; + if (Ctx->EnableBitsCount > 0) { + RtlZeroMemory(Ctx->EnableBitMask, (((Ctx->EnableBitsCount - 1) / 32) + 1) * sizeof(ULONG)); + } + break; + + default: + break; + } + +#ifdef MCGEN_PRIVATE_ENABLE_CALLBACK_V2 + // + // Call user defined callback + // + MCGEN_PRIVATE_ENABLE_CALLBACK_V2( + SourceId, + ControlCode, + Level, + MatchAnyKeyword, + MatchAllKeyword, + FilterData, + CallbackContext + ); +#endif // MCGEN_PRIVATE_ENABLE_CALLBACK_V2 + + return; +} + +#endif // MCGEN_CONTROL_CALLBACK + +#ifndef McGenEventWrite_def +#define McGenEventWrite_def +DECLSPEC_NOINLINE __inline +ULONG __stdcall +McGenEventWrite( + _In_ PMCGEN_TRACE_CONTEXT Context, + _In_ PCEVENT_DESCRIPTOR Descriptor, + _In_opt_ LPCGUID ActivityId, + _In_range_(1, 128) ULONG EventDataCount, + _Inout_updates_(EventDataCount) EVENT_DATA_DESCRIPTOR* EventData + ) +{ + const USHORT UNALIGNED* Traits; + + // Some customized MCGEN_EVENTWRITETRANSFER macros might ignore ActivityId. + UNREFERENCED_PARAMETER(ActivityId); + + Traits = (const USHORT UNALIGNED*)(UINT_PTR)Context->Logger; + + if (Traits == NULL) { + EventData[0].Ptr = 0; + EventData[0].Size = 0; + EventData[0].Reserved = 0; + } else { + EventData[0].Ptr = (ULONG_PTR)Traits; + EventData[0].Size = *Traits; + EventData[0].Reserved = 2; // EVENT_DATA_DESCRIPTOR_TYPE_PROVIDER_METADATA + } + + return MCGEN_EVENTWRITETRANSFER( + Context->RegistrationHandle, + Descriptor, + ActivityId, + NULL, + EventDataCount, + EventData); +} +#endif // McGenEventWrite_def + +#if !defined(McGenEventRegisterUnregister) +#define McGenEventRegisterUnregister + +#pragma warning(push) +#pragma warning(disable:6103) +DECLSPEC_NOINLINE __inline +ULONG __stdcall +McGenEventRegister( + _In_ LPCGUID ProviderId, + _In_opt_ MCGEN_PENABLECALLBACK EnableCallback, + _In_opt_ PVOID CallbackContext, + _Inout_ PREGHANDLE RegHandle + ) +/*++ + +Routine Description: + + This function registers the provider with ETW. + +Arguments: + + ProviderId - Provider ID to register with ETW. + + EnableCallback - Callback to be used. + + CallbackContext - Context for the callback. + + RegHandle - Pointer to registration handle. + +Remarks: + + Should not be called if the provider is already registered (i.e. should not + be called if *RegHandle != 0). Repeatedly registering a provider is a bug + and may indicate a race condition. However, for compatibility with previous + behavior, this function will return SUCCESS in this case. + +--*/ +{ + ULONG Error; + + if (*RegHandle != 0) + { + Error = 0; // ERROR_SUCCESS + } + else + { + Error = MCGEN_EVENTREGISTER(ProviderId, EnableCallback, CallbackContext, RegHandle); + } + + return Error; +} +#pragma warning(pop) + +DECLSPEC_NOINLINE __inline +ULONG __stdcall +McGenEventUnregister(_Inout_ PREGHANDLE RegHandle) +/*++ + +Routine Description: + + Unregister from ETW and set *RegHandle = 0. + +Arguments: + + RegHandle - the pointer to the provider registration handle + +Remarks: + + If provider has not been registered (i.e. if *RegHandle == 0), + return SUCCESS. It is safe to call McGenEventUnregister even if the + call to McGenEventRegister returned an error. + +--*/ +{ + ULONG Error; + + if(*RegHandle == 0) + { + Error = 0; // ERROR_SUCCESS + } + else + { + Error = MCGEN_EVENTUNREGISTER(*RegHandle); + *RegHandle = (REGHANDLE)0; + } + + return Error; +} + +#endif // McGenEventRegisterUnregister + +#endif // MCGEN_DISABLE_PROVIDER_CODE_GENERATION + +//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +// Provider "minifi_unit_test_provider" event count 1 +//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +// Provider GUID = abcdef01-8174-f1ca-87be-da129ff6001b +EXTERN_C __declspec(selectany) const GUID minifi_unit_test_provider = {0xabcdef01, 0x8174, 0xf1ca, {0x87, 0xbe, 0xda, 0x12, 0x9f, 0xf6, 0x00, 0x1b}}; + +#ifndef minifi_unit_test_provider_Traits +#define minifi_unit_test_provider_Traits NULL +#endif // minifi_unit_test_provider_Traits + +// +// Channel +// +#define minifi_unit_test_provider_CHANNEL_minifi_unit_test_provider_Log 0x10 + +// +// Event Descriptors +// +EXTERN_C __declspec(selectany) const EVENT_DESCRIPTOR CustomEvent = {0x2710, 0x1, 0x10, 0x0, 0x0, 0x0, 0x8000000000000000}; +#define CustomEvent_value 0x2710 + +// +// MCGEN_DISABLE_PROVIDER_CODE_GENERATION macro: +// Define this macro to have the compiler skip the generated functions in this +// header. +// +#ifndef MCGEN_DISABLE_PROVIDER_CODE_GENERATION + +// +// Event Enablement Bits +// +EXTERN_C __declspec(selectany) DECLSPEC_CACHEALIGN ULONG minifi_unit_test_providerEnableBits[1]; +EXTERN_C __declspec(selectany) const ULONGLONG minifi_unit_test_providerKeywords[1] = {0x8000000000000000}; +EXTERN_C __declspec(selectany) const unsigned char minifi_unit_test_providerLevels[1] = {0}; + +// +// Provider context +// +EXTERN_C __declspec(selectany) MCGEN_TRACE_CONTEXT minifi_unit_test_provider_Context = {0, (ULONG_PTR)minifi_unit_test_provider_Traits, 0, 0, 0, 0, 0, 0, 1, minifi_unit_test_providerEnableBits, minifi_unit_test_providerKeywords, minifi_unit_test_providerLevels}; + +// +// Provider REGHANDLE +// +#define minifi_unit_test_providerHandle (minifi_unit_test_provider_Context.RegistrationHandle) + +// +// This macro is set to 0, indicating that the EventWrite[Name] macros do not +// have an Activity parameter. This is controlled by the -km and -um options. +// +#define minifi_unit_test_provider_EventWriteActivity 0 + +// +// Register with ETW using the control GUID specified in the manifest. +// Invoke this macro during module initialization (i.e. program startup, +// DLL process attach, or driver load) to initialize the provider. +// Note that if this function returns an error, the error means that +// will not work, but no action needs to be taken -- even if EventRegister +// returns an error, it is generally safe to use EventWrite and +// EventUnregister macros (they will be no-ops if EventRegister failed). +// +#ifndef EventRegisterminifi_unit_test_provider +#define EventRegisterminifi_unit_test_provider() McGenEventRegister(&minifi_unit_test_provider, McGenControlCallbackV2, &minifi_unit_test_provider_Context, &minifi_unit_test_providerHandle) +#endif + +// +// Register with ETW using a specific control GUID (i.e. a GUID other than what +// is specified in the manifest). Advanced scenarios only. +// +#ifndef EventRegisterByGuidminifi_unit_test_provider +#define EventRegisterByGuidminifi_unit_test_provider(Guid) McGenEventRegister(&(Guid), McGenControlCallbackV2, &minifi_unit_test_provider_Context, &minifi_unit_test_providerHandle) +#endif + +// +// Unregister with ETW and close the provider. +// Invoke this macro during module shutdown (i.e. program exit, DLL process +// detach, or driver unload) to unregister the provider. +// Note that you MUST call EventUnregister before DLL or driver unload +// (not optional): failure to unregister a provider before DLL or driver unload +// will result in crashes. +// +#ifndef EventUnregisterminifi_unit_test_provider +#define EventUnregisterminifi_unit_test_provider() McGenEventUnregister(&minifi_unit_test_providerHandle) +#endif + +// +// Enablement check macro for CustomEvent +// +#define EventEnabledCustomEvent() MCGEN_EVENT_BIT_SET(minifi_unit_test_providerEnableBits, 0) + +// +// Event write macros for CustomEvent +// +#define EventWriteCustomEvent(param1, param2, Channel, __binLength, binary) \ + MCGEN_EVENT_ENABLED(CustomEvent) \ + ? McTemplateU0zzzqbr3(&minifi_unit_test_provider_Context, &CustomEvent, param1, param2, Channel, __binLength, binary) : 0 +#define EventWriteCustomEvent_AssumeEnabled(param1, param2, Channel, __binLength, binary) \ + McTemplateU0zzzqbr3(&minifi_unit_test_provider_Context, &CustomEvent, param1, param2, Channel, __binLength, binary) + +#endif // MCGEN_DISABLE_PROVIDER_CODE_GENERATION + +// +// MCGEN_DISABLE_PROVIDER_CODE_GENERATION macro: +// Define this macro to have the compiler skip the generated functions in this +// header. +// +#ifndef MCGEN_DISABLE_PROVIDER_CODE_GENERATION + +// +// Template Functions +// +// +//Template from manifest : CustomTemplate +// +#ifndef McTemplateU0zzzqbr3_def +#define McTemplateU0zzzqbr3_def +ETW_INLINE +ULONG +McTemplateU0zzzqbr3( + _In_ PMCGEN_TRACE_CONTEXT Context, + _In_ PCEVENT_DESCRIPTOR Descriptor, + _In_opt_ PCWSTR _Arg0, + _In_opt_ PCWSTR _Arg1, + _In_opt_ PCWSTR _Arg2, + _In_ const unsigned int _Arg3, + _In_reads_(_Arg3) const unsigned char* _Arg4 + ) +{ +#define McTemplateU0zzzqbr3_ARGCOUNT 5 + + EVENT_DATA_DESCRIPTOR EventData[McTemplateU0zzzqbr3_ARGCOUNT + 1]; + + EventDataDescCreate(&EventData[1], + (_Arg0 != NULL) ? _Arg0 : L"NULL", + (_Arg0 != NULL) ? (ULONG)((wcslen(_Arg0) + 1) * sizeof(WCHAR)) : (ULONG)sizeof(L"NULL")); + + EventDataDescCreate(&EventData[2], + (_Arg1 != NULL) ? _Arg1 : L"NULL", + (_Arg1 != NULL) ? (ULONG)((wcslen(_Arg1) + 1) * sizeof(WCHAR)) : (ULONG)sizeof(L"NULL")); + + EventDataDescCreate(&EventData[3], + (_Arg2 != NULL) ? _Arg2 : L"NULL", + (_Arg2 != NULL) ? (ULONG)((wcslen(_Arg2) + 1) * sizeof(WCHAR)) : (ULONG)sizeof(L"NULL")); + + EventDataDescCreate(&EventData[4],&_Arg3, sizeof(const unsigned int) ); + + EventDataDescCreate(&EventData[5],_Arg4, (ULONG)sizeof(char)*_Arg3); + + return McGenEventWrite(Context, Descriptor, NULL, McTemplateU0zzzqbr3_ARGCOUNT + 1, EventData); +} +#endif // McTemplateU0zzzqbr3_def + +#endif // MCGEN_DISABLE_PROVIDER_CODE_GENERATION + +#if defined(__cplusplus) +}; +#endif From 825a57b4678900b6462cbbf8445e81b717e960c3 Mon Sep 17 00:00:00 2001 From: Adam Debreceni Date: Mon, 18 Jan 2021 12:36:41 +0100 Subject: [PATCH 08/15] MINIFICPP-1448 - Create manifest and register provider as part of the build --- .github/workflows/ci.yml | 22 +- .../windows-event-log/tests/CMakeLists.txt | 10 +- .../tests/CWELCustomProviderTests.cpp | 132 ++++ .../windows-event-log/tests/CWELTestUtils.h | 111 +++ .../tests/ConsumeWindowsEventLogTests.cpp | 249 +------ .../custom-provider/generate-and-register.bat | 62 ++ .../custom-provider/unit-test-provider.man | 36 - .../tests/unit-test-provider.h | 705 ------------------ win_build_vs.bat | 5 +- 9 files changed, 337 insertions(+), 995 deletions(-) create mode 100644 extensions/windows-event-log/tests/CWELCustomProviderTests.cpp create mode 100644 extensions/windows-event-log/tests/CWELTestUtils.h create mode 100644 extensions/windows-event-log/tests/custom-provider/generate-and-register.bat delete mode 100644 extensions/windows-event-log/tests/custom-provider/unit-test-provider.man delete mode 100644 extensions/windows-event-log/tests/unit-test-provider.h diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9bc04ab81b..69e87175ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,18 +46,11 @@ jobs: uses: actions/checkout@v2 - name: Setup PATH uses: microsoft/setup-msbuild@v1.0.2 - - id: create-wel-provider + - id: build run: | - cd extensions\windows-event-log\tests\custom-provider PATH %PATH%;C:\Program Files (x86)\Windows Kits\10\bin\10.0.17763.0\x86 PATH %PATH%;C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\MSBuild\15.0\Bin\Roslyn - mc -css Namespace unit-test-provider.man - rc unit-test-provider.rc - csc /target:library /unsafe /win32res:unit-test-provider.res unit-test-provider.cs - wevtutil im unit-test-provider.man - shell: cmd - - id: build - run: win_build_vs.bat build /CI /S /A + win_build_vs.bat build /CI /S /A shell: cmd windows_VS2019: name: "windows-vs2019" @@ -68,18 +61,11 @@ jobs: uses: actions/checkout@v2 - name: Setup PATH uses: microsoft/setup-msbuild@v1.0.2 - - id: create-wel-provider + - id: build run: | - cd extensions\windows-event-log\tests\custom-provider PATH %PATH%;C:\Program Files (x86)\Windows Kits\10\bin\10.0.19041.0\x64 PATH %PATH%;C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\Roslyn - mc -css Namespace unit-test-provider.man - rc unit-test-provider.rc - csc /target:library /unsafe /win32res:unit-test-provider.res unit-test-provider.cs - wevtutil im unit-test-provider.man - shell: cmd - - id: build - run: win_build_vs.bat build /2019 /64 /CI + win_build_vs.bat build /2019 /64 /CI shell: cmd ubuntu_16_04: name: "ubuntu-16.04" diff --git a/extensions/windows-event-log/tests/CMakeLists.txt b/extensions/windows-event-log/tests/CMakeLists.txt index 4d144ce364..1f8c9f3538 100644 --- a/extensions/windows-event-log/tests/CMakeLists.txt +++ b/extensions/windows-event-log/tests/CMakeLists.txt @@ -17,7 +17,15 @@ # under the License. # -file(GLOB WEL_INTEGRATION_TESTS "*.cpp") +set(WEL_INTEGRATION_TESTS "BookmarkTests.cpp" "ConsumeWindowsEventLogTests.cpp" "MetadataWalkerTests.cpp") +if (TEST_CUSTOM_WEL_PROVIDER) + execute_process(COMMAND + "${CMAKE_CURRENT_LIST_DIR}/custom-provider/generate-and-register.bat" + "${CMAKE_CURRENT_LIST_DIR}/custom-provider" + ) + list(APPEND WEL_INTEGRATION_TESTS "CWELCustomProviderTests.cpp") +endif() + SET(WEL_TEST_COUNT 0) FOREACH(testfile ${WEL_INTEGRATION_TESTS}) get_filename_component(testfilename "${testfile}" NAME_WE) diff --git a/extensions/windows-event-log/tests/CWELCustomProviderTests.cpp b/extensions/windows-event-log/tests/CWELCustomProviderTests.cpp new file mode 100644 index 0000000000..a788c955af --- /dev/null +++ b/extensions/windows-event-log/tests/CWELCustomProviderTests.cpp @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ConsumeWindowsEventLog.h" + +#include "core/ConfigurableComponent.h" +#include "processors/LogAttribute.h" +#include "processors/PutFile.h" +#include "TestBase.h" +#include "utils/TestUtils.h" +#include "utils/file/FileUtils.h" +#include "rapidjson/document.h" + +#include "CWELTestUtils.h" + +// generated from the manifest file "custom-provider/unit-test-provider.man" +// using the command "mc -um unit-test-provider.man" +#include "custom-provider/unit-test-provider.h" + +namespace { + +struct CustomEventData { + std::wstring first; + std::wstring second; + std::wstring third; + int binary_length; + const unsigned char* binary_data; +}; + +const std::string CUSTOM_PROVIDER_NAME = "minifi_unit_test_provider"; +const std::string CUSTOM_CHANNEL = CUSTOM_PROVIDER_NAME + "/Log"; + +bool dispatchCustomEvent(const CustomEventData& event) { + static auto provider_initialized = [] { + return EventRegisterminifi_unit_test_provider(); + }(); + REQUIRE(provider_initialized == ERROR_SUCCESS); + + auto result = EventWriteCustomEvent( + event.first.c_str(), + event.second.c_str(), + event.third.c_str(), + event.binary_length, + event.binary_data + ); + return result == ERROR_SUCCESS; +} + +} // namespace + +class CustomProviderController : public OutputFormatTestController { + public: + CustomProviderController(std::string format) : OutputFormatTestController(CUSTOM_CHANNEL, "*", std::move(format)) {} + + protected: + void dispatchBookmarkEvent() override { + auto binary = reinterpret_cast("\x0c\x10"); + REQUIRE(dispatchCustomEvent({L"Bookmark", L"Second", L"Third", 2, binary})); + // even though we are using the API, we still have to wait for the event to appear + // for CWEL processor + std::this_thread::sleep_for(std::chrono::seconds{1}); + } + void dispatchCollectedEvent() override { + auto binary = reinterpret_cast("\x09\x01"); + REQUIRE(dispatchCustomEvent({L"Actual event", L"Second", L"Third", 2, binary})); + // even though we are using the API, we still have to wait for the event to appear + // for CWEL processor + std::this_thread::sleep_for(std::chrono::seconds{1}); + } +}; + +const std::string EVENT_DATA_JSON = R"( + [{ + "Type": "Data", + "Content": "Actual event", + "Name": "param1" + }, { + "Type": "Data", + "Content": "Second", + "Name": "param2" + }, { + "Type": "Data", + "Content": "Third", + "Name": "Channel" + }, { + "Type": "Binary", + "Content": "0901", + "Name": "" + }] +)"; + +TEST_CASE("ConsumeWindowsEventLog prints events in JSON::Simple correctly custom provider", "[onTrigger]") { + std::string event = CustomProviderController{"JSON::Simple"}.run(); + verifyJSON(event, R"( + { + "System": { + "Provider": { + "Name": ")" + CUSTOM_PROVIDER_NAME + R"(" + }, + "Channel": ")" + CUSTOM_CHANNEL + R"(" + }, + "EventData": )" + EVENT_DATA_JSON + R"( + } + )"); +} + +TEST_CASE("ConsumeWindowsEventLog prints events in JSON::Flattened correctly custom provider", "[onTrigger]") { + std::string event = CustomProviderController{"JSON::Flattened"}.run(); + verifyJSON(event, R"( + { + "Name": ")" + CUSTOM_PROVIDER_NAME + R"(", + "Channel": ")" + CUSTOM_CHANNEL /* Channel is not overwritten by data named "Channel" */ + R"(", + "EventData": )" + EVENT_DATA_JSON /* EventData is not discarded */ + R"(, + "param1": "Actual event", + "param2": "Second" + } + )"); +} diff --git a/extensions/windows-event-log/tests/CWELTestUtils.h b/extensions/windows-event-log/tests/CWELTestUtils.h new file mode 100644 index 0000000000..8d980c4799 --- /dev/null +++ b/extensions/windows-event-log/tests/CWELTestUtils.h @@ -0,0 +1,111 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "ConsumeWindowsEventLog.h" +#include "processors/PutFile.h" +#include "TestBase.h" +#include "utils/TestUtils.h" +#include "utils/file/FileUtils.h" + +core::Relationship Success{"success", "Everything is fine"}; + +using ConsumeWindowsEventLog = org::apache::nifi::minifi::processors::ConsumeWindowsEventLog; +using PutFile = org::apache::nifi::minifi::processors::PutFile; + +class OutputFormatTestController : public TestController { + public: + OutputFormatTestController(std::string channel, std::string query, std::string output_format) + : channel_(std::move(channel)), + query_(std::move(query)), + output_format_(std::move(output_format)) {} + + std::string run() { + LogTestController::getInstance().setDebug(); + LogTestController::getInstance().setDebug(); + std::shared_ptr test_plan = createPlan(); + + auto cwel_processor = test_plan->addProcessor("ConsumeWindowsEventLog", "cwel"); + test_plan->setProperty(cwel_processor, ConsumeWindowsEventLog::Channel.getName(), channel_); + test_plan->setProperty(cwel_processor, ConsumeWindowsEventLog::Query.getName(), query_); + test_plan->setProperty(cwel_processor, ConsumeWindowsEventLog::OutputFormat.getName(), output_format_); + + auto dir = utils::createTempDir(this); + + auto put_file = test_plan->addProcessor("PutFile", "putFile", Success, true); + test_plan->setProperty(put_file, PutFile::Directory.getName(), dir); + + { + dispatchBookmarkEvent(); + + runSession(test_plan); + } + + test_plan->reset(); + LogTestController::getInstance().resetStream(LogTestController::getInstance().log_output); + + + { + dispatchCollectedEvent(); + + runSession(test_plan); + + auto files = utils::file::list_dir_all(dir, LogTestController::getInstance().getLogger(), false); + REQUIRE(files.size() == 1); + + std::ifstream file{utils::file::concat_path(files[0].first, files[0].second)}; + return {std::istreambuf_iterator{file}, {}}; + } + } + + protected: + virtual void dispatchBookmarkEvent() = 0; + virtual void dispatchCollectedEvent() = 0; + + std::string channel_; + std::string query_; + std::string output_format_; +}; + +// carries out a loose match on objects, i.e. it doesn't matter if the +// actual object has extra fields than expected +void matchJSON(const rapidjson::Value& json, const rapidjson::Value& expected) { + if (expected.IsObject()) { + REQUIRE(json.IsObject()); + for (const auto& expected_member : expected.GetObject()) { + REQUIRE(json.HasMember(expected_member.name)); + matchJSON(json[expected_member.name], expected_member.value); + } + } else if (expected.IsArray()) { + REQUIRE(json.IsArray()); + REQUIRE(json.Size() == expected.Size()); + for (size_t idx{0}; idx < expected.Size(); ++idx) { + matchJSON(json[idx], expected[idx]); + } + } else { + REQUIRE(json == expected); + } +} + +void verifyJSON(const std::string& json_str, const std::string& expected_str) { + rapidjson::Document json, expected; + REQUIRE_FALSE(json.Parse(json_str.c_str()).HasParseError()); + REQUIRE_FALSE(expected.Parse(expected_str.c_str()).HasParseError()); + + matchJSON(json, expected); +} diff --git a/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp b/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp index 7c1d724517..6873e8568e 100644 --- a/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp +++ b/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp @@ -25,9 +25,7 @@ #include "utils/file/FileUtils.h" #include "rapidjson/document.h" -// generated from the manifest file "custom-provider/unit-test-provider.man" -// using the command "mc -um unit-test-provider.man" -#include "unit-test-provider.h" +#include "CWELTestUtils.h" using ConsumeWindowsEventLog = org::apache::nifi::minifi::processors::ConsumeWindowsEventLog; using LogAttribute = org::apache::nifi::minifi::processors::LogAttribute; @@ -37,8 +35,6 @@ using IdGenerator = org::apache::nifi::minifi::utils::IdGenerator; namespace { -core::Relationship Success{"success", "Everything is fine"}; - const std::string APPLICATION_CHANNEL = "Application"; constexpr DWORD CWEL_TESTS_OPCODE = 14985; // random opcode hopefully won't clash with something important @@ -50,41 +46,18 @@ void reportEvent(const std::string& channel, const char* message, WORD log_level ReportEventA(event_source, log_level, 0, CWEL_TESTS_OPCODE, nullptr, 1, 0, &message, nullptr); } -struct CustomEventData { - std::wstring first; - std::wstring second; - std::wstring third; - int binary_length; - const unsigned char* binary_data; -}; - -const std::string CUSTOM_PROVIDER_NAME = "minifi_unit_test_provider"; -const std::string CUSTOM_CHANNEL = CUSTOM_PROVIDER_NAME + "/Log"; - -bool initializeCustomProvider() { - static auto provider_initialized = [] { - return EventRegisterminifi_unit_test_provider(); - }(); - - return provider_initialized == ERROR_SUCCESS; -} - -bool dispatchCustomEvent(const CustomEventData& event) { - REQUIRE(initializeCustomProvider()); - - auto result = EventWriteCustomEvent( - event.first.c_str(), - event.second.c_str(), - event.third.c_str(), - event.binary_length, - event.binary_data - ); - return result == ERROR_SUCCESS; -} +class SimpleFormatTestController : public OutputFormatTestController { + public: + using OutputFormatTestController::OutputFormatTestController; -bool supportsCustomEvents() { - return initializeCustomProvider(); -} + protected: + void dispatchBookmarkEvent() { + reportEvent(APPLICATION_CHANNEL, "Event zero: this is in the past"); + } + void OutputFormatTestController::dispatchCollectedEvent() { + reportEvent(APPLICATION_CHANNEL, "Event one"); + } +}; } // namespace @@ -345,83 +318,12 @@ TEST_CASE("ConsumeWindowsEventLog output format can be set", "[create][output_fo outputFormatSetterTestHelper("InvalidValue", 0); } -namespace { - -class OutputFormatTestController : public TestController { - public: - OutputFormatTestController(std::string channel, std::string query, std::string output_format) - : channel_(std::move(channel)), - query_(std::move(query)), - output_format_(std::move(output_format)) {} - - std::string run() { - LogTestController::getInstance().setDebug(); - LogTestController::getInstance().setDebug(); - std::shared_ptr test_plan = createPlan(); - - auto cwel_processor = test_plan->addProcessor("ConsumeWindowsEventLog", "cwel"); - test_plan->setProperty(cwel_processor, ConsumeWindowsEventLog::Channel.getName(), channel_); - test_plan->setProperty(cwel_processor, ConsumeWindowsEventLog::Query.getName(), query_); - test_plan->setProperty(cwel_processor, ConsumeWindowsEventLog::OutputFormat.getName(), output_format_); - - auto dir = utils::createTempDir(this); - - auto put_file = test_plan->addProcessor("PutFile", "putFile", Success, true); - test_plan->setProperty(put_file, PutFile::Directory.getName(), dir); - - { - dispatchBookmarkEvent(); - - runSession(test_plan); - } - - test_plan->reset(); - LogTestController::getInstance().resetStream(LogTestController::getInstance().log_output); - - - { - dispatchCollectedEvent(); - - runSession(test_plan); - - auto files = utils::file::list_dir_all(dir, LogTestController::getInstance().getLogger(), false); - REQUIRE(files.size() == 1); - - std::ifstream file{utils::file::concat_path(files[0].first, files[0].second)}; - return {std::istreambuf_iterator{file}, {}}; - } - } - - protected: - virtual void dispatchBookmarkEvent() = 0; - virtual void dispatchCollectedEvent() = 0; - - std::string channel_; - std::string query_; - std::string output_format_; -}; - -} // namespace - // NOTE(fgerlits): I don't know how to unit test this, as my manually published events all result in an empty string if OutputFormat is Plaintext // but it does seem to work, based on manual tests reading system logs // TEST_CASE("ConsumeWindowsEventLog prints events in plain text correctly", "[onTrigger]") TEST_CASE("ConsumeWindowsEventLog prints events in XML correctly", "[onTrigger]") { - class XMLFormat : public OutputFormatTestController { - public: - XMLFormat() : OutputFormatTestController(APPLICATION_CHANNEL, QUERY, "XML") {} - - protected: - void dispatchBookmarkEvent() override { - reportEvent(APPLICATION_CHANNEL, "Event zero: this is in the past"); - } - void dispatchCollectedEvent() override { - reportEvent(APPLICATION_CHANNEL, "Event one"); - } - } test_controller; - - std::string event = test_controller.run(); + std::string event = SimpleFormatTestController{APPLICATION_CHANNEL, QUERY, "XML"}.run(); REQUIRE(event.find(R"()") != std::string::npos); REQUIRE(event.find(R"(14985)") != std::string::npos); @@ -437,52 +339,8 @@ TEST_CASE("ConsumeWindowsEventLog prints events in XML correctly", "[onTrigger]" REQUIRE(event.find(R"(Event one)") != std::string::npos); } -namespace { -// carries out a loose match on objects, i.e. it doesn't matter if the -// actual object has extra fields than expected -void matchJSON(const rapidjson::Value& json, const rapidjson::Value& expected) { - if (expected.IsObject()) { - REQUIRE(json.IsObject()); - for (const auto& expected_member : expected.GetObject()) { - REQUIRE(json.HasMember(expected_member.name)); - matchJSON(json[expected_member.name], expected_member.value); - } - } else if (expected.IsArray()) { - REQUIRE(json.IsArray()); - REQUIRE(json.Size() == expected.Size()); - for (size_t idx{0}; idx < expected.Size(); ++idx) { - matchJSON(json[idx], expected[idx]); - } - } else { - REQUIRE(json == expected); - } -} - -void verifyJSON(const std::string& json_str, const std::string& expected_str) { - rapidjson::Document json, expected; - REQUIRE_FALSE(json.Parse(json_str.c_str()).HasParseError()); - REQUIRE_FALSE(expected.Parse(expected_str.c_str()).HasParseError()); - - matchJSON(json, expected); -} - -class JSONOutputController : public OutputFormatTestController { - public: - JSONOutputController(std::string format) : OutputFormatTestController(APPLICATION_CHANNEL, "*", std::move(format)) {} - - protected: - void dispatchBookmarkEvent() override { - reportEvent(APPLICATION_CHANNEL, "Event zero: this is in the past"); - } - void dispatchCollectedEvent() override { - reportEvent(APPLICATION_CHANNEL, "Event one"); - } -}; - -} // namespace - TEST_CASE("ConsumeWindowsEventLog prints events in JSON::Simple correctly", "[onTrigger]") { - std::string event = JSONOutputController{"JSON::Simple"}.run(); + std::string event = SimpleFormatTestController{APPLICATION_CHANNEL, "*", "JSON::Simple"}.run(); verifyJSON(event, R"json( { "System": { @@ -501,7 +359,7 @@ TEST_CASE("ConsumeWindowsEventLog prints events in JSON::Simple correctly", "[on } TEST_CASE("ConsumeWindowsEventLog prints events in JSON::Flattened correctly", "[onTrigger]") { - std::string event = JSONOutputController{"JSON::Flattened"}.run(); + std::string event = SimpleFormatTestController{APPLICATION_CHANNEL, "*", "JSON::Flattened"}.run(); verifyJSON(event, R"json( { "Name": "Application", @@ -516,7 +374,7 @@ TEST_CASE("ConsumeWindowsEventLog prints events in JSON::Flattened correctly", " } TEST_CASE("ConsumeWindowsEventLog prints events in JSON::Raw correctly", "[onTrigger]") { - std::string event = JSONOutputController{"JSON::Raw"}.run(); + std::string event = SimpleFormatTestController{APPLICATION_CHANNEL, "*", "JSON::Raw"}.run(); verifyJSON(event, R"json( [ { @@ -536,81 +394,6 @@ TEST_CASE("ConsumeWindowsEventLog prints events in JSON::Raw correctly", "[onTri )json"); } -class CustomProviderController : public OutputFormatTestController { - public: - CustomProviderController(std::string format) : OutputFormatTestController(CUSTOM_CHANNEL, "*", std::move(format)) {} - - protected: - void dispatchBookmarkEvent() override { - auto binary = reinterpret_cast("\x0c\x10"); - REQUIRE(dispatchCustomEvent({L"Bookmark", L"Second", L"Third", 2, binary})); - // even though we are using the API, we still have to wait for the event to appear - // for CWEL processor - std::this_thread::sleep_for(std::chrono::seconds{1}); - } - void dispatchCollectedEvent() override { - auto binary = reinterpret_cast("\x09\x01"); - REQUIRE(dispatchCustomEvent({L"Actual event", L"Second", L"Third", 2, binary})); - // even though we are using the API, we still have to wait for the event to appear - // for CWEL processor - std::this_thread::sleep_for(std::chrono::seconds{1}); - } -}; - -const std::string EVENT_DATA_JSON = R"( - [{ - "Type": "Data", - "Content": "Actual event", - "Name": "param1" - }, { - "Type": "Data", - "Content": "Second", - "Name": "param2" - }, { - "Type": "Data", - "Content": "Third", - "Name": "Channel" - }, { - "Type": "Binary", - "Content": "0901", - "Name": "" - }] -)"; - -TEST_CASE("ConsumeWindowsEventLog prints events in JSON::Simple correctly custom provider", "[onTrigger]") { - if (!supportsCustomEvents()) { - return; - } - std::string event = CustomProviderController{"JSON::Simple"}.run(); - verifyJSON(event, R"( - { - "System": { - "Provider": { - "Name": ")" + CUSTOM_PROVIDER_NAME + R"(" - }, - "Channel": ")" + CUSTOM_CHANNEL + R"(" - }, - "EventData": )" + EVENT_DATA_JSON + R"( - } - )"); -} - -TEST_CASE("ConsumeWindowsEventLog prints events in JSON::Flattened correctly custom provider", "[onTrigger]") { - if (!supportsCustomEvents()) { - return; - } - std::string event = CustomProviderController{"JSON::Flattened"}.run(); - verifyJSON(event, R"( - { - "Name": ")" + CUSTOM_PROVIDER_NAME + R"(", - "Channel": ")" + CUSTOM_CHANNEL /* Channel is not overwritten by data named "Channel" */ + R"(", - "EventData": )" + EVENT_DATA_JSON /* EventData is not discarded */ + R"(, - "param1": "Actual event", - "param2": "Second" - } - )"); -} - namespace { void batchCommitSizeTestHelper(std::size_t num_events_read, std::size_t batch_commit_size, std::size_t expected_event_count) { TestController test_controller; diff --git a/extensions/windows-event-log/tests/custom-provider/generate-and-register.bat b/extensions/windows-event-log/tests/custom-provider/generate-and-register.bat new file mode 100644 index 0000000000..c74090b0da --- /dev/null +++ b/extensions/windows-event-log/tests/custom-provider/generate-and-register.bat @@ -0,0 +1,62 @@ +@echo off &setlocal enabledelayedexpansion +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. + +cd %1 + +( + echo ^ + echo ^ + echo ^ + echo ^ + echo ^ + echo ^ + echo ^ + echo ^ + echo ^ + echo ^ + echo ^ + echo ^ + echo ^ + echo ^ + echo ^ + echo ^