optimise template load time (#241)

trying to fix #237

according to research
https://github.com/HFTrader/regex-performance
std::regex is extremely slow

x times slower than boost one's

==> changing default regex engine to boost one's
presering ability to build jinja2cpp with standard regex

on the local PC speed up of using boost::regex instead of std::regex shows up to be 5-6x times 20s to 120s

Author: rmorozov

CMakeLists.txt

@@ -15,6 +15,9 @@ endif()
 set(JINJA2CPP_SANITIZERS address+undefined memory)
 set(JINJA2CPP_WITH_SANITIZERS none CACHE STRING "Build with sanitizer")
 set_property(CACHE JINJA2CPP_WITH_SANITIZERS PROPERTY STRINGS ${JINJA2CPP_SANITIZERS})
+set(JINJA2CPP_SUPPORTED_REGEX std boost)
+set(JINJA2CPP_USE_REGEX boost CACHE STRING "Use regex parser in lexer, boost works faster on most platforms")
+set_property(CACHE JINJA2CPP_USE_REGEX PROPERTY STRINGS ${JINJA2CPP_SUPPORTED_REGEX})
 set(JINJA2CPP_WITH_JSON_BINDINGS boost nlohmann rapid all none)
 set(JINJA2CPP_WITH_JSON_BINDINGS all CACHE STRING "Build with json support")
 set_property(CACHE JINJA2CPP_WITH_JSON_BINDINGS PROPERTY STRINGS ${JINJA2CPP_WITH_JSON_BINDINGS})
@@ -210,7 +213,15 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC")
     target_compile_options(${LIB_TARGET_NAME} PRIVATE ${MSVC_CXX_FLAGS})
 endif ()
 
-target_compile_definitions(${LIB_TARGET_NAME} PUBLIC -DBOOST_SYSTEM_NO_DEPRECATED -DBOOST_ERROR_CODE_HEADER_ONLY)
+if ("${JINJA2CPP_USE_REGEX}" STREQUAL "boost")
+    set(_regex_define "-DJINJA2CPP_USE_REGEX_BOOST")
+endif()
+target_compile_definitions(${LIB_TARGET_NAME}
+    PUBLIC
+        -DBOOST_SYSTEM_NO_DEPRECATED
+        -DBOOST_ERROR_CODE_HEADER_ONLY
+        ${_regex_define}
+)
 
 if (JINJA2CPP_BUILD_SHARED)
     target_compile_definitions(${LIB_TARGET_NAME} PRIVATE -DJINJA2CPP_BUILD_AS_SHARED PUBLIC -DJINJA2CPP_LINK_AS_SHARED)

src/template_parser.h

@@ -17,11 +17,28 @@
 #include <nonstd/expected.hpp>
 
 #include <list>
-#include <regex>
 #include <sstream>
 #include <string>
 #include <vector>
 
+#ifdef JINJA2CPP_USE_REGEX_BOOST
+#include <boost/regex.hpp>
+template <typename CharType>
+using BasicRegex = boost::basic_regex<CharType>;
+using Regex = boost::regex;
+using WideRegex = boost::wregex;
+template <typename CharIterator>
+using RegexIterator = boost::regex_iterator<CharIterator>;
+#else
+#include <regex>
+template <typename CharType>
+using BasicRegex = std::basic_regex<CharType>;
+using Regex = std::regex;
+using WideRegex = std::wregex;
+template <typename CharIterator>
+using RegexIterator = std::regex_iterator<CharIterator>;
+#endif
+
 namespace jinja2
 {
 template<typename CharT>
@@ -58,9 +75,9 @@ MultiStringLiteral ParserTraitsBase<T>::s_regexp = UNIVERSAL_STR(
 template<>
 struct ParserTraits<char> : public ParserTraitsBase<>
 {
-    static std::regex GetRoughTokenizer()
-    { return std::regex(s_regexp.GetValueStr<char>()); }
-    static std::regex GetKeywords()
+    static Regex GetRoughTokenizer()
+    { return Regex(s_regexp.GetValueStr<char>()); }
+    static Regex GetKeywords()
     {
         std::string pattern;
         std::string prefix("(^");
@@ -76,7 +93,7 @@ struct ParserTraits<char> : public ParserTraitsBase<>
 
             pattern += prefix + info.name.charValue + postfix;
         }
-        return std::regex(pattern);
+        return Regex(pattern);
     }
     static std::string GetAsString(const std::string& str, CharRange range) { return str.substr(range.startOffset, range.size()); }
     static InternalValue RangeToNum(const std::string& str, CharRange range, Token::Type hint)
@@ -109,9 +126,9 @@ struct ParserTraits<char> : public ParserTraitsBase<>
 template<>
 struct ParserTraits<wchar_t> : public ParserTraitsBase<>
 {
-    static std::wregex GetRoughTokenizer()
-    { return std::wregex(s_regexp.GetValueStr<wchar_t>()); }
-    static std::wregex GetKeywords()
+    static WideRegex GetRoughTokenizer()
+    { return WideRegex(s_regexp.GetValueStr<wchar_t>()); }
+    static WideRegex GetKeywords()
     {
         std::wstring pattern;
         std::wstring prefix(L"(^");
@@ -127,7 +144,7 @@ struct ParserTraits<wchar_t> : public ParserTraitsBase<>
 
             pattern += prefix + info.name.wcharValue + postfix;
         }
-        return std::wregex(pattern);
+        return WideRegex(pattern);
     }
     static std::string GetAsString(const std::wstring& str, CharRange range)
     {
@@ -248,7 +265,7 @@ class TemplateParser : public LexerHelper
 public:
     using string_t = std::basic_string<CharT>;
     using traits_t = ParserTraits<CharT>;
-    using sregex_iterator = std::regex_iterator<typename string_t::const_iterator>;
+    using sregex_iterator = RegexIterator<typename string_t::const_iterator>;
     using ErrorInfo = ErrorInfoTpl<CharT>;
     using ParseResult = nonstd::expected<RendererPtr, std::vector<ErrorInfo>>;
 
@@ -437,7 +454,7 @@ class TemplateParser : public LexerHelper
                     FinishCurrentLine(match.position() + 2);
                     return MakeParseError(ErrorCode::UnexpectedCommentEnd, MakeToken(Token::CommentEnd, { matchStart, matchStart + 2 }));
                 }
-                
+
                 m_currentBlockInfo.range.startOffset = FinishCurrentBlock(matchStart, TextBlockType::RawText);
                 break;
             case RM_ExprBegin:
@@ -925,8 +942,8 @@ class TemplateParser : public LexerHelper
     std::string m_templateName;
     const Settings& m_settings;
     TemplateEnv* m_env = nullptr;
-    std::basic_regex<CharT> m_roughTokenizer;
-    std::basic_regex<CharT> m_keywords;
+    BasicRegex<CharT> m_roughTokenizer;
+    BasicRegex<CharT> m_keywords;
     std::vector<LineInfo> m_lines;
     std::vector<TextBlockInfo> m_textBlocks;
     LineInfo m_currentLineInfo = {};

test/perf_test.cpp

@@ -181,6 +181,26 @@ TEST(PerfTests, ForLoopIfText)
     std::cout << result << std::endl;
 }
 
+TEST(PerfTests, LoadTemplate)
+{
+    std::string source = "{{ title }}";
+    jinja2::Template tpl;
+
+    const int N = 100000;
+
+    for (int i = 0; i < N; i++)
+    {
+        tpl.Load(source);
+    }
+
+    ValuesMap data;
+    data["title"] = "My title";
+    data["t"] = "My List";
+    std::string result = tpl.RenderAsString(data).value();;
+
+    std::cout << result << std::endl;
+}
+
 TEST(PerfTests, DISABLED_TestMatsuhiko)
 {
     std::string source = R"(

thirdparty/CMakeLists.txt

@@ -68,7 +68,7 @@ endif()
 
 if (NOT DEFINED JINJA2_PRIVATE_LIBS_INT)
     set(JINJA2CPP_PRIVATE_LIBS ${JINJA2CPP_PRIVATE_LIBS} Boost::variant
-        Boost::filesystem Boost::algorithm Boost::lexical_cast Boost::json fmt RapidJson)
+        Boost::filesystem Boost::algorithm Boost::lexical_cast Boost::json Boost::regex fmt RapidJson)
 else ()
     set (JINJA2CPP_PRIVATE_LIBS ${JINJA2_PRIVATE_LIBS_INT})
 endif ()

thirdparty/external_boost_deps.cmake

@@ -24,30 +24,37 @@ find_package(boost_filesystem ${FIND_BOOST_PACKAGE_QUIET})
 find_package(boost_json       ${FIND_BOOST_PACKAGE_QUIET})
 find_package(boost_optional   ${FIND_BOOST_PACKAGE_QUIET})
 find_package(boost_variant    ${FIND_BOOST_PACKAGE_QUIET})
+find_package(boost_regex      ${FIND_BOOST_PACKAGE_QUIET})
 
 if (boost_algorithm_FOUND AND
    boost_filesystem_FOUND AND
    boost_json_FOUND AND
    boost_optional_FOUND AND
-   boost_variant_FOUND)
+   boost_variant_FOUND AND boost_regex_FOUND)
    imported_target_alias(boost_algorithm  ALIAS boost_algorithm::boost_algorithm)
    imported_target_alias(boost_filesystem ALIAS boost_filesystem::boost_filesystem)
    imported_target_alias(boost_json       ALIAS boost_json::boost_json)
    imported_target_alias(boost_optional   ALIAS boost_optional::boost_optional)
    imported_target_alias(boost_variant    ALIAS boost_variant::boost_variant)
+   imported_target_alias(boost_regex      ALIAS boost_regex::boost_regex)
 else ()
-    find_package(Boost COMPONENTS system filesystem json ${FIND_BOOST_PACKAGE_QUIET} REQUIRED)
+    find_package(Boost COMPONENTS system filesystem json regex ${FIND_BOOST_PACKAGE_QUIET} REQUIRED)
 
     if (Boost_FOUND)
         imported_target_alias(boost_algorithm  ALIAS Boost::boost)
         imported_target_alias(boost_filesystem ALIAS Boost::filesystem)
         imported_target_alias(boost_json       ALIAS Boost::json)
         imported_target_alias(boost_optional   ALIAS Boost::boost)
         imported_target_alias(boost_variant    ALIAS Boost::boost)
+        imported_target_alias(boost_regex      ALIAS Boost::regex)
     endif ()
 endif ()
 
-install(TARGETS boost_algorithm boost_filesystem boost_json boost_optional boost_variant
+set(_additional_boost_install_targets)
+if ("${JINJA2CPP_USE_REGEX}" STREQUAL "boost")
+set(_additional_boost_install_targets "boost_regex")
+endif()
+install(TARGETS boost_algorithm boost_filesystem boost_json boost_optional boost_variant ${_additional_boost_install_targets}
         EXPORT InstallTargets
         RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
         LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}