Add Meta comments parsing logic

cgen-nnparser/src/main/kotlin/ce/parser/domain/usecase/MetaTokenizerUseCaseImpl.kt

@@ -29,7 +29,8 @@ class MetaTokenizerUseCaseImpl @Inject constructor(
         }
         if (namespaceParseMode > 0 && buffer.nextIs("\"")) {
             // What about escaped \" symbols ?
-            val quoteOperator = staticDictionaries.getOperators().search("\"")!!
+            val quoteOperator = staticDictionaries.getOperators().search("\"") ?:
+                throw IllegalStateException("Can't find \" symbol in Meta dictionaries. ")
             result.add(quoteOperator)
             debugLine1.append('"')
             debugLine2.append("${quoteOperator.id}, ")

cgen-nnparser/src/test/kotlin/ce/parser/domain/TestDictionary.kt

@@ -52,6 +52,31 @@ object TestDictionary {
         )
     )
 
+    val metaDictionaries = StaticDictionariesImpl(
+        spaces = spaces,
+        comments = StaticDictionary(
+            listOf(
+                RegexWord(name = "addBlockComment\\(\"(.*)\"\\)", id = 4000, type = Type.COMMENTS),
+            ),
+            sortBySize = false
+        ),
+        keywords = keywordDict,
+        digits = StaticDictionary(
+            listOf(
+                RegexWord(name = "0x[\\dABCDEFabcdef]+", id = 3000, type = Type.DIGIT),
+                RegexWord(name = "\\d+\\.*\\d*f*", id = 3001, type = Type.DIGIT)
+            ),
+            sortBySize = false
+        ),
+        operators = operatorsDict,
+        strings = StaticDictionary(
+            listOf(
+                ClikeLiteralWord(name = "SimpleString", id = 4000, type = Type.STRING_LITERAL)
+            ),
+            sortBySize = false
+        )
+    )
+
     fun prepareDynamicDictionaries(): DynamicDictionaries =
         DynamicDictionariesImpl(
             digitsDictionary = NamesDictionary(

cgen-nnparser/src/test/kotlin/ce/parser/domain/usecase/MetaTokenizerUseCaseImplTest.kt

@@ -33,7 +33,7 @@ class MetaTokenizerUseCaseImplTest {
         val src = SourceBuffer("namespace(\"a.b.c\")")
         val result = env.tokenizer(buffer = src,
             dynamicDictionaries = env.dynamicDictionaries,
-            dictionaries = TestDictionary.dictionaries,
+            dictionaries = TestDictionary.metaDictionaries,
             debugFindings = true
             )
         assertEquals(10, result.words.size)
@@ -43,15 +43,37 @@ class MetaTokenizerUseCaseImplTest {
     fun metaAdd() {
         val env = prepareTokenizer()
         // expected
-        // <keyword namespace><operator (><operator "><literal varName>
+        // <keyword add><operator (><operator "><name varName>
         // <operator "><operator )>
         val src = SourceBuffer("add(\"varName\")")
         val result = env.tokenizer(buffer = src,
             dynamicDictionaries = env.dynamicDictionaries,
-            dictionaries = TestDictionary.dictionaries,
+            dictionaries = TestDictionary.metaDictionaries,
             debugFindings = true
         )
         assertEquals(6, result.words.size)
         assertEquals(Type.KEYWORD, result.words[0].type)
+        assertEquals(Type.OPERATOR, result.words[1].type)
+        assertEquals(Type.OPERATOR, result.words[2].type)
+        assertEquals(Type.NAME, result.words[3].type)
+        assertEquals(Type.OPERATOR, result.words[4].type)
+        assertEquals(Type.OPERATOR, result.words[5].type)
+
+    }
+
+    @Test
+    fun metaAddBlockComment() {
+        val env = prepareTokenizer()
+        // expected
+        // <keyword addBlockComment><operator (><operator "><comment comment>
+        // <operator "><operator )>
+        val src = SourceBuffer("addBlockComment(\"comment\")")
+        val result = env.tokenizer(buffer = src,
+            dynamicDictionaries = env.dynamicDictionaries,
+            dictionaries = TestDictionary.metaDictionaries,
+            debugFindings = true
+        )
+        assertEquals(1, result.words.size)
+        assertEquals(Type.COMMENTS, result.words[0].type)
     }
 }

test/parser/dictionaries/meta_comments.groovy

@@ -0,0 +1,4 @@
+import ce.parser.nnparser.RegexWord
+import ce.parser.nnparser.Type
+
+words.add(new RegexWord("addBlockComment\\(\"(.*)\"\\)", 3000, Type.COMMENTS))