Detect incorrectly delimited strings

Strings should be delimited on both ends by double quotes, but this
is currently not being been detected, and content is simply being
incorrectly trimmed.

This commit adds a check for each string to verify it starts and ends
with a double quote character, issuing a warning/error if that's not the
case (and fixing it as appropriate).

A few new test cases have been added to check that the lack of double
quotes to delimit strings issues errors as expected.

Signed-off-by: Rodrigo Tobar <rtobar@icrar.org>

Author: rtobar

babel/messages/pofile.py

@@ -278,6 +278,18 @@ def _process_keyword_line(self, lineno, line, obsolete=False) -> None:
 
         self.obsolete = obsolete
 
+        def _normalized_string(s):
+            # whole lines are already stripped on both ends
+            s = s.lstrip()
+            assert s, "string to normalize shouldn't be empty"
+            if s[0] != '"':
+                self._invalid_pofile(line, lineno, "String must be delimited on the left by double quotes")
+                s = '"' + s
+            if s[-1] != '"':
+                self._invalid_pofile(line, lineno, "String must be delimited on the right by double quotes")
+                s += '"'
+            return _NormalizedString(s)
+
         # The line that has the msgid is stored as the offset of the msg
         # should this be the msgctxt if it has one?
         if keyword == 'msgid':
@@ -286,7 +298,7 @@ def _process_keyword_line(self, lineno, line, obsolete=False) -> None:
         if keyword in ['msgid', 'msgid_plural']:
             self.in_msgctxt = False
             self.in_msgid = True
-            self.messages.append(_NormalizedString(arg))
+            self.messages.append(_normalized_string(arg))
 
         elif keyword == 'msgstr':
             self.in_msgid = False
@@ -306,13 +318,13 @@ def _process_keyword_line(self, lineno, line, obsolete=False) -> None:
                 if len(msg) < 2:
                     self._invalid_pofile(line, lineno, "msgstr plural doesn't have a message")
                     msg = '""'
-                self.translations.append([idx, _NormalizedString(msg)])
+                self.translations.append([idx, _normalized_string(msg)])
             else:
-                self.translations.append([0, _NormalizedString(arg)])
+                self.translations.append([0, _normalized_string(arg)])
 
         elif keyword == 'msgctxt':
             self.in_msgctxt = True
-            self.context = _NormalizedString(arg)
+            self.context = _normalized_string(arg)
 
     def _process_string_continuation_line(self, line, lineno) -> None:
         if self.in_msgid:
@@ -324,6 +336,10 @@ def _process_string_continuation_line(self, line, lineno) -> None:
         else:
             self._invalid_pofile(line, lineno, "Got line starting with \" but not in msgid, msgstr or msgctxt")
             return
+        assert line[0] == '"'
+        if line[-1] != '"':
+            self._invalid_pofile(line, lineno, "Continuation string must end with double quotes")
+            line += '"'
         s.append(line)
 
     def _process_comment(self, line) -> None:

tests/messages/test_pofile.py

@@ -505,6 +505,7 @@ def test_with_location(self):
 
 
     def test_abort_invalid_po_file(self):
+        # right double quote missing on msgstr, continuation missing both double quotes
         invalid_po = '''
             msgctxt ""
             "{\"checksum\": 2148532640, \"cxt\": \"collector_thankyou\", \"id\": "
@@ -517,6 +518,7 @@ def test_abort_invalid_po_file(self):
             Pour toute question, veuillez communiquer avec Fulano  à nadie@blah.com
             "
         '''
+        # as above, but contains ascii-only characters
         invalid_po_2 = '''
             msgctxt ""
             "{\"checksum\": 2148532640, \"cxt\": \"collector_thankyou\", \"id\": "
@@ -529,20 +531,29 @@ def test_abort_invalid_po_file(self):
             Pour toute question, veuillez communiquer avec Fulano a fulano@blah.com
             "
             '''
-        # Catalog not created, throws Unicode Error
-        buf = StringIO(invalid_po)
-        output = pofile.read_po(buf, locale='fr', abort_invalid=False)
-        assert isinstance(output, Catalog)
-
-        # Catalog not created, throws PoFileError
-        buf = StringIO(invalid_po_2)
-        with pytest.raises(pofile.PoFileError):
-            pofile.read_po(buf, locale='fr', abort_invalid=True)
-
-        # Catalog is created with warning, no abort
-        buf = StringIO(invalid_po_2)
-        output = pofile.read_po(buf, locale='fr', abort_invalid=False)
-        assert isinstance(output, Catalog)
+        # left double quote missing in msgid
+        invalid_po_3 = '''
+            msgid *A"
+            msgstr "B"
+        '''
+        # right double quote missing in msgstr
+        invalid_po_4 = '''
+            msgid "A"
+            msgstr "B*
+        '''
+        # right double quote missing in msgstr continuation
+        invalid_po_5 = '''
+            msgid "A"
+            msgstr ""
+            "*
+        '''
+        for incorrectly_delimited_po_string in (invalid_po, invalid_po_2, invalid_po_3, invalid_po_4, invalid_po_5):
+            buf = StringIO(incorrectly_delimited_po_string)
+            with pytest.raises(pofile.PoFileError):
+                pofile.read_po(buf, locale='fr', abort_invalid=True)
+            buf.seek(0)
+            output = pofile.read_po(buf, locale='fr', abort_invalid=False)
+            assert isinstance(output, Catalog)
 
     def test_invalid_msgstr_plural(self):
         # msgstr plural broken
@@ -563,7 +574,12 @@ def test_invalid_msgstr_plural(self):
         msgid "A"
         msgstr[0]
         '''
-        for incorrectly_plural in (invalid_1, invalid_2, invalid_3, invalid_4):
+        # not correctly delimited
+        invalid_5 = '''
+        msgid "A"
+        msgstr[0] not delimited
+        '''
+        for incorrectly_plural in (invalid_1, invalid_2, invalid_3, invalid_4, invalid_5):
             buf = StringIO(incorrectly_plural)
             with pytest.raises(pofile.PoFileError):
                 pofile.read_po(buf, locale='fr', abort_invalid=True)