'Clean up' a few aspects of the code

Author: chrisjsewell

markdown_it/normalize_url.py markdown_it/common/normalize_url.pymarkdown_it/normalize_url.py renamed to markdown_it/common/normalize_url.py

@@ -1,9 +1,14 @@
 import html
 import re
+from typing import Callable
 from urllib.parse import urlparse, urlunparse, quote, unquote  # noqa: F401
 
-# TODO
-# replaced: parsed = mdurl.parse(url, True)
+# TODO below we port the use of the JS packages:
+# var mdurl        = require('mdurl')
+# var punycode     = require('punycode')
+
+# e.g. mdurl: parsed = mdurl.parse(url, True)
+#
 # but need to check these fixes from https://www.npmjs.com/package/mdurl:
 #
 # Parse url string. Similar to node's url.parse,
@@ -56,7 +61,15 @@ def normalize_uri(uri):
 
 
 def normalizeLink(url):
+    """Normalize destination URLs in links::
+
+        [label]:   destination   'title'
+                ^^^^^^^^^^^
+    """
+    url_unescaped = unescape_string(url)
+    return normalize_uri(url_unescaped)
 
+    # markdown-it code:
     # parsed = urlparse(url)
 
     # if parsed.hostname:
@@ -71,13 +84,18 @@ def normalizeLink(url):
     #             parsed.hostname = punycode.toASCII(parsed.hostname)
     #         except Exception:
     #             pass
-    # quote(urlunparse(parsed))
-    return normalize_uri(unescape_string(url))
+    # return quote(urlunparse(parsed))
 
 
 def normalizeLinkText(title):
-    """Normalize autolinks """
+    """Normalize autolink content::
 
+        <destination>
+         ~~~~~~~~~~~
+    """
+    return unquote(unescape_string(title))
+
+    # markdown-it code:
     # parsed = urlparse(url)
 
     # if parsed.hostname:
@@ -92,24 +110,26 @@ def normalizeLinkText(title):
     #             parsed.hostname = punycode.toUnicode(parsed.hostname)
     #         except Exception:
     #             pass
-    return unquote(unescape_string(title))  # unquote(urlunparse(parsed))
+    # return unquote(urlunparse(parsed))
 
 
-################################################################################
-#
-# This validator can prohibit more than really needed to prevent XSS. It's a
-# tradeoff to keep code simple and to be secure by default.
-#
-# If you need different setup - override validator method as you wish. Or
-# replace it with dummy function and use external sanitizer.
-#
-
 BAD_PROTO_RE = re.compile(r"^(vbscript|javascript|file|data):")
 GOOD_DATA_RE = re.compile(r"^data:image\/(gif|png|jpeg|webp);")
 
 
-def validateLink(url: str):
-    """url should be normalized at this point, and existing entities are decoded."""
+def validateLink(url: str, validator: Callable = None):
+    """Validate URL link is allowed in output.
+
+    This validator can prohibit more than really needed to prevent XSS.
+    It's a tradeoff to keep code simple and to be secure by default.
+
+    If you need different setup - override validator method as you wish.
+    Or replace it with dummy function and use external sanitizer.
+
+    Note: url should be normalized at this point, and existing entities decoded.
+    """
+    if validator is not None:
+        return validator(url)
     url = url.strip().lower()
     return (
         (True if GOOD_DATA_RE.search(url) else False)

markdown_it/common/utils.py

@@ -1,10 +1,12 @@
-"""Utilities
+"""Utilities for parsing source text
 """
 import html
 import re
 
 from .entities import entities
 
+# from .normalize_url import unescape_string
+
 
 def charCodeAt(src: str, pos: int):
     """
@@ -117,7 +119,7 @@ def replaceEntityPattern(match, name):
     """
     ::
         In [2]: from markdown_it import MarkdownIt
-           ...: md = MarkdownIt("working")
+           ...: md = MarkdownIt()
            ...: md.render("![](https://www.google.com)")
         Out[2]: '<p><img src="https%3A//www.google.com" alt=""></p>\n'
     """
@@ -134,11 +136,10 @@ def replaceEntityPattern(match, name):
     return match
 
 
-# function replaceEntities(string) {
-#   if (string.indexOf('&') < 0) { return string; }
-
+# def replaceEntities(string):
+#   if (string.indexOf('&') < 0):
+#       return string
 #   return string.replace(ENTITY_RE, replaceEntityPattern)
-# }
 
 
 def unescapeMd(string: str):
@@ -148,23 +149,7 @@ def unescapeMd(string: str):
 
 
 def unescapeAll(string: str):
-    if "\\" in string and "&" in string:
-        return string
-
-    # TODO here we use the built-in python method
-    # check this is ok?
-    return html.escape(string).replace("&#x27;", "'")
-
-    def func(match):
-        # TODO how to get escaped?
-        escaped = False
-        entity = match.group()
-        if escaped:
-            return escaped
-        return replaceEntityPattern(match, entity)
-
-    string, _ = UNESCAPE_ALL_RE.subn(func, string)
-    return string
+    return html.unescape(string)
 
 
 # //////////////////////////////////////////////////////////////////////////////
@@ -341,30 +326,3 @@ def normalizeReference(string: str) -> str:
     # most notably, `__proto__`)
     #
     return string.lower().upper()
-
-
-#########################################################################
-
-# Re-export libraries commonly used in both markdown-it and its plugins,
-# so plugins won't have to depend on them explicitly, which reduces their
-# bundled size (e.g. a browser build).
-#
-# exports.lib                 = {}
-# exports.lib.mdurl           = require('mdurl')
-# exports.lib.ucmicro         = require('uc.micro')
-
-# exports.assign              = assign
-# exports.has                 = has
-# exports.unescapeMd          = unescapeMd
-# exports.unescapeAll         = unescapeAll
-# exports.isValidEntityCode   = isValidEntityCode
-# exports.fromCodePoint       = fromCodePoint
-# // exports.replaceEntities     = replaceEntities
-# exports.escapeHtml          = escapeHtml
-# exports.arrayReplaceAt      = arrayReplaceAt
-# exports.isSpace             = isSpace
-# exports.isWhiteSpace        = isWhiteSpace
-# exports.isMdAsciiPunct      = isMdAsciiPunct
-# exports.isPunctChar         = isPunctChar
-# exports.escapeRE            = escapeRE
-# exports.normalizeReference  = normalizeReference

markdown_it/helpers/__init__.py

@@ -1,5 +1,4 @@
-"""
-Just a shortcut for bulk export
+"""Functions for parsing Links
 """
 from .parse_link_label import parseLinkLabel  # noqa: F401
 from .parse_link_destination import parseLinkDestination  # noqa: F401

markdown_it/main.py

@@ -1,7 +1,6 @@
 from typing import Callable, List, Optional, Union
 
 from . import helpers, presets  # noqa F401
-from .normalize_url import normalizeLink, normalizeLinkText, validateLink
 from .common import utils  # noqa F401
 from .parser_core import ParserCore  # noqa F401
 from .parser_block import ParserBlock  # noqa F401
@@ -10,16 +9,12 @@
 from .renderer import RendererHTML
 from .utils import AttrDict
 
-# var LinkifyIt    = require('linkify-it')
-# var mdurl        = require('mdurl')
-# var punycode     = require('punycode')
 
 config = AttrDict(
     {
         "default": presets.default.presets,
         "zero": presets.zero.presets,
         "commonmark": presets.commonmark.presets,
-        "working": presets.working.presets,
     }
 )
 
@@ -46,11 +41,9 @@ def __init__(
         self.block = ParserBlock()
         self.core = ParserCore()
         self.renderer = RendererHTML() if renderer is None else renderer
+        # var LinkifyIt    = require('linkify-it')
         # self.linkify = LinkifyIt()  # TODO maybe see https://github.com/Suor/autolink
 
-        self.validateLink = validateLink
-        self.normalizeLink = normalizeLink
-        self.normalizeLinkText = normalizeLinkText
         self.utils = utils
         self.helpers = helpers
         self.options = {}

markdown_it/port.yaml

@@ -2,17 +2,34 @@
   commit: f798bea9623277bbf89b9621cf7fb283c693fcab
   date: Mar 12, 2020
   notes:
-    - Rename variables that use python built-in names
+    - Rename variables that use python built-in names, e.g.
       - `max` -> `maximum`
       - `len` -> `length`
-    - Convert JS for loops -> while loops
+      - `str` -> `string`
+    - |
+      Convert JS for loops -to while loops
+      this is generally the main difference between the codes,
+      because in python you can't do e.g. `for {i=1;i<x;i++} {}`
     - Use python version of `charCodeAt`
-    - allow custom renderer to be passed to `MarkdownIt`
     - |
-      render method signatures
+      Remove indirect references within `MarkdownIt`;
+
+        self.validateLink = validateLink
+        self.normalizeLink = normalizeLink
+        self.normalizeLinkText = normalizeLinkText
+
+      in favour of using them directly through:
+
+        from markdown_it.common.normalize_url import normalizeLinkText
+
+    - Allow custom renderer to be passed to `MarkdownIt`
+    - |
+      change render method signatures
       `func(tokens, idx, options, env, slf)` to
       `func(self, tokens, idx, options, env)`
     - |
-      Extension add render methods by format
+      Extensions add render methods by format
       `MarkdownIt.add_render_rule(name, function, fmt="html")`,
-      and renderers should declare a class property `__output__ = "html"`
+      rather than `MarkdownIt.renderer.rules[name] = function`
+      and renderers should declare a class property `__output__ = "html"`.
+      This allows for extensability to more than just HTML renderers

markdown_it/presets/__init__.py

@@ -1 +1 @@
-from . import commonmark, default, working, zero  # noqa: F401
+from . import commonmark, default, zero  # noqa: F401

markdown_it/presets/commonmark.py

@@ -3,27 +3,32 @@
 
 presets = {
     "options": {
-        "html": True,  # Enable HTML tags in source
-        "xhtmlOut": True,  # Use '/' to close single tags (<br />)
-        "breaks": False,  # Convert '\n' in paragraphs into <br>
-        "langPrefix": "language-",  # CSS language prefix for fenced blocks
+        "maxNesting": 20,  # Internal protection, recursion limit
+        "html": True,  # Enable HTML tags in source,
+        # this is just a shorthand for .enable(["html_inline", "html_block"])
+        # used by the linkify rule:
         "linkify": False,  # autoconvert URL-like texts to links
+        # used by the replacements and smartquotes rules
         # Enable some language-neutral replacements + quotes beautification
         "typographer": False,
+        # used by the smartquotes rule:
         # Double + single quotes replacement pairs, when typographer enabled,
         # and smartquotes on. Could be either a String or an Array.
         #
         # For example, you can use '«»„“' for Russian, '„“‚‘' for German,
         # and ['«\xA0', '\xA0»', '‹\xA0', '\xA0›'] for French (including nbsp).
         "quotes": "\u201c\u201d\u2018\u2019",  # /* “”‘’ */
+        # Renderer specific; these options are used directly in the HTML renderer
+        "xhtmlOut": True,  # Use '/' to close single tags (<br />)
+        "breaks": False,  # Convert '\n' in paragraphs into <br>
+        "langPrefix": "language-",  # CSS language prefix for fenced blocks
         # Highlighter function. Should return escaped HTML,
         # or '' if the source string is not changed and should be escaped externally.
         # If result starts with <pre... internal wrapper is skipped.
         #
         # function (/*str, lang*/) { return ''; }
         #
         "highlight": None,
-        "maxNesting": 20,  # Internal protection, recursion limit
     },
     "components": {
         "core": {"rules": ["normalize", "block", "inline"]},

markdown_it/presets/default.py

@@ -3,27 +3,31 @@
 
 presets = {
     "options": {
+        "maxNesting": 100,  # Internal protection, recursion limit
         "html": False,  # Enable HTML tags in source
-        "xhtmlOut": False,  # Use '/' to close single tags (<br />)
-        "breaks": False,  # Convert '\n' in paragraphs into <br>
-        "langPrefix": "language-",  # CSS language prefix for fenced blocks
+        # this is just a shorthand for .disable(["html_inline", "html_block"])
+        # used by the linkify rule:
         "linkify": False,  # autoconvert URL-like texts to links
+        # used by the replacements and smartquotes rules:
         # Enable some language-neutral replacements + quotes beautification
         "typographer": False,
+        # used by the smartquotes rule:
         # Double + single quotes replacement pairs, when typographer enabled,
         # and smartquotes on. Could be either a String or an Array.
-        #
         # For example, you can use '«»„“' for Russian, '„“‚‘' for German,
         # and ['«\xA0', '\xA0»', '‹\xA0', '\xA0›'] for French (including nbsp).
         "quotes": "\u201c\u201d\u2018\u2019",  # /* “”‘’ */
+        # Renderer specific; these options are used directly in the HTML renderer
+        "xhtmlOut": False,  # Use '/' to close single tags (<br />)
+        "breaks": False,  # Convert '\n' in paragraphs into <br>
+        "langPrefix": "language-",  # CSS language prefix for fenced blocks
         # Highlighter function. Should return escaped HTML,
         # or '' if the source string is not changed and should be escaped externaly.
         # If result starts with <pre... internal wrapper is skipped.
         #
         # function (/*str, lang*/) { return ''; }
         #
         "highlight": None,
-        "maxNesting": 100,  # Internal protection, recursion limit
     },
     "components": {"core": {}, "block": {}, "inline": {}},
 }