Merge pull request #89 from scrapinghub/parse-cleanup

cleanup: extract repeated html loading code to functions

Author: kmike

extruct/_extruct.py

@@ -1,16 +1,13 @@
 import logging
 import warnings
 
-from lxml.html import fromstring
-
 from extruct.jsonld import JsonLdExtractor
 from extruct.rdfa import RDFaExtractor
 from extruct.w3cmicrodata import MicrodataExtractor
 from extruct.opengraph import OpenGraphExtractor
 from extruct.microformat import MicroformatExtractor
-from extruct.xmldom import XmlDomHTMLParser
 from extruct.uniform import _umicrodata_microformat, _uopengraph
-
+from extruct.utils import parse_xmldom_html
 
 logger = logging.getLogger(__name__)
 SYNTAXES = ['microdata', 'opengraph', 'json-ld', 'microformat', 'rdfa']
@@ -52,8 +49,7 @@ def extract(htmlstring, base_url=None, encoding="UTF-8",
     if errors not in ['log', 'ignore', 'strict']:
         raise ValueError('Invalid error command, valid values are either "log"'
                          ', "ignore" or "strict"')
-    domparser = XmlDomHTMLParser(encoding=encoding)
-    tree = fromstring(htmlstring, parser=domparser)
+    tree = parse_xmldom_html(htmlstring, encoding=encoding)
     processors = []
     if 'microdata' in syntaxes:
         processors.append(('microdata', MicrodataExtractor(add_html_node=return_html_node).extract_items, tree))

extruct/jsonld.py

@@ -7,8 +7,8 @@
 import re
 
 import lxml.etree
-import lxml.html
 
+from extruct.utils import parse_html
 
 HTML_OR_JS_COMMENTLINE = re.compile('^\s*(//.*|<!--.*-->)')
 
@@ -17,9 +17,8 @@ class JsonLdExtractor(object):
     _xp_jsonld = lxml.etree.XPath('descendant-or-self::script[@type="application/ld+json"]')
 
     def extract(self, htmlstring, base_url=None, encoding="UTF-8"):
-        parser = lxml.html.HTMLParser(encoding=encoding)
-        lxmldoc = lxml.html.fromstring(htmlstring, parser=parser)
-        return self.extract_items(lxmldoc, base_url=base_url)
+        tree = parse_html(htmlstring, encoding=encoding)
+        return self.extract_items(tree, base_url=base_url)
 
     def extract_items(self, document, base_url=None):
         return [item for items in map(self._extract_items,

extruct/opengraph.py

@@ -1,5 +1,6 @@
 import re
-import lxml.html
+
+from extruct.utils import parse_html
 
 
 _PREFIX_PATTERN = re.compile(r'\s*(\w+):\s*([^\s]+)')
@@ -17,9 +18,8 @@ class OpenGraphExtractor(object):
     """OpenGraph extractor following extruct API."""
 
     def extract(self, htmlstring, base_url=None, encoding='UTF-8'):
-        parser = lxml.html.HTMLParser(encoding=encoding)
-        doc = lxml.html.fromstring(htmlstring, parser=parser)
-        return list(self.extract_items(doc, base_url=base_url))
+        tree = parse_html(htmlstring, encoding=encoding)
+        return list(self.extract_items(tree, base_url=base_url))
 
     def extract_items(self, document, base_url=None):
         # OpenGraph defines a web page as a single rich object.

extruct/rdfa.py

@@ -6,15 +6,15 @@
 """
 import json
 import logging
+
 rdflib_logger = logging.getLogger('rdflib')
 rdflib_logger.setLevel(logging.ERROR)
 
-from lxml.html import fromstring
 from rdflib import Graph, logger as rdflib_logger
 from rdflib.plugins.parsers.pyRdfa import pyRdfa as PyRdfa, Options, logger as pyrdfa_logger
 from rdflib.plugins.parsers.pyRdfa.initialcontext import initial_context
 
-from extruct.xmldom import XmlDomHTMLParser
+from extruct.utils import parse_xmldom_html
 
 
 # silence rdflib/PyRdfa INFO logs
@@ -31,9 +31,7 @@ class RDFaExtractor(object):
 
     def extract(self, htmlstring, base_url=None, encoding="UTF-8",
                 expanded=True):
-
-        domparser = XmlDomHTMLParser(encoding=encoding)
-        tree = fromstring(htmlstring, parser=domparser)
+        tree = parse_xmldom_html(htmlstring, encoding=encoding)
         return self.extract_items(tree, base_url=base_url, expanded=expanded)
 
     def extract_items(self, document, base_url=None, expanded=True):

extruct/tool.py

@@ -1,10 +1,12 @@
 import argparse
 import json
 import requests
+
 import extruct
 from extruct import SYNTAXES
 
-def metadata_from_url(url, syntaxes=SYNTAXES, uniform=False, 
+
+def metadata_from_url(url, syntaxes=SYNTAXES, uniform=False,
                       schema_context='http://schema.org', errors='strict'):
     resp = requests.get(url, timeout=30)
     result = {'url': url, 'status': '{} {}'.format(resp.status_code, resp.reason)}

extruct/utils.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+import lxml.html
+
+from extruct.xmldom import XmlDomHTMLParser
+
+
+def parse_html(html, encoding):
+    """ Parse HTML using lxml.html.HTMLParser, return a tree """
+    parser = lxml.html.HTMLParser(encoding=encoding)
+    return lxml.html.fromstring(html, parser=parser)
+
+
+def parse_xmldom_html(html, encoding):
+    """ Parse HTML using XmlDomHTMLParser, return a tree """
+    parser = XmlDomHTMLParser(encoding=encoding)
+    return lxml.html.fromstring(html, parser=parser)

extruct/w3cmicrodata.py

@@ -16,9 +16,10 @@
     from urllib.parse import urljoin
 
 import lxml.etree
-import lxml.html
 from w3lib.html import strip_html5_whitespace
 
+from extruct.utils import parse_html
+
 
 class LxmlMicrodataExtractor(object):
     _xp_item = lxml.etree.XPath('descendant-or-self::*[@itemscope]')
@@ -42,9 +43,8 @@ def get_docid(self, node):
         return int(self._xp_item_docid(node))
 
     def extract(self, htmlstring, base_url=None, encoding="UTF-8"):
-        parser = lxml.html.HTMLParser(encoding=encoding)
-        lxmldoc = lxml.html.fromstring(htmlstring, parser=parser)
-        return self.extract_items(lxmldoc, base_url)
+        tree = parse_html(htmlstring, encoding=encoding)
+        return self.extract_items(tree, base_url)
 
     def extract_items(self, document, base_url):
         items_seen = set()

extruct/xmldom.py

@@ -3,7 +3,8 @@
 from xml.dom import Node
 from xml.dom.minidom import Attr, NamedNodeMap
 
-from lxml.etree import ElementBase, _ElementStringResult, _ElementUnicodeResult, XPath, tostring
+from lxml.etree import (ElementBase, _ElementStringResult,
+                        _ElementUnicodeResult, XPath, tostring)
 from lxml.html import HTMLParser, HtmlElementClassLookup