[pull] master from sqlmapproject:master

data/txt/sha256sums.txt

@@ -171,7 +171,7 @@ f6062e324fdeaacf9df0a289fc3f12f755143e3876a70cb65b38aa2e690f73c1  lib/core/commo
 39ea62d4224be860befeffb3843c150f2343b64555ad8c438a400222056f6cc0  lib/core/convert.py
 ae500647c4074681749735a4f3b17b7eca44868dd3f39f9cab0a575888ba04a1  lib/core/data.py
 ffae7cfe9f9afb92e887b9a8dbc1630d0063e865f35984ae417b04a4513e5024  lib/core/datatype.py
-322978f03cd69f7c98f2ea2cbe7567ab4f386b6c0548dcdf09064a6e9c393383  lib/core/decorators.py
+253309dc355ae27cd275e7de5a068e7e22feba603c4fe3429e2b69f8a51c0d13  lib/core/decorators.py
 d573a37bb00c8b65f75b275aa92549683180fb209b75fd0ff3870e3848939900  lib/core/defaults.py
 bb7e6521edad1cbfffa89fd7d5e255ed4ff148d984ffadbeac8d42baa2d76dea  lib/core/dicts.py
 20a6edda1d57a7564869e366f57ed7b2ab068dd8716cf7a10ef4a02d154d6c80  lib/core/dump.py
@@ -188,12 +188,12 @@ c4bfb493a03caf84dd362aec7c248097841de804b7413d0e1ecb8a90c8550bc0  lib/core/readl
 d1bd70c1a55858495c727fbec91e30af267459c8f64d50fabf9e4ee2c007e920  lib/core/replication.py
 1d0f80b0193ac5204527bfab4bde1a7aee0f693fd008e86b4b29f606d1ef94f3  lib/core/revision.py
 d2eb8e4b05ac93551272b3d4abfaf5b9f2d3ac92499a7704c16ed0b4f200db38  lib/core/session.py
-c42265c888448e115be0ea6ba6fdc86c86cbd00cdbc3a635c21b2a06949920d6  lib/core/settings.py
+eabd2afcdc5b6efc7824789e400acc387df4760587de98367149cf28f24c53c4  lib/core/settings.py
 1c5eab9494eb969bc9ce118a2ea6954690c6851cbe54c18373c723b99734bf09  lib/core/shell.py
 4eea6dcf023e41e3c64b210cb5c2efc7ca893b727f5e49d9c924f076bb224053  lib/core/subprocessng.py
 cdd352e1331c6b535e780f6edea79465cb55af53aa2114dcea0e8bf382e56d1a  lib/core/target.py
 6cf11d8b00fa761046686437fe90565e708809f793e88a3f02527d0e49c4d2a8  lib/core/testing.py
-2a179b7601026a8da092271b30ad353cdb6decd658e2614fa51983aaf6dd80e7  lib/core/threads.py
+2194ffd7891a9c6c012fb93e76222e33e85e49e6f1d351cd7664c5d306ebc675  lib/core/threads.py
 6f61e7946e368ee1450c301aaf5a26381a8ae31fc8bffa28afc9383e8b1fbc3f  lib/core/unescaper.py
 8919863be7a86f46d2c41bd30c0114a55a55c5931be48e3cfc66dfa96b7109c8  lib/core/update.py
 cba481f8c79f4a75bd147b9eb5a1e6e61d70422fceadd12494b1dbaa4f1d27f4  lib/core/wordlist.py
@@ -210,7 +210,7 @@ d7082e4a5937f65cbb4862701bad7d4fbc096a826621ba7eab92e52e48ebd6d7  lib/parse/site
 0f52f3c1d1f1322a91c98955bd8dc3be80964d8b3421d453a0e73a523c9cfcbf  lib/request/basicauthhandler.py
 48bdb0f5f05ece57e6e681801f7ed765739ebe537f9fa5a0465332d4f3f91c06  lib/request/basic.py
 fdb4a9f2ca9d01480c3eb115f6fdf8d89f8ff0506c56a223421b395481527670  lib/request/chunkedhandler.py
-c56a2c170507861403e0ddebd68a111bcf3a5f5fddc7334a9de4ecd572fdcc2f  lib/request/comparison.py
+d14df1d76f4ae239654bf360cb8a2410ed08d020cc50f68d06a4efdf1b1b20b9  lib/request/comparison.py
 cfa172dbc459a3250db7fbaadb62b282b62d56b4f290c585d3abec01597fcd40  lib/request/connect.py
 a890be5dee3fb4f5cb8b5f35984017a5c172d587722cf0c690bf50e338deebfa  lib/request/direct.py
 a53fa3513431330ce1725a90e7e3d20f223e14605d699e1f66b41625f04439c7  lib/request/dns.py
@@ -247,7 +247,7 @@ af67d25e8c16b429a5b471d3c629dc1da262262320bf7cd68465d151c02def16  lib/utils/brut
 56b93ba38f127929346f54aa75af0db5f46f9502b16acfe0d674a209de6cad2d  lib/utils/deps.py
 3aca7632d53ab2569ddef876a1b90f244640a53e19b304c77745f8ddb15e6437  lib/utils/getch.py
 4979120bbbc030eaef97147ee9d7d564d9683989059b59be317153cdaa23d85b  lib/utils/har.py
-00135cf61f1cfe79d7be14c526f84a841ad22e736db04e4fe087baeb4c22dc0d  lib/utils/hashdb.py
+af047a6efc1719a3d166fac0b7ff98ab3d29af7b676ff977e98c31c80e9e883e  lib/utils/hashdb.py
 8c9caffbd821ad9547c27095c8e55c398ea743b2e44d04b3572e2670389ccf5b  lib/utils/hash.py
 ba862f0c96b1d39797fb21974599e09690d312b17a85e6639bee9d1db510f543  lib/utils/httpd.py
 4608f21a4333c162ab3c266c903fda4793cc5834de30d06affe9b7566dd09811  lib/utils/__init__.py
@@ -477,7 +477,7 @@ cbc7684de872fac4baeabd1fce3938bc771316c36e54d69ac6a301e8a99f07b2  plugins/generi
 535ab6ac8b8441a3758cee86df3e68abec8b43eee54e32777967252057915acc  sqlmapapi.py
 168309215af7dd5b0b71070e1770e72f1cbb29a3d8025143fb8aa0b88cd56b62  sqlmapapi.yaml
 a40607ce164eb2d21865288d24b863edb1c734b56db857e130ac1aef961c80b9  sqlmap.conf
-1beb0711d15e38956759fbffa5331bde763c568a1baa8e32a04ebe5bc7a27e87  sqlmap.py
+c3a4c520df0a3396ed9e0f88fea0c9e0f420f779eff7e3d213603bd3f250f927  sqlmap.py
 82caac95182ac5cae02eb7d8a2dc07e71389aeae6b838d3d3f402c9597eb086a  tamper/0eunion.py
 bc8f5e638578919e4e75a5b01a84b47456bac0fd540e600975a52408a3433460  tamper/apostrophemask.py
 c9c3d71f11de0140906d7b4f24fadb9926dc8eaf5adab864f8106275f05526ce  tamper/apostrophenullencode.py

lib/core/decorators.py

@@ -42,28 +42,34 @@ def cachedmethod(f):
 
     @functools.wraps(f)
     def _f(*args, **kwargs):
-        parts = (
-            f.__module__ + "." + f.__name__,
-            "^".join(repr(a) for a in args),
-            "^".join("%s=%r" % (k, kwargs[k]) for k in sorted(kwargs))
-        )
         try:
-            key = struct.unpack(">Q", hashlib.md5("`".join(parts).encode(UNICODE_ENCODING)).digest()[:8])[0] & 0x7fffffffffffffff
-        except (struct.error, ValueError):  # https://github.com/sqlmapproject/sqlmap/issues/4281 (NOTE: non-standard Python behavior where hexdigest returns binary value)
-            result = f(*args, **kwargs)
-        else:
-            lock, cache = _method_locks[f], _cache[f]
-
-            with lock:
-                if key in cache:
-                    return cache[key]
+            # NOTE: fast-path
+            if kwargs:
+                key = hash((f, args, tuple(map(type, args)), frozenset(kwargs.items()))) & 0x7fffffffffffffff
+            else:
+                key = hash((f, args, tuple(map(type, args)))) & 0x7fffffffffffffff
+        except TypeError:
+            # NOTE: failback slow-path
+            parts = (
+                f.__module__ + "." + f.__name__,
+                "^".join(repr(a) for a in args),
+                "^".join("%s=%r" % (k, kwargs[k]) for k in sorted(kwargs))
+            )
+            try:
+                key = struct.unpack("<Q", hashlib.md5("`".join(parts).encode(UNICODE_ENCODING)).digest()[:8])[0] & 0x7fffffffffffffff
+            except (struct.error, ValueError):
+                return f(*args, **kwargs)
+
+        lock, cache = _method_locks[f], _cache[f]
 
-            result = f(*args, **kwargs)
+        with lock:
+            if key in cache:
+                return cache[key]
 
-            with lock:
-                cache[key] = result
+        result = f(*args, **kwargs)
 
-            return result
+        with lock:
+            cache[key] = result
 
         return result
 

lib/core/settings.py

@@ -19,7 +19,7 @@
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.9.12.32"
+VERSION = "1.9.12.37"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -191,7 +191,7 @@
 MAX_BUFFERED_PARTIAL_UNION_LENGTH = 1024
 
 # Maximum size of cache used in @cachedmethod decorator
-MAX_CACHE_ITEMS = 256
+MAX_CACHE_ITEMS = 1024
 
 # Suffix used for naming meta databases in DBMS(es) without explicit database name
 METADB_SUFFIX = "_masterdb"
@@ -703,8 +703,11 @@
 # Github OAuth token used for creating an automatic Issue for unhandled exceptions
 GITHUB_REPORT_OAUTH_TOKEN = "wxqc7vTeW8ohIcX+1wK55Mnql2Ex9cP+2s1dqTr/mjlZJVfLnq24fMAi08v5vRvOmuhVZQdOT/lhIRovWvIJrdECD1ud8VMPWpxY+NmjHoEx+VLK1/vCAUBwJe"
 
-# Skip unforced HashDB flush requests below the threshold number of cached items
-HASHDB_FLUSH_THRESHOLD = 32
+# Flush HashDB threshold number of cached items
+HASHDB_FLUSH_THRESHOLD_ITEMS = 200
+
+# Flush HashDB threshold "dirty" time
+HASHDB_FLUSH_THRESHOLD_TIME = 5
 
 # Number of retries for unsuccessful HashDB flush attempts
 HASHDB_FLUSH_RETRIES = 3
@@ -716,7 +719,7 @@
 HASHDB_END_TRANSACTION_RETRIES = 3
 
 # Unique milestone value used for forced deprecation of old HashDB values (e.g. when changing hash/pickle mechanism)
-HASHDB_MILESTONE_VALUE = "OdqjeUpBLc"  # python -c 'import random, string; print "".join(random.sample(string.ascii_letters, 10))'
+HASHDB_MILESTONE_VALUE = "GpqxbkWTfz"  # python -c 'import random, string; print "".join(random.sample(string.ascii_letters, 10))'
 
 # Pickle protocl used for storage of serialized data inside HashDB (https://docs.python.org/3/library/pickle.html#data-stream-format)
 PICKLE_PROTOCOL = 2

lib/core/threads.py

@@ -52,6 +52,8 @@ def reset(self):
         self.lastComparisonHeaders = None
         self.lastComparisonCode = None
         self.lastComparisonRatio = None
+        self.lastPageTemplateCleaned = None
+        self.lastPageTemplate = None
         self.lastErrorPage = tuple()
         self.lastHTTPError = None
         self.lastRedirectMsg = None
@@ -255,7 +257,7 @@ def _threadFunction():
                     pass
 
         if conf.get("hashDB"):
-            conf.hashDB.flush(True)
+            conf.hashDB.flush()
 
         if cleanupFunction:
             cleanupFunction()

lib/request/comparison.py

@@ -106,7 +106,11 @@ def _comparison(page, headers, code, getRatioValue, pageLength):
         # Dynamic content lines to be excluded before comparison
         if not kb.nullConnection:
             page = removeDynamicContent(page)
-            seqMatcher.set_seq1(removeDynamicContent(kb.pageTemplate))
+            if threadData.lastPageTemplate != kb.pageTemplate:
+                threadData.lastPageTemplateCleaned = removeDynamicContent(kb.pageTemplate)
+                threadData.lastPageTemplate = kb.pageTemplate
+
+            seqMatcher.set_seq1(threadData.lastPageTemplateCleaned)
 
         if not pageLength:
             pageLength = len(page)

lib/utils/hashdb.py

@@ -8,6 +8,7 @@
 import hashlib
 import os
 import sqlite3
+import struct
 import threading
 import time
 
@@ -22,10 +23,10 @@
 from lib.core.exception import SqlmapConnectionException
 from lib.core.settings import HASHDB_END_TRANSACTION_RETRIES
 from lib.core.settings import HASHDB_FLUSH_RETRIES
-from lib.core.settings import HASHDB_FLUSH_THRESHOLD
+from lib.core.settings import HASHDB_FLUSH_THRESHOLD_ITEMS
+from lib.core.settings import HASHDB_FLUSH_THRESHOLD_TIME
 from lib.core.settings import HASHDB_RETRIEVE_RETRIES
 from lib.core.threads import getCurrentThreadData
-from lib.core.threads import getCurrentThreadName
 from thirdparty import six
 
 class HashDB(object):
@@ -34,13 +35,14 @@ def __init__(self, filepath):
         self._write_cache = {}
         self._cache_lock = threading.Lock()
         self._connections = []
+        self._last_flush_time = time.time()
 
     def _get_cursor(self):
         threadData = getCurrentThreadData()
 
         if threadData.hashDBCursor is None:
             try:
-                connection = sqlite3.connect(self.filepath, timeout=3, isolation_level=None)
+                connection = sqlite3.connect(self.filepath, timeout=10, isolation_level=None, check_same_thread=False)
                 self._connections.append(connection)
                 threadData.hashDBCursor = connection.cursor()
                 threadData.hashDBCursor.execute("CREATE TABLE IF NOT EXISTS storage (id INTEGER PRIMARY KEY, value TEXT)")
@@ -80,13 +82,13 @@ def closeAll(self):
     @staticmethod
     def hashKey(key):
         key = getBytes(key if isinstance(key, six.text_type) else repr(key), errors="xmlcharrefreplace")
-        retVal = int(hashlib.md5(key).hexdigest(), 16) & 0x7fffffffffffffff  # Reference: http://stackoverflow.com/a/4448400
+        retVal = struct.unpack("<Q", hashlib.md5(key).digest()[:8])[0] & 0x7fffffffffffffff
         return retVal
 
     def retrieve(self, key, unserialize=False):
         retVal = None
 
-        if key and (self._write_cache or os.path.isfile(self.filepath)):
+        if key and (self._write_cache or self._connections or os.path.isfile(self.filepath)):
             hash_ = HashDB.hashKey(key)
             retVal = self._write_cache.get(hash_)
             if not retVal:
@@ -123,28 +125,26 @@ def retrieve(self, key, unserialize=False):
     def write(self, key, value, serialize=False):
         if key:
             hash_ = HashDB.hashKey(key)
-            self._cache_lock.acquire()
-            self._write_cache[hash_] = getUnicode(value) if not serialize else serializeObject(value)
-            self._cache_lock.release()
+            with self._cache_lock:
+                self._write_cache[hash_] = getUnicode(value) if not serialize else serializeObject(value)
+                cache_size = len(self._write_cache)
+                time_since_flush = time.time() - self._last_flush_time
 
-        if getCurrentThreadName() in ('0', "MainThread"):
-            self.flush()
+            if cache_size >= HASHDB_FLUSH_THRESHOLD_ITEMS or time_since_flush >= HASHDB_FLUSH_THRESHOLD_TIME:
+                self.flush()
 
-    def flush(self, forced=False):
-        if not self._write_cache:
-            return
+    def flush(self):
+        with self._cache_lock:
+            if not self._write_cache:
+                return
 
-        if not forced and len(self._write_cache) < HASHDB_FLUSH_THRESHOLD:
-            return
-
-        self._cache_lock.acquire()
-        _ = self._write_cache
-        self._write_cache = {}
-        self._cache_lock.release()
+            flush_cache = self._write_cache
+            self._write_cache = {}
+            self._last_flush_time = time.time()
 
         try:
             self.beginTransaction()
-            for hash_, value in _.items():
+            for hash_, value in flush_cache.items():
                 retries = 0
                 while True:
                     try:
@@ -160,7 +160,8 @@ def flush(self, forced=False):
                             logger.debug(debugMsg)
                             break
 
-                        if retries == 0:
+                        # NOTE: skipping the retries == 0 for graceful resolution of multi-threaded runs
+                        if retries == 1:
                             warnMsg = "there has been a problem while writing to "
                             warnMsg += "the session file ('%s')" % getSafeExString(ex)
                             logger.warning(warnMsg)

sqlmap.py

@@ -588,7 +588,7 @@ def main():
                     pass
 
         if conf.get("hashDB"):
-            conf.hashDB.flush(True)
+            conf.hashDB.flush()
             conf.hashDB.close()         # NOTE: because of PyPy
 
         if conf.get("harFile"):