diff --git a/data/txt/sha256sums.txt b/data/txt/sha256sums.txt index 2bec0c3857..62886ac578 100644 --- a/data/txt/sha256sums.txt +++ b/data/txt/sha256sums.txt @@ -171,7 +171,7 @@ f6062e324fdeaacf9df0a289fc3f12f755143e3876a70cb65b38aa2e690f73c1 lib/core/commo 39ea62d4224be860befeffb3843c150f2343b64555ad8c438a400222056f6cc0 lib/core/convert.py ae500647c4074681749735a4f3b17b7eca44868dd3f39f9cab0a575888ba04a1 lib/core/data.py ffae7cfe9f9afb92e887b9a8dbc1630d0063e865f35984ae417b04a4513e5024 lib/core/datatype.py -322978f03cd69f7c98f2ea2cbe7567ab4f386b6c0548dcdf09064a6e9c393383 lib/core/decorators.py +253309dc355ae27cd275e7de5a068e7e22feba603c4fe3429e2b69f8a51c0d13 lib/core/decorators.py d573a37bb00c8b65f75b275aa92549683180fb209b75fd0ff3870e3848939900 lib/core/defaults.py bb7e6521edad1cbfffa89fd7d5e255ed4ff148d984ffadbeac8d42baa2d76dea lib/core/dicts.py 20a6edda1d57a7564869e366f57ed7b2ab068dd8716cf7a10ef4a02d154d6c80 lib/core/dump.py @@ -188,12 +188,12 @@ c4bfb493a03caf84dd362aec7c248097841de804b7413d0e1ecb8a90c8550bc0 lib/core/readl d1bd70c1a55858495c727fbec91e30af267459c8f64d50fabf9e4ee2c007e920 lib/core/replication.py 1d0f80b0193ac5204527bfab4bde1a7aee0f693fd008e86b4b29f606d1ef94f3 lib/core/revision.py d2eb8e4b05ac93551272b3d4abfaf5b9f2d3ac92499a7704c16ed0b4f200db38 lib/core/session.py -c42265c888448e115be0ea6ba6fdc86c86cbd00cdbc3a635c21b2a06949920d6 lib/core/settings.py +eabd2afcdc5b6efc7824789e400acc387df4760587de98367149cf28f24c53c4 lib/core/settings.py 1c5eab9494eb969bc9ce118a2ea6954690c6851cbe54c18373c723b99734bf09 lib/core/shell.py 4eea6dcf023e41e3c64b210cb5c2efc7ca893b727f5e49d9c924f076bb224053 lib/core/subprocessng.py cdd352e1331c6b535e780f6edea79465cb55af53aa2114dcea0e8bf382e56d1a lib/core/target.py 6cf11d8b00fa761046686437fe90565e708809f793e88a3f02527d0e49c4d2a8 lib/core/testing.py -2a179b7601026a8da092271b30ad353cdb6decd658e2614fa51983aaf6dd80e7 lib/core/threads.py +2194ffd7891a9c6c012fb93e76222e33e85e49e6f1d351cd7664c5d306ebc675 lib/core/threads.py 6f61e7946e368ee1450c301aaf5a26381a8ae31fc8bffa28afc9383e8b1fbc3f lib/core/unescaper.py 8919863be7a86f46d2c41bd30c0114a55a55c5931be48e3cfc66dfa96b7109c8 lib/core/update.py cba481f8c79f4a75bd147b9eb5a1e6e61d70422fceadd12494b1dbaa4f1d27f4 lib/core/wordlist.py @@ -210,7 +210,7 @@ d7082e4a5937f65cbb4862701bad7d4fbc096a826621ba7eab92e52e48ebd6d7 lib/parse/site 0f52f3c1d1f1322a91c98955bd8dc3be80964d8b3421d453a0e73a523c9cfcbf lib/request/basicauthhandler.py 48bdb0f5f05ece57e6e681801f7ed765739ebe537f9fa5a0465332d4f3f91c06 lib/request/basic.py fdb4a9f2ca9d01480c3eb115f6fdf8d89f8ff0506c56a223421b395481527670 lib/request/chunkedhandler.py -c56a2c170507861403e0ddebd68a111bcf3a5f5fddc7334a9de4ecd572fdcc2f lib/request/comparison.py +d14df1d76f4ae239654bf360cb8a2410ed08d020cc50f68d06a4efdf1b1b20b9 lib/request/comparison.py cfa172dbc459a3250db7fbaadb62b282b62d56b4f290c585d3abec01597fcd40 lib/request/connect.py a890be5dee3fb4f5cb8b5f35984017a5c172d587722cf0c690bf50e338deebfa lib/request/direct.py a53fa3513431330ce1725a90e7e3d20f223e14605d699e1f66b41625f04439c7 lib/request/dns.py @@ -247,7 +247,7 @@ af67d25e8c16b429a5b471d3c629dc1da262262320bf7cd68465d151c02def16 lib/utils/brut 56b93ba38f127929346f54aa75af0db5f46f9502b16acfe0d674a209de6cad2d lib/utils/deps.py 3aca7632d53ab2569ddef876a1b90f244640a53e19b304c77745f8ddb15e6437 lib/utils/getch.py 4979120bbbc030eaef97147ee9d7d564d9683989059b59be317153cdaa23d85b lib/utils/har.py -00135cf61f1cfe79d7be14c526f84a841ad22e736db04e4fe087baeb4c22dc0d lib/utils/hashdb.py +af047a6efc1719a3d166fac0b7ff98ab3d29af7b676ff977e98c31c80e9e883e lib/utils/hashdb.py 8c9caffbd821ad9547c27095c8e55c398ea743b2e44d04b3572e2670389ccf5b lib/utils/hash.py ba862f0c96b1d39797fb21974599e09690d312b17a85e6639bee9d1db510f543 lib/utils/httpd.py 4608f21a4333c162ab3c266c903fda4793cc5834de30d06affe9b7566dd09811 lib/utils/__init__.py @@ -477,7 +477,7 @@ cbc7684de872fac4baeabd1fce3938bc771316c36e54d69ac6a301e8a99f07b2 plugins/generi 535ab6ac8b8441a3758cee86df3e68abec8b43eee54e32777967252057915acc sqlmapapi.py 168309215af7dd5b0b71070e1770e72f1cbb29a3d8025143fb8aa0b88cd56b62 sqlmapapi.yaml a40607ce164eb2d21865288d24b863edb1c734b56db857e130ac1aef961c80b9 sqlmap.conf -1beb0711d15e38956759fbffa5331bde763c568a1baa8e32a04ebe5bc7a27e87 sqlmap.py +c3a4c520df0a3396ed9e0f88fea0c9e0f420f779eff7e3d213603bd3f250f927 sqlmap.py 82caac95182ac5cae02eb7d8a2dc07e71389aeae6b838d3d3f402c9597eb086a tamper/0eunion.py bc8f5e638578919e4e75a5b01a84b47456bac0fd540e600975a52408a3433460 tamper/apostrophemask.py c9c3d71f11de0140906d7b4f24fadb9926dc8eaf5adab864f8106275f05526ce tamper/apostrophenullencode.py diff --git a/lib/core/decorators.py b/lib/core/decorators.py index 201abac75b..930bdc418a 100644 --- a/lib/core/decorators.py +++ b/lib/core/decorators.py @@ -42,28 +42,34 @@ def cachedmethod(f): @functools.wraps(f) def _f(*args, **kwargs): - parts = ( - f.__module__ + "." + f.__name__, - "^".join(repr(a) for a in args), - "^".join("%s=%r" % (k, kwargs[k]) for k in sorted(kwargs)) - ) try: - key = struct.unpack(">Q", hashlib.md5("`".join(parts).encode(UNICODE_ENCODING)).digest()[:8])[0] & 0x7fffffffffffffff - except (struct.error, ValueError): # https://github.com/sqlmapproject/sqlmap/issues/4281 (NOTE: non-standard Python behavior where hexdigest returns binary value) - result = f(*args, **kwargs) - else: - lock, cache = _method_locks[f], _cache[f] - - with lock: - if key in cache: - return cache[key] + # NOTE: fast-path + if kwargs: + key = hash((f, args, tuple(map(type, args)), frozenset(kwargs.items()))) & 0x7fffffffffffffff + else: + key = hash((f, args, tuple(map(type, args)))) & 0x7fffffffffffffff + except TypeError: + # NOTE: failback slow-path + parts = ( + f.__module__ + "." + f.__name__, + "^".join(repr(a) for a in args), + "^".join("%s=%r" % (k, kwargs[k]) for k in sorted(kwargs)) + ) + try: + key = struct.unpack("...) -VERSION = "1.9.12.32" +VERSION = "1.9.12.37" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) @@ -191,7 +191,7 @@ MAX_BUFFERED_PARTIAL_UNION_LENGTH = 1024 # Maximum size of cache used in @cachedmethod decorator -MAX_CACHE_ITEMS = 256 +MAX_CACHE_ITEMS = 1024 # Suffix used for naming meta databases in DBMS(es) without explicit database name METADB_SUFFIX = "_masterdb" @@ -703,8 +703,11 @@ # Github OAuth token used for creating an automatic Issue for unhandled exceptions GITHUB_REPORT_OAUTH_TOKEN = "wxqc7vTeW8ohIcX+1wK55Mnql2Ex9cP+2s1dqTr/mjlZJVfLnq24fMAi08v5vRvOmuhVZQdOT/lhIRovWvIJrdECD1ud8VMPWpxY+NmjHoEx+VLK1/vCAUBwJe" -# Skip unforced HashDB flush requests below the threshold number of cached items -HASHDB_FLUSH_THRESHOLD = 32 +# Flush HashDB threshold number of cached items +HASHDB_FLUSH_THRESHOLD_ITEMS = 200 + +# Flush HashDB threshold "dirty" time +HASHDB_FLUSH_THRESHOLD_TIME = 5 # Number of retries for unsuccessful HashDB flush attempts HASHDB_FLUSH_RETRIES = 3 @@ -716,7 +719,7 @@ HASHDB_END_TRANSACTION_RETRIES = 3 # Unique milestone value used for forced deprecation of old HashDB values (e.g. when changing hash/pickle mechanism) -HASHDB_MILESTONE_VALUE = "OdqjeUpBLc" # python -c 'import random, string; print "".join(random.sample(string.ascii_letters, 10))' +HASHDB_MILESTONE_VALUE = "GpqxbkWTfz" # python -c 'import random, string; print "".join(random.sample(string.ascii_letters, 10))' # Pickle protocl used for storage of serialized data inside HashDB (https://docs.python.org/3/library/pickle.html#data-stream-format) PICKLE_PROTOCOL = 2 diff --git a/lib/core/threads.py b/lib/core/threads.py index 57411b03a7..df1e91610a 100644 --- a/lib/core/threads.py +++ b/lib/core/threads.py @@ -52,6 +52,8 @@ def reset(self): self.lastComparisonHeaders = None self.lastComparisonCode = None self.lastComparisonRatio = None + self.lastPageTemplateCleaned = None + self.lastPageTemplate = None self.lastErrorPage = tuple() self.lastHTTPError = None self.lastRedirectMsg = None @@ -255,7 +257,7 @@ def _threadFunction(): pass if conf.get("hashDB"): - conf.hashDB.flush(True) + conf.hashDB.flush() if cleanupFunction: cleanupFunction() diff --git a/lib/request/comparison.py b/lib/request/comparison.py index 1730f2ccaf..55cb36cb36 100644 --- a/lib/request/comparison.py +++ b/lib/request/comparison.py @@ -106,7 +106,11 @@ def _comparison(page, headers, code, getRatioValue, pageLength): # Dynamic content lines to be excluded before comparison if not kb.nullConnection: page = removeDynamicContent(page) - seqMatcher.set_seq1(removeDynamicContent(kb.pageTemplate)) + if threadData.lastPageTemplate != kb.pageTemplate: + threadData.lastPageTemplateCleaned = removeDynamicContent(kb.pageTemplate) + threadData.lastPageTemplate = kb.pageTemplate + + seqMatcher.set_seq1(threadData.lastPageTemplateCleaned) if not pageLength: pageLength = len(page) diff --git a/lib/utils/hashdb.py b/lib/utils/hashdb.py index 3748905879..51753e4083 100644 --- a/lib/utils/hashdb.py +++ b/lib/utils/hashdb.py @@ -8,6 +8,7 @@ import hashlib import os import sqlite3 +import struct import threading import time @@ -22,10 +23,10 @@ from lib.core.exception import SqlmapConnectionException from lib.core.settings import HASHDB_END_TRANSACTION_RETRIES from lib.core.settings import HASHDB_FLUSH_RETRIES -from lib.core.settings import HASHDB_FLUSH_THRESHOLD +from lib.core.settings import HASHDB_FLUSH_THRESHOLD_ITEMS +from lib.core.settings import HASHDB_FLUSH_THRESHOLD_TIME from lib.core.settings import HASHDB_RETRIEVE_RETRIES from lib.core.threads import getCurrentThreadData -from lib.core.threads import getCurrentThreadName from thirdparty import six class HashDB(object): @@ -34,13 +35,14 @@ def __init__(self, filepath): self._write_cache = {} self._cache_lock = threading.Lock() self._connections = [] + self._last_flush_time = time.time() def _get_cursor(self): threadData = getCurrentThreadData() if threadData.hashDBCursor is None: try: - connection = sqlite3.connect(self.filepath, timeout=3, isolation_level=None) + connection = sqlite3.connect(self.filepath, timeout=10, isolation_level=None, check_same_thread=False) self._connections.append(connection) threadData.hashDBCursor = connection.cursor() threadData.hashDBCursor.execute("CREATE TABLE IF NOT EXISTS storage (id INTEGER PRIMARY KEY, value TEXT)") @@ -80,13 +82,13 @@ def closeAll(self): @staticmethod def hashKey(key): key = getBytes(key if isinstance(key, six.text_type) else repr(key), errors="xmlcharrefreplace") - retVal = int(hashlib.md5(key).hexdigest(), 16) & 0x7fffffffffffffff # Reference: http://stackoverflow.com/a/4448400 + retVal = struct.unpack("= HASHDB_FLUSH_THRESHOLD_ITEMS or time_since_flush >= HASHDB_FLUSH_THRESHOLD_TIME: + self.flush() - def flush(self, forced=False): - if not self._write_cache: - return + def flush(self): + with self._cache_lock: + if not self._write_cache: + return - if not forced and len(self._write_cache) < HASHDB_FLUSH_THRESHOLD: - return - - self._cache_lock.acquire() - _ = self._write_cache - self._write_cache = {} - self._cache_lock.release() + flush_cache = self._write_cache + self._write_cache = {} + self._last_flush_time = time.time() try: self.beginTransaction() - for hash_, value in _.items(): + for hash_, value in flush_cache.items(): retries = 0 while True: try: @@ -160,7 +160,8 @@ def flush(self, forced=False): logger.debug(debugMsg) break - if retries == 0: + # NOTE: skipping the retries == 0 for graceful resolution of multi-threaded runs + if retries == 1: warnMsg = "there has been a problem while writing to " warnMsg += "the session file ('%s')" % getSafeExString(ex) logger.warning(warnMsg) diff --git a/sqlmap.py b/sqlmap.py index 9698d5db3b..14f2eb9892 100755 --- a/sqlmap.py +++ b/sqlmap.py @@ -588,7 +588,7 @@ def main(): pass if conf.get("hashDB"): - conf.hashDB.flush(True) + conf.hashDB.flush() conf.hashDB.close() # NOTE: because of PyPy if conf.get("harFile"):