Add --prove, opt-in --auto-tamper WAF bypass, and blindbinary/infoschema2innodb tampers

Author: stamparm

data/txt/sha256sums.txt

@@ -160,10 +160,10 @@ ca86d61d3349ed2d94a6b164d4648cff9701199b5e32378c3f40fca0f517b128  extra/shutils/
 df768bcb9838dc6c46dab9b4a877056cb4742bd6cfaaf438c4a3712c5cc0d264  extra/shutils/recloak.sh
 1972990a67caf2d0231eacf60e211acf545d9d0beeb3c145a49ba33d5d491b3f  extra/shutils/strip.sh
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  extra/vulnserver/__init__.py
-072a2c19162cc4e76476cf474134f18a5ec45cce9a4e4d216dad8e7a71ece048  extra/vulnserver/vulnserver.py
-b8411d1035bb49b073476404e61e1be7f4c61e205057730e2f7880beadcd5f60  lib/controller/action.py
-6da812281a69c8b7a5181c2f76374dc695e4727b2936042651bacbeda4e6bcc9  lib/controller/checks.py
-969737ac9cd3fa7bac8b582a85016bd348ba2087daa3644a570a9127e686363b  lib/controller/controller.py
+63657c00a046ca0fb28fd069407ab6305bd7b95c42f26a96ed083fd05b152252  extra/vulnserver/vulnserver.py
+3abecaec1a9c59645a4821463a2d761235f7a4f763a491f188a41a083bbddd98  lib/controller/action.py
+6574ed70c7fe0ac305dbc85ed7102f648b6a3f42fe2fe6b89172d69717327149  lib/controller/checks.py
+dcd4adcd7a2447a624ca7927541941d25767a4581af2d762c3197dc93790f4df  lib/controller/controller.py
 d69e84f1648cdb907f5d2dd454f03874a4613752b07867510145d51d84b3c56f  lib/controller/handler.py
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/controller/__init__.py
 b36b085ff1b5797e375c1e2ca3b12c7ab4204f48acd1a1efb075cff8302d9750  lib/core/agent.py
@@ -177,30 +177,30 @@ c03dc585f89642cfd81b087ac2723e3e1bb3bfa8c60e6f5fe58ef3b0113ebfe6  lib/core/data.
 147823c37596bd6a56d677697781f34b8d1d1671d5a2518fbc9468d623c6d07d  lib/core/defaults.py
 2f44a1bfe6f18aafe64147b99e69aa93cf438c0e7befe59f4e2aee9065c8b7b6  lib/core/dicts.py
 2592b0fd38c272c0b0d49878f4449437eb8ba8ff7536bb39b2ac9a2511010f7c  lib/core/dump.py
-6b9932d9c789a0e2ac28a493fb7914f49100a1c91de989bcdb20df9d40648522  lib/core/enums.py
+e4f92e09737ff0dda7ec30e0db1912570e252853b3af9b8f2b9f68ad33cf09fe  lib/core/enums.py
 5387168e5dfedd94ae22af7bb255f27d6baaca50b24179c6b98f4f325f5cc7b4  lib/core/exception.py
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/core/__init__.py
 914a13ee21fd610a6153a37cbe50830fcbd1324c7ebc1e7fc206d5e598b0f7ad  lib/core/log.py
-3ec59b5eb336d9808d28496f1cbbad716b4a0e276b5399023142826e460e3fd2  lib/core/optiondict.py
-b61676f0aa44798aaf9be72ff37550e2b78ed6ad3c71fbcad54f8c8bf7b34096  lib/core/option.py
+06651cff25422dcb84c159f80faf8dc377d82ddd451b5910f12c4c6a3ebe1e94  lib/core/optiondict.py
+e3a3729a24306b7ecace614fe27a8123c0becb0c5283ca519e5bcf376af2c711  lib/core/option.py
 ccc4a717e887652b1fcce073d9409d9c59a3b28548c703a9e453d15845f90cd7  lib/core/patch.py
 49c0fa7e3814dfda610d665ee02b12df299b28bc0b6773815b4395514ddf8dec  lib/core/profiling.py
 03db48f02c3d07a047ddb8fe33a757b6238867352d8ddda2a83e4fec09a98d04  lib/core/readlineng.py
 48797d6c34dd9bb8a53f7f3794c85f4288d82a9a1d6be7fcf317d388cb20d4b3  lib/core/replication.py
 0b8c38a01bb01f843d94a6c5f2075ee47520d0c4aa799cecea9c3e2c5a4a23a6  lib/core/revision.py
 888daba83fd4a34e9503fe21f01fef4cc730e5cde871b1d40e15d4cbc847d56c  lib/core/session.py
-222177a7a8e4c16ec4eae9f9542794ebf46a34b29390e967fe9fc26189261372  lib/core/settings.py
+f01361d999b0cf89b8418265c4a4962924fcc03a6b87e15b39c0836788725e85  lib/core/settings.py
 cd5a66deee8963ba8e7e9af3dd36eb5e8127d4d68698811c29e789655f507f82  lib/core/shell.py
 bcb5d8090d5e3e0ef2a586ba09ba80eef0c6d51feb0f611ed25299fbb254f725  lib/core/subprocessng.py
 70ea3768f1b3062b22d20644df41c86238157ec80dd43da40545c620714273c6  lib/core/target.py
-40b703993441fcd10ab06545b7dbe4a4762ab1ff517592a7e104a52785e62586  lib/core/testing.py
+c39dae0602b356d42f55df369c05614bbfb00c2abf2f0419fefe2ae781aa3098  lib/core/testing.py
 e3e653364d08d04d7492aa40a2bd29c6a28f4d78fecdd6c10f21f6cb28b98b4c  lib/core/threads.py
 b9aacb840310173202f79c2ba125b0243003ee6b44c92eca50424f2bdfc83c02  lib/core/unescaper.py
 53e396902cb2546eaa09e77073fcba8be8827ee9ce055cfc899e81b0e6ad4d6d  lib/core/update.py
 2400e465fa4d13e4c32795910878c71ff212e4361b46428d57ce43983f5e997c  lib/core/wordlist.py
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/__init__.py
 54bfd31ebded3ffa5848df1c644f196eb704116517c7a3d860b5d081e984d821  lib/parse/banner.py
-053079fe796dfce09cf94ac6f094043f2dfa393b5631387fadb4f735cf1ac6a4  lib/parse/cmdline.py
+14b2fcfa2d6c3a155e3b85f093929c6129893ad191d1988a717daa1ffbb422e7  lib/parse/cmdline.py
 02d82e4069bd98c52755417f8b8e306d79945672656ac24f1a45e7a6eff4b158  lib/parse/configfile.py
 c5b258be7485089fac9d9cd179960e774fbd85e62836dc67cce76cc028bb6aeb  lib/parse/handler.py
 5c9a9caee948843d5537745640cc7b98d70a0412cc0949f59d4ebe8b2907c06c  lib/parse/headers.py
@@ -254,6 +254,7 @@ a94958be0ec3e9d28d8171813a6a90655a9ad7e6aa33c661e8d8ebbfcf208dbb  lib/utils/deps
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/utils/__init__.py
 e7d31de0e268c129ee11c590eb618f73a85e1022c08b8ed1f77753043c949214  lib/utils/pivotdumptable.py
 c1dfc3bed0fed9b181f612d1d747955dd2b506dbe99bc9fd481495602371473a  lib/utils/progress.py
+0aeb890fb6b0783f25df7c1ba7c9d0098325b4f7a677ff0151e411be24760f04  lib/utils/prove.py
 2cd84db16edef8c9948e197a51d870cf1c338f4a89037b4d422de990f4a45237  lib/utils/purge.py
 f635872093a12cd63a72d77adf88e8f8cd4084a5cc64384f12966cd75a499bdf  lib/utils/safe2bin.py
 de4be7e291db0962cd59f9c04b3f7259f846e315df1fd9b323954f89fae0b2db  lib/utils/search.py
@@ -262,6 +263,7 @@ de4be7e291db0962cd59f9c04b3f7259f846e315df1fd9b323954f89fae0b2db  lib/utils/sear
 f0e5525a92fe971defc8f74c27942ff9138b1e8251f2e0d9a8bd59285b656084  lib/utils/timeout.py
 f821dc39a75ea48dccfa758788de15d38b9ca6a780a98f59935fb6610f75508c  lib/utils/tui.py
 e430db49aa768ff2cdba76932e30871c366054599c44d91580dde459ab9b6fef  lib/utils/versioncheck.py
+b3c5109394f6c3cdd73a524a737b36cca7ecc56619f2a5f801eb1e7f1bfdb78b  lib/utils/wafbypass.py
 1b439fc59fd202c21c74978ed9f36d1c309533226c77907eae159461525f9fef  lib/utils/xrange.py
 b1bbb62f5b272a6247d442d5e4f644a5bca7138e70776539ec84a5a90433fd13  LICENSE
 6b1828a80ae3472f1adb53a540dee0835eccac14f8cfc4bf73962c4e49a49557  plugins/dbms/access/connector.py
@@ -501,6 +503,7 @@ cf26bc8006519bd25ce06d347f72770cd75b61575cf65e5812274e8ab9392eb4  tamper/apostro
 11ad15d66c43f32f5d0a39052e5f623a4752ad4fb275d642f2e4cd841ff82b41  tamper/base64encode.py
 1b55b7c59c623411c8cf328fff9e7de96a2dfc48ef4e5455325bfd41aebbbc13  tamper/between.py
 6e72b92662185a56847cca235106bc354bd6a10e3e89a135b9ea8fa09cd8eb34  tamper/binary.py
+3fb1a7f8a37d8a49fb88fa880e163ff75a2b224c4a7799abe29bec1a367d5273  tamper/blindbinary.py
 f833cfbb53e6849ed1b3b554ec1c973f85e6d41ebd62f94f8e0dcf0ba5da2f49  tamper/bluecoat.py
 69c7eb987dec666da227ee1024c31b89ad324a3f7cab287ada6dade7f51c8a36  tamper/chardoubleencode.py
 c7892bff56b2b85dfdf9f24c783c569edac57a3fd5a254cf4554987a374206c9  tamper/charencode.py
@@ -524,6 +527,7 @@ d05dafb86e82807e75bb8f54dcd6afbb4a08ba3b83b35562fee7f7022a75dbd7  tamper/if2case
 55092820a856f583cf1b661001b60216886d172cb7d0008920bf4ab3df88aff0  tamper/ifnull2casewhenisnull.py
 eeda2b2fd54a4aa5fcf5630f8bfae43e0a38a840ae908e2f6b0878959067413c  tamper/ifnull2ifisnull.py
 94fe273bee7df27c9b4f1ee043779d06e4553169d9aec30c301d469275883dd1  tamper/informationschemacomment.py
+ff07320cb134520c3be99407b5c1e67528f944c6a12838ab583716622e877a95  tamper/infoschema2innodb.py
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  tamper/__init__.py
 017c91ba64c669382aa88ce627f925b00101a81c1a37a23dba09bfa2bfaf42ae  tamper/least.py
 d762543ef6d90fd6ce8b897fdfb864e0461d2941922d331d97a334aefdbbe291  tamper/lowercase.py
@@ -606,6 +610,7 @@ b3e13febe9e0ff6f97334f2868655bfdbaa18755e464a6dc4c6d424f513bad02  tests/test_tar
 4b646f513c6da1e33200184ed6eabe0aa345eb2e2a19598dc123e191168591bf  tests/test_urls.py
 23ffd75b5aec33066e6d6aad01ab2c9c1b12ee20c1a0990f8f1be81f1ad16161  tests/_testutils.py
 2364db35025a53ea4e5a0a80c034997642785f7e6d1566d0d0f1db959fe3c82e  tests/test_utils.py
+93ef9944effc62d4f744c57bd643137c90fd92205c6a6cbe891e0e99efb80a7f  tests/test_wafbypass.py
 81bb6d7449f224fa337734ae361c1a340bf9a51768a854d6a1a6e718ed1263ca  tests/test_wordlist.py
 55eaefc664bd8598329d535370612351ec8443c52465f0a37172ea46a97c458a  thirdparty/ansistrm/ansistrm.py
 e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855  thirdparty/ansistrm/__init__.py

extra/vulnserver/vulnserver.py

@@ -24,6 +24,7 @@
 DEBUG = False
 
 if PY3:
+    from http.client import FORBIDDEN
     from http.client import INTERNAL_SERVER_ERROR
     from http.client import NOT_FOUND
     from http.client import OK
@@ -35,6 +36,7 @@
 else:
     from BaseHTTPServer import BaseHTTPRequestHandler
     from BaseHTTPServer import HTTPServer
+    from httplib import FORBIDDEN
     from httplib import INTERNAL_SERVER_ERROR
     from httplib import NOT_FOUND
     from httplib import OK
@@ -157,6 +159,53 @@ def finish_request(self, *args, **kwargs):
             if DEBUG:
                 traceback.print_exc()
 
+# Primitive (CRS-style) WAF/IPS emulator used to exercise the automatic WAF/IPS bypass. The request
+# surface is normalized like a real WAF (lowercase, comments->space, whitespace compressed) BEFORE
+# a cumulative anomaly score is summed; when the score reaches the per-level threshold the request
+# is blocked (403 + marker). The rules are shaped so that camouflage tampers (case/whitespace/
+# comments) are normalized away and a *structural* substitution (e.g. 'between'/'equaltolike',
+# which removes the scored '=' operator) is the genuine bypass - matching real-world behavior.
+#
+# The emulator also models the OTHER real-world dimension: a scanner-fingerprint rule (mirroring
+# CRS 913100) adds a constant score for a recognizable scanner User-Agent that *stacks* with the
+# payload score. Its weight is below every threshold, so the scanner UA alone never blocks (benign
+# browsing passes), but it tips an otherwise-permitted payload over the threshold - so neutralizing
+# the request fingerprint (a non-scanner User-Agent) is itself a genuine bypass, with no SQL tamper.
+WAF_NUMERIC_COMPARISON = r"\d+\s*=\s*\d+"       # numeric self-comparison (boolean payloads); the structural lever 'between'/'equaltolike' removes it
+WAF_RULES = (
+    (r"\bunion\b.{0,40}\bselect\b", 6),
+    (r"\binformation_schema\b", 5),
+    (r"\b(sleep|benchmark|extractvalue|updatexml|xp_cmdshell|waitfor)\b", 5),
+    (r"\b(select|insert|update|delete|drop)\b", 3),
+    (WAF_NUMERIC_COMPARISON, 4),
+    (r"<script", 6),
+)
+WAF_THRESHOLD = {1: 6, 2: 4, 3: 2, 4: 8, 5: 5}      # security_level -> cumulative score that triggers a block
+WAF_SCANNER_UA = r"(?i)\b(?:sqlmap|nikto|nessus|acunetix|nmap|masscan|w3af|havij|wpscan|dirbuster|arachni)\b"
+WAF_SCANNER_UA_WEIGHT = 3       # CRS 913100-style: constant score for a scanner User-Agent, stacked with the payload score
+
+# Levels 4-5 model a libinjection-class WAF (e.g. OWASP CRS rule 942100): ANY boolean-comparison
+# fingerprint scores a flat amount REGARDLESS of operator, so '=','LIKE','BETWEEN','IN' are all
+# caught equally - structural tampers (between/equaltolike) do NOT help. There, neutralizing the
+# scanner fingerprint is the only payload-preserving bypass (level 4); when even that is not enough
+# the search must bail honestly (level 5). This mirrors the hardest real-world case.
+WAF_LIBINJECTION_LEVELS = (4, 5)
+WAF_LIBINJECTION_WEIGHT = 5
+WAF_LIBINJECTION = r"(?i)\b(?:and|or)\b.{0,40}(?:=|>|<|\blike\b|\bbetween\b|\bin\b|\brlike\b|\bregexp\b)"
+
+def waf_score(value, ua=None, level=0):
+    value = (value or "").lower()
+    value = re.sub(r"/\*.*?\*/", " ", value)        # t:replaceComments (note: -> single space, not empty)
+    value = re.sub(r"(?:--|#)[^\n]*", " ", value)   # t:removeComments (line comments)
+    value = re.sub(r"\s+", " ", value)              # t:compressWhitespace
+    libinjection = level in WAF_LIBINJECTION_LEVELS
+    retVal = sum(weight for (pattern, weight) in WAF_RULES if not (libinjection and pattern == WAF_NUMERIC_COMPARISON) and re.search(pattern, value))
+    if libinjection and re.search(WAF_LIBINJECTION, value):     # operator-agnostic comparison score (tampers cannot remove it)
+        retVal += WAF_LIBINJECTION_WEIGHT
+    if ua and re.search(WAF_SCANNER_UA, ua):        # scanner-fingerprint score, stacked with the payload score
+        retVal += WAF_SCANNER_UA_WEIGHT
+    return retVal
+
 class ReqHandler(BaseHTTPRequestHandler):
     def do_REQUEST(self):
         path, query = self.path.split('?', 1) if '?' in self.path else (self.path, "")
@@ -198,6 +247,22 @@ def do_REQUEST(self):
 
         self.url, self.params = path, params
 
+        # primitive WAF/IPS emulator (opt-in via 'security_level' param; 0/absent = off)
+        try:
+            level = int(self.params.get("security_level", 0) or 0)
+        except (TypeError, ValueError):
+            level = 0
+
+        if level > 0:
+            surface = "%s %s" % (unquote_plus(query), getattr(self, "data", "") or "")
+            if waf_score(surface, ua=self.params.get("user-agent"), level=level) >= WAF_THRESHOLD.get(level, 2):
+                self.send_response(FORBIDDEN)
+                self.send_header("Content-type", "text/html; charset=%s" % UNICODE_ENCODING)
+                self.send_header("Connection", "close")
+                self.end_headers()
+                self.wfile.write(b"<html><body>Request blocked: security policy violation (WAF)</body></html>")
+                return
+
         if self.url == "/csrf":
             if self.params.get("csrf_token") == _csrf_token:
                 self.url = "/"

lib/controller/action.py

@@ -8,11 +8,14 @@
 from lib.controller.handler import setHandler
 from lib.core.common import Backend
 from lib.core.common import Format
+from lib.core.common import hashDBWrite
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
 from lib.core.data import paths
 from lib.core.enums import CONTENT_TYPE
+from lib.core.enums import DBMS
+from lib.core.enums import HASHDB_KEYS
 from lib.core.exception import SqlmapNoneDataException
 from lib.core.exception import SqlmapUnsupportedDBMSException
 from lib.core.settings import SUPPORTED_DBMS
@@ -30,8 +33,41 @@ def action():
 
     # First of all we have to identify the back-end database management
     # system to be able to go ahead with the injection
+    # automatic WAF-bypass: if a WAF/IPS is present and the back-end DBMS is already indicated by the error
+    # page or the heuristic checks, skip active fingerprinting (the WAF would just block its payloads
+    # and flood the run with 403s) and assume that DBMS, so the user gets a usable result
+    if kb.wafBypass and not conf.forceDbms:
+        fallback = Backend.getErrorParsedDBMSes() or ([kb.heuristicDbms] if kb.heuristicDbms else [])
+        fallback = next((_ for _ in fallback if _ and _.lower() in SUPPORTED_DBMS), None)
+        if fallback:
+            logger.warning("skipping active back-end DBMS fingerprinting behind the WAF/IPS and assuming '%s' from error/heuristic detection" % fallback)
+            conf.forceDbms = fallback
+
     setHandler()
 
+    if kb.wafBypass and Backend.getDbms():      # persist the assumed DBMS so a resumed run restores it instead of re-fingerprinting (and dead-ending) behind the WAF
+        hashDBWrite(HASHDB_KEYS.DBMS, Backend.getDbms())
+
+    # automatic WAF-bypass: with MySQL behind the WAF, make data retrieval AND table enumeration survive a
+    # libinjection-class WAF (e.g. OWASP CRS), verified end-to-end through ModSecurity/CRS:
+    #   * fingerprinting was skipped, so flag has_information_schema (modern MySQL >=5.0 always has it) -
+    #     otherwise enumeration wrongly assumes 'MySQL < 5.0' and bails with "no tables";
+    #   * 'blindbinary' reshapes the single-character read ORD(MID())->RIGHT(LEFT())>BINARY 0x.. (sheds the
+    #     ORD/MID function names scored by 942151/942190);
+    #   * 'infoschema2innodb' moves table enumeration off 'information_schema' (scored by 942140) onto
+    #     'mysql.innodb_table_stats', which is not on those blocklists.
+    # (blindbinary also reshapes PostgreSQL, but full extraction through the CRS proxy garbles there - an
+    #  open issue - so PG is not auto-applied; it stays available as manual '--tamper=blindbinary'.)
+    if kb.wafBypass and Backend.getIdentifiedDbms() == DBMS.MYSQL:
+        kb.data.has_information_schema = True
+        if not conf.tamper:
+            from lib.utils.wafbypass import loadTamper
+            for _name in ("blindbinary", "infoschema2innodb"):
+                function = loadTamper(_name)
+                if function is not None and function not in (kb.tamperFunctions or []):
+                    kb.tamperFunctions = (kb.tamperFunctions or []) + [function]
+            logger.info("using tamper scripts 'blindbinary' and 'infoschema2innodb' so data retrieval and table enumeration can pass the WAF/IPS")
+
     if not Backend.getDbms() or not conf.dbmsHandler:
         htmlParsed = Format.getErrorParsedDBMSes()
 

lib/controller/checks.py

@@ -1351,6 +1351,10 @@ def checkWaf():
             warnMsg = "previous heuristics detected that the target "
             warnMsg += "is protected by some kind of WAF/IPS"
             logger.critical(warnMsg)
+            if hashDBRetrieve(HASHDB_KEYS.CHECK_WAF_BYPASS, True):    # re-apply a previously accepted automatic bypass
+                from lib.utils.wafbypass import neutralizeFingerprint
+                kb.wafBypass = True
+                neutralizeFingerprint()
         return _
 
     if not kb.originalPage:
@@ -1393,6 +1397,7 @@ def checkWaf():
 
     hashDBWrite(HASHDB_KEYS.CHECK_WAF_RESULT, retVal, True)
 
+
     if retVal:
         if not kb.identifiedWafs:
             warnMsg = "heuristics detected that the target "
@@ -1406,9 +1411,19 @@ def checkWaf():
         if not choice:
             raise SqlmapUserQuitException
         else:
-            if not conf.tamper:
-                warnMsg = "please consider usage of tamper scripts (option '--tamper')"
-                singleTimeWarnMessage(warnMsg)
+            if not conf.tamper and not kb.tamperFunctions:
+                message = "do you want sqlmap to try to automatically bypass the WAF/IPS during "
+                message += "the run (e.g. by using a non-scanner User-Agent and tamper script(s))? [Y/n] "
+                kb.wafBypass = readInput(message, default='Y', boolean=True)
+                hashDBWrite(HASHDB_KEYS.CHECK_WAF_BYPASS, kb.wafBypass, True)
+                if kb.wafBypass:
+                    # apply it up-front so the whole run (detection included) avoids the scanner
+                    # fingerprint, instead of getting blocked first and only then retrying
+                    from lib.utils.wafbypass import neutralizeFingerprint
+                    neutralizeFingerprint()
+                    logger.info("using a random (non-scanner) User-Agent and browser-like headers to bypass the WAF/IPS")
+                else:
+                    singleTimeWarnMessage("please consider manual usage of tamper scripts (option '--tamper')")
 
     return retVal