[pull] master from sqlmapproject:master

lib/controller/checks.py

@@ -166,9 +166,11 @@ def checkSqlInjection(place, parameter, value):
                     # keyword-free fallback: heuristicCheckDbms() above uses SELECT/quote payloads
                     # and is skipped when the WAF/IPS is dropping requests; the operator-dialect
                     # probes carry no SELECT/quote/schema name, so they can still narrow the DBMS in
-                    # that case (or when it was inconclusive), using the now-calibrated boolean oracle
-                    if not Backend.getIdentifiedDbms() and kb.heuristicDbms is None:
-                        kb.heuristicDbms = dialectCheckDbms(injection)
+                    # that case (or when it was inconclusive), using the now-calibrated boolean oracle.
+                    # It feeds the lower-confidence heuristicExtendedDbms (UNION FROM / handler hint),
+                    # deliberately NOT heuristicDbms, so it never drives reduceTests (skipping payloads)
+                    if not Backend.getIdentifiedDbms() and kb.heuristicDbms is None and kb.heuristicExtendedDbms is None:
+                        kb.heuristicExtendedDbms = dialectCheckDbms(injection)
 
                 # If the DBMS has already been fingerprinted (via DBMS-specific
                 # error message, simple heuristic check or via DBMS-specific
@@ -730,15 +732,16 @@ def genCmpPayload():
                             if len(kb.dbmsFilter or []) == 1:
                                 Backend.forceDbms(kb.dbmsFilter[0])
                             elif not Backend.getIdentifiedDbms():
-                                if kb.heuristicDbms is None:
+                                heuristicDbms = kb.heuristicDbms or kb.heuristicExtendedDbms
+                                if heuristicDbms is None:
                                     if kb.heuristicTest == HEURISTIC_TEST.POSITIVE or injection.data:
                                         warnMsg = "using unescaped version of the test "
                                         warnMsg += "because of zero knowledge of the "
                                         warnMsg += "back-end DBMS. You can try to "
                                         warnMsg += "explicitly set it with option '--dbms'"
                                         singleTimeWarnMessage(warnMsg)
                                 else:
-                                    Backend.forceDbms(kb.heuristicDbms)
+                                    Backend.forceDbms(heuristicDbms)
 
                             if unionExtended:
                                 infoMsg = "automatically extending ranges for UNION "

lib/core/agent.py

@@ -133,7 +133,7 @@ def payload(self, place=None, parameter=None, value=None, newValue=None, where=N
             origValue = origValue.split(kb.customInjectionMark)[0]
             if kb.postHint in (POST_HINT.SOAP, POST_HINT.XML):
                 origValue = re.split(r"['\">]", origValue)[-1]
-            elif kb.postHint in (POST_HINT.JSON, POST_HINT.JSON_LIKE):
+            elif kb.postHint in (POST_HINT.JSON, POST_HINT.JSON_LIKE, POST_HINT.GRPC_WEB):
                 match = re.search(r"['\"]", origValue)
                 quote = match.group(0) if match else '"'
                 origValue = extractRegexResult(r"%s\s*:\s*(?P<result>\d+)\Z" % quote, origValue) or extractRegexResult(r"(?P<result>[^%s]*)\Z" % quote, origValue)
@@ -206,7 +206,7 @@ def payload(self, place=None, parameter=None, value=None, newValue=None, where=N
         if place in (PLACE.URI, PLACE.CUSTOM_POST, PLACE.CUSTOM_HEADER):
             _ = "%s%s" % (_origValue if base64Encoding else origValue, kb.customInjectionMark)
 
-            if kb.postHint == POST_HINT.JSON and isNumber(origValue) and not isNumber(newValue) and '"%s"' % _ not in paramString:
+            if kb.postHint in (POST_HINT.JSON, POST_HINT.GRPC_WEB) and isNumber(origValue) and not isNumber(newValue) and '"%s"' % _ not in paramString:
                 newValue = '"%s"' % self.addPayloadDelimiters(newValue)
             elif kb.postHint == POST_HINT.JSON_LIKE and isNumber(origValue) and not isNumber(newValue) and re.search(r"['\"]%s['\"]" % re.escape(_), paramString) is None:
                 newValue = "'%s'" % self.addPayloadDelimiters(newValue)

lib/core/dicts.py

@@ -294,13 +294,15 @@
     DBMS.ACCESS: "CVAR(NULL)",
     DBMS.MAXDB: "ALPHA(NULL)",
     DBMS.MSSQL: "PARSENAME(NULL,NULL)",
+    DBMS.SYBASE: "STR_REPLACE(NULL,'x','x')",  # ASE extension (MSSQL has REPLACE); doc-derived, not live-tested. Also SAP IQ / SQL Anywhere (not in DBMS)
     DBMS.MYSQL: "IFNULL(QUARTER(NULL),NULL XOR NULL)",  # NOTE: previous form (i.e., QUARTER(NULL XOR NULL)) was bad as some optimization engines wrongly evaluate QUARTER(NULL XOR NULL) to 0
     DBMS.ORACLE: "INSTR2(NULL,NULL)",
     DBMS.PGSQL: "QUOTE_IDENT(NULL)",
     DBMS.SQLITE: "JULIANDAY(NULL)",
     DBMS.H2: "STRINGTOUTF8(NULL)",
     DBMS.MONETDB: "CODE(NULL)",
-    DBMS.DERBY: "NULLIF(USER,SESSION_USER)",
+    DBMS.DERBY: "NULLIF(USER,SESSION_USER)",  # not Derby-specific; safe only because DB2 (shares this dummy) is tested first
+    DBMS.DB2: "MULTIPLY_ALT(NULL,NULL)",  # DB2-unique (Derby shares the dummy, not this function)
     DBMS.VERTICA: "BITSTRING_TO_BINARY(NULL)",
     DBMS.MCKOI: "TONUMBER(NULL)",
     DBMS.PRESTO: "FROM_HEX(NULL)",
@@ -314,7 +316,7 @@
     DBMS.VIRTUOSO: "__MAX_NOTNULL(NULL)",
     DBMS.CLICKHOUSE: "halfMD5(NULL)",
     DBMS.SNOWFLAKE: "BOOLNOT(NULL)",
-    DBMS.SPANNER: "FARM_FINGERPRINT(NULL)",
+    DBMS.SPANNER: "FARM_FINGERPRINT(NULL)",  # also BigQuery GoogleSQL (not in DBMS)
     DBMS.HANA: "MAP(NULL,NULL,NULL,NULL,NULL)",
 }
 
@@ -383,6 +385,7 @@
 POST_HINT_CONTENT_TYPES = {
     POST_HINT.JSON: "application/json",
     POST_HINT.JSON_LIKE: "application/json",
+    POST_HINT.GRPC_WEB: "application/grpc-web-text",
     POST_HINT.MULTIPART: "multipart/form-data",
     POST_HINT.SOAP: "application/soap+xml",
     POST_HINT.XML: "application/xml",

lib/core/enums.py

@@ -146,6 +146,7 @@ class POST_HINT(object):
     MULTIPART = "MULTIPART"
     XML = "XML (generic)"
     ARRAY_LIKE = "Array-like"
+    GRPC_WEB = "gRPC-Web"
 
 class HTTPMETHOD(object):
     GET = "GET"

lib/core/option.py

@@ -2295,6 +2295,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.permissionFlag = False
     kb.place = None
     kb.postHint = None
+    kb.grpcWeb = None
     kb.postSpaceToPlus = False
     kb.postUrlEncode = True
     kb.prependFlag = False

lib/core/settings.py

@@ -20,7 +20,7 @@
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.7.110"
+VERSION = "1.10.7.117"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -46,7 +46,6 @@
 
 # Minimum distance of ratio from kb.matchRatio to result in True
 DIFF_TOLERANCE = 0.05
-CONSTANT_RATIO = 0.9
 
 # Ratio used in heuristic check for WAF/IPS protected targets
 IPS_WAF_CHECK_RATIO = 0.5
@@ -339,7 +338,7 @@
 ORACLE_SYSTEM_DBS = ("ADAMS", "ANONYMOUS", "APEX_030200", "APEX_PUBLIC_USER", "APPQOSSYS", "AURORA$ORB$UNAUTHENTICATED", "AWR_STAGE", "BI", "BLAKE", "CLARK", "CSMIG", "CTXSYS", "DBSNMP", "DEMO", "DIP", "DMSYS", "DSSYS", "EXFSYS", "FLOWS_%", "FLOWS_FILES", "HR", "IX", "JONES", "LBACSYS", "MDDATA", "MDSYS", "MGMT_VIEW", "OC", "OE", "OLAPSYS", "ORACLE_OCM", "ORDDATA", "ORDPLUGINS", "ORDSYS", "OUTLN", "OWBSYS", "PAPER", "PERFSTAT", "PM", "SCOTT", "SH", "SI_INFORMTN_SCHEMA", "SPATIAL_CSW_ADMIN_USR", "SPATIAL_WFS_ADMIN_USR", "SYS", "SYSMAN", "SYSTEM", "TRACESVR", "TSMSYS", "WK_TEST", "WKPROXY", "WKSYS", "WMSYS", "XDB", "XS$NULL")
 SQLITE_SYSTEM_DBS = ("sqlite_master", "sqlite_temp_master")
 ACCESS_SYSTEM_DBS = ("MSysAccessObjects", "MSysACEs", "MSysObjects", "MSysQueries", "MSysRelationships", "MSysAccessStorage", "MSysAccessXML", "MSysModules", "MSysModules2", "MSysNavPaneGroupCategories", "MSysNavPaneGroups", "MSysNavPaneGroupToObjects", "MSysNavPaneObjectIDs")
-FIREBIRD_SYSTEM_DBS = ("RDB$BACKUP_HISTORY", "RDB$CHARACTER_SETS", "RDB$CHECK_CONSTRAINTS", "RDB$COLLATIONS", "RDB$DATABASE", "RDB$DEPENDENCIES", "RDB$EXCEPTIONS", "RDB$FIELDS", "RDB$FIELD_DIMENSIONS", " RDB$FILES", "RDB$FILTERS", "RDB$FORMATS", "RDB$FUNCTIONS", "RDB$FUNCTION_ARGUMENTS", "RDB$GENERATORS", "RDB$INDEX_SEGMENTS", "RDB$INDICES", "RDB$LOG_FILES", "RDB$PAGES", "RDB$PROCEDURES", "RDB$PROCEDURE_PARAMETERS", "RDB$REF_CONSTRAINTS", "RDB$RELATIONS", "RDB$RELATION_CONSTRAINTS", "RDB$RELATION_FIELDS", "RDB$ROLES", "RDB$SECURITY_CLASSES", "RDB$TRANSACTIONS", "RDB$TRIGGERS", "RDB$TRIGGER_MESSAGES", "RDB$TYPES", "RDB$USER_PRIVILEGES", "RDB$VIEW_RELATIONS")
+FIREBIRD_SYSTEM_DBS = ("RDB$BACKUP_HISTORY", "RDB$CHARACTER_SETS", "RDB$CHECK_CONSTRAINTS", "RDB$COLLATIONS", "RDB$DATABASE", "RDB$DEPENDENCIES", "RDB$EXCEPTIONS", "RDB$FIELDS", "RDB$FIELD_DIMENSIONS", "RDB$FILES", "RDB$FILTERS", "RDB$FORMATS", "RDB$FUNCTIONS", "RDB$FUNCTION_ARGUMENTS", "RDB$GENERATORS", "RDB$INDEX_SEGMENTS", "RDB$INDICES", "RDB$LOG_FILES", "RDB$PAGES", "RDB$PROCEDURES", "RDB$PROCEDURE_PARAMETERS", "RDB$REF_CONSTRAINTS", "RDB$RELATIONS", "RDB$RELATION_CONSTRAINTS", "RDB$RELATION_FIELDS", "RDB$ROLES", "RDB$SECURITY_CLASSES", "RDB$TRANSACTIONS", "RDB$TRIGGERS", "RDB$TRIGGER_MESSAGES", "RDB$TYPES", "RDB$USER_PRIVILEGES", "RDB$VIEW_RELATIONS")
 MAXDB_SYSTEM_DBS = ("SYSINFO", "DOMAIN")
 SYBASE_SYSTEM_DBS = ("master", "model", "sybsystemdb", "sybsystemprocs", "tempdb")
 DB2_SYSTEM_DBS = ("NULLID", "SQLJ", "SYSCAT", "SYSFUN", "SYSIBM", "SYSIBMADM", "SYSIBMINTERNAL", "SYSIBMTS", "SYSPROC", "SYSPUBLIC", "SYSSTAT", "SYSTOOLS", "SYSDEBUG", "SYSINST")
@@ -795,15 +794,6 @@
 # Payload used for checking of existence of WAF/IPS (dummier the better)
 IPS_WAF_CHECK_PAYLOAD = "AND 1=1 UNION ALL SELECT 1,NULL,'<script>alert(\"XSS\")</script>',table_name FROM information_schema.tables WHERE 2>1--/**/; EXEC xp_cmdshell('cat ../../../etc/passwd')#"
 
-# Vectors used for provoking specific WAF/IPS behavior(s)
-WAF_ATTACK_VECTORS = (
-    "",  # NIL
-    "search=<script>alert(1)</script>",
-    "file=../../../../etc/passwd",
-    "q=<invalid>foobar",
-    "id=1 %s" % IPS_WAF_CHECK_PAYLOAD
-)
-
 # Used for status representation in dictionary attack phase
 ROTATING_CHARS = ('\\', '|', '|', '/', '-')
 
@@ -892,7 +882,7 @@
 HASH_ATTACK_TIME_LIMIT = 300
 
 # Regular expression used for automatic hex conversion and hash cracking of (RAW) binary column values
-HASH_BINARY_COLUMNS_REGEX = r"(?i)pass|psw|hash"
+HASH_BINARY_COLUMNS_REGEX = r"(?i)pass|psw|hash|secret|digest"
 
 # Regular expression matching (declared) binary column types, used to auto-hex their values during dumping
 # so raw bytes (e.g. password hashes stored in binary form) are not silently truncated at NUL / mangled by
@@ -1121,6 +1111,7 @@
     ("Freemarker", r"freemarker\.(?:core|template|extract|cache)\.\w+|ParseException|InvalidReferenceException|TemplateException"),
     ("Velocity", r"org\.apache\.velocity\.(?:runtime|exception)\.\w+|ParseErrorException|MethodInvocationException|ResourceNotFoundException"),
     ("Spring EL / Thymeleaf", r"org\.springframework\.expression\.\w+|org\.thymeleaf\.\w+|SpelEvaluationException|TemplateProcessingException|ExpressionParsingException"),
+    ("Struts2 (OGNL)", r"ognl\.(?:OgnlException|NoSuchPropertyException|MethodFailedException|InappropriateExpressionException|ExpressionSyntaxException)|com\.opensymphony\.xwork2|org\.apache\.struts2|There is no Action mapped for|Struts (?:Problem Report|has detected an unhandled exception)"),
     ("ERB", r"\(erb\):\d+|NameError.*undefined local variable"),
     ("Pug/Jade", r"pug|jade|ParseError"),
     ("Handlebars", r"handlebars|Handlebars|Parse error on line"),
@@ -1341,7 +1332,7 @@
 MAX_CONNECT_RETRIES = 100
 
 # Strings for detecting formatting errors
-FORMAT_EXCEPTION_STRINGS = ("Type mismatch", "Error converting", "Please enter a", "Conversion failed", "String or binary data would be truncated", "Failed to convert", "unable to interpret text value", "Input string was not in a correct format", "System.FormatException", "java.lang.NumberFormatException", "ValueError: invalid literal", "TypeMismatchException", "CF_SQL_INTEGER", "CF_SQL_NUMERIC", " for CFSQLTYPE ", "cfqueryparam cfsqltype", "InvalidParamTypeException", "Invalid parameter type", "Attribute validation error for tag", "is not of type numeric", "<cfif Not IsNumeric(", "invalid input syntax for integer", "invalid input syntax for type", "invalid number", "character to number conversion error", "String was not recognized as a valid", "Convert.ToInt", "cannot be converted to a ", "InvalidDataException", "Arguments are of the wrong type", "Invalid conversion")
+FORMAT_EXCEPTION_STRINGS = ("Type mismatch", "Error converting", "Please enter a", "Conversion failed", "String or binary data would be truncated", "Failed to convert", "unable to interpret text value", "Input string was not in a correct format", "System.FormatException", "java.lang.NumberFormatException", "ValueError: invalid literal", "TypeMismatchException", "CF_SQL_INTEGER", "CF_SQL_NUMERIC", " for CFSQLTYPE ", "cfqueryparam cfsqltype", "InvalidParamTypeException", "Invalid parameter type", "Attribute validation error for tag", "is not of type numeric", "<cfif Not IsNumeric(", "invalid input syntax for integer", "invalid input syntax for type", "invalid number", "character to number conversion error", "String was not recognized as a valid", "Convert.ToInt", "cannot be converted to a ", "InvalidDataException", "Arguments are of the wrong type", "Invalid conversion", "Incorrect integer value", "Truncated incorrect", "Out of range value", "datatype mismatch")
 
 # Regular expression used for extracting ASP.NET view state values
 VIEWSTATE_REGEX = r'(?i)(?P<name>__VIEWSTATE[^"]*)[^>]+value="(?P<result>[^"]+)'
@@ -1379,9 +1370,6 @@
 # Format used for representing invalid unicode characters
 INVALID_UNICODE_CHAR_FORMAT = r"\x%02x"
 
-# Minimum supported version of httpx library (for --http2)
-MIN_HTTPX_VERSION = "0.28"
-
 # Regular expression for XML POST data
 XML_RECOGNITION_REGEX = r"(?s)\A\s*<[^>]+>(.+>)?\s*\Z"
 

lib/core/target.py

@@ -77,6 +77,8 @@
 from lib.core.settings import USER_AGENT_ALIASES
 from lib.core.settings import XML_RECOGNITION_REGEX
 from lib.core.threads import getCurrentThreadData
+from lib.utils.grpcweb import CONTENT_TYPE as grpcWebContentType
+from lib.utils.grpcweb import decodeBody as grpcDecodeBody
 from lib.utils.hashdb import HashDB
 from thirdparty import six
 from collections import OrderedDict
@@ -112,6 +114,16 @@ def _setRequestParams():
     if conf.data is not None:
         conf.method = conf.method or HTTPMETHOD.POST
 
+        # probe (side-effect-free) whether this is a gRPC-Web body; if so, expose its string fields as a
+        # JSON view so the standard JSON injection path can process it. The skeleton is committed to
+        # kb.grpcWeb (and the JSON view kept) ONLY on user acceptance below; if declined, the original
+        # body is restored so it is sent verbatim (never the JSON surrogate)
+        kb.grpcWeb = None
+        _grpcOriginalData = conf.data
+        _grpcView, _grpcSkeleton = grpcDecodeBody(conf.data)
+        if _grpcView is not None:
+            conf.data = _grpcView
+
         def process(match, repl):
             retVal = match.group(0)
 
@@ -145,14 +157,26 @@ def process(match, repl):
                     kb.testOnlyCustom = True
 
         if re.search(JSON_RECOGNITION_REGEX, conf.data):
-            message = "JSON data found in %s body. " % conf.method
-            message += "Do you want to process it? [Y/n/q] "
+            if _grpcSkeleton:
+                message = "gRPC-Web text data found in %s body. Do you want to process its detected string fields? [Y/n/q] " % conf.method
+            else:
+                message = "JSON data found in %s body. Do you want to process it? [Y/n/q] " % conf.method
             choice = readInput(message, default='Y').upper()
 
             if choice == 'Q':
                 raise SqlmapUserQuitException
             elif choice == 'Y':
-                kb.postHint = POST_HINT.JSON
+                kb.postHint = POST_HINT.GRPC_WEB if _grpcSkeleton else POST_HINT.JSON
+                if _grpcSkeleton:
+                    kb.grpcWeb = _grpcSkeleton  # commit only on acceptance
+                    # force a grpc-web-text response (the only kind this cut decodes) by REPLACING any
+                    # existing Accept - a captured 'Accept: */*', or an explicit 'q=0' we cannot honor
+                    for _index, (_header, _value) in enumerate(conf.httpHeaders or []):
+                        if _header.lower() == HTTP_HEADER.ACCEPT.lower():
+                            conf.httpHeaders[_index] = (_header, grpcWebContentType)
+                            break
+                    else:
+                        conf.httpHeaders.append((HTTP_HEADER.ACCEPT, grpcWebContentType))
                 if not (kb.processUserMarks and kb.customInjectionMark in conf.data):
                     conf.data = getattr(conf.data, UNENCODED_ORIGINAL_VALUE, conf.data)
                     conf.data = conf.data.replace(kb.customInjectionMark, ASTERISK_MARKER)
@@ -245,6 +269,11 @@ def _attr(m):
                     conf.data = re.sub(r"(?si)(Content-Disposition:[^\n]+\s+name=\"(?P<name>[^\"]+)\"(?:[^f|^b]|f(?!ilename=)|b(?!oundary=))*?)((%s)--)" % ("\r\n" if "\r\n" in conf.data else '\n'),
                                        functools.partial(process, repl=r"\g<1>%s\g<3>" % kb.customInjectionMark), conf.data)
 
+        # a gRPC-Web body that was NOT accepted as GRPC_WEB (declined prompt) must be sent verbatim,
+        # not as the JSON surrogate that was temporarily swapped into conf.data for detection
+        if _grpcSkeleton is not None and kb.postHint != POST_HINT.GRPC_WEB:
+            conf.data = _grpcOriginalData
+
         if not kb.postHint:
             if kb.customInjectionMark in conf.data:  # later processed
                 pass