Lots more NOT matches for generic secret pattern

Author: aegilops

generic/passwords.js

@@ -2,8 +2,9 @@ var password="itisaSEKRIT";
 
 var secret="thisisalsonottobetold";
 
-let ok="notasecret";
+let ok=secret;
 
 if (secret == "hardcodedsecret") {
 	do_admin();
 }
+

generic/patterns.yml

@@ -7,22 +7,69 @@ patterns:
     regex:
       version: 0.4
       pattern: |
-        [a-zA-Z0-9\t !.,$%&*+?^_`{|}()[\]\\/~-]+
+        [a-zA-Z0-9!.,$%&*+?^_`{|}()[\]\\/~-][a-zA-Z0-9\t !.,$%&*+?^_`{|}()[\]\\/~-]*
       start: |
-        (?i)(?:api|jwt|mysql)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)?|pwd|secret|token)[\t ]*[=:][\t ]*["']?
+        (?i)(?:api|jwt|mysql)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)?|pwd?|secret)[\t ]*(={1,3}|:)[\t ]*["']?
       end: |
         (\z|[\r\n'"])
-      additional_not_match:
-        - \b((?i)(?:pass?(wo?r?d|code|phrase)?|pwd|secret|token|write|read|on|off|true|false|placeholder|dummy|none)\b
-        - ^[\t \r\n"'()${}].*
-
+    additional_not_match:
+      # something that means "password" or a placeholder name - either a variable or a placeholder
+      - ^(?i)(?:[A-Za-z0-9_.]*,\s*)?(?:str\()?[[<(]?(?:(?:user|key)_?)?(?:pass?(wo?r?d|code|phrase)|pass|pwd|secret|token|tok|redacted|placeholder|dummy|pw|thephrase),?[\]>)]?\\?$
+      - ^.*token.*$
+      - ^[a-zA-Z0-9._]+[_.](?:password|passphrase|secret|key).*$
+      - ^.* passphrase .*$
+      # a literal value or config switch
+      - ^(?i)(?:[a-zA-Z0-9_.]*,\s*)?[[<(]?(?:write|read|on|off|true|false|none|null|nil|undefined|eof|ignore|eol|git),?[\]>)]?(?:\)\s*\{)?\\?$
+      - ^\s*%[sr]\s*$                 # format string placeholder
+      - ^\s*$                         # whitespace
+      - ^\s*(?:int|str|Any|None|bytes|bool)\s*([,|].*)?\s*$  # Python basic type hints
+      # Python container type hints
+      - ^\s*(?:[Tt]uple|[Ll]ist|[Dd]ict|Callable|Iterable|Sequence|Optional)\[.*\]\s*$
+      - ^\s*\.\.\.\s*$                # ...
+      - ^\s*\\\s*$                    # a bare backslash
+      - ^\s*,s*$                      # a bare comma
+      - ^\\0$                         # literal \0
+      # function definitions, e.g. Javascript
+      - ^function\s*\([^)]*\)\s*{\s*
+      - ^\([^)]*\)\s*=>\s*(?:{\s*|[^;)]+[;)])$
+      # short numbers, possibly followed by a comment (some kind of tokenisation)
+      - ^\s*[0-9]{1,4}(?:\s*(?:/\*|#|//).*)?$       
+      # function calls or a new variable
+      - ^(?:new )?[a-zA-Z0-9_.]+\(.*$
+      # reference to a member variable
+      - ^\s*(?:self|this)\.[a-zA-Z_][a-zA-Z0-9_]+[,[]?\s*$
+      # index into a variable
+      - ^\s*[a-zA-Z0-9_.]+\[(?:[a-zA-Z0-9_.]+)?\]?\s*$
+      # directories
+      - ^\s*(?:~|/tmp|\.\.|\.)\s*$
+      # regex
+      - ^\\{1,2}w\+/g,( \\?)?$
+      # a variable substitution
+      - ^\s*\$\{[^}]+}\s*$
+      # surrounded by {}
+      - ^\s*\{[^}]*\}\s*$
+      # surrounded by []
+      - ^\s*\[[^\]]*\]\s*$
+      # - ^["'()${}].*                  # check this!
+      # stray single chars
+      - ^[,()[\]{}`.]\\?$
+      # urllib demo passphrase
+      - ^geheim\$parole$
+      # docs
+      - ^\s*\([Oo]ptional\).*$
+      # smilies
+      - ^-[)(]$ 
+    
     expected:
       - name: passwords.js
         start_offset: 14
         end_offset: 25
       - name: passwords.js
         start_offset: 41
         end_offset: 62
+      - name: passwords.js
+        start_offset: 97
+        end_offset: 112
 
     comments:
       - "`password`, `secret`, `key`, or password like prefix (fuzzy)"
@@ -35,7 +82,16 @@ patterns:
     regex:
       version: 0.1
       pattern: |
-        [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}
+        (?i)[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}
+      start: |
+        \A|[^0-9A-Fa-f-]
+      end: |
+        \z|[^0-9A-Fa-f-]
+    additional_not_match:
+      - ^12345678-1234-5678-1234-567812345678$
+      - ^00000000-0000-0000-0000-000000000000$
+      - ^(?i)00010203-0405-0607-0809-0a0b0c0d0e0f$
+      - ^(?i)12345678-1234-1234-1234-123456789abc$
     expected:
       - name: uuids.txt
         start_offset: 12
@@ -46,3 +102,4 @@ patterns:
       - name: uuids.txt
         start_offset: 131
         end_offset: 167
+

testing/test.py

@@ -46,6 +46,7 @@ def __init__(self, name: str, description: str, start: str, pattern: str, end: s
                                       ] if additional_not_matches is not None else []
         self.additional_matches = [add_match.strip() for add_match in additional_matches
                                   ] if additional_matches is not None else []
+
         self.expected = expected
 
     def regex_string(self) -> bytes:
@@ -199,7 +200,7 @@ def pcre_result_match(pattern: Pattern,
 
         if not dry_run:
             if not any([path_offsets_match(file_details, loc) for loc in pattern.expected]):
-                if not quietn:
+                if not quiet:
                     LOG.log(logging.ERROR if pattern.expected else logging.INFO,
                             "%s result '%s' for '%s' in path '%s'; %s:%d-%d",
                             "❌ unexpected" if pattern.expected else "ℹ️  found", parts['pattern'], pattern.name,