Fix NuGetApiKey detection to not require Guid bytes (#101)

* Fix NuGetApiKey detection to not require Guid bytes

This also simplifies the regex to just expect oy2 and then 43 lowercase base32 characters.

* Improve comments

* Remove tests per feedback

* Add more true positive examples

* Find NuGet API keys when bounded by non-alphanumeric, find uppercase keys which are non-standard but accepted

* Address comment

GeneratedRegexPatterns/HighConfidenceSecurityModels.json

@@ -318,11 +318,12 @@
     "DetectionMetadata": "Identifiable"
   },
   {
-    "Pattern": "(^|[^0-9a-z])(?<refine>oy2[a-p][0-9a-z]{15}[aq][0-9a-z]{11}[eu][bdfhjlnprtvxz357][a-p][0-9a-z]{11}[aeimquy4])([^aeimquy4]|$)",
+    "Pattern": "(?i)(^|[^a-z0-9])(?<refine>oy2[a-z2-7]{43})([^a-z0-9]|$)",
     "Id": "SEC101/031",
     "Name": "NuGetApiKey",
     "Signatures": [
-      "oy2"
+      "oy2",
+      "OY2"
     ],
     "DetectionMetadata": "HighEntropy, FixedSignature, HighConfidence"
   },

GeneratedRegexPatterns/PreciselyClassifiedSecurityKeys.json

@@ -318,11 +318,12 @@
     "DetectionMetadata": "Identifiable"
   },
   {
-    "Pattern": "(^|[^0-9a-z])(?<refine>oy2[a-p][0-9a-z]{15}[aq][0-9a-z]{11}[eu][bdfhjlnprtvxz357][a-p][0-9a-z]{11}[aeimquy4])([^aeimquy4]|$)",
+    "Pattern": "(?i)(^|[^a-z0-9])(?<refine>oy2[a-z2-7]{43})([^a-z0-9]|$)",
     "Id": "SEC101/031",
     "Name": "NuGetApiKey",
     "Signatures": [
-      "oy2"
+      "oy2",
+      "OY2"
     ],
     "DetectionMetadata": "HighEntropy, FixedSignature, HighConfidence"
   },

src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_031_NuGetApiKey.cs

@@ -1,24 +1,61 @@
 // Copyright (c) Microsoft. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
 using System;
 using System.Collections.Generic;
+using System.Linq;
 
 namespace Microsoft.Security.Utilities
 {
     public class NuGetApiKey : RegexPattern
     {
+        private const string Base32 = "abcdefghijklmnopqrstuvwxyz234567";
+
         public NuGetApiKey()
         {
             Id = "SEC101/031";
             Name = nameof(NuGetApiKey);
-            DetectionMetadata = DetectionMetadata.FixedSignature | DetectionMetadata.HighEntropy| DetectionMetadata.HighConfidence;
-            Pattern = "(^|[^0-9a-z])(?<refine>oy2[a-p][0-9a-z]{15}[aq][0-9a-z]{11}[eu][bdfhjlnprtvxz357][a-p][0-9a-z]{11}[aeimquy4])([^aeimquy4]|$)";
-            Signatures = new HashSet<string>(new[] { "oy2" });
+            DetectionMetadata = DetectionMetadata.FixedSignature | DetectionMetadata.HighEntropy | DetectionMetadata.HighConfidence;
+
+            // This is the ApiKeyV4 format implemented here:
+            // https://github.com/NuGet/NuGetGallery/blob/main/src/NuGetGallery.Services/Authentication/ApiKeyV4.cs
+            Pattern = "(?i)(^|[^a-z0-9])(?<refine>oy2[a-z2-7]{43})([^a-z0-9]|$)";
+
+            Signatures = new HashSet<string>(new[] { "oy2", "OY2" });
+        }
+
+        public override Tuple<string, string> GetMatchIdAndName(string match)
+        {
+            if (match.Any(char.IsLower) && match.Any(char.IsUpper))
+            {
+                // The API key is not all uppercase, which is non-standard but accepted by the service.
+                // Nullify the match if there is a mix of upper and lowercase to improve redaction.
+                // A match with a mix of case is more likely to be base64 than a real API key.
+                return null;
+            }
+
+            return base.GetMatchIdAndName(match);
         }
 
         public override IEnumerable<string> GenerateTruePositiveExamples()
         {
+            // matches the older, GUID-based entropy source
             yield return $"oy2a{WellKnownRegexPatterns.RandomLowercase(15)}a{WellKnownRegexPatterns.RandomLowercase(11)}e7a{WellKnownRegexPatterns.RandomLowercase(11)}a";
+
+            // matches the newer, PRNG-based entropy source
+            yield return $"oy2{WellKnownRegexPatterns.GenerateString(Base32, 43)}";
+
+            // uppercase API key, which is non-standard but accepted by the service
+            yield return $"OY2{WellKnownRegexPatterns.GenerateString(Base32, 43).ToUpperInvariant()}";
+
+            // repeat a single base32 character 43 times, for an obviously contrived example
+            yield return $"oy2{new string(WellKnownRegexPatterns.GenerateString(Base32, 1)[0], 43)}";
+        }
+
+        public override IEnumerable<string> GenerateFalsePositiveExamples()
+        {
+            // mixed case, excluded by GetMatchIdAndName
+            yield return $"oy2{WellKnownRegexPatterns.GenerateString(Base32, 10)}{WellKnownRegexPatterns.GenerateString(Base32, 33).ToUpperInvariant()}";
         }
     }
 }