Edits for vso integration (#114)

* More analysis tweaks for ADO integration.

* Update ADO integration API changes. Fix false positives in UrlCredentials check.

* Update release notes.

* Update generated regexes.

GeneratedRegexPatterns/MediumConfidenceSecurityModels.json

@@ -20,7 +20,7 @@
     "DetectionMetadata": "HighEntropy, MediumConfidence"
   },
   {
-    "Pattern": "(ftps?|https?):\\/\\/(?:[^:@]+):(?<refine>[^:@?]+)@",
+    "Pattern": "(ftps?|https?):\\/\\/(?<refine>[^:@\\/]+:[^:@?\\/]+)@",
     "Id": "SEC101/127",
     "Name": "UrlCredentials",
     "Signatures": [

GeneratedRegexPatterns/UnclassifiedPotentialSecurityKeys.json

@@ -11,7 +11,7 @@
     "DetectionMetadata": "HighEntropy, MediumConfidence"
   },
   {
-    "Pattern": "(ftps?|https?):\\/\\/(?:[^:@]+):(?<refine>[^:@?]+)@",
+    "Pattern": "(ftps?|https?):\\/\\/(?<refine>[^:@\\/]+:[^:@?\\/]+)@",
     "Id": "SEC101/127",
     "Name": "UrlCredentials",
     "Signatures": [

docs/ReleaseHistory.md

@@ -12,13 +12,17 @@
 - FNS => False negative reduction in static analysis.
 
 # UNRELEASED
+- BRK: Update `SEC101/127.UrlCredentials` match refinement to include both the account name and password. This is a breaking change as the correlating id will differ.
 - BUG: Merge multiple calls to `DateTime.UtcNow` in `GenerateCommonAnnotatedKey`, forcing year and month to agree. Add overload to provide an arbitrary allocation time, with bound checks (year 2024 to 2085).
 - BUG: Mark `SecretMasker(SecretMasker)` copy contructor as protected to make it callable by derived classes.
+
 - BUG: Mark `SecretMasker.Clone` as public virtual, to make it overridable by derived classes.
 - BUG: Update `SEC101/127.UrlCredentials` visibility to public to make it independently creatable.
+- BUG: Mark `SecretMasker.LiteralEncoders`, `SecretMasker.EncodedSecretLiterals` and `SecretMasker.ExplicitlyAddedSecretLiterals` as public.
 - BUG: Update `SEC101/154.AzureCacheForRedisIdentifiableKey` test example production to call base class (which generates test keys consisting of repeated characters in the randomized component).
 - BUG: Short-circuit `SecretMasker.DetectSecret(string)` operation if there are no configured regexes, encoded, or explicitly added secret literals.
-- FNS: Update `SEC101/127` regex to detect ftp(s) credentials.
+- FPS: Update `SEC101/127.UrlCredentials` regex to not fire on use of colon within URL path component.
+- FNS: Update `SEC101/127.UrlCredentials` regex to detect ftp(s) credentials.
 
 # 1.9.1 - 11/18/2024
 - DEP: Removed dependency of the `base-62` crate in the Rust codebase, since it depended on the `failure` crate which has a known [vulnerability](https://github.com/advisories/GHSA-jq66-xh47-j9f3).

src/Microsoft.Security.Utilities.Core/SecretMasker.cs

@@ -32,6 +32,10 @@ internal static Version RetrieveVersion()
         return new Version(version.Major, version.Minor, version.Build);
     }
 
+    public SecretMasker() : this (default, default, default, default, default)
+    {
+    }
+
     public SecretMasker(IEnumerable<RegexPattern>? regexSecrets,
                         bool generateCorrelatingIds = false,
                         IRegexEngine? regexEngine = default,
@@ -56,11 +60,6 @@ public SecretMasker(IEnumerable<RegexPattern>? regexSecrets,
         DefaultLiteralRedactionToken = defaultLiteralRedactionToken ?? SecretLiteral.FallbackRedactionToken;
     }
 
-    public SecretMasker()
-        : this(new HashSet<RegexPattern>())
-    {
-    }
-
     // We don't permit secrets great than 5 characters in length to be
     // skipped at masking time. The secrets that will be ignored when
     // masking will N - 1 of this property value.
@@ -413,19 +412,19 @@ protected virtual void Dispose(bool disposing)
 
     public void AddPatterns(IEnumerable<RegexPattern> regexPatterns)
     {
-        foreach(var regexPattern in regexPatterns)
+        foreach (var regexPattern in regexPatterns)
         {
             AddRegex(regexPattern);
-        } 
+        }
     }
 
-
 
     private readonly bool m_generateCorrelatingIds;
-    protected readonly HashSet<LiteralEncoder> LiteralEncoders;
-    protected readonly HashSet<SecretLiteral> EncodedSecretLiterals;
-    protected readonly HashSet<SecretLiteral> ExplicitlyAddedSecretLiterals;
-    protected readonly ReaderWriterLockSlim SyncObject = new (LockRecursionPolicy.NoRecursion);
+    public HashSet<LiteralEncoder> LiteralEncoders { get; }
+    public HashSet<SecretLiteral> EncodedSecretLiterals { get; }
+    public HashSet<SecretLiteral> ExplicitlyAddedSecretLiterals { get; }
+
+    public ReaderWriterLockSlim SyncObject = new (LockRecursionPolicy.NoRecursion);
 
     private bool m_disposed;
 }

src/Microsoft.Security.Utilities.Core/UrlCredentials.cs

@@ -14,7 +14,7 @@ public UrlCredentials()
 
         Name = nameof(UrlCredentials);
 
-        Pattern = @"(ftps?|https?):\/\/(?:[^:@]+):(?<refine>[^:@?]+)@";
+        Pattern = @"(ftps?|https?):\/\/(?<refine>[^:@\/]+:[^:@?\/]+)@";
 
         DetectionMetadata = DetectionMetadata.MediumConfidence;
 
@@ -38,7 +38,18 @@ public override IEnumerable<string> GenerateTruePositiveExamples()
             $"http://{Guid.NewGuid()}:{Guid.NewGuid()}@example.com/",
             $"https://user:[email protected]",
             $"ftp://{Guid.NewGuid()}:{Guid.NewGuid()}@example.com/",
-            $"ftps://user:[email protected]"
+            $"ftps://user:[email protected]",
+            $"http://{Guid.NewGuid()}:{Guid.NewGuid()}@example.com/embedded:colon",
+            $"ftp://{Guid.NewGuid()}:{Guid.NewGuid()}@example.com/embedded:colon",
+        };
+    }
+
+    public override IEnumerable<string> GenerateFalsePositiveExamples()
+    {
+        return new[]
+        {
+            $"http://example.com/embedded:colon",
+            $"ftp://@example.com/embedded:colon",
         };
     }
 }

src/Tests.Microsoft.Security.Utilities.Core/SecretMaskerTests.cs

@@ -251,7 +251,7 @@ public void SecretMasker_UrlCredentialsAreMasked()
     {
         using var secretMasker = InitializeTestMasker();
         string input = "https://user:[email protected]";
-        string expected = "https://user:[email protected]";
+        string expected = "https://[email protected]";
 
         string actual = secretMasker.MaskSecrets(input);
         Assert.AreEqual(expected, actual);
@@ -265,7 +265,7 @@ public void IsUserInfoWithSpecialCharactersMaskedCorrectly()
         using var secretMasker = InitializeTestMasker();
 
         string input = @"https://user:pass4';.!&*()=,[email protected]";
-        string expected = "https://user:[email protected]";
+        string expected = "https://[email protected]";
         string actual = secretMasker.MaskSecrets(input);
 
         Assert.AreEqual(expected, actual);
@@ -277,7 +277,7 @@ public void IsUserInfoWithDigitsInNameMaskedCorrectly()
     {
         using var secretMasker = InitializeTestMasker();
         string input = @"https://username123:[email protected]";
-        string expected = "https://username123:[email protected]";
+        string expected = "https://[email protected]";
         string actual = secretMasker.MaskSecrets(input);
 
         Assert.AreEqual(expected, actual);
@@ -289,7 +289,7 @@ public void IsUserInfoWithLongPasswordAndNameMaskedCorrectly()
     {
         using var secretMasker = InitializeTestMasker();
         string input = @"https://username_loooooooooooooooooooooooooooooooooooooooooong:password_looooooooooooooooooooooooooooooooooooooooooooooooong@example.com";
-        string expected = "https://username_loooooooooooooooooooooooooooooooooooooooooong:[email protected]";
+        string expected = "https://[email protected]";
         string actual = secretMasker.MaskSecrets(input);
 
         Assert.AreEqual(expected, actual);
@@ -301,7 +301,7 @@ public void IsUserInfoWithEncodedCharactersInNameMaskedCorrectly()
     {
         using var secretMasker = InitializeTestMasker();
         string input = @"https://username%10%A3%F6:[email protected]";
-        string expected = "https://username%10%A3%F6:[email protected]";
+        string expected = "https://[email protected]";
         string actual = secretMasker.MaskSecrets(input);
 
         Assert.AreEqual(expected, actual);
@@ -313,7 +313,7 @@ public void IsUserInfoWithEncodedAndEscapedCharactersInNameMaskedCorrectly()
     {
         using var secretMasker = InitializeTestMasker();
         string input = @"https://username%AZP2510%AZP25A3%AZP25F6:[email protected]";
-        string expected = "https://username%AZP2510%AZP25A3%AZP25F6:[email protected]";
+        string expected = "https://[email protected]";
         string actual = secretMasker.MaskSecrets(input);
 
         Assert.AreEqual(expected, actual);