API changes pending ADO pipeline agent integration. (#113)

* More API changes pending ADO pipeline agent integration.:

* Add new tests

* Update generated rules files.

* Update release notes.

* Remove short secrets filtering.:

GeneratedRegexPatterns/MediumConfidenceSecurityModels.json

@@ -20,10 +20,11 @@
     "DetectionMetadata": "HighEntropy, MediumConfidence"
   },
   {
-    "Pattern": "https?:\\/\\/(?:[^:@]+):(?<refine>[^:@?]+)@",
+    "Pattern": "(ftps?|https?):\\/\\/(?:[^:@]+):(?<refine>[^:@?]+)@",
     "Id": "SEC101/127",
     "Name": "UrlCredentials",
     "Signatures": [
+      "ftp",
       "http"
     ],
     "DetectionMetadata": "MediumConfidence"

GeneratedRegexPatterns/UnclassifiedPotentialSecurityKeys.json

@@ -11,10 +11,11 @@
     "DetectionMetadata": "HighEntropy, MediumConfidence"
   },
   {
-    "Pattern": "https?:\\/\\/(?:[^:@]+):(?<refine>[^:@?]+)@",
+    "Pattern": "(ftps?|https?):\\/\\/(?:[^:@]+):(?<refine>[^:@?]+)@",
     "Id": "SEC101/127",
     "Name": "UrlCredentials",
     "Signatures": [
+      "ftp",
       "http"
     ],
     "DetectionMetadata": "MediumConfidence"

docs/ReleaseHistory.md

@@ -13,6 +13,12 @@
 
 # UNRELEASED
 - BUG: Merge multiple calls to `DateTime.UtcNow` in `GenerateCommonAnnotatedKey`, forcing year and month to agree. Add overload to provide an arbitrary allocation time, with bound checks (year 2024 to 2085).
+- BUG: Mark `SecretMasker(SecretMasker)` copy contructor as protected to make it callable by derived classes.
+- BUG: Mark `SecretMasker.Clone` as public virtual, to make it overridable by derived classes.
+- BUG: Update `SEC101/127.UrlCredentials` visibility to public to make it independently creatable.
+- BUG: Update `SEC101/154.AzureCacheForRedisIdentifiableKey` test example production to call base class (which generates test keys consisting of repeated characters in the randomized component).
+- BUG: Short-circuit `SecretMasker.DetectSecret(string)` operation if there are no configured regexes, encoded, or explicitly added secret literals.
+- FNS: Update `SEC101/127` regex to detect ftp(s) credentials.
 
 # 1.9.1 - 11/18/2024
 - DEP: Removed dependency of the `base-62` crate in the Rust codebase, since it depended on the `failure` crate which has a known [vulnerability](https://github.com/advisories/GHSA-jq66-xh47-j9f3).

src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_154_AzureCacheForRedisIdentifiableKey.cs

@@ -30,13 +30,8 @@ public override Tuple<string, string> GetMatchIdAndName(string match)
 
         public override IEnumerable<string> GenerateTruePositiveExamples()
         {
-            while (true)
-            {
-                string key =
-                    IdentifiableSecrets.GenerateStandardBase64Key(IdentifiableMetadata.AzureCacheForRedisChecksumSeed,
-                                                                  32,
-                                                                  IdentifiableMetadata.AzureCacheForRedisSignature);
-
+            foreach (string key in base.GenerateTruePositiveExamples())
+            { 
                 if (key.Contains("/") || key.Contains("+"))
                 {
                     continue;

src/Microsoft.Security.Utilities.Core/SecretMasker.cs

@@ -46,9 +46,9 @@ public SecretMasker(IEnumerable<RegexPattern>? regexSecrets,
 
         m_generateCorrelatingIds = generateCorrelatingIds;
 
-        m_explicitlyAddedSecretLiterals = new HashSet<SecretLiteral>();
-        m_encodedSecretLiterals = new HashSet<SecretLiteral>();
-        m_literalEncoders = new HashSet<LiteralEncoder>();
+        ExplicitlyAddedSecretLiterals = new HashSet<SecretLiteral>();
+        EncodedSecretLiterals = new HashSet<SecretLiteral>();
+        LiteralEncoders = new HashSet<LiteralEncoder>();
 
         _regexEngine = regexEngine ??= CachedDotNetRegex.Instance;
 
@@ -66,25 +66,25 @@ public SecretMasker()
     // masking will N - 1 of this property value.
     public static int MinimumSecretLengthCeiling { get; set; }
 
-    private SecretMasker(SecretMasker copy)
+    protected SecretMasker(SecretMasker copy)
     {
         // Read section.
         try
         {
-            copy.m_lock.EnterReadLock();
+            copy.SyncObject.EnterReadLock();
             MinimumSecretLength = copy.MinimumSecretLength;
             DefaultRegexRedactionToken = copy.DefaultRegexRedactionToken;
             DefaultLiteralRedactionToken = copy.DefaultLiteralRedactionToken;
             RegexPatterns = new HashSet<RegexPattern>(copy.RegexPatterns);
-            m_literalEncoders = new HashSet<LiteralEncoder>(copy.m_literalEncoders);
-            m_encodedSecretLiterals = new HashSet<SecretLiteral>(copy.m_encodedSecretLiterals);
-            m_explicitlyAddedSecretLiterals = new HashSet<SecretLiteral>(copy.m_explicitlyAddedSecretLiterals);
+            LiteralEncoders = new HashSet<LiteralEncoder>(copy.LiteralEncoders);
+            EncodedSecretLiterals = new HashSet<SecretLiteral>(copy.EncodedSecretLiterals);
+            ExplicitlyAddedSecretLiterals = new HashSet<SecretLiteral>(copy.ExplicitlyAddedSecretLiterals);
         }
         finally
         {
-            if (copy.m_lock.IsReadLockHeld)
+            if (copy.SyncObject.IsReadLockHeld)
             {
-                copy.m_lock.ExitReadLock();
+                copy.SyncObject.ExitReadLock();
             }
         }
     }
@@ -111,14 +111,14 @@ public virtual void AddRegex(RegexPattern regexSecret)
         // Write section.
         try
         {
-            m_lock.EnterWriteLock();
+            SyncObject.EnterWriteLock();
             _ = RegexPatterns.Add(regexSecret);
         }
         finally
         {
-            if (m_lock.IsWriteLockHeld)
+            if (SyncObject.IsWriteLockHeld)
             {
-                m_lock.ExitWriteLock();
+                SyncObject.ExitWriteLock();
             }
         }
     }
@@ -231,22 +231,22 @@ public void AddValue(string value)
         LiteralEncoder[] literalEncoders;
         try
         {
-            m_lock.EnterReadLock();
+            SyncObject.EnterReadLock();
 
             // Test whether already added.
-            if (m_explicitlyAddedSecretLiterals.Contains(secretLiterals[0]))
+            if (ExplicitlyAddedSecretLiterals.Contains(secretLiterals[0]))
             {
                 return;
             }
 
             // Read the value encoders.
-            literalEncoders = m_literalEncoders.ToArray();
+            literalEncoders = LiteralEncoders.ToArray();
         }
         finally
         {
-            if (m_lock.IsReadLockHeld)
+            if (SyncObject.IsReadLockHeld)
             {
-                m_lock.ExitReadLock();
+                SyncObject.ExitReadLock();
             }
         }
 
@@ -263,20 +263,20 @@ public void AddValue(string value)
         // Write section.
         try
         {
-            m_lock.EnterWriteLock();
+            SyncObject.EnterWriteLock();
 
             // Add the values.
-            _ = m_explicitlyAddedSecretLiterals.Add(secretLiterals[0]);
+            _ = ExplicitlyAddedSecretLiterals.Add(secretLiterals[0]);
             foreach (SecretLiteral secretLiteral in secretLiterals)
             {
-                _ = m_encodedSecretLiterals.Add(secretLiteral);
+                _ = EncodedSecretLiterals.Add(secretLiteral);
             }
         }
         finally
         {
-            if (m_lock.IsWriteLockHeld)
+            if (SyncObject.IsWriteLockHeld)
             {
-                m_lock.ExitWriteLock();
+                SyncObject.ExitWriteLock();
             }
         }
     }
@@ -291,21 +291,21 @@ public void AddLiteralEncoder(LiteralEncoder encoder)
         // Read section.
         try
         {
-            m_lock.EnterReadLock();
+            SyncObject.EnterReadLock();
 
-            if (m_literalEncoders.Contains(encoder))
+            if (LiteralEncoders.Contains(encoder))
             {
                 return;
             }
 
             // Read the original value secrets.
-            originalSecrets = m_explicitlyAddedSecretLiterals.ToArray();
+            originalSecrets = ExplicitlyAddedSecretLiterals.ToArray();
         }
         finally
         {
-            if (m_lock.IsReadLockHeld)
+            if (SyncObject.IsReadLockHeld)
             {
-                m_lock.ExitReadLock();
+                SyncObject.ExitReadLock();
             }
         }
 
@@ -323,22 +323,22 @@ public void AddLiteralEncoder(LiteralEncoder encoder)
         // Write section.
         try
         {
-            m_lock.EnterWriteLock();
+            SyncObject.EnterWriteLock();
 
             // Add the encoder.
-            _ = m_literalEncoders.Add(encoder);
+            _ = LiteralEncoders.Add(encoder);
 
             // Add the values.
             foreach (SecretLiteral encodedSecret in encodedSecrets)
             {
-                _ = m_encodedSecretLiterals.Add(encodedSecret);
+                _ = EncodedSecretLiterals.Add(encodedSecret);
             }
         }
         finally
         {
-            if (m_lock.IsWriteLockHeld)
+            if (SyncObject.IsWriteLockHeld)
             {
-                m_lock.ExitWriteLock();
+                SyncObject.ExitWriteLock();
             }
         }
     }
@@ -351,15 +351,16 @@ public IEnumerable<Detection> DetectSecrets(string input)
         }
 
         if (RegexPatterns.Count == 0 &&
-            m_explicitlyAddedSecretLiterals.Count == 0)
+            EncodedSecretLiterals.Count == 0 &&
+            ExplicitlyAddedSecretLiterals.Count == 0)
         {
             yield break;
         }
 
         // Read section.
         try
         {
-            m_lock.EnterReadLock();
+            SyncObject.EnterReadLock();
             var stopwatch = Stopwatch.StartNew();
 
             // Get indexes and lengths of all substrings that will be replaced.
@@ -371,7 +372,7 @@ public IEnumerable<Detection> DetectSecrets(string input)
                 }
             }
 
-            foreach (SecretLiteral secretLiteral in m_encodedSecretLiterals)
+            foreach (SecretLiteral secretLiteral in EncodedSecretLiterals)
             {
                 foreach (var detection in secretLiteral.GetDetections(input, DefaultLiteralRedactionToken))
                 {
@@ -383,9 +384,9 @@ public IEnumerable<Detection> DetectSecrets(string input)
         }
         finally
         {
-            if (m_lock.IsReadLockHeld)
+            if (SyncObject.IsReadLockHeld)
             {
-                m_lock.ExitReadLock();
+                SyncObject.ExitReadLock();
             }
         }
     }
@@ -396,7 +397,7 @@ public void Dispose()
         GC.SuppressFinalize(this);
     }
 
-    internal SecretMasker Clone()
+    public virtual SecretMasker Clone()
     {
         return new SecretMasker(this);
     }
@@ -405,7 +406,7 @@ protected virtual void Dispose(bool disposing)
     {
         if (disposing && !m_disposed)
         {
-            m_lock.Dispose();
+            SyncObject.Dispose();
             m_disposed = true;
         }
     }
@@ -418,11 +419,13 @@ public void AddPatterns(IEnumerable<RegexPattern> regexPatterns)
         } 
     }
 
+
+
     private readonly bool m_generateCorrelatingIds;
-    private readonly HashSet<LiteralEncoder> m_literalEncoders;
-    private readonly HashSet<SecretLiteral> m_encodedSecretLiterals;
-    private readonly HashSet<SecretLiteral> m_explicitlyAddedSecretLiterals;
-    private readonly ReaderWriterLockSlim m_lock = new(LockRecursionPolicy.NoRecursion);
+    protected readonly HashSet<LiteralEncoder> LiteralEncoders;
+    protected readonly HashSet<SecretLiteral> EncodedSecretLiterals;
+    protected readonly HashSet<SecretLiteral> ExplicitlyAddedSecretLiterals;
+    protected readonly ReaderWriterLockSlim SyncObject = new (LockRecursionPolicy.NoRecursion);
 
     private bool m_disposed;
 }

src/Microsoft.Security.Utilities.Core/UrlCredentials.cs

@@ -6,31 +6,39 @@
 
 namespace Microsoft.Security.Utilities;
 
-internal sealed class UrlCredentials : RegexPattern
+public sealed class UrlCredentials : RegexPattern
 {
     public UrlCredentials()
     {
         Id = "SEC101/127";
 
         Name = nameof(UrlCredentials);
 
-        Pattern = @"https?:\/\/(?:[^:@]+):(?<refine>[^:@?]+)@";
+        Pattern = @"(ftps?|https?):\/\/(?:[^:@]+):(?<refine>[^:@?]+)@";
 
         DetectionMetadata = DetectionMetadata.MediumConfidence;
 
         Signatures = new HashSet<string>(new[]
         {
+            "ftp",
             "http"
         });
     }
 
+    public override Tuple<string, string> GetMatchIdAndName(string match)
+    {
+        return base.GetMatchIdAndName(match);
+    }
+
     public override IEnumerable<string> GenerateTruePositiveExamples()
     {
         // https://tools.ietf.org/html/rfc3986#section-3.2)
         return new[]
         {
             $"http://{Guid.NewGuid()}:{Guid.NewGuid()}@example.com/",
-            $"https://user:[email protected]"
+            $"https://user:[email protected]",
+            $"ftp://{Guid.NewGuid()}:{Guid.NewGuid()}@example.com/",
+            $"ftps://user:[email protected]"
         };
     }
 }