C#: Add binlog support to buildless with source generator support

csharp/autobuilder/Semmle.Autobuild.CSharp/BUILD.bazel

@@ -13,6 +13,7 @@ codeql_csharp_binary(
         "//csharp/autobuilder/Semmle.Autobuild.Shared",
         "//csharp/extractor/Semmle.Extraction.CSharp",
         "//csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching",
+        "//csharp/extractor/Semmle.Extraction.CSharp.Driver:bin/Semmle.Extraction.CSharp.Driver",
         "//csharp/extractor/Semmle.Extraction.CSharp.Standalone:bin/Semmle.Extraction.CSharp.Standalone",
         "//csharp/extractor/Semmle.Util",
         "@paket.main//microsoft.build",

csharp/autobuilder/Semmle.Autobuild.CSharp/CSharpAutobuilder.cs

@@ -13,8 +13,10 @@ public class CSharpAutobuildOptions : AutobuildOptionsShared
     {
         private const string buildModeEnvironmentVariable = "CODEQL_EXTRACTOR_CSHARP_BUILD_MODE";
         internal const string ExtractorOptionBuildless = "CODEQL_EXTRACTOR_CSHARP_OPTION_BUILDLESS";
+        internal const string ExtractorOptionBinlog = "CODEQL_EXTRACTOR_CSHARP_OPTION_BINLOG";
 
         public bool Buildless { get; }
+        public string? Binlog { get; }
 
         public override Language Language => Language.CSharp;
 
@@ -29,7 +31,7 @@ public CSharpAutobuildOptions(IBuildActions actions) : base(actions)
                 actions.GetEnvironmentVariable(ExtractorOptionBuildless).AsBool("buildless", false) ||
                 actions.GetEnvironmentVariable(buildModeEnvironmentVariable)?.ToLower() == "none";
 
-
+            Binlog = actions.GetEnvironmentVariable(ExtractorOptionBinlog);
         }
     }
 

csharp/autobuilder/Semmle.Autobuild.CSharp/Semmle.Autobuild.CSharp.csproj

@@ -6,6 +6,7 @@
     <ProjectReference Include="..\..\extractor\Semmle.Util\Semmle.Util.csproj" />
     <ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp\Semmle.Extraction.CSharp.csproj" />
     <ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp.Standalone\Semmle.Extraction.CSharp.Standalone.csproj" />
+    <ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp.Driver\Semmle.Extraction.CSharp.Driver.csproj" />
     <ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp.DependencyFetching\Semmle.Extraction.CSharp.DependencyFetching.csproj" />
     <ProjectReference Include="..\Semmle.Autobuild.Shared\Semmle.Autobuild.Shared.csproj" />
   </ItemGroup>

csharp/autobuilder/Semmle.Autobuild.CSharp/StandaloneBuildRule.cs

@@ -10,7 +10,9 @@ internal class StandaloneBuildRule : IBuildRule<CSharpAutobuildOptions>
     {
         public BuildScript Analyse(IAutobuilder<CSharpAutobuildOptions> builder, bool auto)
         {
-            return BuildScript.Create(_ => Semmle.Extraction.CSharp.Standalone.Program.Main([]));
+            return builder.Options.Binlog is string binlog
+                ? BuildScript.Create(_ => Semmle.Extraction.CSharp.Driver.Main(["--binlog", binlog]))
+                : BuildScript.Create(_ => Semmle.Extraction.CSharp.Standalone.Program.Main([]));
         }
     }
 }

csharp/codeql-extractor.yml

@@ -65,3 +65,9 @@ options:
             - progress+++
         type: string
         pattern: "^(off|errors|warnings|(info|progress)|(debug|progress\\+)|(trace|progress\\+\\+)|progress\\+\\+\\+)$"
+  binlog:
+    title: Binlog
+    description: >
+      [EXPERIMENTAL] The value is a path to the MsBuild binary log file that should be extracted.
+      This option only works when `--build-mode none` is also specified.
+    type: string

csharp/extractor/Semmle.Extraction.CSharp.Driver/BUILD.bazel

@@ -8,7 +8,7 @@ codeql_csharp_binary(
     srcs = glob([
         "*.cs",
     ]),
-    visibility = ["//csharp:__pkg__"],
+    visibility = ["//csharp:__subpackages__"],
     deps = [
         "//csharp/extractor/Semmle.Extraction.CSharp",
     ],

csharp/extractor/Semmle.Extraction.CSharp/BUILD.bazel

@@ -19,6 +19,7 @@ codeql_csharp_library(
         "//csharp/extractor/Semmle.Extraction",
         "//csharp/extractor/Semmle.Extraction.CSharp.Util",
         "//csharp/extractor/Semmle.Util",
+        "@paket.main//basic.compilerlog.util",
         "@paket.main//microsoft.build",
         "@paket.main//microsoft.codeanalysis.csharp",
     ],

csharp/extractor/Semmle.Extraction.CSharp/Entities/File.cs

@@ -34,7 +34,14 @@ public override void Populate(TextWriter trapFile)
                         lineCounts.Total++;
 
                     trapFile.numlines(this, lineCounts);
-                    Context.TrapWriter.Archive(originalPath, TransformedPath, text.Encoding ?? System.Text.Encoding.Default);
+                    if (BinaryLogExtractionContext.GetAdjustedPath(Context.ExtractionContext, originalPath) is not null)
+                    {
+                        Context.TrapWriter.ArchiveContent(rawText, TransformedPath);
+                    }
+                    else
+                    {
+                        Context.TrapWriter.Archive(originalPath, TransformedPath, text.Encoding ?? System.Text.Encoding.Default);
+                    }
                 }
             }
             else if (IsPossiblyTextFile())

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs

@@ -185,7 +185,8 @@ private void DoExtractTree(SyntaxTree tree)
             {
                 var stopwatch = new Stopwatch();
                 stopwatch.Start();
-                var sourcePath = tree.FilePath;
+                var sourcePath = BinaryLogExtractionContext.GetAdjustedPath(ExtractionContext, tree.FilePath) ?? tree.FilePath;
+
                 var transformedSourcePath = PathTransformer.Transform(sourcePath);
 
                 var trapPath = transformedSourcePath.GetTrapPath(Logger, options.TrapCompression);

csharp/extractor/Semmle.Extraction.CSharp/Extractor/BinaryLogAnalyser.cs

@@ -0,0 +1,29 @@
+using System.Collections.Generic;
+using Microsoft.CodeAnalysis.CSharp;
+using Semmle.Util;
+using Semmle.Util.Logging;
+
+namespace Semmle.Extraction.CSharp
+{
+    public class BinaryLogAnalyser : Analyser
+    {
+        public BinaryLogAnalyser(IProgressMonitor pm, ILogger logger, PathTransformer pathTransformer, IPathCache pathCache, bool addAssemblyTrapPrefix)
+            : base(pm, logger, pathTransformer, pathCache, addAssemblyTrapPrefix)
+        {
+        }
+
+        public void Initialize(
+            string cwd, string[] args, string outputPath, CSharpCompilation compilation,
+            IEnumerable<Microsoft.CodeAnalysis.SyntaxTree> generatedSyntaxTrees,
+            string compilationIdentifier, CommonOptions options)
+        {
+            base.compilation = compilation;
+            ExtractionContext = new BinaryLogExtractionContext(
+                cwd, args, outputPath, generatedSyntaxTrees, compilationIdentifier,
+                Logger, PathTransformer, options.QlTest);
+            this.options = options;
+            LogExtractorInfo();
+            SetReferencePaths();
+        }
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs

@@ -8,6 +8,7 @@
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
+using Basic.CompilerLog.Util;
 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.CSharp;
 using Microsoft.CodeAnalysis.Text;
@@ -102,55 +103,154 @@
 
             try
             {
-                if (options.ProjectsToLoad.Any())
+                var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
+                var pathTransformer = new PathTransformer(canonicalPathCache);
+
+                if (options.BinaryLogPath is string binlogPath)
                 {
-                    AddSourceFilesFromProjects(options.ProjectsToLoad, options.CompilerArguments, logger);
+                    logger.LogInfo(" Running binary log analysis.");
+                    return RunBinaryLogAnalysis(analyzerStopwatch, options, binlogPath, logger, canonicalPathCache, pathTransformer);
                 }
-
-                var compilerVersion = new CompilerVersion(options);
-                if (compilerVersion.SkipExtraction)
+                else
                 {
-                    logger.LogWarning($"  Unrecognized compiler '{compilerVersion.SpecifiedCompiler}' because {compilerVersion.SkipReason}");
-                    return ExitCode.Ok;
+                    logger.LogInfo(" Running tracing analysis.");
+                    return RunTracingAnalysis(analyzerStopwatch, options, logger, canonicalPathCache, pathTransformer);
                 }
+            }
+            catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
+            {
+                logger.LogError($"  Unhandled exception: {ex}");
+                return ExitCode.Errors;
+            }
+        }
 
-                var workingDirectory = Directory.GetCurrentDirectory();
-                var compilerArgs = options.CompilerArguments.ToArray();
+        private static ExitCode RunBinaryLogAnalysis(Stopwatch stopwatch, Options options, string binlogPath, ILogger logger, CanonicalPathCache canonicalPathCache, PathTransformer pathTransformer)
+        {
+            logger.LogInfo($"Reading compiler calls from binary log {binlogPath}");
+            try
+            {
+                using var fileStream = new FileStream(binlogPath, FileMode.Open, FileAccess.Read, FileShare.Read);
+                using var reader = BinaryLogReader.Create(fileStream);
 
-                var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
-                var pathTransformer = new PathTransformer(canonicalPathCache);
+                // Filter out compiler calls that aren't interesting for examination
+                static bool filter(CompilerCall compilerCall)
+                {
+                    return compilerCall.IsCSharp &&
+                        compilerCall.Kind == CompilerCallKind.Regular;
+                }
 
-                using var analyser = new TracingAnalyser(new LogProgressMonitor(logger), logger, pathTransformer, canonicalPathCache, options.AssemblySensitiveTrap);
+                var allCompilationData = reader.ReadAllCompilationData(filter);
+                var allFailed = true;
 
-                var compilerArguments = CSharpCommandLineParser.Default.Parse(
-                    compilerVersion.ArgsWithResponse,
-                    workingDirectory,
-                    compilerVersion.FrameworkPath,
-                    compilerVersion.AdditionalReferenceDirectories
-                    );
+                logger.LogInfo($"  Found {allCompilationData.Count} compilations in binary log");
 
-                if (compilerArguments is null)
+                foreach (var compilationData in allCompilationData)
                 {
-                    var sb = new StringBuilder();
-                    sb.Append("  Failed to parse command line: ").AppendList(" ", compilerArgs);
-                    logger.LogError(sb.ToString());
-                    ++analyser.CompilationErrors;
-                    return ExitCode.Failed;
-                }
+                    if (compilationData.GetCompilationAfterGenerators() is not CSharpCompilation compilation)
+                    {
+                        logger.LogError("  Compilation data is not C#");
+                        continue;
+                    }
 
-                if (!analyser.BeginInitialize(compilerVersion.ArgsWithResponse))
-                {
-                    logger.LogInfo("Skipping extraction since files have already been extracted");
-                    return ExitCode.Ok;
+                    var compilerCall = compilationData.CompilerCall;
+                    var diagnosticName = compilerCall.GetDiagnosticName();
+                    logger.LogInfo($"  Processing compilation {diagnosticName} at {compilerCall.ProjectDirectory}");
+                    var compilerArgs = compilerCall.GetArguments();
+
+                    var compilationIdentifierPath = string.Empty;
+                    try
+                    {
+                        compilationIdentifierPath = FileUtils.ConvertPathToSafeRelativePath(
+                            Path.GetRelativePath(Directory.GetCurrentDirectory(), compilerCall.ProjectDirectory));
+                    }
+                    catch (ArgumentException exc)
+                    {
+                        logger.LogWarning($"  Failed to get relative path for {compilerCall.ProjectDirectory} from current working directory {Directory.GetCurrentDirectory()}: {exc.Message}");
+                    }
+
+                    var args = reader.ReadCommandLineArguments(compilerCall);
+                    var generatedSyntaxTrees = compilationData.GetGeneratedSyntaxTrees();
+
+                    using var analyser = new BinaryLogAnalyser(new LogProgressMonitor(logger), logger, pathTransformer, canonicalPathCache, options.AssemblySensitiveTrap);
+
+                    var exit = Analyse(stopwatch, analyser, options,
+                        references => [() => compilation.References.ForEach(r => references.Add(r))],
+                        (analyser, syntaxTrees) => [() => syntaxTrees.AddRange(compilation.SyntaxTrees)],
+                        (syntaxTrees, references) => compilation,
+                        (compilation, options) => analyser.Initialize(
+                            compilerCall.ProjectDirectory,
+                            compilerArgs?.ToArray() ?? [],
+                            TracingAnalyser.GetOutputName(compilation, args),
+                            compilation,
+                            generatedSyntaxTrees,
+                            Path.Combine(compilationIdentifierPath, diagnosticName),
+                            options),
+                        () => { });
+
+                    switch (exit)
+                    {
+                        case ExitCode.Ok:
+                            allFailed &= false;
+                            logger.LogInfo($"  Compilation {diagnosticName} succeeded");
+                            break;
+                        case ExitCode.Errors:
+                            allFailed &= false;
+                            logger.LogWarning($"  Compilation {diagnosticName} had errors");
+                            break;
+                        case ExitCode.Failed:
+                            logger.LogWarning($"  Compilation {diagnosticName} failed");
+                            break;
+                    }
                 }
+                return allFailed ? ExitCode.Failed : ExitCode.Ok;
+            }
+            catch (IOException ex)
+            {
+                logger.LogError($"Failed to open binary log: {ex.Message}");
+                return ExitCode.Failed;
+            }
+        }
 
-                return AnalyseTracing(workingDirectory, compilerArgs, analyser, compilerArguments, options, analyzerStopwatch);
+        private static ExitCode RunTracingAnalysis(Stopwatch analyzerStopwatch, Options options, ILogger logger, CanonicalPathCache canonicalPathCache, PathTransformer pathTransformer)
+        {
+            if (options.ProjectsToLoad.Any())
+            {
+                AddSourceFilesFromProjects(options.ProjectsToLoad, options.CompilerArguments, logger);
             }
-            catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
+
+            var compilerVersion = new CompilerVersion(options);
+            if (compilerVersion.SkipExtraction)
             {
-                logger.LogError($"  Unhandled exception: {ex}");
-                return ExitCode.Errors;
+                logger.LogWarning($"  Unrecognized compiler '{compilerVersion.SpecifiedCompiler}' because {compilerVersion.SkipReason}");
+                return ExitCode.Ok;
+            }
+
+            var workingDirectory = Directory.GetCurrentDirectory();
+            var compilerArgs = options.CompilerArguments.ToArray();
+            using var analyser = new TracingAnalyser(new LogProgressMonitor(logger), logger, pathTransformer, canonicalPathCache, options.AssemblySensitiveTrap);
+            var compilerArguments = CSharpCommandLineParser.Default.Parse(
+                compilerVersion.ArgsWithResponse,
+                workingDirectory,
+                compilerVersion.FrameworkPath,
+                compilerVersion.AdditionalReferenceDirectories
+                );
+
+            if (compilerArguments is null)
+            {
+                var sb = new StringBuilder();
+                sb.Append("  Failed to parse command line: ").AppendList(" ", compilerArgs);
+                logger.LogError(sb.ToString());
+                ++analyser.CompilationErrors;
+                return ExitCode.Failed;
+            }
+
+            if (!analyser.BeginInitialize(compilerVersion.ArgsWithResponse))
+            {
+                logger.LogInfo("Skipping extraction since files have already been extracted");
+                return ExitCode.Ok;
             }
+
+            return AnalyseTracing(workingDirectory, compilerArgs, analyser, compilerArguments, options, analyzerStopwatch);
         }
 
         private static void AddSourceFilesFromProjects(IEnumerable<string> projectsToLoad, IList<string> compilerArguments, ILogger logger)

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Options.cs

@@ -32,6 +32,11 @@ public sealed class Options : CommonOptions
         /// </summary>
         public bool AssemblySensitiveTrap { get; private set; } = false;
 
+        /// <summary>
+        /// The path to the binary log file, or null if unspecified.
+        /// </summary>
+        public string? BinaryLogPath { get; set; }
+
         public static Options CreateWithEnvironment(string[] arguments)
         {
             var options = new Options();
@@ -65,6 +70,9 @@ public override bool HandleOption(string key, string value)
                 case "load-sources-from-project":
                     ProjectsToLoad.Add(value);
                     return true;
+                case "binlog":
+                    BinaryLogPath = value;
+                    return true;
                 default:
                     return base.HandleOption(key, value);
             }

csharp/extractor/Semmle.Extraction.CSharp/Extractor/TracingAnalyser.cs

@@ -107,11 +107,8 @@ private bool LogRoslynArgs(IEnumerable<string> roslynArgs)
         /// <summary>
         /// Determine the path of the output dll/exe.
         /// </summary>
-        /// <param name="compilation">Information about the compilation.</param>
-        /// <param name="cancel">Cancellation token required.</param>
-        /// <returns>The filename.</returns>
-        private static string GetOutputName(CSharpCompilation compilation,
-            CSharpCommandLineArguments commandLineArguments)
+        internal static string GetOutputName(CSharpCompilation compilation,
+            CommandLineArguments commandLineArguments)
         {
             // There's no apparent way to access the output filename from the compilation,
             // so we need to re-parse the command line arguments.

csharp/extractor/Semmle.Extraction.CSharp/paket.references

@@ -1,3 +1,3 @@
 Microsoft.Build
 Microsoft.CodeAnalysis.CSharp
-
+Basic.CompilerLog.Util

csharp/extractor/Semmle.Extraction/Entities/File.cs

@@ -8,7 +8,8 @@ protected File(Context cx, string path)
             : base(cx, path)
         {
             originalPath = path;
-            transformedPathLazy = new Lazy<PathTransformer.ITransformedPath>(() => Context.ExtractionContext.PathTransformer.Transform(originalPath));
+            var adjustedPath = BinaryLogExtractionContext.GetAdjustedPath(Context.ExtractionContext, originalPath) ?? path;
+            transformedPathLazy = new Lazy<PathTransformer.ITransformedPath>(() => Context.ExtractionContext.PathTransformer.Transform(adjustedPath));
         }
 
         protected readonly string originalPath;