Define a concept of Artifact Visibility

Documentation/ArcadeSdk.md

@@ -290,6 +290,7 @@ These tools are only used for build operations performed outside of the reposito
 Customization of Authenticode signing process.
 
 Configurable item groups:
+
 - `ItemsToSign`
   List of files to sign in-place, during the build. May list individual files to sign (e.g. .dll, .exe, .ps1, etc.) as well as container files (.nupkg, .vsix, .zip, etc.). All files embedded in a container file are signed (recursively) unless specified otherwise.
 - `ItemsToSignPostBuild`
@@ -302,8 +303,12 @@ Configurable item groups:
   Specifies Authenticode certificate properties, such as whether a certificate allows dual signing.
 - `StrongNameSignInfo`
   Strong Name key to use to sign a specific managed assembly.
+- `Artifact`
+  List of files to sign and publish, either in-place or post-build depending on `PostBuildSign`. May list packages and blobs to publish.
+  More documentation available in the section on Publishing.props.
 
 Properties:
+
 - `AllowEmptySignList`
   True to allow ItemsToSign to be empty (the repository doesn't have any file to sign in-build).
 - `AllowEmptyPostBuildSignList`
@@ -321,6 +326,16 @@ To change the key used for strong-naming assemblies see `StrongNameKeyId` proper
 
 Customization of publishing process.
 
+Configurable item groups:
+
+- `Artifact`: List of files to publish. May list packages and blobs to publish.
+  Supported Metadata:
+  - `Visibility`
+    - Visibility of the artifact. Default is `External`. Visibility options are listed below:
+      - `External`: The artifact is visible outside of the build. It will be uploaded to AzDO artifacts and published to [the Build Asset Registry (BAR)](./Maestro/BuildAssetRegistry.md) as an artifact of the build.
+      - `Internal`: The artifact is visible only within the build. It will be uploaded to AzDO artifacts but not published to BAR.
+      - `Vertical`: The artifact is only visible within a given vertical in a Vertical Build. It will be copied into the local artifacts folder for the repository on disk during a vertical build. It will not be uploaded to AzDO artifacts, nor published to BAR. Any artifacts marked with `Vertical` visibility are also not available in any other Join Points in a Unified Build. See [here](./UnifiedBuild/Unified-Build-Join-Point.md) for more information. `Vertical` visibility is only available in a Vertical Build.
+
 ### /eng/AfterSolutionBuild.targets (optional)
 
 Targets executed in a step right after the solution is built.

src/Microsoft.DotNet.Arcade.Sdk/tools/Publish.proj

@@ -216,10 +216,32 @@
       <ItemsToPushToBlobFeed Include="@(Artifact)" Condition="'%(Artifact.SkipPublish)' != 'true' and '%(Extension)' != '.nupkg'" />
     </ItemGroup>
 
+    <ItemGroup>
+      <ItemsToPushToBlobFeed>
+        <!-- Default artifact visibility is External -->
+        <Visibility Condition="'%(ItemsToPushToBlobFeed.Visibility)' == ''">External</Visibility>
+        <!-- Default to IsShipping=true -->
+        <IsShipping Condition="'%(ItemsToPushToBlobFeed.IsShipping)' == ''">true</IsShipping>
+      </ItemsToPushToBlobFeed>
+    </ItemGroup>
+
     <ItemGroup>
       <ItemsToPushToBlobFeed>
         <ManifestArtifactData Condition="'%(ItemsToPushToBlobFeed.IsShipping)' == 'false'">%(ItemsToPushToBlobFeed.ManifestArtifactData);NonShipping=true</ManifestArtifactData>
-        <ManifestArtifactData Condition="'%(ItemsToPushToBlobFeed.IsShipping)' != 'false' and '$(ProducesDotNetReleaseShippingAssets)' == 'true'">%(ItemsToPushToBlobFeed.ManifestArtifactData);DotNetReleaseShipping=true</ManifestArtifactData>
+        <ManifestArtifactData Condition="'%(ItemsToPushToBlobFeed.IsShipping)' == 'true' and '$(ProducesDotNetReleaseShippingAssets)' == 'true'">%(ItemsToPushToBlobFeed.ManifestArtifactData);DotNetReleaseShipping=true</ManifestArtifactData>
+      </ItemsToPushToBlobFeed>
+    </ItemGroup>
+
+    <Error Condition="'@(ItemsToPublishToBlobFeed->WithMetadataValue('IsShipping','true')->AnyHaveMetadataValue('Visibility','Vertical'))' == 'true'"
+           Text="Visibility 'Vertical' is not supported for shipping artifacts." />
+    <Error Condition="'@(ItemsToPublishToBlobFeed->WithMetadataValue('IsShipping','true')->AnyHaveMetadataValue('Visibility','Internal'))' == 'true'"
+           Text="Visibility 'Internal' is not supported for shipping artifacts." />
+    <Error Condition="'@(ItemsToPublishToBlobFeed->AnyHaveMetadataValue('Visibility','Vertical'))' == 'true' and '$(DotNetBuild)' != 'true'"
+           Text="Visibility 'Vertical' is only supported in vertical builds." />
+
+    <ItemGroup>
+      <ItemsToPushToBlobFeed>
+        <ManifestArtifactData Condition="'%(ItemsToPushToBlobFeed.Visibility)' != ''">%(ItemsToPushToBlobFeed.ManifestArtifactData);Visibility=%(ItemsToPushToBlobFeed.Visibility)</ManifestArtifactData>
       </ItemsToPushToBlobFeed>
     </ItemGroup>
 

src/Microsoft.DotNet.Arcade.Sdk/tools/Sign.props

@@ -11,6 +11,7 @@
     <Artifact>
       <PublishFlatContainer>true</PublishFlatContainer>
       <IsShipping>true</IsShipping>
+      <Visibility>External</Visibility>
     </Artifact>
   </ItemDefinitionGroup>
 

src/Microsoft.DotNet.Build.Tasks.Feed/src/PushToBuildStorage.cs

@@ -218,6 +218,22 @@ public bool ExecuteTask(IFileSystem fileSystem,
                             .Where(blob => blob != null);
                     }
 
+                    // Filter out any "Internal" artifacts from the asset manifest.
+                    // We don't want to publish these outside of the build, so we exclude them from the manifest but still publish them.
+                    // We obviously want to include "External" artifacts as they are the ones we want to publish externally.
+                    packageArtifacts = packageArtifacts.Where(p => p.Visibility != ArtifactVisibility.Internal);
+                    blobArtifacts = blobArtifacts.Where(p => p.Visibility != ArtifactVisibility.Internal);
+
+                    if (!PushToLocalStorage)
+                    {
+                        // Inside the VMR, we publish to local storage to pass artifacts between repo sub-builds.
+                        // Outside the VMR, we publish to Azure DevOps artifacts storage.
+                        // "Vertical" artifacts should only ever be available within a build on a single machine, and should never be uploaded
+                        // to Azure DevOps artifacts storage.
+                        packageArtifacts = packageArtifacts.Where(p => p.Visibility != ArtifactVisibility.Vertical);
+                        blobArtifacts = blobArtifacts.Where(p => p.Visibility != ArtifactVisibility.Vertical);
+                    }
+
                     PublishingInfraVersion targetPublishingVersion = PublishingInfraVersion.Latest;
 
                     if (!string.IsNullOrEmpty(PublishingVersion))

src/VersionTools/Microsoft.DotNet.VersionTools.Tests/BuildManifest/ManifestModelTests.cs

@@ -569,6 +569,25 @@ public void PackageArtifactModelEquals_ReturnsFalseWhenObjectsAreDifferentTypes(
             Assert.False(packageArtifact.Equals("thisIsNotAPackageArtifact!"));
         }
 
+        [InlineData("Vertical")]
+        [InlineData("External")]
+        [InlineData("Internal")]
+        [InlineData("")]
+        [Theory]
+        public void PackageArtifactModel_Visibility(string visibility)
+        {
+            PackageArtifactModel packageArtifact = new PackageArtifactModel
+            {
+                Id = "AssetName",
+                Attributes = new Dictionary<string, string>
+                {
+                    { "Visibility", visibility }
+                }
+            };
+
+            Assert.Equal(visibility is "" ? ArtifactVisibility.External : (ArtifactVisibility)Enum.Parse(typeof(ArtifactVisibility), visibility), packageArtifact.Visibility);
+        }
+
         [Fact]
         public void BlobArtifactModelEquals_ReturnsTrueWhenTwoObjectsHaveMatchingAttributes()
         {
@@ -656,6 +675,25 @@ public void BlobArtifactModelEquals_ReturnsFalseWhenObjectsAreDifferentTypes()
             Assert.False(blobArtifact.Equals("thisIsNotABlobArtifact!"));
         }
 
+        [InlineData("Vertical")]
+        [InlineData("External")]
+        [InlineData("Internal")]
+        [InlineData("")]
+        [Theory]
+        public void BlobArtifactModel_Visibility(string visibility)
+        {
+            BlobArtifactModel blobArtifact = new BlobArtifactModel
+            {
+                Id = "AssetName",
+                Attributes = new Dictionary<string, string>
+                {
+                    { "Visibility", visibility }
+                }
+            };
+
+            Assert.Equal(visibility is "" ? ArtifactVisibility.External : (ArtifactVisibility)Enum.Parse(typeof(ArtifactVisibility), visibility), blobArtifact.Visibility);
+        }
+
         private BuildModel CreatePackageOnlyBuildManifestModel()
         {
             return new BuildModel(new BuildIdentity { Name = "SimpleBuildManifest", BuildId = "123" })

src/VersionTools/Microsoft.DotNet.VersionTools/BuildManifest/Model/ArtifactVisibility.cs

@@ -0,0 +1,30 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace Microsoft.DotNet.VersionTools.BuildManifest.Model
+{
+    /// <summary>
+    /// Description of how an artifact should be published between different builds in various different scenarios.
+    /// </summary>
+    public enum ArtifactVisibility
+    {
+        /// <summary>
+        /// The artifact should be published for external usage, whether as a shipping or non-shipping package.
+        /// </summary>
+        External,
+        /// <summary>
+        /// The artifact is used by different jobs within the same overall build, vertical or not.
+        /// The artifact should be uploaded to build artifacts, but should not be published to any NuGet/blob feeds.
+        /// </summary>
+        /// <remarks>
+        /// This visibility should be used for artifacts that must be flowed to jobs in a later build pass but not published.
+        /// </remarks>
+        Internal,
+        /// <summary>
+        /// The artifact is used by other repositories targeting the same target RID/platform.
+        /// In vertical builds, it should be published to the "on-disk" locations, but not to any NuGet/blob feeds.
+        /// In non-vertical builds, it should be treated as a non-shipping package with <see cref="External"/> visibility.
+        /// </summary>
+        Vertical,
+    }
+}

src/VersionTools/Microsoft.DotNet.VersionTools/BuildManifest/Model/BlobArtifactModel.cs

@@ -43,6 +43,30 @@ public string RepoOrigin
             set => Attributes[nameof(RepoOrigin)] = value;
         }
 
+        public ArtifactVisibility Visibility
+        {
+            get
+            {
+                string val = Attributes.GetOrDefault(nameof(Visibility));
+                if (string.IsNullOrEmpty(val))
+                {
+                    return ArtifactVisibility.External;
+                }
+                else if (Enum.TryParse(val, out ArtifactVisibility visibility))
+                {
+                    return visibility;
+                }
+                else
+                {
+                    throw new ArgumentException($"Invalid value for {nameof(Visibility)}: {val}");
+                }
+            }
+            set
+            {
+                Attributes[nameof(Visibility)] = value.ToString();
+            }
+        }
+
         public override string ToString() => $"Blob {Id}";
 
         public override bool Equals(object obj)

src/VersionTools/Microsoft.DotNet.VersionTools/BuildManifest/Model/PackageArtifactModel.cs

@@ -55,6 +55,30 @@ public bool NonShipping
             }
         }
 
+        public ArtifactVisibility Visibility
+        {
+            get
+            {
+                string val = Attributes.GetOrDefault(nameof(Visibility));
+                if (string.IsNullOrEmpty(val))
+                {
+                    return ArtifactVisibility.External;
+                }
+                else if (Enum.TryParse(val, out ArtifactVisibility visibility))
+                {
+                    return visibility;
+                }
+                else
+                {
+                    throw new ArgumentException($"Invalid value for {nameof(Visibility)}: {val}");
+                }
+            }
+            set
+            {
+                Attributes[nameof(Visibility)] = value.ToString();
+            }
+        }
+
         public string RepoOrigin
         {
             get => Attributes.GetOrDefault(nameof(RepoOrigin));