From 173e2d6b689d3c457574ed3ca6872c0592b3afcf Mon Sep 17 00:00:00 2001 From: Harry Li <110055355+harryli0108@users.noreply.github.com> Date: Tue, 26 Mar 2024 15:21:12 -0700 Subject: [PATCH] Moved 1ES compliant pipeline templates to a separate folder temporarily (#2397) * Convert Bases Image pipelines to 1ES Template * Update node * Update node * Remove empty pipelineArtifact * Convert Oryx-PlatformBinary-Php * Convert more PlatformBinaries * Add python base image * Fix merge conflicts * Temporarily moved 1ES pipeline templates to separate folder and kept old templates --------- Co-authored-by: Dan Vouaux Co-authored-by: daniv-msft <48293037+daniv-msft@users.noreply.github.com> --- .../PlatformBinaries/dotnetcore.yml | 120 ++++++++ .../1ESPipelines/PlatformBinaries/golang.yml | 106 +++++++ .../1ESPipelines/PlatformBinaries/java.yml | 162 +++++++++++ .../1ESPipelines/PlatformBinaries/node.yml | 120 ++++++++ .../1ESPipelines/PlatformBinaries/php.yml | 120 ++++++++ .../1ESPipelines/PlatformBinaries/python.yml | 120 ++++++++ .../1ESPipelines/PlatformBinaries/ruby.yml | 106 +++++++ vsts/pipelines/1ESPipelines/agentCleanUp.yml | 25 ++ .../1ESPipelines/baseImages/dotnetcore.yml | 67 +++++ .../1ESPipelines/baseImages/node.yml | 65 +++++ .../1ESPipelines/baseImages/php-fpm.yml | 67 +++++ .../pipelines/1ESPipelines/baseImages/php.yml | 67 +++++ .../1ESPipelines/baseImages/python.yml | 58 ++++ vsts/pipelines/1ESPipelines/ci-detector.yml | 115 ++++++++ vsts/pipelines/1ESPipelines/ci.yml | 269 ++++++++++++++++++ vsts/pipelines/1ESPipelines/ciDebug.yml | 196 +++++++++++++ .../1ESPipelines/copyAllBlobsToProd.yml | 54 ++++ .../copySdksFromProdToStorageAccount.yml | 62 ++++ vsts/pipelines/1ESPipelines/nightly.yml | 235 +++++++++++++++ .../1ESPipelines/publishSdkToProd.yml | 75 +++++ .../templates/_agentCleanUpJobTemplate.yml | 13 + .../1ESPipelines/templates/_buildTemplate.yml | 211 ++++++++++++++ .../templates/_buildTemplateDetector.yml | 52 ++++ .../templates/_builderTemplate.yml | 41 +++ .../templates/_buildimageBasesJobTemplate.yml | 40 +++ .../templates/_cleanImageCacheTemplate.yml | 18 ++ .../templates/_copyImgFromDevToTest.yml | 246 ++++++++++++++++ .../templates/_integrationJobTemplate.yml | 17 ++ .../_platformBinariesReleaseTemplate.yml | 32 +++ .../templates/_platformBinariesTemplate.yml | 27 ++ .../_releaseBaseImagesJobTemplate.yml | 61 ++++ .../_releaseBaseImagesStepTemplate.yml | 54 ++++ .../templates/_releaseJobTemplate.yml | 57 ++++ .../templates/_releaseStepTemplate.yml | 124 ++++++++ .../templates/_securityChecks.yml | 41 +++ .../1ESPipelines/templates/_setReleaseTag.yml | 18 ++ .../1ESPipelines/templates/_signBinary.yml | 82 ++++++ .../templates/_signBinaryDetector.yml | 71 +++++ .../_dotnetcoreIntegrationJobTemplate.yml | 51 ++++ .../_golangIntegrationJobTemplate.yml | 50 ++++ .../_nodeIntegrationJobTemplate.yml | 49 ++++ .../_phpIntegrationJobTemplate.yml | 49 ++++ .../_pythonIntegrationJobTemplate.yml | 57 ++++ .../1ESPipelines/testACRImgUpload.yml | 28 ++ .../1ESPipelines/validation-detector.yml | 22 ++ vsts/pipelines/1ESPipelines/validation.yml | 171 +++++++++++ 46 files changed, 3891 insertions(+) create mode 100644 vsts/pipelines/1ESPipelines/PlatformBinaries/dotnetcore.yml create mode 100644 vsts/pipelines/1ESPipelines/PlatformBinaries/golang.yml create mode 100644 vsts/pipelines/1ESPipelines/PlatformBinaries/java.yml create mode 100644 vsts/pipelines/1ESPipelines/PlatformBinaries/node.yml create mode 100644 vsts/pipelines/1ESPipelines/PlatformBinaries/php.yml create mode 100644 vsts/pipelines/1ESPipelines/PlatformBinaries/python.yml create mode 100644 vsts/pipelines/1ESPipelines/PlatformBinaries/ruby.yml create mode 100644 vsts/pipelines/1ESPipelines/agentCleanUp.yml create mode 100644 vsts/pipelines/1ESPipelines/baseImages/dotnetcore.yml create mode 100644 vsts/pipelines/1ESPipelines/baseImages/node.yml create mode 100644 vsts/pipelines/1ESPipelines/baseImages/php-fpm.yml create mode 100644 vsts/pipelines/1ESPipelines/baseImages/php.yml create mode 100644 vsts/pipelines/1ESPipelines/baseImages/python.yml create mode 100644 vsts/pipelines/1ESPipelines/ci-detector.yml create mode 100644 vsts/pipelines/1ESPipelines/ci.yml create mode 100644 vsts/pipelines/1ESPipelines/ciDebug.yml create mode 100644 vsts/pipelines/1ESPipelines/copyAllBlobsToProd.yml create mode 100644 vsts/pipelines/1ESPipelines/copySdksFromProdToStorageAccount.yml create mode 100644 vsts/pipelines/1ESPipelines/nightly.yml create mode 100644 vsts/pipelines/1ESPipelines/publishSdkToProd.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/_agentCleanUpJobTemplate.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/_buildTemplate.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/_buildTemplateDetector.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/_builderTemplate.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/_buildimageBasesJobTemplate.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/_cleanImageCacheTemplate.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/_copyImgFromDevToTest.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/_integrationJobTemplate.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/_platformBinariesReleaseTemplate.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/_platformBinariesTemplate.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/_releaseBaseImagesJobTemplate.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/_releaseBaseImagesStepTemplate.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/_releaseJobTemplate.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/_releaseStepTemplate.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/_securityChecks.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/_setReleaseTag.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/_signBinary.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/_signBinaryDetector.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/integrationTests/_dotnetcoreIntegrationJobTemplate.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/integrationTests/_golangIntegrationJobTemplate.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/integrationTests/_nodeIntegrationJobTemplate.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/integrationTests/_phpIntegrationJobTemplate.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/integrationTests/_pythonIntegrationJobTemplate.yml create mode 100644 vsts/pipelines/1ESPipelines/testACRImgUpload.yml create mode 100644 vsts/pipelines/1ESPipelines/validation-detector.yml create mode 100644 vsts/pipelines/1ESPipelines/validation.yml diff --git a/vsts/pipelines/1ESPipelines/PlatformBinaries/dotnetcore.yml b/vsts/pipelines/1ESPipelines/PlatformBinaries/dotnetcore.yml new file mode 100644 index 0000000000..c61bf3d570 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/PlatformBinaries/dotnetcore.yml @@ -0,0 +1,120 @@ +variables: +- group: Oryx +parameters: +- name: destinationStorageAccountName + displayName: Destination Storage Account Name + type: string + default: oryxsdksstaging +trigger: + batch: true + branches: + include: + - main + paths: + exclude: + - /* + include: + - platforms/dotnet + - vsts/PlatformBinaries/dotnetcore.yml +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + os: windows + customBuildTags: + - ES365AIMigrationTooling + stages: + - stage: Build + jobs: + - job: DotNetCore_Bookworm + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'dotnet' + debianFlavor: 'bookworm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: DotNetCore_Bullseye + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'dotnet' + debianFlavor: 'bullseye' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: DotNetCore_Buster + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'dotnet' + debianFlavor: 'buster' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: DotNetCore_Stretch + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'dotnet' + debianFlavor: 'stretch' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: DotNetCore_Ubuntu + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'dotnet' + debianFlavor: 'focal-scm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - stage: Release + dependsOn: Build + jobs: + - job: Publish_Platform_Binaries + timeoutInMinutes: 250 + displayName: Publish to Azure Blob Storage + steps: + - template: /vsts/pipelines/templates/_platformBinariesReleaseTemplate.yml@self + parameters: + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/PlatformBinaries/golang.yml b/vsts/pipelines/1ESPipelines/PlatformBinaries/golang.yml new file mode 100644 index 0000000000..4fc2046c69 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/PlatformBinaries/golang.yml @@ -0,0 +1,106 @@ +variables: +- group: Oryx +parameters: +- name: destinationStorageAccountName + displayName: Destination Storage Account Name + type: string + default: oryxsdksstaging +trigger: + batch: true + branches: + include: + - main + paths: + exclude: + - /* + include: + - platforms/golang + - vsts/PlatformBinaries/golang.yml +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + os: windows + customBuildTags: + - ES365AIMigrationTooling + stages: + - stage: Build + jobs: + - job: Golang_Bullseye + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'golang' + debianFlavor: 'bullseye' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Golang_Buster + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'golang' + debianFlavor: 'buster' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Golang_Stretch + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'golang' + debianFlavor: 'stretch' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Golang_Ubuntu + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'golang' + debianFlavor: 'focal-scm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - stage: Release + dependsOn: Build + jobs: + - job: Publish_Platform_Binaries + timeoutInMinutes: 250 + displayName: Publish to Azure Blob Storage + steps: + - template: /vsts/pipelines/templates/_platformBinariesReleaseTemplate.yml@self + parameters: + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/PlatformBinaries/java.yml b/vsts/pipelines/1ESPipelines/PlatformBinaries/java.yml new file mode 100644 index 0000000000..cd6513896b --- /dev/null +++ b/vsts/pipelines/1ESPipelines/PlatformBinaries/java.yml @@ -0,0 +1,162 @@ +variables: +- group: Oryx +parameters: +- name: destinationStorageAccountName + displayName: Destination Storage Account Name + type: string + default: oryxsdksstaging +trigger: + batch: true + branches: + include: + - main + paths: + exclude: + - /* + include: + - platforms/java + - vsts/PlatformBinaries/java.yml +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + os: windows + customBuildTags: + - ES365AIMigrationTooling + stages: + - stage: Build + jobs: + - job: Java_Bullseye + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'java' + debianFlavor: 'bullseye' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Java_Buster + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'java' + debianFlavor: 'buster' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Java_Stretch + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'java' + debianFlavor: 'stretch' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Java_Ubuntu + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'java' + debianFlavor: 'focal-scm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Maven_Bullseye + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'maven' + debianFlavor: 'bullseye' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Maven_Buster + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'maven' + debianFlavor: 'buster' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Maven_Stretch + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'maven' + debianFlavor: 'stretch' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Maven_Ubuntu + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'maven' + debianFlavor: 'focal-scm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - stage: Release + dependsOn: Build + jobs: + - job: Publish_Platform_Binaries + timeoutInMinutes: 250 + displayName: Publish to Azure Blob Storage + steps: + - template: /vsts/pipelines/templates/_platformBinariesReleaseTemplate.yml@self + parameters: + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/PlatformBinaries/node.yml b/vsts/pipelines/1ESPipelines/PlatformBinaries/node.yml new file mode 100644 index 0000000000..cc9ec7f03a --- /dev/null +++ b/vsts/pipelines/1ESPipelines/PlatformBinaries/node.yml @@ -0,0 +1,120 @@ +variables: +- group: Oryx +parameters: +- name: destinationStorageAccountName + displayName: Destination Storage Account Name + type: string + default: oryxsdksstaging +trigger: + batch: true + branches: + include: + - main + paths: + exclude: + - /* + include: + - platforms/nodejs + - vsts/PlatformBinaries/node.yml +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + os: windows + customBuildTags: + - ES365AIMigrationTooling + stages: + - stage: Build + jobs: + - job: Node_Bookworm + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'nodejs' + debianFlavor: 'bookworm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Node_Bullseye + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'nodejs' + debianFlavor: 'bullseye' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Node_Buster + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'nodejs' + debianFlavor: 'buster' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Node_Stretch + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'nodejs' + debianFlavor: 'stretch' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Node_Ubuntu + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'nodejs' + debianFlavor: 'focal-scm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - stage: Release + dependsOn: Build + jobs: + - job: Publish_Platform_Binaries + timeoutInMinutes: 250 + displayName: Publish to Azure Blob Storage + steps: + - template: /vsts/pipelines/templates/_platformBinariesReleaseTemplate.yml@self + parameters: + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/PlatformBinaries/php.yml b/vsts/pipelines/1ESPipelines/PlatformBinaries/php.yml new file mode 100644 index 0000000000..b6d58465b9 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/PlatformBinaries/php.yml @@ -0,0 +1,120 @@ +variables: +- group: Oryx +parameters: +- name: destinationStorageAccountName + displayName: Destination Storage Account Name + type: string + default: oryxsdksstaging +trigger: + batch: true + branches: + include: + - main + paths: + exclude: + - /* + include: + - platforms/php + - vsts/PlatformBinaries/php.yml +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + os: windows + customBuildTags: + - ES365AIMigrationTooling + stages: + - stage: Build + jobs: + - job: Php_Bookworm_SDK + timeoutInMinutes: 1440 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'php' + debianFlavor: 'bookworm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Php_Bullseye_SDK + timeoutInMinutes: 1440 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'php' + debianFlavor: 'bullseye' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Php_Stretch_SDK + timeoutInMinutes: 1440 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'php' + debianFlavor: 'stretch' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Php_Buster_SDK + timeoutInMinutes: 1440 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'php' + debianFlavor: 'buster' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Php_Ubuntu_SDK + timeoutInMinutes: 1440 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'php' + debianFlavor: 'focal-scm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - stage: Release + dependsOn: Build + jobs: + - job: Publish_Platform_Binaries + timeoutInMinutes: 250 + displayName: Publish to Azure Blob Storage + steps: + - template: /vsts/pipelines/templates/_platformBinariesReleaseTemplate.yml@self + parameters: + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/PlatformBinaries/python.yml b/vsts/pipelines/1ESPipelines/PlatformBinaries/python.yml new file mode 100644 index 0000000000..38cfa613b1 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/PlatformBinaries/python.yml @@ -0,0 +1,120 @@ +variables: +- group: Oryx +parameters: +- name: destinationStorageAccountName + displayName: Destination Storage Account Name + type: string + default: oryxsdksstaging +trigger: + batch: true + branches: + include: + - main + paths: + exclude: + - /* + include: + - platforms/python + - vsts/PlatformBinaries/python.yml +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + os: windows + customBuildTags: + - ES365AIMigrationTooling + stages: + - stage: Build + jobs: + - job: Python_Bookworm_SDK + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'python' + debianFlavor: 'bookworm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Python_Bullseye_SDK + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'python' + debianFlavor: 'bullseye' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Python_Buster_SDK + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'python' + debianFlavor: 'buster' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Python_Stretch_SDK + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'python' + debianFlavor: 'stretch' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Python_Ubuntu_SDK + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'python' + debianFlavor: 'focal-scm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - stage: Release + dependsOn: Build + jobs: + - job: Publish_Platform_Binaries + timeoutInMinutes: 250 + displayName: Publish to Azure Blob Storage + steps: + - template: /vsts/pipelines/templates/_platformBinariesReleaseTemplate.yml@self + parameters: + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/PlatformBinaries/ruby.yml b/vsts/pipelines/1ESPipelines/PlatformBinaries/ruby.yml new file mode 100644 index 0000000000..3c16e028cd --- /dev/null +++ b/vsts/pipelines/1ESPipelines/PlatformBinaries/ruby.yml @@ -0,0 +1,106 @@ +variables: +- group: Oryx +parameters: +- name: destinationStorageAccountName + displayName: Destination Storage Account Name + type: string + default: oryxsdksstaging +trigger: + batch: true + branches: + include: + - main + paths: + exclude: + - /* + include: + - platforms/ruby + - vsts/PlatformBinaries/ruby.yml +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + os: windows + customBuildTags: + - ES365AIMigrationTooling + stages: + - stage: Build + jobs: + - job: Ruby_Bullseye + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'ruby' + debianFlavor: 'bullseye' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Ruby_Buster + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'ruby' + debianFlavor: 'buster' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Ruby_Stretch + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'ruby' + debianFlavor: 'stretch' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Ruby_Ubuntu + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'ruby' + debianFlavor: 'focal-scm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - stage: Release + dependsOn: Build + jobs: + - job: Publish_Platform_Binaries + timeoutInMinutes: 250 + displayName: Publish to Azure Blob Storage + steps: + - template: /vsts/pipelines/templates/_platformBinariesReleaseTemplate.yml@self + parameters: + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/agentCleanUp.yml b/vsts/pipelines/1ESPipelines/agentCleanUp.yml new file mode 100644 index 0000000000..379f067a0e --- /dev/null +++ b/vsts/pipelines/1ESPipelines/agentCleanUp.yml @@ -0,0 +1,25 @@ +parameters: +- name: poolName + displayName: Pool Name + type: string + default: Azure Pipelines + values: + - Azure Pipelines +- name: agentName + displayName: Agent Name + type: string + +variables: + skipComponentGovernanceDetection: true + +stages: +- stage: CleanAgents + displayName: Clean Oryx Agents + jobs: + - template: templates/_agentCleanUpJobTemplate.yml + parameters: + agentName: ${{ parameters.agentName }} + poolName: ${{ parameters.poolName }} + +# trigger is set using vsts UI, cleanup build will run every sundays 10 PM PST/PDT +trigger: none \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/baseImages/dotnetcore.yml b/vsts/pipelines/1ESPipelines/baseImages/dotnetcore.yml new file mode 100644 index 0000000000..0695591eed --- /dev/null +++ b/vsts/pipelines/1ESPipelines/baseImages/dotnetcore.yml @@ -0,0 +1,67 @@ +variables: +- group: Oryx +- name: ascName + value: oryx-new-service-connection +- name: acrName + value: oryxdevmcr.azurecr.io +- name: skipComponentGovernanceDetection + value: true +- name: Packaging.EnableSBOMSigning + value: true +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + os: windows + customBuildTags: + - ES365AIMigrationTooling-BulkMigrated + stages: + - stage: stage + jobs: + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build DotNetCore runtime buster base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: dotnetcore + imageDebianFlavor: buster + artifactsFileName: dotnetcore-runtimeimage-bases-buster.txt + jobName: Build_DotNetCore_BaseImage_Buster + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build DotNetCore runtime bullseye base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: dotnetcore + imageDebianFlavor: bullseye + artifactsFileName: dotnetcore-runtimeimage-bases-bullseye.txt + jobName: Build_DotNetCore_BaseImage_Bullseye + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build DotNetCore runtime bookworm base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: dotnetcore + imageDebianFlavor: bookworm + artifactsFileName: dotnetcore-runtimeimage-bases-bookworm.txt + jobName: Build_DotNetCore_BaseImage_Bookworm + - job: Release_DotNetCoreRuntimeBaseImage + dependsOn: + - Build_DotNetCore_BaseImage_Buster + - Build_DotNetCore_BaseImage_Bullseye + - Build_DotNetCore_BaseImage_Bookworm + displayName: Push images to MCR + timeoutInMinutes: 250 + steps: + - template: /vsts/pipelines/templates/_releaseBaseImagesStepTemplate.yml@self + parameters: + baseImageName: 'dotnetcore' \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/baseImages/node.yml b/vsts/pipelines/1ESPipelines/baseImages/node.yml new file mode 100644 index 0000000000..7e89a2cd0c --- /dev/null +++ b/vsts/pipelines/1ESPipelines/baseImages/node.yml @@ -0,0 +1,65 @@ +variables: +- group: Oryx +- name: ascName + value: oryx-new-service-connection +- name: acrName + value: oryxdevmcr.azurecr.io +- name: skipComponentGovernanceDetection + value: true +- name: Packaging.EnableSBOMSigning + value: true +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + os: windows + customBuildTags: + - ES365AIMigrationTooling-BulkMigrated + stages: + - stage: stage + jobs: + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build Node runtime buster based images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: node + imageDebianFlavor: buster + artifactsFileName: node-runtimeimage-bases-buster.txt + jobName: Build_Buster_BaseImages + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build Node runtime bullseye base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: node + imageDebianFlavor: bullseye + artifactsFileName: node-runtimeimage-bases-bullseye.txt + jobName: Build_Bullseye_BaseImages + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build Node runtime bookworm base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: node + imageDebianFlavor: bookworm + artifactsFileName: node-runtimeimage-bases-bookworm.txt + jobName: Build_Bookworm_BaseImages + - job: Release_NodeRuntimeBaseImage + dependsOn: + - Build_Buster_BaseImages + displayName: Push images to MCR + timeoutInMinutes: 250 + steps: + - template: /vsts/pipelines/templates/_releaseBaseImagesStepTemplate.yml@self + parameters: + baseImageName: 'node' \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/baseImages/php-fpm.yml b/vsts/pipelines/1ESPipelines/baseImages/php-fpm.yml new file mode 100644 index 0000000000..95b62a21ca --- /dev/null +++ b/vsts/pipelines/1ESPipelines/baseImages/php-fpm.yml @@ -0,0 +1,67 @@ +variables: +- group: Oryx +- name: ascName + value: oryx-new-service-connection +- name: acrName + value: oryxdevmcr.azurecr.io +- name: skipComponentGovernanceDetection + value: true +- name: Packaging.EnableSBOMSigning + value: true +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + os: windows + customBuildTags: + - ES365AIMigrationTooling-BulkMigrated + stages: + - stage: stage + jobs: + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build php buster runtime base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: php-fpm + imageDebianFlavor: buster + artifactsFileName: php-fpm-runtimeimage-bases-buster.txt + jobName: Build_PHP_FPM_Buster_Base + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build php bullseye runtime base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: php-fpm + imageDebianFlavor: bullseye + artifactsFileName: php-fpm-runtimeimage-bases-bullseye.txt + jobName: Build_PHP_Fpm_Bullseye_Base + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build php bookworm runtime base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: php-fpm + imageDebianFlavor: bookworm + artifactsFileName: php-fpm-runtimeimage-bases-bookworm.txt + jobName: Build_PHP_Fpm_Bookworm_Base + - job: Release_PhpRuntimeBaseImage + dependsOn: + - Build_Php_Fpm_Buster_Base + - Build_PHP_Fpm_Bullseye_Base + - Build_PHP_Fpm_Bookworm_Base + displayName: Push images to MCR + timeoutInMinutes: 250 + steps: + - template: /vsts/pipelines/templates/_releaseBaseImagesStepTemplate.yml@self + parameters: + baseImageName: 'php-fpm' diff --git a/vsts/pipelines/1ESPipelines/baseImages/php.yml b/vsts/pipelines/1ESPipelines/baseImages/php.yml new file mode 100644 index 0000000000..1ca3afd92b --- /dev/null +++ b/vsts/pipelines/1ESPipelines/baseImages/php.yml @@ -0,0 +1,67 @@ +variables: +- group: Oryx +- name: ascName + value: oryx-new-service-connection +- name: acrName + value: oryxdevmcr.azurecr.io +- name: skipComponentGovernanceDetection + value: true +- name: Packaging.EnableSBOMSigning + value: true +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + os: windows + customBuildTags: + - ES365AIMigrationTooling-BulkMigrated + stages: + - stage: stage + jobs: + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build php runtime buster base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: php + imageDebianFlavor: buster + artifactsFileName: php-runtimeimage-bases-buster.txt + jobName: Build_Buster_BaseImages + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build php runtime bullseye base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: php + imageDebianFlavor: bullseye + artifactsFileName: php-runtimeimage-bases-bullseye.txt + jobName: Build_Bullseye_BaseImages + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build php runtime bookworm base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: php + imageDebianFlavor: bookworm + artifactsFileName: php-runtimeimage-bases-bookworm.txt + jobName: Build_Bookworm_BaseImages + - job: Release_PhpRuntimeBaseImage + dependsOn: + - Build_Buster_BaseImages + - Build_Bullseye_BaseImages + - Build_Bookworm_BaseImages + displayName: Push images to MCR + timeoutInMinutes: 250 + steps: + - template: /vsts/pipelines/templates/_releaseBaseImagesStepTemplate.yml@self + parameters: + baseImageName: 'php' diff --git a/vsts/pipelines/1ESPipelines/baseImages/python.yml b/vsts/pipelines/1ESPipelines/baseImages/python.yml new file mode 100644 index 0000000000..86a0b8f73e --- /dev/null +++ b/vsts/pipelines/1ESPipelines/baseImages/python.yml @@ -0,0 +1,58 @@ +variables: +- group: Oryx +- name: ascName + value: oryx-new-service-connection +- name: acrName + value: oryxdevmcr.azurecr.io +- name: skipComponentGovernanceDetection + value: true +- name: Packaging.EnableSBOMSigning + value: true +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + os: windows + customBuildTags: + - ES365AIMigrationTooling-BulkMigrated + stages: + - stage: stage + jobs: + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build python runtime bullseye base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: python + imageDebianFlavor: bullseye + artifactsFileName: python-runtimeimage-bases-bullseye.txt + jobName: Build_Bullseye_BaseImages + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build python runtime bookworm base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: python + imageDebianFlavor: bookworm + artifactsFileName: python-runtimeimage-bases-bookworm.txt + jobName: Build_Bookworm_BaseImages + - job: Release_PythonRuntimeBaseImage + dependsOn: + - Build_Bullseye_BaseImages + - Build_Bookworm_BaseImages + displayName: Push images to MCR + timeoutInMinutes: 250 + steps: + - template: /vsts/pipelines/templates/_releaseBaseImagesStepTemplate.yml@self + parameters: + baseImageName: 'python' \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/ci-detector.yml b/vsts/pipelines/1ESPipelines/ci-detector.yml new file mode 100644 index 0000000000..39220d3c54 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/ci-detector.yml @@ -0,0 +1,115 @@ +resources: +- repo: self + +variables: +- group: Oryx + +stages: + - stage: ValidatePipeline + displayName: Validate Pipeline + jobs: + - job: ValidatePipeline + displayName: Validate pipeline + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + skipComponentGovernanceDetection: true + steps: + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection - OSS Compliance' + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/tests' + + - script: | + set -x + if [ "$(Build.Reason)" != "Manual" ]; then + echo + echo "Invalid configuration." + echo "Build should be Manual" + exit 1 + fi + if [ "$(Build.DefinitionName)" != "Oryx-Detector-CI" ]; then + echo + echo "Invalid configuration." + echo "Build definition name should be Oryx-Detector-CI." + exit 1 + fi + if [[ "$(Build.SourceBranch)" != "refs/heads/main" ]] \ + && [[ "$(Build.SourceBranch)" != refs/heads/patch* ]] \ + && [[ "$(Build.SourceBranch)" != refs/heads/exp* ]]; then + echo + echo "Invalid configuration." + echo 'Only 'main', 'patch/*' and 'exp/*' branches are allowed.' + exit 1 + fi + displayName: 'Validate pipeline run' + + - stage: Build + displayName: Build Stage + dependsOn: ValidatePipeline + jobs: + - job: Job_SignBinariesAndPackages + displayName: Sign Detector Binaries and NuGet packages + pool: + name: VSEngSS-MicroBuild2022-1ES + demands: + - msbuild + - visualstudio + variables: + skipComponentGovernanceDetection: true + steps: + - template: templates/_signBinaryDetector.yml + - job: Job_BuildAndTestDetector + displayName: Build and test detector + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: templates/_buildTemplateDetector.yml + + - stage: Release + displayName: Release Stage + dependsOn: Build + jobs: + - job: Release_GitHub + displayName: Create GitHub release + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + skipComponentGovernanceDetection: true + steps: + - checkout: self + clean: true + - task: DownloadPipelineArtifact@2 + displayName: 'Download artifacts to publish to release' + inputs: + artifactName: 'detector_signed' + path: $(Build.ArtifactStagingDirectory)/detector_signed + - bash: | + . build/detector/__detectorNugetPackagesVersions.sh + echo "Setting release name to '$releaseName'..." + echo "##vso[task.setvariable variable=RELEASE_NAME;]$VERSION" + displayName: 'Set relase name environment variable' + - bash: | + dotnet nuget push \ + $(Build.ArtifactStagingDirectory)/detector_signed/packages/*.nupkg \ + -s https://api.nuget.org/v3/index.json \ + --api-key $NUGETORG_API_KEY + env: + # Map the value that we got from keyvault/variablegroup to environment variable + NUGETORG_API_KEY: $(NUGETORG-API-KEY) + displayName: 'Push nuget packages to nuget.org' + - task: GitHubRelease@0 + displayName: 'GitHub release (create)' + inputs: + gitHubConnection: 'Oryx-GitHub' + repositoryName: microsoft/oryx + tagSource: manual + tag: '$(RELEASE_NAME)' + assets: $(Build.ArtifactStagingDirectory)/detector_signed/packages/*.nupkg +trigger: none \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/ci.yml b/vsts/pipelines/1ESPipelines/ci.yml new file mode 100644 index 0000000000..94195443a0 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/ci.yml @@ -0,0 +1,269 @@ +parameters: +- name: storageAccountUrl + displayName: SDK storage account URL for production images and testing + type: string + default: https://oryx-cdn.microsoft.io + values: + - https://oryx-cdn.microsoft.io +- name: buildImages + type: object + default: + - key: Latest + value: latest + - key: Jamstack + value: jamstack + - key: GithubActions + value: githubactions + - key: Full + value: full + - key: Cli + value: cli-stretch + - key: CliBuster + value: cli-buster + - key: CliBullseye + value: cli-bullseye + - key: CliBuilderBullseye + value: cli-builder-bullseye + - key: Buildpack + value: buildpack +resources: + repositories: + - repository: MicroBuildTemplate + type: git + name: 1ESPipelineTemplates/MicroBuildTemplate + ref: refs/tags/release +variables: + group: Oryx + Packaging.EnableSBOMSigning: true +trigger: none +extends: + template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate + parameters: + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + image: AzurePipelinesWindows2022compliantGPT + os: windows + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + customBuildTags: + - ES365AIMigrationTooling-BulkMigrated + stages: + - stage: CreateReleaseTag + jobs: + - job: CreateReleaseTag + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + skipComponentGovernanceDetection: true + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish artifact file having the release tag name' + targetPath: $(Build.SourcesDirectory)/artifacts/releaseTag.txt + artifactName: releaseTag + steps: + - checkout: self + clean: true + - task: ShellScript@2 + inputs: + scriptPath: ./vsts/scripts/createReleaseTag.sh + displayName: 'Create release tag' + - script: | + set -ex + sourcesArtifactsDir="$(Build.SourcesDirectory)/artifacts" + mkdir -p "$sourcesArtifactsDir" + echo "$(RELEASE_TAG_NAME)" > "$sourcesArtifactsDir/releaseTag.txt" + displayName: 'Write release tag name to an artifact file' + - stage: Build + displayName: Build Stage + jobs: + - job: Job_Security + displayName: Security + condition: succeeded() + pool: + name: Azure Pipelines + vmImage: windows-2022 + steps: + - template: /vsts/pipelines/templates/_securityChecks.yml@self + - job: Job_SignBinaries + displayName: Sign Oryx Binaries + pool: + name: VSEngSS-MicroBuild2022-1ES + demands: + - msbuild + - visualstudio + variables: + SignType: 'test' + skipComponentGovernanceDetection: true + templateContext: + mb: + signing: + enabled: true + signType: '$(setSignTypeVariable.SignType)' + outputs: + - output: pipelineArtifact + displayName: 'Publish Pipeline Artifact' + condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) + artifactName: 'signed_binaries' + targetPath: '$(Build.ArtifactStagingDirectory)' + steps: + - task: DownloadPipelineArtifact@2 + displayName: 'Download artifact file having release tag name' + inputs: + targetPath: $(Build.SourcesDirectory)/artifacts/releaseTag + artifactName: 'releaseTag' + - powershell: | + $file = "$env:BUILD_SOURCESDIRECTORY\artifacts\releaseTag\releaseTag.txt" + if (Test-Path $file) { + $tagName = [IO.File]::ReadAllText($file) + Write-Host "Content is $tagName" + echo "##vso[task.setvariable variable=RELEASE_TAG_NAME;]$tagName" + } + displayName: 'Set release tag name as environment variable' + - template: /vsts/pipelines/templates/_signBinary.yml@self + - ${{ each buildImage in parameters.buildImages }}: + - job: Job_BuildImage_${{ buildImage.key }} + displayName: 'Build & Test ${{ buildImage.key }} Build image' + condition: succeeded() + timeoutInMinutes: 480 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + SignType: $[ dependencies.Job_SignBinaries.outputs['setSignTypeVariable.SignType'] ] + skipComponentGovernanceDetection: true + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]true" + echo "##vso[task.setvariable variable=TestBuildImages;]true" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=PushBuildImages;]true" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + if [[ "${{ buildImage.value }}" =~ "cli-builder" ]]; then + echo "##vso[task.setvariable variable=PushBuilderImages;]true" + fi + displayName: 'Set variables' + - template: /vsts/pipelines/templates/_setReleaseTag.yml@self + - template: /vsts/pipelines/templates/_buildTemplate.yml@self + parameters: + imageType: ${{ buildImage.value }} + - job: Job_Buster_RuntimeImages + displayName: Build and Test Buster Runtime Images + dependsOn: Job_SignBinaries + condition: succeeded() + timeoutInMinutes: 480 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + SignType: $[ dependencies.Job_SignBinaries.outputs['setSignTypeVariable.SignType'] ] + skipComponentGovernanceDetection: true + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]true" + echo "##vso[task.setvariable variable=TestRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: /vsts/pipelines/templates/_setReleaseTag.yml@self + - template: /vsts/pipelines/templates/_buildTemplate.yml@self + parameters: + imageType: buster + - job: Job_Bullseye_RuntimeImages + displayName: Build and Test Bullseye Runtime Images + dependsOn: Job_SignBinaries + condition: succeeded() + timeoutInMinutes: 480 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + SignType: $[ dependencies.Job_SignBinaries.outputs['setSignTypeVariable.SignType'] ] + skipComponentGovernanceDetection: true + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]true" + echo "##vso[task.setvariable variable=TestRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: /vsts/pipelines/templates/_setReleaseTag.yml@self + - template: /vsts/pipelines/templates/_buildTemplate.yml@self + parameters: + imageType: bullseye + - job: Job_Bookworm_RuntimeImages + displayName: Build and Test Bookworm Runtime Images + dependsOn: Job_SignBinaries + condition: succeeded() + timeoutInMinutes: 480 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + SignType: $[ dependencies.Job_SignBinaries.outputs['setSignTypeVariable.SignType'] ] + skipComponentGovernanceDetection: true + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]true" + echo "##vso[task.setvariable variable=TestRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: /vsts/pipelines/templates/_setReleaseTag.yml@self + - template: /vsts/pipelines/templates/_buildTemplate.yml@self + parameters: + imageType: bookworm + - template: /vsts/pipelines/templates/_integrationJobTemplate.yml@self + parameters: + storageAccountUrl: ${{ parameters.storageAccountUrl }} + - job: Job_ProductionStorageAccountTest + displayName: Test Production Storage Account + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + skipComponentGovernanceDetection: true + steps: + - task: UseDotNet@2 + displayName: 'Use .NET Core sdk 7.x' + inputs: + version: 7.0.306 + - task: ShellScript@2 + displayName: 'Test production storage account' + env: + ORYX_TEST_SDK_STORAGE_URL: ${{ parameters.storageAccountUrl }} + inputs: + scriptPath: ./build/testIntegration.sh + args: StorageAccountTests=Prod + - stage: Release + displayName: Release Stage + dependsOn: Build + condition: > + and(succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'],'refs/heads/patch/'), startsWith(variables['Build.SourceBranch'],'refs/heads/exp/'))) + jobs: + - template: /vsts/pipelines/templates/_releaseJobTemplate.yml@self \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/ciDebug.yml b/vsts/pipelines/1ESPipelines/ciDebug.yml new file mode 100644 index 0000000000..8ddcbcd0a2 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/ciDebug.yml @@ -0,0 +1,196 @@ +resources: +- repo: self + +variables: +- group: Oryx + +stages: + - stage: CreateReleaseTag + jobs: + - job: CreateReleaseTag + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + skipComponentGovernanceDetection: true + steps: + - task: ShellScript@2 + inputs: + scriptPath: ./vsts/scripts/createReleaseTag.sh + displayName: 'Create release tag' + - script: | + set -ex + sourcesArtifactsDir="$(Build.SourcesDirectory)/artifacts" + mkdir -p "$sourcesArtifactsDir" + echo "$(RELEASE_TAG_NAME)" > "$sourcesArtifactsDir/releaseTag.txt" + displayName: 'Write release tag name to an artifact file' + - task: PublishPipelineArtifact@1 + displayName: 'Publish artifact file having the release tag name' + inputs: + targetPath: $(Build.SourcesDirectory)/artifacts/releaseTag.txt + artifactName: releaseTag + + - stage: Build + displayName: Build Stage + jobs: + - job: Job_Security + displayName: Security + condition: succeeded() + pool: + name: Azure Pipelines + vmImage: windows-2022 + steps: + - template: templates/_securityChecks.yml + + - job: Job_BuildImage + displayName: Build and Test Build Image + timeoutInMinutes: 300 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + SignType: 'test' + skipComponentGovernanceDetection: true + + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]true" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=PushBuildImages;]true" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + displayName: 'Set variables' + + - template: templates/_setReleaseTag.yml + + - template: templates/_buildTemplate.yml + + - job: Job_Buster_RuntimeImages + displayName: Build and Test Buster Runtime Images + timeoutInMinutes: 300 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + SignType: 'test' + skipComponentGovernanceDetection: true + + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]true" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + displayName: 'Set variables' + + - template: templates/_setReleaseTag.yml + + - template: templates/_buildTemplate.yml + parameters: + imageType: buster + + - job: Job_Bullseye_RuntimeImages + displayName: Build and Test Bullseye Runtime Images + timeoutInMinutes: 300 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + SignType: 'test' + skipComponentGovernanceDetection: true + + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]true" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + displayName: 'Set variables' + + - template: templates/_setReleaseTag.yml + + - template: templates/_buildTemplate.yml + parameters: + imageType: bullseye + + - job: Job_Bookworm_RuntimeImages + displayName: Build and Test Bookworm Runtime Images + timeoutInMinutes: 300 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + SignType: 'test' + skipComponentGovernanceDetection: true + + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]true" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + displayName: 'Set variables' + + - template: templates/_setReleaseTag.yml + + - template: templates/_buildTemplate.yml + parameters: + imageType: bookworm + + - stage: Release + displayName: Release Stage + dependsOn: Build + condition: succeeded() + + jobs: + - job: Release_BuildImage + displayName: Push Build Image to MCR + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + skipComponentGovernanceDetection: true + timeoutInMinutes: 300 + steps: + - script: | + echo "##vso[task.setvariable variable=ReleaseBuildImages;]true" + echo "##vso[task.setvariable variable=ReleaseRuntimeImages;]false" + displayName: 'Set variables' + + - template: templates/_releaseStepTemplate.yml + + - job: Release_RuntimeImages + displayName: Push Runtime Images to MCR + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + skipComponentGovernanceDetection: true + timeoutInMinutes: 400 + steps: + - script: | + echo "##vso[task.setvariable variable=ReleaseBuildImages;]false" + echo "##vso[task.setvariable variable=ReleaseRuntimeImages;]true" + displayName: 'Set variables' + + - template: templates/_releaseStepTemplate.yml + +trigger: none \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/copyAllBlobsToProd.yml b/vsts/pipelines/1ESPipelines/copyAllBlobsToProd.yml new file mode 100644 index 0000000000..01151a1d48 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/copyAllBlobsToProd.yml @@ -0,0 +1,54 @@ +variables: +- group: Oryx + +parameters: +- name: sourceStorageAccount + displayName: Source Storage Account + type: string + default: https://oryxsdkssandbox.blob.core.windows.net + values: + - https://oryxsdkssandbox.blob.core.windows.net + - https://oryxsdksstaging.blob.core.windows.net +- name: dryRun + displayName: Dry Run? + type: boolean + default: true + +trigger: none + +jobs: + - job: CreateReleaseTag + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - checkout: self + clean: true + + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection - OSS Compliance' + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/tests' + + - task: UseDotNet@2 + displayName: 'Use .NET Core sdk 7.x' + inputs: + version: 7.0.306 + + - task: ShellScript@2 + displayName: 'Copy all blobs from a source storage account to the prod storage account' + env: + DEV_STORAGE_SAS_TOKEN: $(DEV-STORAGE-SAS-TOKEN) + ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN: $(ORYX-SDK-STAGING-PRIVATE-SAS-TOKEN) + SANDBOX_STORAGE_SAS_TOKEN: $(SANDBOX-STORAGE-SAS-TOKEN) + PROD_STORAGE_SAS_TOKEN: $(PROD-STORAGE-SAS-TOKEN) + inputs: + scriptPath: ./vsts/scripts/copyAllBlobsToProdStorageAccount.sh + args: ${{ parameters.sourceStorageAccount }} ${{ parameters.dryRun }} + + - task: ShellScript@2 + displayName: 'Test Prod storage account' + inputs: + scriptPath: ./build/testIntegration.sh + args: StorageAccountTests=Prod \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/copySdksFromProdToStorageAccount.yml b/vsts/pipelines/1ESPipelines/copySdksFromProdToStorageAccount.yml new file mode 100644 index 0000000000..75de2dbc92 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/copySdksFromProdToStorageAccount.yml @@ -0,0 +1,62 @@ +variables: +- group: Oryx + +parameters: +- name: destinationStorageAccountName + displayName: Destination Storage Account Name + type: string + default: oryxsdksprodbackup +- name: dryRun + displayName: Dry Run? + type: boolean + default: false +# If overwrite is false, the only time SDKs will be copied is when the destination +# account does not have the existing container (e.g. dotnet, nodejs, ruby) +- name: overwrite + displayName: Copy new SDKs to existing containers? + type: boolean + default: true + +jobs: + - job: CopySDKsToStorageAccount + timeoutInMinutes: 300 # sometimes copying a fresh storage account takes longer than the default 60 minute timeout + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - checkout: self + clean: true + + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection - OSS Compliance' + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/tests' + + - task: UseDotNet@2 + displayName: 'Use .NET Core sdk 7.x' + inputs: + version: 7.0.306 + + - task: ShellScript@2 + displayName: 'Copy SDKs from the prod storage account to a destination storage account' + env: + DEV_STORAGE_SAS_TOKEN: $(DEV-STORAGE-SAS-TOKEN) + ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN: $(ORYX-SDK-STAGING-PRIVATE-SAS-TOKEN) + SANDBOX_STORAGE_SAS_TOKEN: $(SANDBOX-STORAGE-SAS-TOKEN) + PROD_STORAGE_SAS_TOKEN: $(PROD-STORAGE-SAS-TOKEN) + PROD_BACKUP_STORAGE_SAS_TOKEN: $(PROD-BACKUP-STORAGE-SAS-TOKEN) + PERSONAL_STORAGE_SAS_TOKEN: $(${{ upper(parameters.destinationStorageAccountName) }}-PERSONAL-STORAGE-SAS-TOKEN) + inputs: + scriptPath: ./vsts/scripts/copySdksFromProdToStorageAccount.sh + args: ${{ parameters.destinationStorageAccountName }} ${{ parameters.dryRun }} ${{ parameters.overwrite }} + +trigger: none +pr: none +schedules: +- cron: "0 12 * * 0" + displayName: Weekly Sunday Copy to Backup Prod + branches: + include: + - main + always: true # whether to always run the pipeline even if there have been source code changes since the last run \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/nightly.yml b/vsts/pipelines/1ESPipelines/nightly.yml new file mode 100644 index 0000000000..cbef3404df --- /dev/null +++ b/vsts/pipelines/1ESPipelines/nightly.yml @@ -0,0 +1,235 @@ +parameters: + - name: storageAccountUrl + displayName: SDK storage account URL for testing + type: string + default: https://oryxsdksstaging.blob.core.windows.net + - name: buildImages + type: object + default: + - + key: Latest + value: latest + - + key: LtsVersions + value: ltsversions + - + key: Jamstack + value: jamstack + - + key: GithubActions + value: githubactions + - + key: VsoFocal + value: vso-focal + - + key: VsoBullseye + value: vso-bullseye + - + key: Full + value: full + - + key: Cli + value: cli-stretch + - + key: CliBuster + value: cli-buster + - + key: CliBullseye + value: cli-bullseye + - + key: CliBuilderBullseye + value: cli-builder-bullseye + - + key: Buildpack + value: buildpack + +resources: +- repo: self + fetchDepth: 15 + +variables: + group: Oryx + Packaging.EnableSBOMSigning: true + +stages: + - stage: CreateReleaseTag + jobs: + - job: CreateReleaseTag + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + skipComponentGovernanceDetection: true + steps: + - task: ShellScript@2 + inputs: + scriptPath: ./vsts/scripts/createReleaseTag.sh + displayName: 'Create release tag' + - script: | + set -ex + sourcesArtifactsDir="$(Build.SourcesDirectory)/artifacts" + mkdir -p "$sourcesArtifactsDir" + echo "$(RELEASE_TAG_NAME)" > "$sourcesArtifactsDir/releaseTag.txt" + displayName: 'Write release tag name to an artifact file' + - task: PublishPipelineArtifact@1 + displayName: 'Publish artifact file having the release tag name' + inputs: + targetPath: $(Build.SourcesDirectory)/artifacts/releaseTag.txt + artifactName: releaseTag + + - stage: Build + displayName: Build Stage + jobs: + - job: Job_Security + displayName: Security + condition: succeeded() + pool: + name: Azure Pipelines + vmImage: windows-2022 + steps: + - template: templates/_securityChecks.yml + + # Job loop for BuildImages + - ${{ each buildImage in parameters.buildImages }}: + - job: Job_BuildImage_${{ buildImage.key }} + displayName: 'Build & Test ${{ buildImage.key }} Build image' + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + timeoutInMinutes: 480 + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]true" + echo "##vso[task.setvariable variable=TestBuildImages;]true" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=PushBuildImages;]true" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + echo "##vso[task.setvariable variable=RELEASE_TAG_NAME;]$(Build.BuildNumber)" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + if [[ "${{ buildImage.value }}" =~ "cli-builder" ]]; then + echo "##vso[task.setvariable variable=PushBuilderImages;]true" + fi + displayName: 'Set variables' + - template: templates/_buildTemplate.yml + parameters: + imageType: ${{ buildImage.value }} + +# commented out temporarily + # - job: Job_BuilderImages + # displayName: Build Builder Images + # pool: + # name: AzurePipelines-EO + # demands: + # - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + # timeoutInMinutes: 480 + # steps: + # - template: templates/_builderTemplate.yml + # dependsOn: Job_BuildImage_CliBuilderBullseye + + - job: Job_Buster_RuntimeImages + displayName: Build and Test Buster Runtime Images + condition: succeeded() + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + # Building runtime images can take a long time due to our PHP images + timeoutInMinutes: 480 + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]true" + echo "##vso[task.setvariable variable=TestRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + echo "##vso[task.setvariable variable=RELEASE_TAG_NAME;]$(Build.BuildNumber)" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: templates/_buildTemplate.yml + parameters: + imageType: buster + + - job: Job_Bullseye_RuntimeImages + displayName: Build and Test Bullseye Runtime Images + condition: succeeded() + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + # Building runtime images can take a long time due to our PHP images + timeoutInMinutes: 480 + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]true" + echo "##vso[task.setvariable variable=TestRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + echo "##vso[task.setvariable variable=RELEASE_TAG_NAME;]$(Build.BuildNumber)" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: templates/_buildTemplate.yml + parameters: + imageType: bullseye + + - job: Job_Bookworm_RuntimeImages + displayName: Build and Test Bookworm Runtime Images + condition: succeeded() + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + # Building runtime images can take a long time due to our PHP images + timeoutInMinutes: 480 + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]true" + echo "##vso[task.setvariable variable=TestRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + echo "##vso[task.setvariable variable=RELEASE_TAG_NAME;]$(Build.BuildNumber)" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: templates/_buildTemplate.yml + parameters: + imageType: bookworm + + - template: templates/_integrationJobTemplate.yml + parameters: + storageAccountUrl: ${{ parameters.storageAccountUrl }} + + - job: Job_StagingStorageAccountTest + displayName: Test Staging Storage Account + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + skipComponentGovernanceDetection: true + steps: + - task: UseDotNet@2 + displayName: 'Use .NET Core sdk 7.x' + inputs: + version: 7.0.306 + + - task: ShellScript@2 + displayName: 'Test staging storage account' + env: + ORYX_TEST_SDK_STORAGE_URL: ${{ parameters.storageAccountUrl }} + ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN: $(ORYX-SDK-STAGING-PRIVATE-SAS-TOKEN) + inputs: + scriptPath: ./build/testIntegration.sh + args: StorageAccountTests=Staging + +trigger: none \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/publishSdkToProd.yml b/vsts/pipelines/1ESPipelines/publishSdkToProd.yml new file mode 100644 index 0000000000..a983e52417 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/publishSdkToProd.yml @@ -0,0 +1,75 @@ +variables: +- group: Oryx + +stages: + - stage: DryRun + jobs: + - job: DryRun + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - checkout: self + clean: true + + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection - OSS Compliance' + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/tests' + + - task: UseDotNet@2 + displayName: 'Use .NET Core sdk 7.x' + inputs: + version: 7.0.306 + + - task: ShellScript@2 + displayName: '(Dry run) Publish SDKs from dev to prod storage account' + env: + DEV_STORAGE_SAS_TOKEN: $(DEV-STORAGE-SAS-TOKEN) + ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN: $(ORYX-SDK-STAGING-PRIVATE-SAS-TOKEN) + PROD_STORAGE_SAS_TOKEN: $(PROD-STORAGE-SAS-TOKEN) + inputs: + scriptPath: ./vsts/scripts/publishSdksFromStagingToProdStorageAccount.sh + args: True # dry run + + - stage: PublishSDKsToProduction + jobs: + - deployment: PublishSDKsToProduction + environment: oryx-prod # maps to the oryx-prod ADO environment, which requires manual approval to begin this stage + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + strategy: + runOnce: + deploy: + steps: + - checkout: self + clean: true + + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection - OSS Compliance' + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/tests' + + - task: UseDotNet@2 + displayName: 'Use .NET Core sdk 7.x' + inputs: + version: 7.0.306 + + - task: ShellScript@2 + displayName: 'Publish SDKs from dev to prod storage account' + env: + DEV_STORAGE_SAS_TOKEN: $(DEV-STORAGE-SAS-TOKEN) + ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN: $(ORYX-SDK-STAGING-PRIVATE-SAS-TOKEN) + PROD_STORAGE_SAS_TOKEN: $(PROD-STORAGE-SAS-TOKEN) + inputs: + scriptPath: ./vsts/scripts/publishSdksFromStagingToProdStorageAccount.sh + args: False # no dry run + + - task: ShellScript@2 + displayName: 'Test prod storage account' + inputs: + scriptPath: ./build/testIntegration.sh + args: StorageAccountTests=Prod \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/_agentCleanUpJobTemplate.yml b/vsts/pipelines/1ESPipelines/templates/_agentCleanUpJobTemplate.yml new file mode 100644 index 0000000000..5e01904f8e --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/_agentCleanUpJobTemplate.yml @@ -0,0 +1,13 @@ +parameters: + agentName: '' + +jobs: +- job: Cleaning_${{ parameters.agentName }} + displayName: ${{ parameters.agentName }} + pool: + name: ${{ parameters.poolName }} + demands: + - agent.name -equals ${{ parameters.agentName }} + timeoutInMinutes: 50 + steps: + - template: _cleanImageCacheTemplate.yml diff --git a/vsts/pipelines/1ESPipelines/templates/_buildTemplate.yml b/vsts/pipelines/1ESPipelines/templates/_buildTemplate.yml new file mode 100644 index 0000000000..000593473b --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/_buildTemplate.yml @@ -0,0 +1,211 @@ +parameters: + ascName: oryx-new-service-connection + acrName: oryxdevmcr.azurecr.io + imageName: oryxdevmcr.azurecr.io/public/oryx + imageType: null +steps: +- script: | + if [ "$(BuildBuildImages)" != "true" ] && [ "$(BuildRuntimeImages)" != "true" ] && [ "$(TestIntegration)" != "true" ] + then + echo "Invalid configuration." + echo "Variable 'BuildBuildImages' or 'BuildRuntimeImages' needs to be 'true' to run this build." + exit 1 + fi + displayName: 'Validate pipeline run' +- script: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + docker images && docker system prune -fa && docker images && echo + displayName: 'clean docker images' + condition: or(eq(variables['TestBuildImages'], 'true'), eq(variables['TestRuntimeImages'], 'true')) +- checkout: self + clean: true +- task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact' + inputs: + pipelineId: 'Oryx-CI' + artifactName: 'signed_binaries' + targetPath: '$(Build.SourcesDirectory)' + condition: > + and(succeeded(), or(in(variables['SIGNTYPE'], 'real', 'Real'), in(variables['SignType'], 'real', 'Real')), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'],'refs/heads/patch/'), startsWith(variables['Build.SourceBranch'],'refs/heads/exp/'))) +- task: UseDotNet@2 + displayName: 'Use .NET Core sdk 7.x' + inputs: + version: 7.0.306 +- script: | + dotnet --version && dotnet msbuild -version && echo + displayName: 'Print dotnet and msbuild version' +- task: ShellScript@2 + displayName: 'Build Oryx.sln' + inputs: + scriptPath: ./build/buildSln.sh + args: Oryx.sln +- task: Docker@1 + displayName: Container registry logout + inputs: + command: logout + azureSubscriptionEndpoint: ${{ parameters.ascName }} + azureContainerRegistry: ${{ parameters.acrName }} + +- task: Docker@1 + displayName: Container registry login + inputs: + command: login + azureSubscriptionEndpoint: ${{ parameters.ascName }} + azureContainerRegistry: ${{ parameters.acrName }} +- script: | + docker images && docker system prune -fa && df -h && echo + displayName: 'Free up space for oryx layers' +- task: ShellScript@2 + displayName: 'Test build script generator' + inputs: + scriptPath: ./build/testBuildScriptGenerator.sh + condition: and(succeeded(), eq(variables['TestBuildImages'], 'true')) +- task: ShellScript@2 + displayName: 'Test startup script generators' + inputs: + scriptPath: ./build/testStartupScriptGenerators.sh + condition: and(succeeded(), eq(variables['TestRuntimeImages'], 'true')) +- task: ShellScript@2 + displayName: 'Build build images' + env: + ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN: $(ORYX-SDK-STAGING-PRIVATE-SAS-TOKEN) + inputs: + scriptPath: ./build/buildBuildImages.sh + args: -t ${{ parameters.imageType }} -s $(storageAccountUrl) + condition: and(succeeded(), eq(variables['BuildBuildImages'], 'true')) +- task: ShellScript@2 + displayName: 'Build runtime images' + env: + ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN: $(ORYX-SDK-STAGING-PRIVATE-SAS-TOKEN) + inputs: + scriptPath: ./build/buildRunTimeImages.sh + args: -s $(storageAccountUrl) ${{ parameters.imageType }} + condition: and(succeeded(), eq(variables['BuildRuntimeImages'], 'true')) +- script: | + docker images && docker system prune -f && df -h && echo + displayName: 'clean docker unknown layers' +- script: | + docker -v + echo "Installing Docker CE version 24.0.5." + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt update + sudo apt-get install docker-ce=5:24.0.5~3-0~ubuntu-focal docker-ce-cli=5:24.0.5~3-0~ubuntu-focal containerd.io + echo "After installing ." + docker -v + displayName: 'Install Docker 24.0.5' + condition: true +- task: ShellScript@2 + displayName: 'Test build images' + env: + ORYX_TEST_SDK_STORAGE_URL: $(storageAccountUrl) + ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN: $(ORYX-SDK-STAGING-PRIVATE-SAS-TOKEN) + inputs: + scriptPath: ./build/testBuildImages.sh + args: skipBuildingImages ${{ parameters.imageName }} $(Build.DefinitionName).$(RELEASE_TAG_NAME) ${{ parameters.imageType }} + condition: and(succeeded(), eq(variables['TestBuildImages'], 'true')) +- task: ShellScript@2 + displayName: 'Test runtime images' + env: + ORYX_TEST_SDK_STORAGE_URL: $(storageAccountUrl) + ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN: $(ORYX-SDK-STAGING-PRIVATE-SAS-TOKEN) + inputs: + scriptPath: ./build/testRunTimeImages.sh + args: skipBuildingImages ${{ parameters.imageName }} $(Build.DefinitionName).$(RELEASE_TAG_NAME) category=runtime-${{ parameters.imageType }} + condition: and(succeeded(), eq(variables['TestRuntimeImages'], 'true')) +- task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: drop + condition: and(succeeded(), eq(variables['TestIntegration'], 'true')) +- task: Shellpp@0 + displayName: 'Pull and Retag recently built oryx build and runtime images' + inputs: + type: FilePath + scriptPath: ./vsts/scripts/pullAndTag.sh + args: $(System.ArtifactsDirectory)/drop/images/build-images-acr.txt + condition: and(succeeded(), eq(variables['TestIntegration'], 'true')) +- task: Shellpp@0 + displayName: 'Build a build image for tests' + inputs: + type: FilePath + scriptPath: ./build/buildTestBuildImages.sh + condition: and(succeeded(), eq(variables['TestIntegration'], 'true')) +- task: ShellScript@2 + displayName: 'Test integration' + inputs: + scriptPath: ./build/testIntegration.sh + args: $(TestIntegrationCaseFilter) ${{ parameters.imageName }} $(Build.DefinitionName).$(RELEASE_TAG_NAME) + env: + SQLSERVER_DATABASE_HOST: $(SQLSERVER-DATABASE-HOST) + SQLSERVER_DATABASE_NAME: $(SQLSERVER-DATABASE-NAME) + SQLSERVER_DATABASE_USERNAME: $(SQLSERVER-DATABASE-USERNAME) + SQLSERVER_DATABASE_PASSWORD: $(SQLSERVER-DATABASE-PASSWORD) + ORYX_TEST_SDK_STORAGE_URL: $(storageAccountUrl) + ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN: $(ORYX-SDK-STAGING-PRIVATE-SAS-TOKEN) + condition: and(succeeded(), eq(variables['TestIntegration'], 'true')) +- task: CopyFiles@2 + displayName: 'Copy source projects output to artifacts folder' + inputs: + sourceFolder: $(Build.SourcesDirectory) + contents: src/**/bin/**/*.* + targetFolder: $(Build.ArtifactStagingDirectory) + cleanTargetFolder: true + overWrite: true + flattenFolders: true + condition: true +- task: CopyFiles@2 + displayName: 'Copy artifacts from source repo to agent artifacts folder' + inputs: + sourceFolder: '$(Build.SourcesDirectory)/artifacts' + contents: '**/*.*' + targetFolder: $(Build.ArtifactStagingDirectory) + overWrite: true + condition: true +- task: PublishTestResults@2 + inputs: + testRunner: 'xUnit' + testResultsFiles: '$(Build.ArtifactStagingDirectory)/testResults/*.xml' + mergeTestResults: true + condition: true +- task: 1ES.PushContainerImage@1 + displayName: 'Push build images to ACR' + inputs: + image: ${{ parameters.acrName }}/${{ parameters.ascName }} + remoteImage: ${{ parameters.acrName }}/${{ parameters.ascName }}:${{ parameters.acrName }} +- task: 1ES.PushContainerImage@1 + displayName: 'Push runtime images to ACR' + inputs: + image: ${{ parameters.acrName }}/${{ parameters.acrName }}:${{ parameters.imageType }} + remoteImage: ${{ parameters.acrName }}/${{ parameters.acrName }}:${{ parameters.imageType }} + condition: and(succeeded(), eq(variables['PushRuntimeImages'], 'true'), eq(variables['BuildRuntimeImages'], 'true')) +- task: ShellScript@2 + displayName: 'Clean up Docker containers and images' + inputs: + scriptPath: ./vsts/scripts/cleanDocker.sh + condition: or(eq(variables['TestBuildImages'], 'true'), eq(variables['TestRuntimeImages'], 'true'), eq(variables['TestIntegration'], 'true')) +- task: ShellScript@2 + displayName: 'Clean up Docker mounted directories' + inputs: + scriptPath: ./vsts/scripts/removeDockerArtifacts.sh + condition: or(eq(variables['TestBuildImages'], 'true'), eq(variables['TestRuntimeImages'], 'true'), eq(variables['TestIntegration'], 'true')) +- script: | + docker images && docker system prune -fa && df -h && docker images && echo + displayName: 'clean docker images and stopped containers' +- task: ShellScript@2 + displayName: 'Generate release notes' + inputs: + scriptPath: ./vsts/scripts/generateReleaseNotes.sh + condition: and(succeeded(), eq(variables['PushBuildImages'], 'true'), eq(variables['BuildBuildImages'], 'true')) +- task: ArchiveFiles@2 + displayName: 'Archive docker files and scripts for Oryx build and runtime images' + inputs: + rootFolderOrFile: images + archiveFile: '$(Build.ArtifactStagingDirectory)/images/dockerFiles.zip' + condition: true +- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 + displayName: 'Clean Agent Directories' + condition: true \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/_buildTemplateDetector.yml b/vsts/pipelines/1ESPipelines/templates/_buildTemplateDetector.yml new file mode 100644 index 0000000000..bb554f5fbd --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/_buildTemplateDetector.yml @@ -0,0 +1,52 @@ +steps: +- checkout: self + clean: true + +- task: UseDotNet@2 + displayName: 'Use .NET Core SDK 7.x' + inputs: + version: 7.0.306 + +- task: ShellScript@2 + displayName: 'Build Detector.sln' + inputs: + scriptPath: ./build/buildSln.sh + args: Detector.sln + +- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection - OSS Compliance' + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/tests' + +- task: ShellScript@2 + displayName: 'Testing Detector' + inputs: + scriptPath: ./build/testDetector.sh + +- task: CopyFiles@2 + displayName: 'Copy content from artifacts folder in repo to agent artifacts folder' + inputs: + sourceFolder: '$(Build.SourcesDirectory)/artifacts' + contents: '**/*.*' + targetFolder: $(Build.ArtifactStagingDirectory) + overWrite: true + condition: succeededOrFailed() + +- task: PublishTestResults@2 + displayName: 'Publish test results' + inputs: + testRunner: 'xUnit' + testResultsFiles: '$(Build.ArtifactStagingDirectory)/testResults/*.xml' + mergeTestResults: true + condition: succeededOrFailed() + +- task: PublishBuildArtifacts@1 + displayName: 'Publish build artifacts' + inputs: + pathToPublish: $(Build.ArtifactStagingDirectory) + artifactName: 'detector' + condition: succeededOrFailed() + +- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 + displayName: 'Clean Agent Directories' + condition: succeededOrFailed() \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/_builderTemplate.yml b/vsts/pipelines/1ESPipelines/templates/_builderTemplate.yml new file mode 100644 index 0000000000..0890946854 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/_builderTemplate.yml @@ -0,0 +1,41 @@ +parameters: + ascName: oryx-new-service-connection + acrName: oryxdevmcr.azurecr.io +steps: +- script: | + curl -sSL "https://github.com/buildpacks/pack/releases/download/v0.28.0/pack-v0.28.0-linux.tgz" | tar -C /usr/local/bin/ --no-same-owner -xzv pack + displayName: 'Install Pack CLI' + condition: true +- task: Docker@1 + displayName: Container registry login + inputs: + command: login + azureSubscriptionEndpoint: ${{ parameters.ascName }} + azureContainerRegistry: ${{ parameters.acrName }} +- task: ShellScript@2 + displayName: 'Build base builder images' + inputs: + scriptPath: ./builders/base/buildBaseBuilder.sh + args: -f ${{ parameters.acrName }} -r "public/oryx/builder" -v $(Build.DefinitionName).$(Build.BuildNumber) +- task: ShellScript@2 + displayName: 'Build container apps builder images' + inputs: + scriptPath: ./builders/container-apps/buildCappsBuilder.sh + args: -f ${{ parameters.acrName }} -r "public/oryx/builder" -t "capps-$(Build.DefinitionName).$(Build.BuildNumber)" -b "${{ parameters.acrName }}/public/oryx/builder:$(Build.DefinitionName).$(Build.BuildNumber)" +- task: CopyFiles@2 + displayName: 'Copy artifacts from source repo to agent artifacts folder' + inputs: + sourceFolder: '$(Build.SourcesDirectory)/artifacts' + contents: '**/*.*' + targetFolder: $(Build.ArtifactStagingDirectory) + overWrite: true +- task: 1ES.PushContainerImage@1 + displayName: 'Push base build images to ACR' + inputs: + image: ${{ parameters.acrName }}/${{ parameters.acrName }}:$(Build.BuildNumber) + remoteImage: ${{ parameters.acrName }}/${{ parameters.acrName }}:$(Build.BuildNumber) +- task: 1ES.PushContainerImage@1 + displayName: 'Push container apps builder images to ACR' + inputs: + image: ${{ parameters.acrName }}/${{ parameters.ascName }}:$(Build.ArtifactStagingDirectory)/images/capps-builder-images-acr.txt + remoteImage: ${{ parameters.acrName }}/${{ parameters.ascName }}:$(Build.ArtifactStagingDirectory)/images/capps-builder-images-acr.txt \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/_buildimageBasesJobTemplate.yml b/vsts/pipelines/1ESPipelines/templates/_buildimageBasesJobTemplate.yml new file mode 100644 index 0000000000..812bc2ced5 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/_buildimageBasesJobTemplate.yml @@ -0,0 +1,40 @@ +jobs: +- job: ${{ parameters.jobName }} + displayName: ${{ parameters.displayName }} + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - task: ShellScript@2 + displayName: Build images + inputs: + scriptPath: ${{ parameters.scriptPath }} + args: ${{ parameters.imageDir }} ${{ parameters.imageDebianFlavor }} + env: + ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN: $(ORYX-SDK-STAGING-PRIVATE-SAS-TOKEN) + DOTNET_PRIVATE_STORAGE_ACCOUNT_ACCESS_TOKEN: $(DotnetPrivateStorageAccountAccessToken) + - task: CopyFiles@2 + displayName: Copy artifacts to staging directory + inputs: + sourceFolder: '$(Build.SourcesDirectory)/artifacts' + contents: '**/*.*' + targetFolder: $(Build.ArtifactStagingDirectory) + overWrite: true + condition: true + - task: 1ES.PushContainerImage@1 + displayName: Push built base images to dev ACR + inputs: + image: '$(Build.ArtifactStagingDirectory)/images/${{ parameters.artifactsFileName }}' + remoteImage: $(acrname)/$(Build.ArtifactStagingDirectory)/images/${{ parameters.artifactsFileName }} + - task: ShellScript@2 + displayName: 'Clean up Docker containers and images' + inputs: + scriptPath: ./vsts/scripts/cleanDocker.sh + - task: ShellScript@2 + displayName: 'Clean up Docker containers and images' + inputs: + scriptPath: ./vsts/scripts/cleanDocker.sh + condition: true \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/_cleanImageCacheTemplate.yml b/vsts/pipelines/1ESPipelines/templates/_cleanImageCacheTemplate.yml new file mode 100644 index 0000000000..fe96f9f98e --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/_cleanImageCacheTemplate.yml @@ -0,0 +1,18 @@ +steps: +- script: | + echo "List of containers dead/alive ..." + docker ps -a + echo "List of images ..." + docker images + echo "Cleaning containers and images ..." + docker system prune -af + echo "List of containers dead/alive ..." + docker ps -a + echo "List of images ..." + docker images + displayName: 'Remove all existing docker images from machine' + +- script: | + echo "Restarting in 1 minutes" + sudo shutdown -r +1 + displayName: 'Restart machine after cleaning up' diff --git a/vsts/pipelines/1ESPipelines/templates/_copyImgFromDevToTest.yml b/vsts/pipelines/1ESPipelines/templates/_copyImgFromDevToTest.yml new file mode 100644 index 0000000000..030e22230f --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/_copyImgFromDevToTest.yml @@ -0,0 +1,246 @@ +parameters: + - name: ascName + type: string + default: oryx-new-service-connection + - name: acrDevName + type: string + default: oryxdevmcr.azurecr.io + - name: acrTestName + type: string + default: oryxtestmcr.azurecr.io + - name: testImageName + type: string + default: oryxtestmcr.azurecr.io/public/oryx + - name: acrDevContainerName + type: string + default: oryxdevmcr.azurecr.io/public/oryx + - name: acrTestContainerName + type: string + default: oryxtestmcr.azurecr.io/public/oryx + - name: nightlyTag + type: string + - name: cliImages + type: object + default: + - name: 'cli-buster' + sourceTag: 'debian-buster-Oryx-Nightly' + destinationTag: 'debian-buster' + - name: 'cli-bullseye' + sourceTag: 'debian-bullseye-Oryx-Nightly' + destinationTag: 'debian-bullseye' + - name: 'cli-stretch' + sourceTag: 'debian-stretch-Oryx-Nightly' + destinationTag: 'debian-stretch' + - name: ltsImages + type: object + default: + - name: 'lts-buster' + sourceTag: 'lts-versions-debian-buster-Oryx-Nightly' + destinationTag: 'lts-versions-debian-buster' + - name: 'lts-stretch' + sourceTag: 'lts-versions-debian-stretch-Oryx-Nightly' + destinationTag: 'lts-versions-debian-stretch' + - name: vsoImages + type: object + default: + - name: 'vso-ubuntu' + sourceTag: 'vso-ubuntu-focal-Oryx-Nightly' + destinationTag: 'vso-ubuntu-focal' + - name: 'vso-bullseye' + sourceTag: 'vso-debian-bullseye-Oryx-Nightly' + destinationTag: 'vso-debian-bullseye' + - name: fullImages + type: object + default: + - name: 'full-buster' + sourceTag: 'full-debian-buster-Oryx-Nightly' + destinationTag: 'full-debian-buster' + - name: 'full-bullseye' + sourceTag: 'full-debian-bullseye-Oryx-Nightly' + destinationTag: 'full-debian-bullseye' + - name: githubActions + type: object + default: + - name: 'github-actions-bookworm' + sourceTag: 'github-actions-debian-bookworm-Oryx-Nightly' + destinationTag: 'github-actions-debian-bookworm' + - name: 'github-actions-bullseye' + sourceTag: 'github-actions-debian-bullseye-Oryx-Nightly' + destinationTag: 'github-actions-debian-bullseye' + - name: 'github-actions-buster' + sourceTag: 'github-actions-debian-buster-Oryx-Nightly' + destinationTag: 'github-actions-debian-buster' + - name: 'github-actions-stretch' + sourceTag: 'github-actions-debian-stretch-Oryx-Nightly' + destinationTag: 'github-actions-debian-stretch' + - name: dotnet + type: object + default: + - name: '3.0-debian-buster' + sourceTag: '3.0-debian-buster-Oryx-Nightly' + destinationTag: '3.0-debian-buster' + - name: '3.1-debian-buster' + sourceTag: '3.1-debian-buster-Oryx-Nightly' + destinationTag: '3.1-debian-buster' + - name: '5.0-debian-buster' + sourceTag: '5.0-debian-buster-Oryx-Nightly' + destinationTag: '5.0-debian-buster' + - name: '6.0-debian-buster' + sourceTag: '6.0-debian-buster-Oryx-Nightly' + destinationTag: '6.0-debian-buster' + - name: '7.0-debian-buster' + sourceTag: '7.0-debian-buster-Oryx-Nightly' + destinationTag: '7.0-debian-buster' + - name: node + type: object + default: + - name: '18-debian-bullseye' + sourceTag: '18-debian-bullseye-Oryx-Nightly' + destinationTag: '18-debian-bullseye' + - name: '16-debian-buster' + sourceTag: '16-debian-buster-Oryx-Nightly' + destinationTag: '16-debian-buster' + - name: '14-debian-buster' + sourceTag: '14-debian-buster-Oryx-Nightly' + destinationTag: '14-debian-buster' + - name: python + type: object + default: + - name: '3.7-debian-bullseye' + sourceTag: '3.7-debian-bullseye-Oryx-Nightly' + destinationTag: '3.7-debian-bullseye' + - name: '3.8-debian-bullseye' + sourceTag: '3.8-debian-bullseye-Oryx-Nightly' + destinationTag: '3.8-debian-bullseye' + - name: '3.9-debian-buster' + sourceTag: '3.9-debian-buster-Oryx-Nightly' + destinationTag: '3.9-debian-buster' + - name: '3.10-debian-bullseye' + sourceTag: '3.10-debian-bullseye-Oryx-Nightly' + destinationTag: '3.10-debian-bullseye' + - name: '3.11-debian-bullseye' + sourceTag: '3.11-debian-bullseye-Oryx-Nightly' + destinationTag: '3.11-debian-bullseye' + + + +steps: + +- checkout: self + clean: true + +- task: Docker@1 + displayName: Dev container registry login + inputs: + command: login + azureSubscriptionEndpoint: ${{ parameters.ascName }} + azureContainerRegistry: ${{ parameters.acrDevName }} + +- ${{ each image in parameters.cliImages }}: + - script: | + docker pull "${{ parameters.acrDevContainerName }}/cli:${{ image.sourceTag }}.${{ parameters.nightlyTag }}" + docker tag "${{ parameters.acrDevContainerName }}/cli:${{ image.sourceTag }}.${{ parameters.nightlyTag }}" "${{ parameters.acrTestContainerName }}/cli:${{ image.destinationTag }}.${{ parameters.nightlyTag }}" + displayName: 'Pull CLI Images and Retag' + +- ${{ each image in parameters.ltsImages }}: + - script: | + docker pull "${{ parameters.acrDevContainerName }}/build:${{ image.sourceTag }}.${{ parameters.nightlyTag }}" + docker tag "${{ parameters.acrDevContainerName }}/build:${{ image.sourceTag }}.${{ parameters.nightlyTag }}" "${{ parameters.acrTestContainerName }}/build:${{ image.destinationTag }}.${{ parameters.nightlyTag }}" + displayName: 'Pull and Retag lts Images' + +- ${{ each image in parameters.vsoImages }}: + - script: | + docker pull "${{ parameters.acrDevContainerName }}/build:${{ image.sourceTag }}.${{ parameters.nightlyTag }}" + docker tag "${{ parameters.acrDevContainerName }}/build:${{ image.sourceTag }}.${{ parameters.nightlyTag }}" "${{ parameters.acrTestContainerName }}/build:${{ image.destinationTag }}.${{ parameters.nightlyTag }}" + displayName: 'Pull and Retag Vso Images' + +- ${{ each image in parameters.fullImages }}: + - script: | + docker pull "${{ parameters.acrDevContainerName }}/build:${{ image.sourceTag }}.${{ parameters.nightlyTag }}" + docker tag "${{ parameters.acrDevContainerName }}/build:${{ image.sourceTag }}.${{ parameters.nightlyTag }}" "${{ parameters.acrTestContainerName }}/build:${{ image.destinationTag }}.${{ parameters.nightlyTag }}" + displayName: 'Pull and Retag Full Images' + +- ${{ each image in parameters.githubActions }}: + - script: | + docker pull "${{ parameters.acrDevContainerName }}/build:${{ image.sourceTag }}.${{ parameters.nightlyTag }}" + docker tag "${{ parameters.acrDevContainerName }}/build:${{ image.sourceTag }}.${{ parameters.nightlyTag }}" "${{ parameters.acrTestContainerName }}/build:${{ image.destinationTag }}.${{ parameters.nightlyTag }}" + displayName: 'Pull and Retag Github Action Images' + +- ${{ each image in parameters.dotnet }}: + - script: | + docker pull "${{ parameters.acrDevContainerName }}/dotnetcore:${{ image.sourceTag }}.${{ parameters.nightlyTag }}" + docker tag "${{ parameters.acrDevContainerName }}/dotnetcore:${{ image.sourceTag }}.${{ parameters.nightlyTag }}" "${{ parameters.acrTestContainerName }}/dotnetcore:${{ image.destinationTag }}.${{ parameters.nightlyTag }}" + displayName: 'Pull and Retag dotnet Images' + +- ${{ each image in parameters.node }}: + - script: | + docker pull "${{ parameters.acrDevContainerName }}/node:${{ image.sourceTag }}.${{ parameters.nightlyTag }}" + docker tag "${{ parameters.acrDevContainerName }}/node:${{ image.sourceTag }}.${{ parameters.nightlyTag }}" "${{ parameters.acrTestContainerName }}/node:${{ image.destinationTag }}.${{ parameters.nightlyTag }}" + displayName: 'Pull and Retag Node Images' + +- ${{ each image in parameters.python }}: + - script: | + docker pull "${{ parameters.acrDevContainerName }}/python:${{ image.sourceTag }}.${{ parameters.nightlyTag }}" + docker tag "${{ parameters.acrDevContainerName }}/python:${{ image.sourceTag }}.${{ parameters.nightlyTag }}" "${{ parameters.acrTestContainerName }}/python:${{ image.destinationTag }}.${{ parameters.nightlyTag }}" + displayName: 'Pull and Retag Python Images' + +- task: Docker@1 + displayName: Dev Container registry logout + inputs: + command: logout + azureSubscriptionEndpoint: ${{ parameters.ascName }} + azureContainerRegistry: ${{ parameters.acrDevName }} + +- task: Docker@1 + displayName: Test container registry login + inputs: + command: login + azureSubscriptionEndpoint: ${{ parameters.ascName }} + azureContainerRegistry: ${{ parameters.acrTestName }} + +- ${{ each image in parameters.cliImages }}: + - script: | + docker push "${{ parameters.acrTestContainerName }}/cli:${{ image.destinationTag }}.${{ parameters.nightlyTag }}" + displayName: 'Push CLI Images to Test ACR' + +- ${{ each image in parameters.ltsImages }}: + - script: | + docker push "${{ parameters.acrTestContainerName }}/build:${{ image.destinationTag }}.${{ parameters.nightlyTag }}" + displayName: 'Push lts Images to Test ACR' + +- ${{ each image in parameters.vsoImages }}: + - script: | + docker push "${{ parameters.acrTestContainerName }}/build:${{ image.destinationTag }}.${{ parameters.nightlyTag }}" + displayName: 'Push Vso Images to Test ACR' + +- ${{ each image in parameters.fullImages }}: + - script: | + docker push "${{ parameters.acrTestContainerName }}/build:${{ image.destinationTag }}.${{ parameters.nightlyTag }}" + displayName: 'Push Full Images to Test ACR' + +- ${{ each image in parameters.githubActions }}: + - script: | + docker push "${{ parameters.acrTestContainerName }}/build:${{ image.destinationTag }}.${{ parameters.nightlyTag }}" + displayName: 'Push Github Action Images to Test ACR' + +- ${{ each image in parameters.dotnet }}: + - script: | + docker push "${{ parameters.acrTestContainerName }}/dotnetcore:${{ image.destinationTag }}.${{ parameters.nightlyTag }}" + displayName: 'Push dotnet Images to Test ACR' + +- ${{ each image in parameters.node }}: + - script: | + docker push "${{ parameters.acrTestContainerName }}/node:${{ image.destinationTag }}.${{ parameters.nightlyTag }}" + displayName: 'Push Node Images to Test ACR' + +- ${{ each image in parameters.python }}: + - script: | + docker push "${{ parameters.acrTestContainerName }}/python:${{ image.destinationTag }}.${{ parameters.nightlyTag }}" + displayName: 'Push Python Images to Test ACR' + +- task: Docker@1 + displayName: Test Container registry logout + inputs: + command: logout + azureSubscriptionEndpoint: ${{ parameters.ascName }} + azureContainerRegistry: ${{ parameters.acrTestName }} diff --git a/vsts/pipelines/1ESPipelines/templates/_integrationJobTemplate.yml b/vsts/pipelines/1ESPipelines/templates/_integrationJobTemplate.yml new file mode 100644 index 0000000000..7f14fafcb1 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/_integrationJobTemplate.yml @@ -0,0 +1,17 @@ +parameters: +- name: storageAccountUrl + type: string + default: https://oryxsdksstaging.blob.core.windows.net +jobs: +- template: /vsts/pipelines/templates/integrationTests/_pythonIntegrationJobTemplate.yml@self + parameters: + storageAccountUrl: ${{ parameters.storageAccountUrl }} +- template: /vsts/pipelines/templates/integrationTests/_dotnetcoreIntegrationJobTemplate.yml@self + parameters: + storageAccountUrl: ${{ parameters.storageAccountUrl }} +- template: /vsts/pipelines/templates/integrationTests/_nodeIntegrationJobTemplate.yml@self + parameters: + storageAccountUrl: ${{ parameters.storageAccountUrl }} +- template: /vsts/pipelines/templates/integrationTests/_phpIntegrationJobTemplate.yml@self + parameters: + storageAccountUrl: ${{ parameters.storageAccountUrl }} \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/_platformBinariesReleaseTemplate.yml b/vsts/pipelines/1ESPipelines/templates/_platformBinariesReleaseTemplate.yml new file mode 100644 index 0000000000..461a771033 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/_platformBinariesReleaseTemplate.yml @@ -0,0 +1,32 @@ +parameters: + destinationSdkStorageAccountName: 'oryxsdksstaging' +steps: +- checkout: self + clean: true +- task: DownloadBuildArtifacts@0 + displayName: 'Download Artifacts' + inputs: + artifactName: drop +- task: ShellScript@2 + displayName: Upload files to Azure Storage + env: + DEV_STORAGE_SAS_TOKEN: $(DEV-STORAGE-SAS-TOKEN) + ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN: $(ORYX-SDK-STAGING-PRIVATE-SAS-TOKEN) + SANDBOX_STORAGE_SAS_TOKEN: $(SANDBOX-STORAGE-SAS-TOKEN) + PERSONAL_STORAGE_SAS_TOKEN: $(${{ upper(parameters.destinationSdkStorageAccountName) }}-PERSONAL-STORAGE-SAS-TOKEN) + PRIVATE_STORAGE_SAS_TOKEN: $(ORYX-SDK-PRIVATE-SAS-TOKEN) + inputs: + scriptPath: ./vsts/scripts/publishFilesToAzureStorage.sh + args: ${{ parameters.destinationSdkStorageAccountName }} +- task: UseDotNet@2 + displayName: 'Use .NET Core SDK 7.x' + inputs: + version: 7.0.306 +- task: ShellScript@2 + displayName: 'Test Dev storage account' + env: + ORYX_TEST_SDK_STORAGE_URL: https://${{ parameters.destinationSdkStorageAccountName }}.blob.core.windows.net + ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN: $(ORYX-SDK-STAGING-PRIVATE-SAS-TOKEN) + inputs: + scriptPath: ./build/testIntegration.sh + args: StorageAccountTests=Dev \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/_platformBinariesTemplate.yml b/vsts/pipelines/1ESPipelines/templates/_platformBinariesTemplate.yml new file mode 100644 index 0000000000..294e83bf79 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/_platformBinariesTemplate.yml @@ -0,0 +1,27 @@ +parameters: + platformName: '' + debianFlavor: '' + destinationSdkStorageAccountName: '' +steps: +- checkout: self + clean: true +- task: UseDotNet@2 + displayName: 'Use .NET Core sdk 7.x' + inputs: + version: 7.0.306 +- task: ShellScript@2 + displayName: 'Building platform binaries' + env: + ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN: $(ORYX-SDK-STAGING-PRIVATE-SAS-TOKEN) + DOTNET_PRIVATE_STORAGE_ACCOUNT_ACCESS_TOKEN: $(DotnetPrivateStorageAccountAccessToken) + inputs: + scriptPath: ./build/buildPlatformBinaries.sh + args: ${{ parameters.platformName }} ${{ parameters.debianFlavor }} https://${{ parameters.destinationSdkStorageAccountName }}.blob.core.windows.net +- task: CopyFiles@2 + displayName: 'Copy artifacts from source repo to agent artifacts folder' + inputs: + sourceFolder: '$(Build.SourcesDirectory)/artifacts' + contents: '**/*.*' + targetFolder: $(Build.ArtifactStagingDirectory) + overWrite: true + condition: true \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/_releaseBaseImagesJobTemplate.yml b/vsts/pipelines/1ESPipelines/templates/_releaseBaseImagesJobTemplate.yml new file mode 100644 index 0000000000..0c231482a6 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/_releaseBaseImagesJobTemplate.yml @@ -0,0 +1,61 @@ +jobs: +- job: Release_PhpRuntimeBaseImage + displayName: Push PHP Runtime Base Image to MCR + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + timeoutInMinutes: 250 + steps: + - template: _releaseBaseImagesStepTemplate.yml + parameters: + baseImageName: 'php' + +- job: Release_PhpFpmNginxRuntimeBaseImage + displayName: Push PHP-FPM Runtime Base Image to MCR + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + timeoutInMinutes: 250 + steps: + - template: _releaseBaseImagesStepTemplate.yml + parameters: + baseImageName: 'php-fpm' + +- job: Release_DotNetCoreRuntimeBaseImage + displayName: Push DotNetCore Runtime Base Images to MCR + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + timeoutInMinutes: 250 + steps: + - template: _releaseBaseImagesStepTemplate.yml + parameters: + baseImageName: 'dotnetcore' + +- job: Release_NodeBaseImage + displayName: Push Node Base Image to MCR + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + timeoutInMinutes: 300 + steps: + - template: _releaseBaseImagesStepTemplate.yml + parameters: + baseImageName: 'node' + +- job: Release_RubyBaseImage + displayName: Push Ruby Base Image to MCR + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + timeoutInMinutes: 300 + + steps: + - template: _releaseBaseImagesStepTemplate.yml + parameters: + aseImageName: 'ruby' \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/_releaseBaseImagesStepTemplate.yml b/vsts/pipelines/1ESPipelines/templates/_releaseBaseImagesStepTemplate.yml new file mode 100644 index 0000000000..6f050f5dd6 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/_releaseBaseImagesStepTemplate.yml @@ -0,0 +1,54 @@ +parameters: + ascName: oryx-new-service-connection + acrDevName: oryxdevmcr + acrProdName: oryxmcr + acrPmeProdName: oryxprodmcr + acrPmeProdSrvConnection: Oryx-PME-ACR-Production + baseImageName: '' + baseImageRepository: public/oryx/base +steps: +- checkout: self + clean: true +- task: Docker@1 + displayName: Container registry login + inputs: + command: login + azureSubscriptionEndpoint: ${{ parameters.ascName }} + azureContainerRegistry: ${{ parameters.acrDevName }}.azurecr.io +- task: DownloadBuildArtifacts@0 + displayName: 'Download Artifacts for release' + inputs: + artifactName: drop +- task: CopyFiles@2 + displayName: 'Copy Files to: $(Build.ArtifactStagingDirectory)' + inputs: + SourceFolder: '$(System.ArtifactsDirectory)' + TargetFolder: '$(Build.ArtifactStagingDirectory)' +- task: Shellpp@0 + displayName: 'Pull and create release tags for PME staging ACR' + inputs: + type: FilePath + scriptPath: ./vsts/scripts/tagBaseImagesForRelease.sh + args: '${{ parameters.baseImageName }} ${{ parameters.acrPmeProdName }}' +- task: Docker@2 + displayName: Login to production PME ACR + inputs: + command: login + containerRegistry: ${{ parameters.acrPmeProdSrvConnection }} +- script: echo base image is ${{ parameters.baseImageName }} and repository is ${{ parameters.baseImageRepository }} +- task: Shellpp@0 + displayName: 'Push images to PME staging ACR' + inputs: + type: FilePath + scriptPath: ./vsts/scripts/pushBaseImagesToRegistry.sh + args: '$(Build.ArtifactStagingDirectory)/drop/images/${{ parameters.baseImageName }}/${{ parameters.acrPmeProdName }}' +- task: Docker@2 + displayName: Logout from PME ACR + inputs: + command: logout + containerRegistry: ${{ parameters.acrPmeProdSrvConnection }} +- task: ShellScript@2 + displayName: 'Clean up Docker containers and images' + inputs: + scriptPath: ./vsts/scripts/cleanDocker.sh + condition: true \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/_releaseJobTemplate.yml b/vsts/pipelines/1ESPipelines/templates/_releaseJobTemplate.yml new file mode 100644 index 0000000000..39b0528908 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/_releaseJobTemplate.yml @@ -0,0 +1,57 @@ +jobs: +- job: Release_BuildImages + displayName: Push Build Image to MCR + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + skipComponentGovernanceDetection: true + timeoutInMinutes: 300 + steps: + - script: | + echo "##vso[task.setvariable variable=ReleaseBuildImages;]true" + echo "##vso[task.setvariable variable=ReleaseRuntimeImages;]false" + displayName: 'Set variables' + - template: /vsts/pipelines/templates/_releaseStepTemplate.yml@self +- job: Release_RuntimeImages + displayName: Push Runtime Images to MCR + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + skipComponentGovernanceDetection: true + timeoutInMinutes: 400 + steps: + - script: | + echo "##vso[task.setvariable variable=ReleaseBuildImages;]false" + echo "##vso[task.setvariable variable=ReleaseRuntimeImages;]true" + displayName: 'Set variables' + - template: /vsts/pipelines/templates/_releaseStepTemplate.yml@self +- job: Release_GitHub + displayName: Create GitHub release + dependsOn: + - Release_BuildImages + - Release_RuntimeImages + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + skipComponentGovernanceDetection: true + steps: + - task: DownloadPipelineArtifact@2 + displayName: 'Download Oryx binaries to publish to release' + inputs: + artifactName: 'signed_binaries' + path: $(Build.SourcesDirectory)/artifacts + - template: /vsts/pipelines/templates/_setReleaseTag.yml@self + - task: GitHubRelease@0 + displayName: 'GitHub release (create)' + inputs: + gitHubConnection: 'Oryx-GitHub' + repositoryName: Microsoft/Oryx + tagSource: manual + tag: '$(RELEASE_TAG_NAME)' + assets: $(Build.SourcesDirectory)/artifacts/oryx_linux_x64.tar.gz \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/_releaseStepTemplate.yml b/vsts/pipelines/1ESPipelines/templates/_releaseStepTemplate.yml new file mode 100644 index 0000000000..671c653163 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/_releaseStepTemplate.yml @@ -0,0 +1,124 @@ +parameters: + ascName: oryx-new-service-connection + acrDevName: oryxdevmcr.azurecr.io + acrProdName: oryxmcr + acrPmeProdName: oryxprodmcr + acrPmeProdSrvConnection: Oryx-PME-ACR-Production + prodImageName: oryxmcr.azurecr.io/public/oryx +steps: +- script: | + if [ "$(ReleaseBuildImages)" != "true" ] && [ "$(ReleaseRuntimeImages)" != "true" ] + then + echo "Invalid configuration." + echo "Variable 'ReleaseBuildImages' or 'ReleaseRuntimeImages' needs to be 'true' to run this" + exit 1 + fi + displayName: 'Validate release pipeline run' +- script: | + docker -v + echo "Installing Docker CE version 24.0.5." + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt update + sudo apt-get install docker-ce=5:24.0.5~3-0~ubuntu-focal docker-ce-cli=5:24.0.5~3-0~ubuntu-focal containerd.io + echo "After installing ." + docker -v + displayName: 'Install Docker 24.0.5' + condition: true +- script: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + docker images && docker system prune -fa && docker images && echo + displayName: 'clean docker images' +- task: UseDotNet@2 + displayName: 'Use .NET Core sdk 7.x' + inputs: + version: 7.0.306 +- script: | + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt update + sudo apt-get install docker-ce=5:24.0.5~3-0~ubuntu-focal docker-ce-cli=5:24.0.5~3-0~ubuntu-focal containerd.io + docker -v + displayName: 'Install Docker 24.0.5' +- checkout: self + clean: true +- task: Docker@1 + displayName: Container registry login + inputs: + command: login + azureSubscriptionEndpoint: ${{ parameters.ascName }} + azureContainerRegistry: ${{ parameters.acrDevName }} +- task: DownloadBuildArtifacts@0 + displayName: 'Download Artifacts for release' + inputs: + artifactName: drop +- task: CopyFiles@2 + displayName: 'Copy Files to: $(Build.ArtifactStagingDirectory)' + inputs: + SourceFolder: '$(System.ArtifactsDirectory)' + TargetFolder: '$(Build.ArtifactStagingDirectory)' +- template: /vsts/pipelines/templates/_setReleaseTag.yml@self +- task: Shellpp@0 + displayName: 'Pull and create release tags for build images' + inputs: + type: FilePath + scriptPath: ./vsts/scripts/tagBuildImagesForRelease.sh + args: '' + condition: and(succeeded(), eq(variables['ReleaseBuildImages'], 'true')) +- task: Shellpp@0 + displayName: 'Pull and create release tags for runtime images' + inputs: + type: FilePath + scriptPath: ./vsts/scripts/tagRunTimeImagesForRelease.sh + condition: and(succeeded(), eq(variables['ReleaseRuntimeImages'], 'true')) +- task: Shellpp@0 + displayName: 'Pull and create release tags for CLI images' + inputs: + type: FilePath + scriptPath: ./vsts/scripts/tagCliImagesForRelease.sh + condition: and(succeeded(), eq(variables['ReleaseBuildImages'], 'true')) +- task: Docker@1 + displayName: Dev Container registry logout + inputs: + command: logout + azureSubscriptionEndpoint: ${{ parameters.ascName }} + azureContainerRegistry: ${{ parameters.acrDevName }} +- task: Docker@2 + displayName: Login to PME ACR + inputs: + command: login + containerRegistry: ${{ parameters.acrPmeProdSrvConnection }} +- task: Shellpp@0 + displayName: 'Push build images to PME staging ACR' + inputs: + type: FilePath + scriptPath: ./vsts/scripts/pushImagesToRegistry.sh + args: '$(Build.ArtifactStagingDirectory)/drop/images/${{ parameters.acrPmeProdName }}-build-images-mcr.txt' + condition: and(succeeded(), eq(variables['ReleaseBuildImages'], 'true')) +- task: Shellpp@0 + displayName: 'Push CLI image to PME staging ACR' + inputs: + type: FilePath + scriptPath: ./vsts/scripts/pushImagesToRegistry.sh + args: '$(Build.ArtifactStagingDirectory)/drop/images/${{ parameters.acrPmeProdName }}-cli-images-mcr.txt' + condition: and(succeeded(), eq(variables['ReleaseBuildImages'], 'true')) +- task: Shellpp@0 + displayName: 'Push runtime images to PME staging ACR' + inputs: + type: FilePath + scriptPath: ./vsts/scripts/pushImagesToRegistry.sh + args: '$(Build.ArtifactStagingDirectory)/drop/images/${{ parameters.acrPmeProdName }}-runtime-images-mcr.txt' + condition: and(succeeded(), eq(variables['ReleaseRuntimeImages'], 'true')) +- task: Docker@2 + displayName: Logout from PME ACR + inputs: + command: logout + containerRegistry: '${{ parameters.acrPmeProdSrvConnection }}' +- task: ShellScript@2 + displayName: 'Clean up Docker containers and images' + inputs: + scriptPath: ./vsts/scripts/cleanDocker.sh + condition: true \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/_securityChecks.yml b/vsts/pipelines/1ESPipelines/templates/_securityChecks.yml new file mode 100644 index 0000000000..1f9d7e63d9 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/_securityChecks.yml @@ -0,0 +1,41 @@ +steps: +- checkout: self + clean: true +- task: NuGetToolInstaller@1 + inputs: + versionSpec: 5.8.x +- task: UseDotNet@2 + displayName: 'Use .NET Core sdk 7.x' + inputs: + version: 7.0.306 +- script: | + dotnet --version && dotnet msbuild -version && echo + displayName: 'Print dotnet and msbuild version' +- task: NuGetCommand@2 + displayName: 'Run "nuget restore" on Oryx solution' + inputs: + command: 'restore' + restoreSolution: '$(Build.SourcesDirectory)\Oryx.sln' +- task: VSBuild@1 + displayName: 'Build the Oryx solution' + inputs: + solution: '$(Build.SourcesDirectory)\Oryx.sln' + configuration: 'debug' + createLogFile: true + logFileVerbosity: diagnostic +- task: PowerShell@2 + displayName: 'Wait 30 seconds after building solution' + inputs: + targetType: 'inline' + script: 'Start-Sleep -Seconds 30' +- task: AntiMalware@3 + displayName: 'Run Antimalware Scan' + inputs: + InputType: 'Basic' + ScanType: 'CustomScan' + EnableServices: true + TreatSignatureUpdateFailureAs: 'Warning' + SupportLogOnError: false + FileDirPath: '$(Build.StagingDirectory)' + SignatureFreshness: 'UpToDate' + TreatStaleSignatureAs: 'Warning' \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/_setReleaseTag.yml b/vsts/pipelines/1ESPipelines/templates/_setReleaseTag.yml new file mode 100644 index 0000000000..91b71548c2 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/_setReleaseTag.yml @@ -0,0 +1,18 @@ +steps: +- task: DownloadPipelineArtifact@2 + displayName: 'Download artifact file having release tag name' + inputs: + targetPath: $(Build.SourcesDirectory)/artifacts/releaseTag + artifactName: 'releaseTag' +- script: | + set -e + file="$(Build.SourcesDirectory)/artifacts/releaseTag/releaseTag.txt" + echo "Release tag name is:" + cat "$file" + tagName="$(Build.BuildNumber)" + if [ -f "$file" ]; then + tagName=`cat $file` + fi + echo "##vso[task.setvariable variable=RELEASE_TAG_NAME;]$tagName" + displayName: 'Read release tag name from artifact file and set it as environment variable' + name: setReleaseTag \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/_signBinary.yml b/vsts/pipelines/1ESPipelines/templates/_signBinary.yml new file mode 100644 index 0000000000..f9d054900d --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/_signBinary.yml @@ -0,0 +1,82 @@ +steps: +- powershell: | + Write-Host "##vso[task.setvariable variable=SignType;isOutput=true]real" + name: setSignTypeVariable + displayName: 'Set sign binary variable' + condition: > + and(in(variables['Build.Reason'], 'Schedule', 'Manual'), eq(variables['Build.DefinitionName'], 'Oryx-CI'), or( + startsWith(variables['Build.SourceBranch'], 'refs/heads/main'), + startsWith(variables['Build.SourceBranch'], 'refs/heads/patch/'), + startsWith(variables['Build.SourceBranch'],'refs/heads/exp/' ))) +- script: | + echo $(setSignTypeVariable.SignType) + name: SignType +- task: NuGetToolInstaller@0 + displayName: 'Use NuGet 4.6.2' + inputs: + versionSpec: 4.6.2 + condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) +- task: UseDotNet@2 + displayName: 'Use .NET Core sdk 7.x' + inputs: + version: 7.0.306 +- powershell: | + Write-Host "Setting up git_commit and build_number as env variable" + $env:GIT_COMMIT=$(git rev-parse HEAD) + $env:BUILD_NUMBER=$env:BUILD_BUILDNUMBER + dotnet publish -r linux-x64 -c Release src\BuildScriptGeneratorCLI\BuildScriptGeneratorCli.csproj + dotnet publish -r linux-x64 -c Release src\BuildServer\BuildServer.csproj + displayName: 'dotnet publish and after setting git_commit and build_number as env variable' + condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) +- task: VSBuild@1 + displayName: 'Sign Oryx Binaries' + inputs: + solution: 'src/BuildScriptGeneratorCli/Oryx_sign.signproj' + msbuildArgs: '/t:SignFiles /p:RuntimeIdentifier=linux-x64 /p:MicroBuild_SigningEnabled=true' + configuration: '$(BuildConfiguration)' + condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) +- task: VSBuild@1 + displayName: 'Sign Oryx Build Server Binaries' + inputs: + solution: 'src/BuildServer/BuildServer.signproj' + msbuildArgs: '/t:SignFiles /p:RuntimeIdentifier=linux-x64 /p:MicroBuild_SigningEnabled=true' + configuration: '$(BuildConfiguration)' + condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) +- task: ArchiveFiles@2 + displayName: 'Create compressed file of Oryx binaries' + inputs: + rootFolderOrFile: 'src\BuildScriptGeneratorCli\bin\$(BuildConfiguration)\linux-x64\publish' + includeRootFolder: false + archiveType: tar + tarCompression: 'gz' + archiveFile: '$(Build.ArtifactStagingDirectory)/oryx_linux_x64.tar.gz' + replaceExistingArchive: true + verbose: true + condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) +- task: ArchiveFiles@2 + displayName: 'Create compressed file of Oryx Build Server binaries' + inputs: + rootFolderOrFile: 'src\BuildServer\bin\$(BuildConfiguration)\linux-x64\publish' + includeRootFolder: false + archiveType: tar + tarCompression: 'gz' + archiveFile: '$(Build.ArtifactStagingDirectory)/oryx_buildServer_linux_x64.tar.gz' + replaceExistingArchive: true + verbose: true + condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) +- task: CopyFiles@2 + displayName: 'Copy Files to: $(Build.ArtifactStagingDirectory)/binaries' + inputs: + SourceFolder: 'src/BuildScriptGeneratorCli/bin/$(BuildConfiguration)/linux-x64/publish' + Contents: '**\*' + TargetFolder: '$(Build.ArtifactStagingDirectory)/binaries' + CleanTargetFolder: true + condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) +- task: CopyFiles@2 + displayName: 'Copy BuildServer Files to: $(Build.ArtifactStagingDirectory)/binaries' + inputs: + SourceFolder: 'src/BuildServer/bin/$(BuildConfiguration)/linux-x64/publish' + Contents: '**\*' + TargetFolder: '$(Build.ArtifactStagingDirectory)/binaries' + CleanTargetFolder: false + condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/_signBinaryDetector.yml b/vsts/pipelines/1ESPipelines/templates/_signBinaryDetector.yml new file mode 100644 index 0000000000..368d021982 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/_signBinaryDetector.yml @@ -0,0 +1,71 @@ +steps: +- checkout: self + clean: true + +- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection - OSS Compliance' + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/tests' + +- task: ms-vseng.MicroBuildTasks.30666190-6959-11e5-9f96-f56098202fef.MicroBuildSigningPlugin@1 + displayName: 'Install Signing Plugin' + inputs: + signType: 'Real' + +- task: UseDotNet@2 + displayName: 'Use .NET Core SDK 7.x' + inputs: + version: 7.0.306 + +- powershell: | + Write-Host "Setting up git_commit and build_number as env variable" + $env:GIT_COMMIT=$(git rev-parse HEAD) + $env:BUILD_NUMBER=$env:BUILD_BUILDNUMBER + $projectFile="src\Detector\Detector.csproj" + + $env:SIGN_PACKAGE="true" + dotnet build Detector.sln -c Release + dotnet restore src\Detector\Detector.signproj + displayName: 'Build Detector.sln' + +- task: VSBuild@1 + displayName: 'Sign Oryx Binaries' + inputs: + solution: 'src/Detector/Detector.signproj' + msbuildArgs: '/t:SignFiles /p:MicroBuild_SigningEnabled=true' + configuration: '$(BuildConfiguration)' + +- powershell: | + Write-Host "Running 'dotnet pack'..." + dotnet pack --no-build --include-symbols -c Release src\Detector\Detector.csproj + displayName: 'Re-package already generated NuGet packages with signed binaries' + +- task: VSBuild@1 + displayName: 'Sign NuGet packages' + inputs: + solution: 'vsts/nugetpackages.signproj' + msbuildArgs: '/t:SignFiles /p:MicroBuild_SigningEnabled=true' + +- task: NuGetToolInstaller@1 + inputs: + versionSpec: '5.6' + +- powershell: | + . $(Build.SourcesDirectory)\build\detector\__detectorNugetPackagesVersions.ps1 + cd "artifacts\packages" + nuget verify -signature ` + -CertificateFingerprint 3F9001EA83C560D712C24CF213C3D312CB3BFF51EE89435D3430BD06B5D0EECE ` + Microsoft.Oryx.Detector.$VERSION.nupkg ` + -verbosity detailed + displayName: 'Verify nuget packages' + +- powershell: | + Copy-Item -Path artifacts\packages -Destination $(Build.ArtifactStagingDirectory)\packages -Recurse + displayName: 'Copying nuget packages to artifact staging directory' + +- task: PublishPipelineArtifact@0 + displayName: 'Publish Pipeline Artifact' + inputs: + artifactName: 'detector_signed' + targetPath: '$(Build.ArtifactStagingDirectory)' + \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/integrationTests/_dotnetcoreIntegrationJobTemplate.yml b/vsts/pipelines/1ESPipelines/templates/integrationTests/_dotnetcoreIntegrationJobTemplate.yml new file mode 100644 index 0000000000..78b2aed23b --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/integrationTests/_dotnetcoreIntegrationJobTemplate.yml @@ -0,0 +1,51 @@ +parameters: +- name: storageAccountUrl + type: string + default: https://oryxsdksstaging.blob.core.windows.net +- name: testMappings + type: object + default: + - category: '6.0' + buildTag: 'debian-buster' + - category: '6.0' + buildTag: 'github-actions-debian-buster' + - category: '7.0' + buildTag: 'github-actions-debian-buster' + - category: '8.0' + buildTag: 'github-actions-debian-bookworm' + - category: 'dynamic' + buildTag: 'debian-buster' + - category: 'dynamic' + buildTag: 'github-actions-debian-buster' +jobs: +- ${{ each mapping in parameters.testMappings }}: + - job: '' + displayName: 'Run .NET Core ${{ mapping.category }} Integration Tests using build image tag ${{ mapping.buildTag }}' + dependsOn: + - Job_BuildImage_Latest + - Job_BuildImage_GithubActions + - Job_Buster_RuntimeImages + - Job_Bullseye_RuntimeImages + - Job_Bookworm_RuntimeImages + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + skipComponentGovernanceDetection: true + timeoutInMinutes: 300 + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=dotnetcore-${{ mapping.category }}&build-image=${{ mapping.buildTag }}" + echo "##vso[task.setvariable variable=TestIntegration;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: /vsts/pipelines/templates/_setReleaseTag.yml@self + - template: /vsts/pipelines/templates/_buildTemplate.yml@self \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/integrationTests/_golangIntegrationJobTemplate.yml b/vsts/pipelines/1ESPipelines/templates/integrationTests/_golangIntegrationJobTemplate.yml new file mode 100644 index 0000000000..36443745c1 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/integrationTests/_golangIntegrationJobTemplate.yml @@ -0,0 +1,50 @@ +parameters: +- name: storageAccountUrl + type: string + default: https://oryxsdksstaging.blob.core.windows.net +- name: testMappings + type: object + default: + - category: '1.17' + buildTag: 'full-debian-buster' + - category: '1.17' + buildTag: 'full-debian-bullseye' + - category: '1.18' + buildTag: 'full-debian-buster' + - category: '1.18' + buildTag: 'full-debian-bullseye' + - category: '1.19' + buildTag: 'full-debian-buster' + - category: '1.19' + buildTag: 'full-debian-bullseye' +jobs: +- ${{ each mapping in parameters.testMappings }}: + - job: '' + displayName: 'Run Golang ${{ mapping.category }} Integration Tests using build image tag ${{ mapping.buildTag }}' + dependsOn: + - Job_BuildImage_Full + - Job_Buster_RuntimeImages + - Job_Bullseye_RuntimeImages + - Job_Bookworm_RuntimeImages + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + skipComponentGovernanceDetection: true + timeoutInMinutes: 300 + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=golang-${{ mapping.category }}&build-image=${{ mapping.buildTag }}" + echo "##vso[task.setvariable variable=TestIntegration;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: /vsts/pipelines/templates/_setReleaseTag.yml@self + - template: /vsts/pipelines/templates/_buildTemplate.yml@self \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/integrationTests/_nodeIntegrationJobTemplate.yml b/vsts/pipelines/1ESPipelines/templates/integrationTests/_nodeIntegrationJobTemplate.yml new file mode 100644 index 0000000000..d79da61eae --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/integrationTests/_nodeIntegrationJobTemplate.yml @@ -0,0 +1,49 @@ +parameters: +- name: storageAccountUrl + type: string + default: https://oryxsdksstaging.blob.core.windows.net +- name: testMappings + type: object + default: + - category: '14-gh-buster' + buildTag: 'github-actions-debian-buster' + - category: '16' + buildTag: 'debian-buster' + - category: '18' + buildTag: 'github-actions-debian-bullseye' + - category: '20' + buildTag: 'github-actions-debian-bookworm' + - category: '20-bullseye' + buildTag: 'github-actions-debian-bullseye' +jobs: +- ${{ each mapping in parameters.testMappings }}: + - job: '' + displayName: 'Run Node ${{ mapping.category }} Integration Tests using build image tag ${{ mapping.buildTag }}' + dependsOn: + - Job_BuildImage_Latest + - Job_BuildImage_GithubActions + - Job_Buster_RuntimeImages + - Job_Bullseye_RuntimeImages + - Job_Bookworm_RuntimeImages + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + skipComponentGovernanceDetection: true + timeoutInMinutes: 300 + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=node-${{ mapping.category }}&build-image=${{ mapping.buildTag }}" + echo "##vso[task.setvariable variable=TestIntegration;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: /vsts/pipelines/templates/_setReleaseTag.yml@self + - template: /vsts/pipelines/templates/_buildTemplate.yml@self \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/integrationTests/_phpIntegrationJobTemplate.yml b/vsts/pipelines/1ESPipelines/templates/integrationTests/_phpIntegrationJobTemplate.yml new file mode 100644 index 0000000000..871fd35362 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/integrationTests/_phpIntegrationJobTemplate.yml @@ -0,0 +1,49 @@ +parameters: +- name: storageAccountUrl + type: string + default: https://oryxsdksstaging.blob.core.windows.net +- name: testMappings + type: object + default: + - category: '7.4' + buildTag: 'github-actions-debian-buster' + - category: '8.0' + buildTag: 'github-actions-debian-buster' + - category: '8.1' + buildTag: 'github-actions-debian-buster' + - category: '8.2' + buildTag: 'github-actions-debian-buster' + - category: '8.2' + buildTag: 'github-actions-debian-bullseye' +jobs: +- ${{ each mapping in parameters.testMappings }}: + - job: '' + displayName: 'Run PHP ${{ mapping.category }} Integration Tests using build image tag ${{ mapping.buildTag }}' + dependsOn: + - Job_BuildImage_Latest + - Job_BuildImage_GithubActions + - Job_Buster_RuntimeImages + - Job_Bullseye_RuntimeImages + - Job_Bookworm_RuntimeImages + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + skipComponentGovernanceDetection: true + timeoutInMinutes: 300 + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=php-${{ mapping.category }}&build-image=${{ mapping.buildTag }}" + echo "##vso[task.setvariable variable=TestIntegration;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: /vsts/pipelines/templates/_setReleaseTag.yml@self + - template: /vsts/pipelines/templates/_buildTemplate.yml@self \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/integrationTests/_pythonIntegrationJobTemplate.yml b/vsts/pipelines/1ESPipelines/templates/integrationTests/_pythonIntegrationJobTemplate.yml new file mode 100644 index 0000000000..4342f98679 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/integrationTests/_pythonIntegrationJobTemplate.yml @@ -0,0 +1,57 @@ +parameters: +- name: storageAccountUrl + type: string + default: https://oryxsdksstaging.blob.core.windows.net +- name: testMappings + type: object + default: + - category: '3.7' + buildTag: 'github-actions-debian-bullseye' + - category: '3.8' + buildTag: 'github-actions-debian-bullseye' + - category: '3.9' + buildTag: 'debian-buster' + - category: '3.9' + buildTag: 'github-actions-debian-buster' + - category: '3.10' + buildTag: 'github-actions-debian-bullseye' + - category: '3.11' + buildTag: 'github-actions-debian-bullseye' + - category: '3.11' + buildTag: 'github-actions-debian-bookworm' + - category: '3.12' + buildTag: 'github-actions-debian-bullseye' + - category: '3.12' + buildTag: 'github-actions-debian-bookworm' +jobs: +- ${{ each mapping in parameters.testMappings }}: + - job: '' + displayName: 'Run Python ${{ mapping.category }} Integration Tests using build image tag ${{ mapping.buildTag }}' + dependsOn: + - Job_BuildImage_Latest + - Job_BuildImage_GithubActions + - Job_Buster_RuntimeImages + - Job_Bullseye_RuntimeImages + - Job_Bookworm_RuntimeImages + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + skipComponentGovernanceDetection: true + timeoutInMinutes: 300 + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=python-${{ mapping.category }}&build-image=${{ mapping.buildTag }}" + echo "##vso[task.setvariable variable=TestIntegration;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: /vsts/pipelines/templates/_setReleaseTag.yml@self + - template: /vsts/pipelines/templates/_buildTemplate.yml@self \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/testACRImgUpload.yml b/vsts/pipelines/1ESPipelines/testACRImgUpload.yml new file mode 100644 index 0000000000..ff79d03797 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/testACRImgUpload.yml @@ -0,0 +1,28 @@ +parameters: + - name: nightlyTagName + displayName: The tag of image built by nightly pipeline + - name: containerRegistryName + displayName: Container registry name for pushing built image + type: string + default: oryxtestmcr.azurecr.io + +resources: +- repo: self + fetchDepth: 15 + +variables: + group: Oryx + Packaging.EnableSBOMSigning: true + +stages: + - stage: Copy + displayName: Copy Stage + + jobs: + - job: Start_Copy + steps: + - template: templates/_copyImgFromDevToTest.yml + parameters: + nightlyTag: ${{ parameters.nightlyTagName }} + +trigger: none \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/validation-detector.yml b/vsts/pipelines/1ESPipelines/validation-detector.yml new file mode 100644 index 0000000000..3cd46a4153 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/validation-detector.yml @@ -0,0 +1,22 @@ +# specific branch +pr: + paths: + exclude: + - /* + include: + - Detector.sln + - src/Detector/* + - tests/Detector.Tests/* + - vsts/nugetpackages.signproj + - vsts/pipelines/validation-detector.yml + +jobs: +- job: Job_Detector + displayName: Detector Validation + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + timeoutInMinutes: 480 + steps: + - template: templates/_buildTemplateDetector.yml \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/validation.yml b/vsts/pipelines/1ESPipelines/validation.yml new file mode 100644 index 0000000000..a7afdcede7 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/validation.yml @@ -0,0 +1,171 @@ +parameters: + - name: storageAccountUrl + displayName: SDK storage account URL for testing + type: string + default: https://oryxsdksstaging.blob.core.windows.net + - name: buildImages + type: object + default: + - + key: Latest + value: latest + - + key: LtsVersions + value: ltsversions + - + key: Jamstack + value: jamstack + - + key: GithubActions + value: githubactions + - + key: VsoFocal + value: vso-focal + - + key: VsoBullseye + value: vso-bullseye + - + key: Full + value: full + - + key: Cli + value: cli-stretch + - + key: CliBuster + value: cli-buster + - + key: CliBullseye + value: cli-bullseye + - + key: CliBuilderBullseye + value: cli-builder-bullseye + - + key: Buildpack + value: buildpack + +resources: +- repo: self + fetchDepth: 15 + +variables: + group: Oryx + Packaging.EnableSBOMSigning: true + +jobs: +- job: Job_Security + displayName: Security + pool: + name: Azure Pipelines + vmImage: windows-2022 + steps: + - template: templates/_securityChecks.yml + +- ${{ each buildImage in parameters.buildImages }}: + - job: Job_BuildImage_${{ buildImage.key }} + displayName: Build and Test Build ${{ buildImage.key }} Image + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + - group: Oryx + timeoutInMinutes: 480 + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]true" + echo "##vso[task.setvariable variable=TestBuildImages;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + echo "##vso[task.setvariable variable=RELEASE_TAG_NAME;]$(Build.BuildNumber)" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + if [[ "${{ buildImage.value }}" =~ "cli-builder" ]]; then + echo "##vso[task.setvariable variable=PushBuilderImages;]true" + fi + displayName: 'Set variables' + - template: templates/_buildTemplate.yml + parameters: + imageType: ${{ buildImage.value }} + +# commented out temporarily +# - job: Job_BuilderImages +# displayName: Build Builder Images +# pool: +# name: AzurePipelines-EO +# demands: +# - ImageOverride -equals AzurePipelinesUbuntu20.04compliant +# timeoutInMinutes: 480 +# steps: +# - template: templates/_builderTemplate.yml +# dependsOn: Job_BuildImage_CliBuilderBullseye + +- job: Job_Buster_RuntimeImages + displayName: Build and Test Buster Runtime Images + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + - group: Oryx + timeoutInMinutes: 480 + steps: + - script: | + echo "##vso[task.setvariable variable=BuildRuntimeImages;]true" + echo "##vso[task.setvariable variable=TestRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + echo "##vso[task.setvariable variable=RELEASE_TAG_NAME;]$(Build.BuildNumber)" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: templates/_buildTemplate.yml + parameters: + imageType: buster + +- job: Job_Bullseye_RuntimeImages + displayName: Build and Test Bullseye Runtime Images + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + - group: Oryx + timeoutInMinutes: 480 + steps: + - script: | + echo "##vso[task.setvariable variable=BuildRuntimeImages;]true" + echo "##vso[task.setvariable variable=TestRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + echo "##vso[task.setvariable variable=RELEASE_TAG_NAME;]$(Build.BuildNumber)" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: templates/_buildTemplate.yml + parameters: + imageType: bullseye + +- job: Job_Bookworm_RuntimeImages + displayName: Build and Test Bookworm Runtime Images + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + - group: Oryx + timeoutInMinutes: 480 + steps: + - script: | + echo "##vso[task.setvariable variable=BuildRuntimeImages;]true" + echo "##vso[task.setvariable variable=TestRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + echo "##vso[task.setvariable variable=RELEASE_TAG_NAME;]$(Build.BuildNumber)" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: templates/_buildTemplate.yml + parameters: + imageType: bookworm + +trigger: none \ No newline at end of file