WX-983 Prototype Azure e2e workflow test

.github/workflows/run_workflow.yml

@@ -0,0 +1,163 @@
+name: 'Azure e2e - Run Workflow'
+on:
+  workflow_dispatch:
+    inputs:
+      target-branch:
+        description: 'Branch of Cromwell to run tests on'
+        required: true
+        default: 'develop'
+        type: string
+      # Replace user data with seeded data provided by devOps (once available)
+      owner-subject:
+        description: 'Owner of billing project'
+        required: true
+        default: '[email protected]'
+        type: string
+      service-account:
+        description: 'Email address or unique identifier of the Google Cloud service account for which to generate credentials'
+        required: true
+        default: '[email protected]'
+        type: string
+
+env:
+  BEE_NAME: '${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt}}-dev'
+  BROADBOT_TOKEN: '${{ secrets.BROADBOT_GITHUB_TOKEN }}' # github token for access to kick off a job in the private repo
+  RUN_NAME_SUFFIX: '${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt }}'
+
+jobs:
+  init-github-context:
+    runs-on: ubuntu-latest
+    outputs:
+      branch: ${{ steps.extract-inputs.outputs.branch }}
+    steps:
+#       NOTE: below was included in the rawls template but seems redundant due to defined values up on top
+#       Remove if it ends up being unecessary
+      - name: Get inputs or use defaults
+        id: extract-inputs
+        run: |
+          echo "branch=${{ inputs.target-branch || 'develop' }}" >> "$GITHUB_OUTPUT"
+
+  # This job provisions useful parameters for e2e tests, including access tokens.
+  # Please note: access tokens are for use in the same workflow, they cannot be dispatched to remote workflows.
+  params-gen:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    outputs:
+      project-name: ${{ steps.gen.outputs.project_name }}
+    steps:
+      - name: Generate a random billing project name
+        id: 'gen'
+        run: |
+          project_name=$(echo "tmp-billing-project-$(uuidgen)" | cut -c -30)
+          echo "project_name=${project_name}" >> $GITHUB_OUTPUT
+
+  create-bee-workflow:
+    runs-on: ubuntu-latest
+    needs: [init-github-context, params-gen]
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    steps:
+      - name: Dispatch to terra-github-workflows
+        uses: broadinstitute/workflow-dispatch@v3
+        with:
+          workflow: bee-create
+          repo: broadinstitute/terra-github-workflows
+          ref: refs/heads/main
+          token: ${{ env.BROADBOT_TOKEN }}
+          # NOTE: Opting to use "prod" instead of custom tag since I specifically want to test against the current prod state
+          # NOTE: For testing purposes I'm using dev
+          inputs: '{ "bee-name": "${{ env.BEE_NAME }}", "version-template": "dev" }'
+
+  create-and-attach-billing-project-to-landing-zone-workflow:
+    runs-on: ubuntu-latest
+    needs: [create-bee-workflow, params-gen]
+    steps:
+      - name: dispatch to terra-github-workflows
+        uses: broadinstitute/workflow-dispatch@v3
+        with:
+          workflow: attach-billing-project-to-landing-zone.yaml
+          repo: broadinstitute/terra-github-workflows
+          ref: refs/heads/main
+          token: ${{ env.BROADBOT_TOKEN }}
+          inputs: '{
+            "run-name": "attach-billing-project-to-landing-zone-${{ env.RUN_NAME_SUFFIX }}",
+            "bee-name": "${{ env.BEE_NAME }}",
+            "billing-project": "${{ needs.params-gen.outputs.project-name }}",
+            "billing-project-creator": "${{ inputs.owner-subject }}",
+            "service-account": "${{inputs.service-account}}" }'
+
+  run-cromwell-az-e2e:
+    runs-on: ubuntu-latest
+    needs: [params-gen, create-and-attach-billing-project-to-landing-zone-workflow]
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    steps:
+      - uses: 'actions/checkout@v3'
+      - name: Generate OAuth2 2.0 access token for owner
+        id: 'owner_auth'
+        uses: google-github-actions/auth@v1
+        with:
+          token_format: 'access_token'
+          workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
+          service_account: ${{ inputs.service-account }}
+          access_token_scopes: 'profile, email, openid'
+          access_token_subject: ${{ inputs.owner-subject }}
+          export_environment_variables: false
+          create_credentials_file: false
+      - uses: actions/checkout@v3
+        with:
+          #NOTE: replace with ${{inputs.target-branch}} before merge
+          ref: refs/heads/WX-983
+      - name: Install poetry
+        uses: snok/install-poetry@v1     
+      - name: Run e2e test
+        env:
+          BEE_NAME: ${{env.BEE_NAME}}
+          BILLING_PROJECT_NAME: ${{needs.params-gen.outputs.project-name}}
+          BEARER_TOKEN: ${{ steps.owner_auth.outputs.access_token }}
+        run: |
+          poetry run python server/src/test/python/cromwell-az-e2e/tests/az-e2e.py
+
+  delete-billing-project-v2-from-bee-workflow:
+    runs-on: ubuntu-latest
+    needs: [run-cromwell-az-e2e, create-and-attach-billing-project-to-landing-zone-workflow, params-gen]
+    if: always()
+    steps:
+      - name: dispatch to terra-github-workflows
+        uses: broadinstitute/workflow-dispatch@v3
+        with:
+          workflow: .github/workflows/delete-billing-project-v2-from-bee.yaml
+          repo: broadinstitute/terra-github-workflows
+          ref: refs/heads/main
+          token: ${{ env.BROADBOT_TOKEN }}
+          inputs: '{ 
+            "run-name": "delete-billing-project-v2-from-bee-${{ env.RUN_NAME_SUFFIX }}",
+            "bee-name": "${{ env.BEE_NAME }}", 
+            "billing-project": "${{ needs.params-gen.outputs.project-name }}", 
+            "billing-project-owner": "${{ inputs.owner-subject }}", 
+            "service-account": "${{ inputs.service-account }}", 
+            "silent-on-failure": "false" }'
+
+  destroy-bee-workflow:
+    runs-on: ubuntu-latest
+    # QUESTION: Can a billing project be deleted if the bee is deleted?
+    # QUESTION: Can a billing project be deleted if the bee is still tied to it?
+    needs: [create-bee-workflow, delete-billing-project-v2-from-bee-workflow]
+    if: always()
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    steps:
+      - name: dispatch to terra-github-workflows
+        uses: broadinstitute/workflow-dispatch@v3
+        with:
+          workflow: bee-destroy.yaml
+          repo: broadinstitute/terra-github-workflows
+          ref: refs/heads/main
+          token: ${{ env.BROADBOT_TOKEN }}
+          inputs: '{ "bee-name": "${{ env.BEE_NAME }}" }'
+          wait-for-completion: true

.gitignore

@@ -23,6 +23,9 @@ console_output.txt
 expected.json
 run_mode_metadata.json
 
+#bloop files
+/.bloop
+
 # custom config
 cromwell-executions
 cromwell-test-executions
@@ -55,3 +58,6 @@ tesk_application.conf
 **/venv/
 exome_germline_single_sample_v1.3/
 **/*.pyc
+
+# GHA credentials
+gha-creds-*.json