Merge pull request #142 from Barts-Life-Science/upstream-refresh

Upstream refresh

.github/ISSUE_TEMPLATE/bug_report.md

@@ -15,8 +15,8 @@ A clear and concise description of what the bug is.
 **Steps to reproduce**
 
 1.
-1.
-1.
+2.
+3.
 
 **Azure TRE release version (e.g. v0.14.0 or main):**
 

.github/actions/devcontainer_run_command/action.yml

@@ -121,6 +121,14 @@ inputs:
     description: "JSON string containing key/value pairs to injet into the Resource Processor as ENV vars"
     required: false
     default: ""
+  KV_PURGE_PROTECTION_ENABLED:
+    description: "A boolean indicating if the purge protection will be enabled on the core keyvault."
+    required: false
+    default: "true"
+  FIREWALL_SKU:
+    description: "Firewall SKU"
+    required: false
+    default: ""
 
 runs:
   using: composite
@@ -219,6 +227,7 @@ runs:
           -e IS_API_SECURED=${{ inputs.IS_API_SECURED }} \
           -e DOCKER_BUILDKIT=1 \
           -e TF_VAR_stateful_resources_locked=${{ inputs.STATEFUL_RESOURCES_LOCKED }} \
+          -e TF_VAR_kv_purge_protection_enabled="${{ inputs.KV_PURGE_PROTECTION_ENABLED }}" \
           -e TF_VAR_enable_airlock_malware_scanning=${{ inputs.ENABLE_AIRLOCK_MALWARE_SCANNING }} \
           -e CI_CACHE_ACR_NAME="${{ inputs.CI_CACHE_ACR_NAME }}" \
           -e TF_VAR_core_app_service_plan_sku="${{ (inputs.CORE_APP_SERVICE_PLAN_SKU != ''
@@ -229,6 +238,7 @@ runs:
             && inputs.RP_BUNDLE_VALUES) || '{}' }}' \
           -e TF_VAR_resource_processor_number_processes_per_instance="${{ (inputs.RESOURCE_PROCESSOR_NUMBER_PROCESSES_PER_INSTANCE != ''
             && inputs.RESOURCE_PROCESSOR_NUMBER_PROCESSES_PER_INSTANCE) || 5 }}" \
+          -e TF_VAR_firewall_sku=${{ inputs.FIREWALL_SKU }} \
           -e E2E_TESTS_NUMBER_PROCESSES="${{ inputs.E2E_TESTS_NUMBER_PROCESSES }}" \
           '${{ inputs.CI_CACHE_ACR_NAME }}${{ env.ACR_DOMAIN_SUFFIX }}/tredev:${{ inputs.DEVCONTAINER_TAG }}' \
         bash -c "${{ inputs.COMMAND }}"

.github/scripts/build.js

@@ -5,6 +5,7 @@
 // These tests can be run from the dev container using the run-tests.sh script
 //
 const { createHash } = require('crypto');
+const { create } = require('domain');
 
 async function getCommandFromComment({ core, context, github }) {
   const commentUsername = context.payload.comment.user.login;
@@ -292,11 +293,12 @@ function getRefIdForBranch(branchName) {
   return createShortHash(`refs/heads/${branchName}\n`);
 }
 function createShortHash(ref) {
-  const hash = createHash('sha1').update(ref, 'utf8').digest('hex')
+  const hash = createHash('sha512').update(ref, 'utf8').digest('hex');
   return hash.substring(0, 8);
 }
 
 module.exports = {
   getCommandFromComment,
-  labelAsExternalIfAuthorDoesNotHaveWriteAccess
+  labelAsExternalIfAuthorDoesNotHaveWriteAccess,
+  createShortHash
 }

.github/scripts/build.test.js

@@ -1,4 +1,4 @@
-const { getCommandFromComment, labelAsExternalIfAuthorDoesNotHaveWriteAccess } = require('./build.js')
+const { getCommandFromComment, labelAsExternalIfAuthorDoesNotHaveWriteAccess, createShortHash } = require('./build.js')
 const { createGitHubContext, PR_NUMBER, outputFor, toHaveComment } = require('./test-helpers.js')
 
 expect.extend({
@@ -146,7 +146,7 @@ describe('getCommandFromComment', () => {
             owner: 'someOwner',
             repo: 'someRepo',
             issue_number: PR_NUMBER.UPSTREAM_NON_DOCS_CHANGES,
-            bodyMatcher: /Running tests: https:\/\/github.com\/someOwner\/someRepo\/actions\/runs\/11112222 \(with refid `cbce50da`\)/,
+            bodyMatcher: /Running tests: https:\/\/github.com\/someOwner\/someRepo\/actions\/runs\/11112222 \(with refid `291ae84f`\)/,
           });
         });
       });
@@ -326,7 +326,7 @@ describe('getCommandFromComment', () => {
             owner: 'someOwner',
             repo: 'someRepo',
             issue_number: PR_NUMBER.FORK_NON_DOCS_CHANGES,
-            bodyMatcher: /Running tests: https:\/\/github.com\/someOwner\/someRepo\/actions\/runs\/11112222 \(with refid `6db070b1`\)/,
+            bodyMatcher: /Running tests: https:\/\/github.com\/someOwner\/someRepo\/actions\/runs\/11112222 \(with refid `607c7437`\)/,
           });
         });
       })
@@ -355,7 +355,7 @@ describe('getCommandFromComment', () => {
             owner: 'someOwner',
             repo: 'someRepo',
             issue_number: PR_NUMBER.FORK_NON_DOCS_CHANGES,
-            bodyMatcher: /Running tests: https:\/\/github.com\/someOwner\/someRepo\/actions\/runs\/11112222 \(with refid `6db070b1`\)/,
+            bodyMatcher: /Running tests: https:\/\/github.com\/someOwner\/someRepo\/actions\/runs\/11112222 \(with refid `607c7437`\)/,
           });
         });
       })
@@ -381,7 +381,7 @@ describe('getCommandFromComment', () => {
             owner: 'someOwner',
             repo: 'someRepo',
             issue_number: PR_NUMBER.UPSTREAM_NON_DOCS_CHANGES,
-            bodyMatcher: /Running extended tests: https:\/\/github.com\/someOwner\/someRepo\/actions\/runs\/11112222 \(with refid `cbce50da`\)/,
+            bodyMatcher: /Running extended tests: https:\/\/github.com\/someOwner\/someRepo\/actions\/runs\/11112222 \(with refid `291ae84f`\)/,
           });
         });
       });
@@ -407,7 +407,7 @@ describe('getCommandFromComment', () => {
             owner: 'someOwner',
             repo: 'someRepo',
             issue_number: PR_NUMBER.UPSTREAM_NON_DOCS_CHANGES,
-            bodyMatcher: /Running extended AAD tests: https:\/\/github.com\/someOwner\/someRepo\/actions\/runs\/11112222 \(with refid `cbce50da`\)/,
+            bodyMatcher: /Running extended AAD tests: https:\/\/github.com\/someOwner\/someRepo\/actions\/runs\/11112222 \(with refid `291ae84f`\)/,
           });
         });
       });
@@ -433,7 +433,7 @@ describe('getCommandFromComment', () => {
             owner: 'someOwner',
             repo: 'someRepo',
             issue_number: PR_NUMBER.UPSTREAM_NON_DOCS_CHANGES,
-            bodyMatcher: /Running shared service tests: https:\/\/github.com\/someOwner\/someRepo\/actions\/runs\/11112222 \(with refid `cbce50da`\)/,
+            bodyMatcher: /Running shared service tests: https:\/\/github.com\/someOwner\/someRepo\/actions\/runs\/11112222 \(with refid `291ae84f`\)/,
           });
         });
       });
@@ -549,7 +549,7 @@ describe('getCommandFromComment', () => {
             owner: 'someOwner',
             repo: 'someRepo',
             issue_number: PR_NUMBER.FORK_NON_DOCS_CHANGES,
-            bodyMatcher: /Running extended tests: https:\/\/github.com\/someOwner\/someRepo\/actions\/runs\/11112222 \(with refid `6db070b1`\)/,
+            bodyMatcher: /Running extended tests: https:\/\/github.com\/someOwner\/someRepo\/actions\/runs\/11112222 \(with refid `607c7437`\)/,
           });
         });
       })
@@ -679,13 +679,13 @@ goes here`,
       });
 
       test('should set prRefId output', async () => {
-        // Using a PR number of 123 should give a refid of 'cbce50da'
+        // Using a PR number of 123 should give a refid of '291ae84f'
         // Based on running `echo "refs/pull/123/merge" | shasum | cut -c1-8` (as per the original bash scripts)
         const context = createCommentContext({
           pullRequestNumber: PR_NUMBER.UPSTREAM_NON_DOCS_CHANGES
         });
         await getCommandFromComment({ core, context, github });
-        expect(outputFor(mockCoreSetOutput, 'prRefId')).toBe('cbce50da');
+        expect(outputFor(mockCoreSetOutput, 'prRefId')).toBe('291ae84f');
       });
 
       test('should not set branchRefId output for PR from forked repo', async () => {
@@ -701,13 +701,13 @@ goes here`,
 
       test('should set branchRefId for PR from upstream repo', async () => {
         // Using PR 123 which is faked as a PR from the upstream repo
-        // The Using a PR number of 123 should give a refid of '71f7c907'
+        // The Using a PR number of 123 should give a refid of '6b751c8f'
         // Based on running `echo "refs/heads/pr-head-ref" | shasum | cut -c1-8` (as per the original bash scripts)
         const context = createCommentContext({
           pullRequestNumber: PR_NUMBER.UPSTREAM_NON_DOCS_CHANGES
         });
         await getCommandFromComment({ core, context, github });
-        expect(outputFor(mockCoreSetOutput, 'branchRefId')).toBe('71f7c907');
+        expect(outputFor(mockCoreSetOutput, 'branchRefId')).toBe('6b751c8f');
       });
 
       test('should set prHeadSha output', async () => {
@@ -770,4 +770,12 @@ goes here`,
     });
   });
 
+  describe('createShortHash creates a short hash from a long hash', () => {
+    test('should return the first 8 characters of the hash', () => {
+      const longHash = '0123456789abcdef';
+      const shortHash = '1c043fbe';
+      expect(createShortHash(longHash)).toBe(shortHash);
+    }
+    );
+  });
 });

.github/scripts/yarn.lock

@@ -774,12 +774,12 @@ brace-expansion@^1.1.7:
     balanced-match "^1.0.0"
     concat-map "0.0.1"
 
-braces@^3.0.2:
-  version "3.0.2"
-  resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107"
-  integrity sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==
+braces@^3.0.3:
+  version "3.0.3"
+  resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.3.tgz#490332f40919452272d55a8480adc0c441358789"
+  integrity sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==
   dependencies:
-    fill-range "^7.0.1"
+    fill-range "^7.1.1"
 
 browser-process-hrtime@^1.0.0:
   version "1.0.0"
@@ -1109,10 +1109,10 @@ fb-watchman@^2.0.0:
   dependencies:
     bser "2.1.1"
 
-fill-range@^7.0.1:
-  version "7.0.1"
-  resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.0.1.tgz#1919a6a7c75fe38b2c7c77e5198535da9acdda40"
-  integrity sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==
+fill-range@^7.1.1:
+  version "7.1.1"
+  resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.1.1.tgz#44265d3cac07e3ea7dc247516380643754a05292"
+  integrity sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==
   dependencies:
     to-regex-range "^5.0.1"
 
@@ -2479,9 +2479,9 @@ write-file-atomic@^3.0.0:
     typedarray-to-buffer "^3.1.5"
 
 ws@^7.4.6:
-  version "7.5.9"
-  resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.9.tgz#54fa7db29f4c7cec68b1ddd3a89de099942bb591"
-  integrity sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==
+  version "7.5.10"
+  resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.10.tgz#58b5c20dc281633f6c19113f39b349bd8bd558d9"
+  integrity sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ==
 
 xml-name-validator@^3.0.0:
   version "3.0.0"

.github/workflows/deploy_tre_reusable.yml

@@ -353,9 +353,11 @@ jobs:
           APPLICATION_ADMIN_CLIENT_ID: "${{ secrets.APPLICATION_ADMIN_CLIENT_ID }}"
           APPLICATION_ADMIN_CLIENT_SECRET: "${{ secrets.APPLICATION_ADMIN_CLIENT_SECRET }}"
           STATEFUL_RESOURCES_LOCKED: "${{ github.ref == 'refs/heads/main' && inputs.prRef == '' && true || false }}"
+          KV_PURGE_PROTECTION_ENABLED: ${{ vars.KV_PURGE_PROTECTION_ENABLED || true }}
           CORE_APP_SERVICE_PLAN_SKU: ${{ vars.CORE_APP_SERVICE_PLAN_SKU }}
           RESOURCE_PROCESSOR_NUMBER_PROCESSES_PER_INSTANCE: ${{ vars.RESOURCE_PROCESSOR_NUMBER_PROCESSES_PER_INSTANCE }}
           RP_BUNDLE_VALUES: ${{ vars.RP_BUNDLE_VALUES }}
+          FIREWALL_SKU: ${{ vars.FIREWALL_SKU}}
 
       - name: API Healthcheck
         uses: ./.github/actions/devcontainer_run_command
@@ -397,6 +399,8 @@ jobs:
              BUNDLE_DIR: "./templates/workspace_services/databricks"}
           - {BUNDLE_TYPE: "workspace_service",
              BUNDLE_DIR: "./templates/workspace_services/ohdsi"}
+          - {BUNDLE_TYPE: "workspace_service",
+             BUNDLE_DIR: "./templates/workspace_services/azuresql"}
           - {BUNDLE_TYPE: "user_resource",
              BUNDLE_DIR: "./templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm"}
           - {BUNDLE_TYPE: "user_resource",
@@ -556,6 +560,8 @@ jobs:
              BUNDLE_DIR: "./templates/workspace_services/databricks"}
           - {BUNDLE_TYPE: "workspace_service",
              BUNDLE_DIR: "./templates/workspace_services/ohdsi"}
+          - {BUNDLE_TYPE: "workspace_service",
+             BUNDLE_DIR: "./templates/workspace_services/azuresql"}
 
     environment: ${{ inputs.environmentName }}
     steps:
@@ -665,6 +671,7 @@ jobs:
           TEST_ACCOUNT_CLIENT_SECRET: "${{ secrets.TEST_ACCOUNT_CLIENT_SECRET }}"
           TRE_ID: ${{ secrets.TRE_ID }}
           LOCATION: ${{ vars.LOCATION }}
+          FIREWALL_SKU: ${{ vars.FIREWALL_SKU}}
 
       - name: State Store Migrations
         uses: ./.github/actions/devcontainer_run_command

.github/workflows/lets_encrypt.yml

@@ -2,10 +2,16 @@
 name: Renew Lets Encrypt Certificates
 
 on:  # yamllint disable-line rule:truthy
-  schedule:
-    # 3am each month https://crontab.guru/#0_3_1_*_*
-    - cron: "0 3 1 * *"
+  # schedule:
+  #   # 3am each month https://crontab.guru/#0_3_1_*_*
+  #   - cron: "0 3 1 * *"
   workflow_dispatch:
+    inputs:
+      environment:
+        description: The environment to run this workflow in
+        type: environment
+        default: CICD
+        required: true
 
 # This will prevent multiple runs of this entire workflow.
 # We should NOT cancel in progress runs as that can destabilize the environment.
@@ -20,7 +26,7 @@ jobs:
   renew_letsencrypt_certs:
     name: Renew Lets Encrypt Certificates
     runs-on: ubuntu-latest
-    environment: CICD
+    environment: ${{ github.event.inputs.environment || 'CICD' }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -38,13 +44,14 @@ jobs:
         env:
           ARM_CLIENT_ID: ${{ fromJSON(secrets.AZURE_CREDENTIALS).clientId }}
           ARM_CLIENT_SECRET: ${{ fromJSON(secrets.AZURE_CREDENTIALS).clientSecret }}
-          ARM_SUBSCRIPTION_ID: ${{ fromJSON(secrets.AZURE_CREDENTIALS).tenantId }}
-          ARM_TENANT_ID: ${{ fromJSON(secrets.AZURE_CREDENTIALS).subscriptionId }}
+          ARM_SUBSCRIPTION_ID: ${{ fromJSON(secrets.AZURE_CREDENTIALS).subscriptionId }}
+          ARM_TENANT_ID: ${{ fromJSON(secrets.AZURE_CREDENTIALS).tenantId }}
+          AZURE_ENVIRONMENT: ${{ vars.AZURE_ENVIRONMENT }}
           TRE_ID: ${{ secrets.TRE_ID }}
-          TERRAFORM_STATE_CONTAINER_NAME:
-            ${{ secrets.TERRAFORM_STATE_CONTAINER_NAME && secrets.TERRAFORM_STATE_CONTAINER_NAME || 'tfstate' }}
-          MGMT_RESOURCE_GROUP_NAME: ${{ secrets.MGMT_RESOURCE_GROUP_NAME }}
-          MGMT_STORAGE_ACCOUNT_NAME: ${{ secrets.MGMT_STORAGE_ACCOUNT_NAME }}
+          TF_VAR_terraform_state_container_name: ${{ secrets.TERRAFORM_STATE_CONTAINER_NAME || 'tfstate' }}
+          TF_VAR_mgmt_resource_group_name: ${{ secrets.MGMT_RESOURCE_GROUP_NAME }}
+          TF_VAR_mgmt_storage_account_name: ${{ secrets.MGMT_STORAGE_ACCOUNT_NAME }}
+          CUSTOM_DOMAIN: ${{ secrets.CUSTOM_DOMAIN }}
         run: |
           sudo apt-get install -y python3 python3-venv libaugeas0 \
             && python3 -m venv /opt/certbot/ \

.gitignore

@@ -183,6 +183,7 @@ terraform.rc
 **/http-client.private.env.json
 
 # vscode
+.vscode
 
 # node
 node_modules