Migrate cert-renewal process to cloud run job (#79)

Previously, the cert-renewal process was a long standing instance that ran the script once and slept. The problem arises that this sleep eventually breaks and the instance never recovers. This migrates the job to be a cloud run job that runs the script and that is it. The sleep is now handled by a cron schedule. This ensures the instance is always fresh. Fixes #57 Fixes #77
web-platform-tests · Oct 24, 2023 · 1ff0261 · 1ff0261
1 parent 26b6a1b
commit 1ff0261
Show file tree

Hide file tree

Showing 11 changed files with 398 additions and 453 deletions.
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -21,6 +21,6 @@ jobs:
     - name: Set up Terraform
       uses: hashicorp/setup-terraform@v1
       with:
-        terraform_version: '1.2.5'
+        terraform_version: '1.6.2'
     - name: terraform
       run: terraform fmt --check
diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl
diff --git a/README.md b/README.md
@@ -139,7 +139,7 @@ Requirements:
 
 - [Python 3](https://python.org)
 - [Pipenv](https://pipenv.pypa.io/)
-- [Terraform](https://www.terraform.io/) version 1.2.5
+- [Terraform](https://www.terraform.io/) version 1.6.2
 
 The following commands will run the lints:
 

diff --git a/cert-renewer.Dockerfile b/cert-renewer.Dockerfile
@@ -8,32 +8,26 @@ ENV WPT_HOST=wpt.live \
   WPT_ALT_HOST=not-wpt.live \
   WPT_BUCKET=wpt-live
 
-# Pin the versions for repeatable builds
+# Pin the versions of python and google cloud cli for repeatable builds
 # For ubuntu package versions, go to https://packages.ubuntu.com/
 #   Search for the package with the "jammy" distribution (aka 22.04) selected.
-# For Google Cloud, look under https://packages.cloud.google.com/apt/dists/cloud-sdk/main/binary-amd64/Packages
 RUN apt-get -qqy update && \
   apt-get -qqy install \
-    apt-transport-https=2.4.6 \
-    ca-certificates=20211016 \
-    curl=7.81.0-1ubuntu1.3 \
-    gnupg=2.2.27-3ubuntu2.1 \
-    python3=3.10.4-0ubuntu2 \
-    python3-dev=3.10.4-0ubuntu2 \
-    python3-pip=22.0.2+dfsg-1 && \
-  # https://cloud.google.com/storage/docs/gsutil_install
-  echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | \
-    tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \
-  curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | \
-    tee /usr/share/keyrings/cloud.google.gpg && \
-  apt-get -qqy update && \
-  apt-get -qqy install \
-    google-cloud-cli=396.0.0-0 && \
-  rm -rf /var/lib/apt/lists/* && apt-get clean
+    apt-transport-https \
+    ca-certificates \
+    curl \
+    gnupg \
+    python3=3.10.6-1~22.04 \
+    python3-dev=3.10.6-1~22.04 \
+    python3-pip=22.0.2+dfsg-1
+# For Google Cloud, look under https://packages.cloud.google.com/apt/dists/cloud-sdk/main/binary-amd64/Packages
+# https://cloud.google.com/storage/docs/gsutil_install
+# Copy the "Docker Tip" instructions from gsutil_install link and then pin the version
+RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg  add - && apt-get update -y && apt-get install google-cloud-cli=451.0.1-0 -y
 
 # Instructions for certbot installation
 # https://certbot.eff.org/instructions?ws=other&os=pip
-RUN pip install certbot==1.29.0 certbot-dns-google==1.29.0
+RUN pip install acme==1.29.0 certbot==1.29.0 certbot-dns-google==1.29.0
 
 COPY src/cert-store.sh /usr/local/bin/
 
@@ -47,19 +41,16 @@ COPY src/cert-store.sh /usr/local/bin/
 # https://eff-certbot.readthedocs.io/en/stable/using.html?highlight=wildcard#dns-plugins
 
 CMD bash -c '\
-  cert-store.sh fetch ${WPT_BUCKET} ${WPT_HOST}; \
-  while true; do \
-    certbot certonly \
-      -d ${WPT_HOST} \
-      -d *.${WPT_HOST} \
-      -d ${WPT_ALT_HOST} \
-      -d *.${WPT_ALT_HOST} \
-      --dns-google \
-      --dns-google-propagation-seconds 120 \
-      --agree-tos \
-      --non-interactive \
-      --email [email protected] \
-      --server https://acme-v02.api.letsencrypt.org/directory \
-      --deploy-hook "cert-store.sh save ${WPT_BUCKET} ${WPT_HOST}"; \
-    sleep $((60 * 60 * 24)); \
-  done'
+  cert-store.sh fetch ${WPT_BUCKET} ${WPT_HOST} && \
+  certbot certonly \
+    -d ${WPT_HOST} \
+    -d *.${WPT_HOST} \
+    -d ${WPT_ALT_HOST} \
+    -d *.${WPT_ALT_HOST} \
+    --dns-google \
+    --dns-google-propagation-seconds 120 \
+    --agree-tos \
+    --non-interactive \
+    --email [email protected] \
+    --server https://acme-v02.api.letsencrypt.org/directory \
+    --deploy-hook "cert-store.sh save ${WPT_BUCKET} ${WPT_HOST}"'
diff --git a/infrastructure/docker-image/versions.tf b/infrastructure/docker-image/versions.tf
@@ -1,6 +1,6 @@
 
 terraform {
-  required_version = "~> 1.2.5"
+  required_version = "~> 1.6.2"
   required_providers {
     external = {
       source = "hashicorp/external"

diff --git a/infrastructure/web-platform-tests/compute.tf b/infrastructure/web-platform-tests/compute.tf
@@ -154,96 +154,54 @@ resource "google_compute_instance_template" "wpt_server" {
 
 ########################################
 # Cert Renewers
-# These configurations come from: github.com/dcaba/terraform-google-managed-instance-group
-# More information about how it was used previously: https://github.com/web-platform-tests/wpt.live/blob/67dc5976ccce2e64483f2028a35659d4d6e58891/infrastructure/web-platform-tests/main.tf#L139-L178
 ########################################
 
-resource "google_compute_instance_template" "cert_renewers" {
-  name_prefix = "default-"
-
-  machine_type = "f1-micro"
-
-  region = var.region
-
-  tags = ["allow-ssh", "${var.name}-allow"]
-
-  labels = {
-    "${module.cert-renewer-container.vm_container_label_key}" = module.cert-renewer-container.vm_container_label
-  }
-
-  network_interface {
-    network    = var.network_name
-    subnetwork = var.subnetwork_name
-    network_ip = ""
-    access_config {
-      network_tier = "PREMIUM"
+resource "google_cloud_run_v2_job" "cert_renewers" {
+  name         = "${var.name}-cert-renewers"
+  location     = var.region
+  launch_stage = "BETA"
+
+  template {
+    template {
+      containers {
+        image = var.cert_renewer_image
+        env {
+          name  = "WPT_HOST"
+          value = var.host_name
+        }
+        env {
+          name  = "WPT_ALT_HOST"
+          value = var.alt_host_name
+        }
+        env {
+          name  = "WPT_BUCKET"
+          value = local.bucket_name
+        }
+      }
     }
   }
-
-  can_ip_forward = false
-
-  disk {
-    auto_delete  = true
-    boot         = true
-    source_image = module.cert-renewer-container.source_image
-    type         = "PERSISTENT"
-    disk_type    = "pd-ssd"
-    mode         = "READ_WRITE"
-  }
-
-  service_account {
-    email  = "default"
-    scopes = ["cloud-platform"]
-  }
-
-  # startup-script and tf_depends_id comes from the module previously used for cert renewer. (see link at top)
-  # TODO: evaluate if those two should be removed.
-  metadata = {
-    "${module.cert-renewer-container.metadata_key}" = module.cert-renewer-container.metadata_value
-    "startup-script"                                = ""
-    "tf_depends_id"                                 = ""
-    "google-logging-enabled"                        = "true"
-  }
-
-  scheduling {
-    preemptible         = false
-    automatic_restart   = true
-    on_host_maintenance = "MIGRATE"
-  }
-
-  lifecycle {
-    create_before_destroy = true
-  }
 }
 
-resource "google_compute_instance_group_manager" "cert_renewers" {
-  name               = "${var.name}-cert-renewers"
-  description        = "compute VM Instance Group"
-  wait_for_instances = false
-
-  base_instance_name = "${var.name}-cert-renewers"
-
-  version {
-    instance_template = google_compute_instance_template.cert_renewers.self_link
-  }
+data "google_project" "project" {
+}
 
-  zone = var.zone
+resource "google_cloud_scheduler_job" "cert_renewer_schedule" {
+  provider         = google
+  name             = "${var.name}-cert-renewer-schedule"
+  description      = "cert renewal schedule job"
+  schedule         = "0 0 * * *"
+  attempt_deadline = "320s"
+  region           = "us-central1"
 
-  update_policy {
-    type                  = local.update_policy.type
-    minimal_action        = local.update_policy.minimal_action
-    max_unavailable_fixed = local.update_policy.max_unavailable_fixed
+  retry_config {
+    retry_count = 3
   }
 
-  target_pools = []
-
-  target_size = 1
-
-  dynamic "named_port" {
-    for_each = var.cert_renewer_ports
-    content {
-      name = named_port.value["name"]
-      port = named_port.value["port"]
+  http_target {
+    http_method = "POST"
+    uri         = "https://${google_cloud_run_v2_job.cert_renewers.location}-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/${data.google_project.project.number}/jobs/${google_cloud_run_v2_job.cert_renewers.name}:run"
+    oauth_token {
+      service_account_email = var.service_account_email
     }
   }
 }
diff --git a/infrastructure/web-platform-tests/main.tf b/infrastructure/web-platform-tests/main.tf
@@ -36,31 +36,6 @@ module "wpt-server-container" {
   restart_policy = "Always"
 }
 
-module "cert-renewer-container" {
-  source  = "terraform-google-modules/container-vm/google"
-  version = "3.0.0"
-
-  container = {
-    image = var.cert_renewer_image
-    env = [
-      {
-        name  = "WPT_HOST"
-        value = var.host_name
-      },
-      {
-        name  = "WPT_ALT_HOST"
-        value = var.alt_host_name
-      },
-      {
-        name  = "WPT_BUCKET"
-        value = local.bucket_name
-      },
-    ]
-  }
-
-  restart_policy = "Always"
-}
-
 resource "google_storage_bucket" "certificates" {
   name     = local.bucket_name
   location = "US"

diff --git a/infrastructure/web-platform-tests/variables.tf b/infrastructure/web-platform-tests/variables.tf
@@ -89,6 +89,10 @@ variable "wpt_server_ports" {
   ]
 }
 
+variable "service_account_email" {
+  type    = string
+  default = "[email protected]"
+}
 
 variable "cert_renewer_ports" {
   type = list(object({

diff --git a/infrastructure/web-platform-tests/versions.tf b/infrastructure/web-platform-tests/versions.tf
@@ -1,6 +1,6 @@
 
 terraform {
-  required_version = "~> 1.2.5"
+  required_version = "~> 1.6.2"
   required_providers {
     google = {
       source = "hashicorp/google"