Merge branch 'main' into #364-docs-previews

docs/guides/migration-recipe.md

@@ -115,7 +115,7 @@ index 68387c9..7a8ace1 100644
       - Copy and paste the generated commands to create and populate the files on `new-host` 
 4. Restart the `salt-minion` service on the **new host** to pick up the configuration and register with salt-master:
     ```console
-    sudo salt-call service.restart salt-minion
+    sudo service salt-minion restart
     ```
 5. On **`salt-master`**, accept the key for the new-host:
     ```console
@@ -232,7 +232,7 @@ index 68387c9..7a8ace1 100644
     ```
 7.  Restart the salt minion:
     ```console
-    sudo salt-call service.restart salt-minion
+    sudo service salt-minion restart
     ```
 8.  Restart Datadog agent:
     ```console

docs/salt-server-list.rst

@@ -7,26 +7,24 @@
    :header: "Name", "Purpose", "Contact", "Distro", "Datacener"
 
 
-   "backup.sfo1.psf.io", "Automated backup of infrastructure", "Infrastructure staff", "Ubuntu-20.04", "sfo1"
+   "backup.sfo1.psf.io", "Automated backup of infrastructure", "Infrastructure staff", "Ubuntu-24.04", "sfo1"
    "bugs.nyc1.psf.io", "Roundup hosting for CPython, Jython, and Roundup", "Infrastructure staff", "Ubuntu-22.04", "nyc1"
-   "buildbot.nyc1.psf.io", "Hosting for CPython buildbot server", "zware, haypo, pablogsa", "Ubuntu-20.04", "nyc1"
+   "buildbot.nyc1.psf.io", "Hosting for CPython buildbot server", "zware, haypo, pablogsa", "Ubuntu-24.04", "nyc1"
    "cdn-logs.nyc1.psf.io", "Realtime log streaming from Fastly CDN for debug", "Infrastructure Staff", "Ubuntu-20.04", "nyc1"
    "codespeed.nyc1.psf.io", "Hosting for speed.python.org and speed.pypy.org", "", "Ubuntu-22.04", "nyc1"
-   "consul-a.nyc1.psf.io", "Runs `Consul <https://www.consul.io/>`_ discovery service", "Infrastructure Staff", "Ubuntu-20.04", "nyc1"
-   "consul-b.nyc1.psf.io", "Runs `Consul <https://www.consul.io/>`_ discovery service", "Infrastructure Staff", "Ubuntu-20.04", "nyc1"
-   "consul-c.nyc1.psf.io", "Runs `Consul <https://www.consul.io/>`_ discovery service", "Infrastructure Staff", "Ubuntu-20.04", "nyc1"
-   "docs.nyc1.psf.io", "Builds and serves CPython's documentation", "mdk", "Ubuntu-20.04", "nyc1"
-   "downloads.nyc1.psf.io", "Serves python.org downloads", "CPython Release Managers", "Ubuntu-20.04", "nyc1"
+   "consul-1.nyc1.psf.io", "Runs `Consul <https://www.consul.io/>`_ discovery service", "Infrastructure Staff", "Ubuntu-24.04", "nyc1"
+   "consul-2.nyc1.psf.io", "Runs `Consul <https://www.consul.io/>`_ discovery service", "Infrastructure Staff", "Ubuntu-24.04", "nyc1"
+   "consul-3.nyc1.psf.io", "Runs `Consul <https://www.consul.io/>`_ discovery service", "Infrastructure Staff", "Ubuntu-24.04", "nyc1"
+   "docs.nyc1.psf.io", "Builds and serves CPython's documentation", "mdk", "Ubuntu-24.04", "nyc1"
+   "downloads.nyc1.psf.io", "Serves python.org downloads", "CPython Release Managers", "Ubuntu-24.04", "nyc1"
    "gnumailman.nyc1.psf.io", "GNU Mailman Project wiki and lists", "Mark Sapiro", "Ubuntu-20.04", "nyc1"
    "hg.nyc1.psf.io", "Version Control Archives, serves hg.python.org and svn.python.org", "Infrastructure Staff", "Ubuntu-24.04", "nyc1"
    "lb-a.nyc1.psf.io", "Load balancer", "Infrastructure Staff", "Ubuntu-20.04", "nyc1"
    "lb-b.nyc1.psf.io", "Load balancer", "Infrastructure Staff", "Ubuntu-20.04", "nyc1"
    "mail.ams1.psf.io", "Mail and mailman server", "postmasters", "Ubuntu-14.04", "ams1"
    "moin.nyc1.psf.io", "Hosts moin sites for wiki.python.org, wiki.jython.org", "lemburg", "Ubuntu-20.04", "nyc1"
-   "planet-2404.nyc1.psf.io", "Planet Python", "benjamin", "Ubuntu-24.04", "nyc1"
    "planet.nyc1.psf.io", "Planet Python", "benjamin", "Ubuntu-18.04", "nyc1"
-   "pythontest-2404.nyc3.psf.io", "Test resources for CPython's test suite.", "Infrastructure Staff", "Ubuntu-24.04", "nyc3"
-   "pythontest.nyc3.psf.io", "Test resources for CPython's test suite.", "Infrastructure Staff", "Ubuntu-18.04", "nyc3"
+   "pythontest.nyc3.psf.io", "Test resources for CPython's test suite.", "Infrastructure Staff", "Ubuntu-24.04", "nyc3"
    "salt.nyc1.psf.io", "Salt server", "Infrastructure Staff", "Ubuntu-20.04", "nyc1"
 
 ..

pillar/base/firewall/ftp.sls

@@ -4,4 +4,4 @@ firewall:
   ftp-21:
     port: 21
   ftp-incoming:
-    raw: -A INPUT -p tcp --destination-port 10090:10100 -j ACCEPT
+    raw: -A INPUT -p tcp --destination-port 10090:10190 -j ACCEPT

pillar/base/firewall/salt.sls

@@ -13,11 +13,11 @@ firewall:
   {# NOTE: These hosts do not run in the primary DC (NYC1) so firewall holes are poked for access #}
   salt_master_pythontest:
     port: 4505:4506
-    source: 159.89.235.38
+    source: 68.183.26.59
 
-  salt_master_pythontest_noble:
+  salt_master_backup_server:
     port: 4505:4506
-    source: 68.183.26.59
+    source: 159.89.159.168
 
   salt_master_remote_backup:
     port: 4505:4506

pillar/prod/consul.sls

@@ -1,9 +1,9 @@
 consul:
   bootstrap:
     nyc1:
-      - consul-a.nyc1.psf.io
-      - consul-b.nyc1.psf.io
-      - consul-c.nyc1.psf.io
+      - consul-1.nyc1.psf.io
+      - consul-2.nyc1.psf.io
+      - consul-3.nyc1.psf.io
   acl:
     default: deny
     dc: nyc1

pillar/prod/swapfile.sls

@@ -0,0 +1,3 @@
+swap_file:
+  swap_size: 1024
+  swap_path: /swapfile

pillar/prod/top.sls

@@ -10,6 +10,7 @@ base:
     - users
     - postgres.clusters
     - secrets.monitoring.datadog
+    - swapfile
 
   'backup-server':
     - match: nodegroup

salt/backup/client/init.sls

@@ -38,7 +38,11 @@ include:
     - template: jinja
     - context:
         pre_script: '{{ config.get('pre_script', ":") }}'
-        remote_command: '/usr/bin/rdiff-backup --terminal-verbosity 1 {%- for exclude in config.get('exclude', []) %} --exclude {{ exclude }} {%- endfor %} --no-eas --remote-schema "ssh -i /etc/backup/.ssh/id_rsa_{{ backup }} -C %s rdiff-backup --server" {{ config['source_directory'] }} {{ config['target_user'] }}@{{ config['target_host'] }}::{{ config['target_directory'] }}'
+        {% if grains["oscodename"] == "noble" -%}
+        remote_command: '/usr/bin/rdiff-backup --terminal-verbosity 1 --remote-schema "ssh -i /etc/backup/.ssh/id_rsa_{{ backup }} -C %s rdiff-backup server" backup --no-eas {%- for exclude in config.get('exclude', []) %} --exclude {{ exclude }} {%- endfor %} {{ config['source_directory'] }} {{ config['target_user'] }}@{{ config['target_host'] }}::{{ config['target_directory'] }}'
+        {% else %}
+        remote_command: '/usr/bin/rdiff-backup --terminal-verbosity 1 {%- for exclude in config.get('exclude', []) %} --exclude {{ exclude }} {%- endfor %} --no-eas --remote-schema "ssh -i /etc/backup/.ssh/id_rsa_{{ backup }} -C %s rdiff-backup server" {{ config['source_directory'] }} {{ config['target_user'] }}@{{ config['target_host'] }}::{{ config['target_directory'] }}'
+        {% endif %}
         post_script: '{{ config.get('post_script', ":") }}'
         cleanup_script: '{{ config.get('cleanup_script', ":") }}'
 

salt/backup/server/init.sls

@@ -15,7 +15,7 @@ include:
     - names:
       - {{ config['authorized_key'] }}
     - options:
-      - command="rdiff-backup --server"
+      - command="rdiff-backup server"
       - no-pty
       - no-port-forwarding
       - no-agent-forwarding
@@ -41,6 +41,6 @@ include:
     - context:
         cron: '0 3 * * *'
         job_user: root
-        job_command: 'rdiff-backup --terminal-verbosity 2 --force --remove-older-than {{ config['increment_retention'] }} {{ config['directory'] }}'
+        job_command: 'rdiff-backup --terminal-verbosity 1 --force remove increments --older-than {{ config['increment_retention'] }} {{ config['directory'] }}'
 
 {% endfor %}

salt/base/swap.sls

@@ -0,0 +1,18 @@
+{% set swap_file = salt["pillar.get"]("swap_file", {}) %}
+{% set swap_size = swap_file.get("swap_size", "1024") %}
+{% set swap_path = swap_file.get("swap_path") %}
+
+{% if swap_path %}
+{{ swap_path }}:
+  cmd.run:
+    - name: |
+        swapon --show=NAME --noheadings | grep -q "^{{ swap_path }}$" && swapoff {{ swap_path }}
+        rm -f {{ swap_path }}
+        fallocate -l {{ swap_size }}M {{ swap_path }}
+        chmod 0600 {{ swap_path }}
+        mkswap {{ swap_path }}
+    - unless: bash -c "[[ $(($(stat -c %s {{ swap_path }}) / 1024**2)) = {{ swap_size }} ]]"
+
+  mount.swap:
+    - persist: true
+{% endif %}

salt/buildbot/init.sls

@@ -31,6 +31,13 @@ buildbot-user:
     - group: buildbot
     - mode: "0750"
 
+/data/www/buildbot:
+  file.directory:
+    - user: buildbot
+    - group: root
+    - mode: "0755"
+    - makedirs: True
+
 /srv:
   file.directory:
     - user: buildbot

salt/codespeed/init.sls

@@ -11,7 +11,7 @@ codespeed-deps:
       - mercurial
       - python3-dev
       - python3-virtualenv
-      {% if grains["oscodename"] != "jammy" %}
+      {% if grains["oscodename"] not in ["jammy", "noble"] %}
       - python-dev
       - python-virtualenv
       {% endif %}

salt/datadog/init.sls

@@ -1,3 +1,5 @@
+{% set swap_path = salt['pillar.get'](salt['pillar.get']('swap_file:swap_path')) %}
+
 {% if grains["oscodename"] == ["jammy", "noble"] %}
 datadogkey:
   file.managed:
@@ -31,11 +33,25 @@ datadog_repo:
     - source: salt://datadog/files
 
 {% if 'datadog_api_key' in pillar %}
+check_datadog_installation:
+  cmd.run:
+    - name: |
+        if ! dpkg-query -W datadog-agent || ! test -f /etc/datadog-agent/datadog.yaml; then
+          dpkg --remove --force-remove-reinstreq datadog-agent || true
+          apt-get -y --fix-broken install
+          apt-get update
+        fi
+    - hide_output: True
+
 datadog-agent:
   pkg:
     - installed
     - require:
       - pkgrepo: datadog_repo
+        {% if swap_path %}
+      - mount: {{ swap_path }}
+        {% endif %}
+      - cmd: check_datadog_installation
   service:
     - running
     - enable: True

salt/pythontest/config/vsftpd.conf

@@ -38,7 +38,7 @@ connect_from_port_20=YES
 pasv_enable=YES
 pasv_addr_resolve=YES
 pasv_address=www.pythontest.net
-pasv_max_port=10100
+pasv_max_port=10190
 pasv_min_port=10090
 
 # This option should be the name of a directory which is empty.  Also, the

salt/top.sls

@@ -18,6 +18,7 @@ base:
     - rsyslog
     - datadog
     - base.motd
+    - base.swap
 
   'backup-server':
     - match: nodegroup