diff --git a/docs/guides/migration-recipe.md b/docs/guides/migration-recipe.md index d9887b25..7563ec90 100644 --- a/docs/guides/migration-recipe.md +++ b/docs/guides/migration-recipe.md @@ -115,7 +115,7 @@ index 68387c9..7a8ace1 100644 - Copy and paste the generated commands to create and populate the files on `new-host` 4. Restart the `salt-minion` service on the **new host** to pick up the configuration and register with salt-master: ```console - sudo salt-call service.restart salt-minion + sudo service salt-minion restart ``` 5. On **`salt-master`**, accept the key for the new-host: ```console @@ -232,7 +232,7 @@ index 68387c9..7a8ace1 100644 ``` 7. Restart the salt minion: ```console - sudo salt-call service.restart salt-minion + sudo service salt-minion restart ``` 8. Restart Datadog agent: ```console diff --git a/docs/salt-server-list.rst b/docs/salt-server-list.rst index a00ed79b..8fa11147 100644 --- a/docs/salt-server-list.rst +++ b/docs/salt-server-list.rst @@ -7,26 +7,24 @@ :header: "Name", "Purpose", "Contact", "Distro", "Datacener" - "backup.sfo1.psf.io", "Automated backup of infrastructure", "Infrastructure staff", "Ubuntu-20.04", "sfo1" + "backup.sfo1.psf.io", "Automated backup of infrastructure", "Infrastructure staff", "Ubuntu-24.04", "sfo1" "bugs.nyc1.psf.io", "Roundup hosting for CPython, Jython, and Roundup", "Infrastructure staff", "Ubuntu-22.04", "nyc1" - "buildbot.nyc1.psf.io", "Hosting for CPython buildbot server", "zware, haypo, pablogsa", "Ubuntu-20.04", "nyc1" + "buildbot.nyc1.psf.io", "Hosting for CPython buildbot server", "zware, haypo, pablogsa", "Ubuntu-24.04", "nyc1" "cdn-logs.nyc1.psf.io", "Realtime log streaming from Fastly CDN for debug", "Infrastructure Staff", "Ubuntu-20.04", "nyc1" "codespeed.nyc1.psf.io", "Hosting for speed.python.org and speed.pypy.org", "", "Ubuntu-22.04", "nyc1" - "consul-a.nyc1.psf.io", "Runs `Consul `_ discovery service", "Infrastructure Staff", "Ubuntu-20.04", "nyc1" - "consul-b.nyc1.psf.io", "Runs `Consul `_ discovery service", "Infrastructure Staff", "Ubuntu-20.04", "nyc1" - "consul-c.nyc1.psf.io", "Runs `Consul `_ discovery service", "Infrastructure Staff", "Ubuntu-20.04", "nyc1" - "docs.nyc1.psf.io", "Builds and serves CPython's documentation", "mdk", "Ubuntu-20.04", "nyc1" - "downloads.nyc1.psf.io", "Serves python.org downloads", "CPython Release Managers", "Ubuntu-20.04", "nyc1" + "consul-1.nyc1.psf.io", "Runs `Consul `_ discovery service", "Infrastructure Staff", "Ubuntu-24.04", "nyc1" + "consul-2.nyc1.psf.io", "Runs `Consul `_ discovery service", "Infrastructure Staff", "Ubuntu-24.04", "nyc1" + "consul-3.nyc1.psf.io", "Runs `Consul `_ discovery service", "Infrastructure Staff", "Ubuntu-24.04", "nyc1" + "docs.nyc1.psf.io", "Builds and serves CPython's documentation", "mdk", "Ubuntu-24.04", "nyc1" + "downloads.nyc1.psf.io", "Serves python.org downloads", "CPython Release Managers", "Ubuntu-24.04", "nyc1" "gnumailman.nyc1.psf.io", "GNU Mailman Project wiki and lists", "Mark Sapiro", "Ubuntu-20.04", "nyc1" "hg.nyc1.psf.io", "Version Control Archives, serves hg.python.org and svn.python.org", "Infrastructure Staff", "Ubuntu-24.04", "nyc1" "lb-a.nyc1.psf.io", "Load balancer", "Infrastructure Staff", "Ubuntu-20.04", "nyc1" "lb-b.nyc1.psf.io", "Load balancer", "Infrastructure Staff", "Ubuntu-20.04", "nyc1" "mail.ams1.psf.io", "Mail and mailman server", "postmasters", "Ubuntu-14.04", "ams1" "moin.nyc1.psf.io", "Hosts moin sites for wiki.python.org, wiki.jython.org", "lemburg", "Ubuntu-20.04", "nyc1" - "planet-2404.nyc1.psf.io", "Planet Python", "benjamin", "Ubuntu-24.04", "nyc1" "planet.nyc1.psf.io", "Planet Python", "benjamin", "Ubuntu-18.04", "nyc1" - "pythontest-2404.nyc3.psf.io", "Test resources for CPython's test suite.", "Infrastructure Staff", "Ubuntu-24.04", "nyc3" - "pythontest.nyc3.psf.io", "Test resources for CPython's test suite.", "Infrastructure Staff", "Ubuntu-18.04", "nyc3" + "pythontest.nyc3.psf.io", "Test resources for CPython's test suite.", "Infrastructure Staff", "Ubuntu-24.04", "nyc3" "salt.nyc1.psf.io", "Salt server", "Infrastructure Staff", "Ubuntu-20.04", "nyc1" .. diff --git a/pillar/base/firewall/ftp.sls b/pillar/base/firewall/ftp.sls index 1eb904dd..1f0abc28 100644 --- a/pillar/base/firewall/ftp.sls +++ b/pillar/base/firewall/ftp.sls @@ -4,4 +4,4 @@ firewall: ftp-21: port: 21 ftp-incoming: - raw: -A INPUT -p tcp --destination-port 10090:10100 -j ACCEPT + raw: -A INPUT -p tcp --destination-port 10090:10190 -j ACCEPT diff --git a/pillar/base/firewall/salt.sls b/pillar/base/firewall/salt.sls index 3bcdf4e3..f41bd5f7 100644 --- a/pillar/base/firewall/salt.sls +++ b/pillar/base/firewall/salt.sls @@ -13,11 +13,11 @@ firewall: {# NOTE: These hosts do not run in the primary DC (NYC1) so firewall holes are poked for access #} salt_master_pythontest: port: 4505:4506 - source: 159.89.235.38 + source: 68.183.26.59 - salt_master_pythontest_noble: + salt_master_backup_server: port: 4505:4506 - source: 68.183.26.59 + source: 159.89.159.168 salt_master_remote_backup: port: 4505:4506 diff --git a/pillar/prod/consul.sls b/pillar/prod/consul.sls index 3f34658a..460d5882 100644 --- a/pillar/prod/consul.sls +++ b/pillar/prod/consul.sls @@ -1,9 +1,9 @@ consul: bootstrap: nyc1: - - consul-a.nyc1.psf.io - - consul-b.nyc1.psf.io - - consul-c.nyc1.psf.io + - consul-1.nyc1.psf.io + - consul-2.nyc1.psf.io + - consul-3.nyc1.psf.io acl: default: deny dc: nyc1 diff --git a/pillar/prod/swapfile.sls b/pillar/prod/swapfile.sls new file mode 100644 index 00000000..77a7130f --- /dev/null +++ b/pillar/prod/swapfile.sls @@ -0,0 +1,3 @@ +swap_file: + swap_size: 1024 + swap_path: /swapfile diff --git a/pillar/prod/top.sls b/pillar/prod/top.sls index 48835d3f..34ba817a 100644 --- a/pillar/prod/top.sls +++ b/pillar/prod/top.sls @@ -10,6 +10,7 @@ base: - users - postgres.clusters - secrets.monitoring.datadog + - swapfile 'backup-server': - match: nodegroup diff --git a/salt/backup/client/init.sls b/salt/backup/client/init.sls index cdd861ac..75da1929 100644 --- a/salt/backup/client/init.sls +++ b/salt/backup/client/init.sls @@ -38,7 +38,11 @@ include: - template: jinja - context: pre_script: '{{ config.get('pre_script', ":") }}' - remote_command: '/usr/bin/rdiff-backup --terminal-verbosity 1 {%- for exclude in config.get('exclude', []) %} --exclude {{ exclude }} {%- endfor %} --no-eas --remote-schema "ssh -i /etc/backup/.ssh/id_rsa_{{ backup }} -C %s rdiff-backup --server" {{ config['source_directory'] }} {{ config['target_user'] }}@{{ config['target_host'] }}::{{ config['target_directory'] }}' + {% if grains["oscodename"] == "noble" -%} + remote_command: '/usr/bin/rdiff-backup --terminal-verbosity 1 --remote-schema "ssh -i /etc/backup/.ssh/id_rsa_{{ backup }} -C %s rdiff-backup server" backup --no-eas {%- for exclude in config.get('exclude', []) %} --exclude {{ exclude }} {%- endfor %} {{ config['source_directory'] }} {{ config['target_user'] }}@{{ config['target_host'] }}::{{ config['target_directory'] }}' + {% else %} + remote_command: '/usr/bin/rdiff-backup --terminal-verbosity 1 {%- for exclude in config.get('exclude', []) %} --exclude {{ exclude }} {%- endfor %} --no-eas --remote-schema "ssh -i /etc/backup/.ssh/id_rsa_{{ backup }} -C %s rdiff-backup server" {{ config['source_directory'] }} {{ config['target_user'] }}@{{ config['target_host'] }}::{{ config['target_directory'] }}' + {% endif %} post_script: '{{ config.get('post_script', ":") }}' cleanup_script: '{{ config.get('cleanup_script', ":") }}' diff --git a/salt/backup/server/init.sls b/salt/backup/server/init.sls index 157120c8..796d0e65 100644 --- a/salt/backup/server/init.sls +++ b/salt/backup/server/init.sls @@ -15,7 +15,7 @@ include: - names: - {{ config['authorized_key'] }} - options: - - command="rdiff-backup --server" + - command="rdiff-backup server" - no-pty - no-port-forwarding - no-agent-forwarding @@ -41,6 +41,6 @@ include: - context: cron: '0 3 * * *' job_user: root - job_command: 'rdiff-backup --terminal-verbosity 2 --force --remove-older-than {{ config['increment_retention'] }} {{ config['directory'] }}' + job_command: 'rdiff-backup --terminal-verbosity 1 --force remove increments --older-than {{ config['increment_retention'] }} {{ config['directory'] }}' {% endfor %} diff --git a/salt/base/swap.sls b/salt/base/swap.sls new file mode 100644 index 00000000..91896c57 --- /dev/null +++ b/salt/base/swap.sls @@ -0,0 +1,18 @@ +{% set swap_file = salt["pillar.get"]("swap_file", {}) %} +{% set swap_size = swap_file.get("swap_size", "1024") %} +{% set swap_path = swap_file.get("swap_path") %} + +{% if swap_path %} +{{ swap_path }}: + cmd.run: + - name: | + swapon --show=NAME --noheadings | grep -q "^{{ swap_path }}$" && swapoff {{ swap_path }} + rm -f {{ swap_path }} + fallocate -l {{ swap_size }}M {{ swap_path }} + chmod 0600 {{ swap_path }} + mkswap {{ swap_path }} + - unless: bash -c "[[ $(($(stat -c %s {{ swap_path }}) / 1024**2)) = {{ swap_size }} ]]" + + mount.swap: + - persist: true +{% endif %} \ No newline at end of file diff --git a/salt/buildbot/init.sls b/salt/buildbot/init.sls index f548c28a..5a0c99fa 100644 --- a/salt/buildbot/init.sls +++ b/salt/buildbot/init.sls @@ -31,6 +31,13 @@ buildbot-user: - group: buildbot - mode: "0750" +/data/www/buildbot: + file.directory: + - user: buildbot + - group: root + - mode: "0755" + - makedirs: True + /srv: file.directory: - user: buildbot diff --git a/salt/codespeed/init.sls b/salt/codespeed/init.sls index f6f80c45..09f92b9a 100644 --- a/salt/codespeed/init.sls +++ b/salt/codespeed/init.sls @@ -11,7 +11,7 @@ codespeed-deps: - mercurial - python3-dev - python3-virtualenv - {% if grains["oscodename"] != "jammy" %} + {% if grains["oscodename"] not in ["jammy", "noble"] %} - python-dev - python-virtualenv {% endif %} diff --git a/salt/datadog/init.sls b/salt/datadog/init.sls index 19aa38a1..3acb43b6 100644 --- a/salt/datadog/init.sls +++ b/salt/datadog/init.sls @@ -1,3 +1,5 @@ +{% set swap_path = salt['pillar.get'](salt['pillar.get']('swap_file:swap_path')) %} + {% if grains["oscodename"] == ["jammy", "noble"] %} datadogkey: file.managed: @@ -31,11 +33,25 @@ datadog_repo: - source: salt://datadog/files {% if 'datadog_api_key' in pillar %} +check_datadog_installation: + cmd.run: + - name: | + if ! dpkg-query -W datadog-agent || ! test -f /etc/datadog-agent/datadog.yaml; then + dpkg --remove --force-remove-reinstreq datadog-agent || true + apt-get -y --fix-broken install + apt-get update + fi + - hide_output: True + datadog-agent: pkg: - installed - require: - pkgrepo: datadog_repo + {% if swap_path %} + - mount: {{ swap_path }} + {% endif %} + - cmd: check_datadog_installation service: - running - enable: True diff --git a/salt/pythontest/config/vsftpd.conf b/salt/pythontest/config/vsftpd.conf index 05043dc1..b7639cd1 100644 --- a/salt/pythontest/config/vsftpd.conf +++ b/salt/pythontest/config/vsftpd.conf @@ -38,7 +38,7 @@ connect_from_port_20=YES pasv_enable=YES pasv_addr_resolve=YES pasv_address=www.pythontest.net -pasv_max_port=10100 +pasv_max_port=10190 pasv_min_port=10090 # This option should be the name of a directory which is empty. Also, the diff --git a/salt/top.sls b/salt/top.sls index 4bd78f04..ab5dd716 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -18,6 +18,7 @@ base: - rsyslog - datadog - base.motd + - base.swap 'backup-server': - match: nodegroup