Dockerfile.tests: Copy checkout and setup workflow

Copy the git checkout, create the test database, and compile static resources
within the Dockerfile rather than as part of the workflow job steps.

This will speed up each job by reducing the number of redundant steps.

.github/workflows/ci.yml

@@ -42,15 +42,11 @@ jobs:
     runs-on: ubuntu-latest
     container:
       image: metabrainz/musicbrainz-tests:${{ github.event_name == 'pull_request' && format('{0}-{1}', github.base_ref, github.head_ref) || github.ref_name }}
-      options: --user root
+      options: --user root --hostname mbtest
 
     steps:
-      - uses: actions/checkout@v4
-
-      - uses: ./.github/actions/setup-tests-container
-
       - run: |
-          ./docker/musicbrainz-tests/run_js_perl_and_pgtap_tests.sh
+          /home/musicbrainz/musicbrainz-server/docker/musicbrainz-tests/run_js_perl_and_pgtap_tests.sh
 
       - if: always()
         uses: mikepenz/action-junit-report@v5
@@ -64,15 +60,11 @@ jobs:
     runs-on: ubuntu-latest
     container:
       image: metabrainz/musicbrainz-tests:${{ github.event_name == 'pull_request' && format('{0}-{1}', github.base_ref, github.head_ref) || github.ref_name }}
-      options: --user root
+      options: --user root --hostname mbtest
 
     steps:
-      - uses: actions/checkout@v4
-
-      - uses: ./.github/actions/setup-tests-container
-
       - run: |
-          ./docker/musicbrainz-tests/run_selenium_tests.sh
+          /home/musicbrainz/musicbrainz-server/docker/musicbrainz-tests/run_selenium_tests.sh
 
       - if: always()
         uses: actions/upload-artifact@v4

docker/musicbrainz-tests/initialize_tests_image.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+set -e -x
+
+cd "$MBS_ROOT"
+
+sudo -E -H -u musicbrainz mkdir -p junit_output
+
+sudo -E -H -u musicbrainz cp docker/musicbrainz-tests/DBDefs.pm lib/
+
+sudo -E -H -u postgres /usr/lib/postgresql/16/bin/pg_ctl start -D /var/lib/postgresql/data
+
+sudo -E -H -u musicbrainz carton exec -- ./script/create_test_db.sh
+
+cd /var/lib/postgresql
+
+# Create clones of the musicbrainz_test DB used by various tests.
+sudo -E -H -u postgres createdb -O musicbrainz -T musicbrainz_test -U postgres musicbrainz_selenium
+sudo -E -H -u postgres createdb -O musicbrainz -T musicbrainz_test -U postgres musicbrainz_test_json_dump
+sudo -E -H -u postgres createdb -O musicbrainz -T musicbrainz_test -U postgres musicbrainz_test_full_export
+sudo -E -H -u postgres createdb -O musicbrainz -T musicbrainz_test -U postgres musicbrainz_test_sitemaps
+
+cd /home/musicbrainz/sir
+
+service rabbitmq-server start
+
+# Setup the rabbitmq user/vhost used by pg_amqp + sir.
+rabbitmqctl add_user sir sir
+rabbitmqctl add_vhost /sir-test
+rabbitmqctl set_permissions -p /sir-test sir '.*' '.*' '.*'
+
+# Generate and install the sir triggers into musicbrainz_selenium.
+sudo -E -H -u musicbrainz sh -c '. venv/bin/activate; python -m sir amqp_setup; python -m sir extension; python -m sir triggers --broker-id=1'
+
+service rabbitmq-server stop
+
+psql -U postgres -f sql/CreateExtension.sql musicbrainz_selenium
+psql -U musicbrainz -f sql/CreateFunctions.sql musicbrainz_selenium
+psql -U musicbrainz -f sql/CreateTriggers.sql musicbrainz_selenium
+
+cd /home/musicbrainz/artwork-indexer
+
+# Install the artwork_indexer schema into musicbrainz_selenium.
+sudo -E -H -u musicbrainz sh -c '. venv/bin/activate; python indexer.py --setup-schema'
+
+cd "$MBS_ROOT"
+
+sudo -E -H -u musicbrainz make -C po all_quiet deploy
+
+# script/dump_js_type_info.pl needs Redis running.
+redis-server &
+REDIS_PID=$!
+
+# Compile static resources.
+NODE_ENV=test \
+    WEBPACK_MODE=development \
+    MUSICBRAINZ_RUNNING_TESTS=1 \
+    NO_PROGRESS=1 \
+    NO_YARN=1 \
+    sudo -E -H -u musicbrainz carton exec -- ./script/compile_resources.sh default tests
+
+kill "$REDIS_PID"
+
+sudo -E -H -u postgres /usr/lib/postgresql/16/bin/pg_ctl stop -D /var/lib/postgresql/data

docker/musicbrainz-tests/run_js_perl_and_pgtap_tests.sh

@@ -14,14 +14,9 @@ function sv_start_if_down() {
   done
 }
 
-sudo -E -H -u musicbrainz make -C po test_source
+cd "$MBS_ROOT"
 
-MUSICBRAINZ_RUNNING_TESTS=1 \
-    NODE_ENV=test \
-    WEBPACK_MODE=development \
-    NO_PROGRESS=1 \
-    sudo -E -H -u musicbrainz \
-        carton exec -- ./script/compile_resources.sh client server web-tests
+sudo -E -H -u musicbrainz make -C po test_source
 
 echo Checking JavaScript code’s static types with Flow
 sudo -E -H -u musicbrainz ./node_modules/.bin/flow --quiet
@@ -35,7 +30,11 @@ echo Checking translation domain in statistics code
 ! sudo -E -H -u musicbrainz git grep -Pw '(N_)?l[np]?\(' -- 'root/statistics/**.js'
 echo OK
 
-sv_start_if_down chrome
+# GitHub Actions overrides the container entrypoint.
+/sbin/my_init &
+sleep 5
+
+sv_start_if_down chrome postgresql redis
 
 sudo -E -H -u musicbrainz carton exec -- node \
     t/web.js \
@@ -46,10 +45,6 @@ sv kill chrome
 
 ./docker/musicbrainz-tests/add_mbtest_alias.sh
 
-sudo -u postgres createdb -O musicbrainz -T musicbrainz_test -U postgres musicbrainz_test_json_dump
-sudo -u postgres createdb -O musicbrainz -T musicbrainz_test -U postgres musicbrainz_test_full_export
-sudo -u postgres createdb -O musicbrainz -T musicbrainz_test -U postgres musicbrainz_test_sitemaps
-
 sv_start_if_down template-renderer vnu website
 
 export MMD_SCHEMA_ROOT=/home/musicbrainz/mb-solr/mmd-schema

docker/musicbrainz-tests/run_selenium_tests.sh

@@ -16,55 +16,33 @@ function sv_start_if_down() {
 
 cd "$MBS_ROOT"
 
-# Create the musicbrainz_selenium DB.
-pushd /var/lib/postgresql
-sudo -u postgres createdb -O musicbrainz -T musicbrainz_test -U postgres \
-     musicbrainz_selenium
-popd
-
-# Set the open file limit Solr requests on startup, then start Solr.
+# Set the open file limit Solr requests on startup.
 ulimit -n 65000
-sv_start_if_down solr
-
-# Setup the rabbitmq user/vhost used by pg_amqp + sir.
-service rabbitmq-server start
-rabbitmqctl add_user sir sir
-rabbitmqctl add_vhost /sir-test
-rabbitmqctl set_permissions -p /sir-test sir '.*' '.*' '.*'
-
-# Install the sir triggers into musicbrainz_selenium.
-export SIR_DIR=/home/musicbrainz/sir
-cd "$SIR_DIR"
-sudo -E -H -u musicbrainz sh -c '. venv/bin/activate; python -m sir amqp_setup; python -m sir extension; python -m sir triggers --broker-id=1'
-psql -U postgres -f sql/CreateExtension.sql musicbrainz_selenium
-psql -U musicbrainz -f sql/CreateFunctions.sql musicbrainz_selenium
-psql -U musicbrainz -f sql/CreateTriggers.sql musicbrainz_selenium
-
-# Install the artwork_indexer schema into musicbrainz_selenium.
-cd /home/musicbrainz/artwork-indexer
-sudo -E -H -u musicbrainz sh -c '. venv/bin/activate; python indexer.py --setup-schema'
-
-cd "$MBS_ROOT"
-
-# Start the various CAA-related services.
-sv_start_if_down artwork-indexer artwork-redirect ssssss
-
-# Compile static resources.
-NODE_ENV=test \
-     WEBPACK_MODE=development \
-     MUSICBRAINZ_RUNNING_TESTS=1 \
-     NO_PROGRESS=1 \
-     sudo -E -H -u musicbrainz carton exec -- ./script/compile_resources.sh default tests
 
 # Add mbtest host alias to work around NO_PROXY restriction.
 # See add_mbtest_alias.sh for details.
 ./docker/musicbrainz-tests/add_mbtest_alias.sh
 
-sv_start_if_down template-renderer website
+service rabbitmq-server start
 
-# Wait for plackup to start.
+# GitHub Actions overrides the container entrypoint.
+/sbin/my_init &
+sleep 5
+
+sv_start_if_down \
+  artwork-indexer \
+  artwork-redirect \
+  postgresql \
+  redis \
+  solr \
+  ssssss \
+  template-renderer \
+  website
+
+# Wait for services to start.
 sleep 10
 
+export SIR_DIR=/home/musicbrainz/sir
 sudo -E -H -u musicbrainz carton exec -- \
      ./t/selenium.js --browser-binary-path=/opt/chrome-linux64/chrome \
      | tee >(./node_modules/.bin/tap-junit > ./junit_output/selenium.xml) \

docker/templates/Dockerfile.tests.m4

@@ -50,12 +50,14 @@ run_with_apt_cache \
     locale-gen && \
     # Allow the musicbrainz user execute any command with sudo.
     # Primarily needed to run rabbitmqctl.
-    echo 'musicbrainz ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+    echo 'musicbrainz ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers && \
+    sudo -E -H -u musicbrainz mkdir MBS_ROOT
 
 ENV JAVA_HOME=/usr/local/jdk \
     PATH=/usr/local/jdk/bin:$PATH \
     PGHOME=/var/lib/postgresql \
     PGDATA=/var/lib/postgresql/data \
+    RABBITMQ_NODENAME='rabbit@mbtest' \
     TERM=ansi \
     `MBS_ROOT'=MBS_ROOT
 
@@ -142,15 +144,22 @@ RUN sudo -E -H -u musicbrainz git clone https://github.com/metabrainz/artwork-re
 
 COPY docker/musicbrainz-tests/artwork-redirect-config.ini artwork-redirect/config.ini
 
+FROM build AS node_modules
+
+COPY --chown=musicbrainz:musicbrainz .yarnrc.yml package.json yarn.lock MBS_ROOT/
+RUN cd MBS_ROOT && \
+    corepack enable && \
+    sudo -E -H -u musicbrainz yarn
+
 FROM build
 
-COPY --chown=musicbrainz:musicbrainz cpanfile cpanfile.snapshot ./
+COPY --chown=musicbrainz:musicbrainz cpanfile cpanfile.snapshot MBS_ROOT/
 # Install Perl module dependencies for MusicBrainz Server
 RUN with_cpanm_cache \
     chown_mb(``/home/musicbrainz/.cpanm'') && \
     chown_mb(``$PERL_CARTON_PATH'') && \
-    sudo -E -H -u musicbrainz carton install --deployment && \
-    rm cpanfile cpanfile.snapshot
+    cd MBS_ROOT && \
+    sudo -E -H -u musicbrainz carton install --deployment
 
 COPY --from=mb_solr /usr/local/jdk/ /usr/local/jdk/
 RUN update-alternatives --install /usr/bin/java java /usr/local/jdk/bin/java 10000 && \
@@ -188,6 +197,7 @@ COPY --from=pg_amqp --chown=musicbrainz:musicbrainz /home/musicbrainz/pg_amqp/ta
 COPY --from=sir --chown=musicbrainz:musicbrainz /home/musicbrainz/sir/ /home/musicbrainz/sir/
 COPY --from=artwork_indexer --chown=musicbrainz:musicbrainz /home/musicbrainz/artwork-indexer/ /home/musicbrainz/artwork-indexer/
 COPY --from=artwork_redirect --chown=musicbrainz:musicbrainz /home/musicbrainz/artwork-redirect/ /home/musicbrainz/artwork-redirect/
+COPY --from=node_modules --chown=musicbrainz:musicbrainz MBS_ROOT/node_modules/ MBS_ROOT/node_modules/
 COPY --chmod=0755 docker/musicbrainz-tests/service/ /etc/service/
 
 COPY --chmod=0755 \
@@ -206,3 +216,8 @@ RUN --mount=type=bind,source=docker/scripts/setup_services.sh,target=/usr/local/
         template-renderer \
         vnu \
         website
+
+COPY --chown=musicbrainz:musicbrainz ./ MBS_ROOT/
+
+RUN cd MBS_ROOT && \
+    ./docker/musicbrainz-tests/initialize_tests_image.sh