Add extra check for response code fetching last_versions (#432)

.github/workflows/test_and_deploy.yml

@@ -23,8 +23,17 @@ jobs:
     steps:
       - uses: neuroinformatics-unit/actions/check_manifest@v2
 
+  get-week-number:
+    runs-on: ubuntu-latest
+    outputs:
+        WEEK_NUMBER: ${{ steps.get-week-number.outputs.WEEK_NUMBER }}
+    steps:
+    - name: Get week number
+      id: get-week-number
+      run: echo "WEEK_NUMBER=$(date -u '+%V')" >> "$GITHUB_OUTPUT"
+
   test:
-    needs: [linting, manifest]
+    needs: [linting, manifest, get-week-number]
     name: ${{ matrix.os }} py${{ matrix.python-version }}
     runs-on: ${{ matrix.os }}
     strategy:
@@ -43,12 +52,16 @@ jobs:
 
     steps:
       - name: Cache atlases
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: | # ensure we don't cache any interrupted atlas download and extraction, if e.g. we cancel the workflow manually
             ~/.brainglobe
             !~/.brainglobe/atlas.tar.gz
-          key: brainglobe
+          key: brainglobe-${{ runner.os }}-${{ needs.get-week-number.outputs.WEEK_NUMBER }}
+          restore-keys: |
+              brainglobe-${{ runner.os }}-${{ needs.get-week-number.outputs.WEEK_NUMBER }}
+              brainglobe-${{ runner.os }}-
+              brainglobe
 
       - uses: neuroinformatics-unit/actions/test@v2
         with:

brainglobe_atlasapi/utils.py

@@ -3,6 +3,7 @@
 import logging
 import re
 from pathlib import Path
+from time import sleep
 from typing import Callable, Optional
 
 import requests
@@ -302,10 +303,29 @@ def conf_from_url(url) -> configparser.ConfigParser:
     conf object
 
     """
-    text = requests.get(url).text
+    cache_path: Path = config.get_brainglobe_dir() / "last_versions.conf"
+
+    result = requests.get(url)
+    max_tries = 5
+    sleep_time = 0.25
+
+    while max_tries > 0 and result.status_code != 200:
+        result = requests.get(url)
+        max_tries -= 1
+        sleep(sleep_time)
+        sleep_time *= 2
+
+    if result.status_code != 200:
+        print(
+            f"Could not fetch the latest atlas versions: {result.status_code}"
+        )
+        print(f"Using the last cached version from {cache_path}")
+
+        return conf_from_file(cache_path)
+
+    text = result.text
     config_obj = configparser.ConfigParser()
     config_obj.read_string(text)
-    cache_path: Path = config.get_brainglobe_dir() / "last_versions.conf"
 
     try:
         if not cache_path.parent.exists():