Merge pull request #178 from NASA-IMPACT/develop

Release changes
NASA-IMPACT · Jul 17, 2023 · e0f6572 · e0f6572
2 parents 448158c + 22bb657
commit e0f6572
Show file tree

Hide file tree

Showing 40 changed files with 1,743 additions and 313 deletions.
diff --git a/.example.env b/.example.env
@@ -16,4 +16,7 @@ VEDA_DOMAIN_ALT_HOSTED_ZONE_ID=[OPTIONAL SECOND HOSTED ZONE]
 VEDA_DOMAIN_ALT_HOSTED_ZONE_NAME=[OPTIONAL SECOND DOMAIN]
 
 VEDA_RASTER_ENABLE_MOSAIC_SEARCH=TRUE
-VEDA_RASTER_DATA_ACCESS_ROLE_ARN=[OPTIONAL ARN OF IAM ROLE TO BE ASSUMED BY RASTER API] 
+VEDA_RASTER_DATA_ACCESS_ROLE_ARN=[OPTIONAL ARN OF IAM ROLE TO BE ASSUMED BY RASTER API]
+VEDA_RASTER_EXPORT_ASSUME_ROLE_CREDS_AS_ENVS=False
+
+VEDA_DB_PUBLICLY_ACCESSIBLE=TRUE
diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml
@@ -13,7 +13,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.8'
+          python-version: '3.9'
 
       - uses: actions/cache@v3
         with:
@@ -36,8 +36,8 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.8'
-    
+          python-version: '3.9'
+
       - uses: actions/cache@v3
         with:
           path: ${{ env.pythonLocation }}
@@ -47,24 +47,23 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           python -m pip install -e .[dev,deploy,test] 
+
       - name: Launch services
-        run: AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY=${{secrets.AWS_SECRET_ACCESS_KEY}} docker-compose up --build -d stac raster 
+        run: AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY=${{secrets.AWS_SECRET_ACCESS_KEY}} docker compose up --build -d
 
       - name: Ingest Stac Items/Collection
         run: |
-          pypgstac pgready --dsn postgresql://username:[email protected]:5432/postgis
-          pypgstac load collections .github/workflows/data/noaa-emergency-response.json --dsn postgresql://username:[email protected]:5432/postgis --method insert
-          pypgstac load items .github/workflows/data/noaa-eri-nashville2020.json --dsn postgresql://username:[email protected]:5432/postgis --method insert
+          ./scripts/load-data-container.sh
 
       - name: Sleep for 10 seconds
         run: sleep 10s
         shell: bash
-
+  
       - name: Integrations tests
         run: python -m pytest .github/workflows/tests/ -vv -s
 
       - name: Stop services
-        run: docker-compose stop
+        run: docker compose stop
 
   deploy: 
     needs: [test]
@@ -76,7 +75,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.8'
+          python-version: '3.9'
 
       - name: Setup Node
         uses: actions/setup-node@v1

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -13,7 +13,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.8'
+          python-version: '3.9'
 
       - uses: actions/cache@v3
         with:
@@ -36,8 +36,8 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.8'
-    
+          python-version: '3.9'
+
       - uses: actions/cache@v3
         with:
           path: ${{ env.pythonLocation }}
@@ -47,24 +47,23 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           python -m pip install -e .[dev,deploy,test] 
+
       - name: Launch services
-        run: AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY=${{secrets.AWS_SECRET_ACCESS_KEY}} docker-compose up --build -d stac raster 
+        run: AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY=${{secrets.AWS_SECRET_ACCESS_KEY}} docker compose up --build -d
 
       - name: Ingest Stac Items/Collection
         run: |
-          pypgstac pgready --dsn postgresql://username:[email protected]:5432/postgis
-          pypgstac load collections .github/workflows/data/noaa-emergency-response.json --dsn postgresql://username:[email protected]:5432/postgis --method insert
-          pypgstac load items .github/workflows/data/noaa-eri-nashville2020.json --dsn postgresql://username:[email protected]:5432/postgis --method insert
+          ./scripts/load-data-container.sh
 
       - name: Sleep for 10 seconds
         run: sleep 10s
         shell: bash
-
+  
       - name: Integrations tests
         run: python -m pytest .github/workflows/tests/ -vv -s
 
       - name: Stop services
-        run: docker-compose stop
+        run: docker compose stop
 
   deploy: 
     needs: [test]
@@ -76,7 +75,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.8'
+          python-version: '3.9'
 
       - name: Setup Node
         uses: actions/setup-node@v1

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -10,7 +10,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.8'
+          python-version: '3.9'
 
       - uses: actions/cache@v3
         with:
@@ -24,7 +24,7 @@ jobs:
 
       - name: Run pre-commit
         run: pre-commit run --all-files
-    
+
   test:
     needs: [lint]
     runs-on: ubuntu-latest
@@ -33,8 +33,8 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.8'
-    
+          python-version: '3.9'
+
       - uses: actions/cache@v3
         with:
           path: ${{ env.pythonLocation }}
@@ -44,41 +44,41 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           python -m pip install -e .[dev,deploy,test] 
+
       - name: Launch services
-        run: AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY=${{secrets.AWS_SECRET_ACCESS_KEY}} docker-compose up --build -d stac raster 
+        run: AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY=${{secrets.AWS_SECRET_ACCESS_KEY}} docker compose up --build -d
 
       - name: Ingest Stac Items/Collection
         run: |
-          pypgstac pgready --dsn postgresql://username:[email protected]:5432/postgis
-          pypgstac load collections .github/workflows/data/noaa-emergency-response.json --dsn postgresql://username:[email protected]:5432/postgis --method insert
-          pypgstac load items .github/workflows/data/noaa-eri-nashville2020.json --dsn postgresql://username:[email protected]:5432/postgis --method insert
+          ./scripts/load-data-container.sh
 
       - name: Sleep for 10 seconds
         run: sleep 10s
         shell: bash
-
+  
       - name: Integrations tests
         run: python -m pytest .github/workflows/tests/ -vv -s
 
       - name: Stop services
-        run: docker-compose stop
-  predeploy: 
+        run: docker compose stop
+
+  predeploy:
     needs: [test]
 
     runs-on: ubuntu-latest
     steps:
-    
+
       - uses: actions/checkout@v3
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.8'
+          python-version: '3.9'
 
       - name: Setup Node
         uses: actions/setup-node@v1
-        with: 
+        with:
           node-version: 17
-      
+
       - name: Configure awscli
         uses: aws-actions/configure-aws-credentials@v1
         with:
@@ -90,24 +90,24 @@ jobs:
         with:
           path: ~/.npm
           key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
-      
+
       - name: Install CDK
         run: npm install -g aws-cdk@2
-      
+
       - uses: actions/cache@v3
         with:
           path: ${{ env.pythonLocation }}
           key:  ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}
-      
+
       - name: Install python dependencies
         run: |
           python -m pip install --upgrade pip
-          python -m pip install -e .[dev,deploy,test] 
+          python -m pip install -e .[dev,deploy,test]
 
       - name: Get dev environment configuration for develop branch
         run: ./scripts/get-env.sh "veda-backend-uah-dev-env"
 
-      - name: Pre deployment CDK diff 
+      - name: Pre deployment CDK diff
         run: |
           echo $STAGE
-          cdk diff veda-backend-uah-dev
+          cdk diff veda-backend-uah-dev
diff --git a/.github/workflows/tests/test_raster.py b/.github/workflows/tests/test_raster.py
@@ -13,7 +13,7 @@ def test_raster_api():
     )
     assert resp.status_code == 200
     assert resp.headers["content-type"] == "application/json"
-    assert resp.headers["content-encoding"] == "br"
+    assert resp.headers["content-encoding"] in ["br", "gzip"]
 
 
 def test_mosaic_api():

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -6,7 +6,7 @@ repos:
         language_version: python
 
   - repo: https://github.com/PyCQA/isort
-    rev: 5.4.2
+    rev: 5.12.0
     hooks:
       - id: isort
         language_version: python

diff --git a/README.md b/README.md
@@ -40,7 +40,7 @@ To retrieve the variables for a stage that has been previously deployed, the sec
 | --- | --- |
 | `APP_NAME` | Optional app name used to name stack and resources, defaults to `veda-backend` |
 | `STAGE` | **REQUIRED** Deployment stage used to name stack and resources, i.e. `dev`, `staging`, `prod` |
-| `VEDA_DB_PGSTAC_VERSION` | **REQUIRED** version of PgStac database, i.e. 0.5 |
+| `VEDA_DB_PGSTAC_VERSION` | **REQUIRED** version of PgStac database, i.e. 0.7.6 |
 | `VEDA_DB_SCHEMA_VERSION` | **REQUIRED** The version of the custom veda-backend schema, i.e. 0.1.1 |
 | `VEDA_DB_SNAPSHOT_ID` | **Once used always REQUIRED** Optional RDS snapshot identifier to initialize RDS from a snapshot |
 > **Note** See [Advanced Configuration](docs/advanced_configuration.md) for details about custom configuration options.
@@ -114,11 +114,11 @@ Support scripts are provided for manual system operations.
 | [**veda-ui**](https://github.com/NASA-IMPACT/veda-ui) | Dashboard UI for viewing and analysing VEDA assets |
 | [**veda-stac-ingestor**](https://github.com/NASA-IMPACT/veda-stac-ingestor) |  Entry-point for users/services to add new records to database |
 | [**veda-data-pipelines**](https://github.com/NASA-IMPACT/veda-data-pipelines) | Cloud optimize data assets and submit records for publication to veda-stac-ingestor |
-| [**veda-documentation**](https://github.com/NASA-IMPACT/veda-documentation) | Documentation repository for end users of VEDA ecosystem data and tools |
+| [**veda-docs**](https://github.com/NASA-IMPACT/veda-docs) | Documentation repository for end users of VEDA ecosystem data and tools |
 
 ## VEDA usage examples
 
-### [VEDA documentation](https://nasa-impact.github.io/veda-documentation/)
+### [VEDA documentation](https://nasa-impact.github.io/veda-docs)
 
 ### [VEDA dashboard](https://www.earthdata.nasa.gov/dashboard)
 
@@ -128,4 +128,4 @@ Support scripts are provided for manual system operations.
 Radiant Earth's [stac-browser](https://github.com/radiantearth/stac-browser) is a browser for STAC catalogs. The demo version of this browser [radiantearth.github.io/stac-browser](https://radiantearth.github.io/stac-browser/#/) can be used to browse the contents of the veda-backend STAC catalog, paste the veda-backend stac-api URL deployed by this project in the demo and click load. Read more about the recent developments and usage of stac-browser [here](https://medium.com/radiant-earth-insights/the-exciting-future-of-the-stac-browser-2351143aa24b).
 
 # License
-This project is licensed under **Apache 2**, see the [LICENSE](LICENSE) file for more details.
+This project is licensed under **Apache 2**, see the [LICENSE](LICENSE) file for more details.
diff --git a/app.py b/app.py
@@ -71,7 +71,6 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
 
 # TODO this conditional supports deploying a second set of APIs to a separate custom domain and should be removed if no longer necessary
 if veda_app_settings.alt_domain():
-
     alt_domain = DomainConstruct(
         veda_stack,
         "alt-domain",

diff --git a/database/infrastructure/config.py b/database/infrastructure/config.py
@@ -34,11 +34,9 @@ class vedaDBSettings(BaseSettings):
             "**Once used always REQUIRED**"
         ),
     )
-    private_subnets: Optional[bool] = Field(
-        False,
-        description="Boolean deploy database to private subnets",
+    publicly_accessible: Optional[bool] = Field(
+        True, description="Boolean if the RDS should be publicly accessible"
     )
-
     # RDS custom postgres parameters
     max_locks_per_transaction: Optional[str] = Field(
         "1024",
@@ -55,6 +53,10 @@ class vedaDBSettings(BaseSettings):
         description="maximum number of temporary buffers used by each session",
         regex=r"^[1-9]\d*$",
     )
+    use_rds_proxy: Optional[bool] = Field(
+        False,
+        description="Boolean if the RDS should be accessed through a proxy",
+    )
 
     class Config:
         """model config."""