feat(raster)!: update titiler-pgstac from 0.2.3 to 0.8.0 (#239)

ref #234 

This PR aims to update titiler-pgstac from version 0.2.3 to 0.8.0. Doing
so, will also update titiler, rio-tiler, morecantile, fastapi and
pydantic.

Endpoint changes 


- ~~Use `/collections/{collection_id}/items/{item_id}` prefix for Item
endpoint.~~

I was able to keep this as it was
(`{endpoint}/stac/info?collection=collection1&item=item1`)

- `post_process=` -> `algorithm=`

- change tile url path parameter order from
`/tiles/{searchid}/{TileMatrixSetId}/{z}/{x}/{y`} to
`/{searchid}/tiles/{TileMatrixSetId}/{z}/{x}/{y}`

```
# Before
{endpoint}/mosaic/tiles/20200307aC0853900w361030/0/0/0

# Now
{endpoint}/mosaic/20200307aC0853900w361030/tiles/0/0/0
```

- replace `/{searchid}/{z}/{x}/{y}/assets` endpoints by
`/{searchid}/tiles/{z}/{x}/{y}/assets`

- replace - by _ in query parameters

    - coord-crs -> coord_crs
    - dst-crs -> dst_crs

- remove `max_size` default for mosaic's `/statistics [POST]` endpoint
breaking change

- add `/bbox` and `/feature [POST]` optional endpoints

.github/workflows/tests/test_raster.py

@@ -33,15 +33,15 @@ def test_mosaic_api():
     assert list(resp.json()[0]) == ["id", "bbox", "assets", "collection"]
     assert resp.json()[0]["id"] == "20200307aC0853900w361030"
 
-    resp = httpx.get(f"{raster_endpoint}/mosaic/{searchid}/15/8589/12849/assets")
+    resp = httpx.get(f"{raster_endpoint}/mosaic/{searchid}/tiles/15/8589/12849/assets")
     assert resp.status_code == 200
     assert len(resp.json()) == 1
     assert list(resp.json()[0]) == ["id", "bbox", "assets", "collection"]
     assert resp.json()[0]["id"] == "20200307aC0853900w361030"
 
     z, x, y = 15, 8589, 12849
     resp = httpx.get(
-        f"{raster_endpoint}/mosaic/tiles/{searchid}/{z}/{x}/{y}",
+        f"{raster_endpoint}/mosaic/{searchid}/tiles/{z}/{x}/{y}",
         params={"assets": "cog"},
         headers={"Accept-Encoding": "br, gzip"},
         timeout=10.0,

.pre-commit-config.yaml

@@ -12,7 +12,7 @@ repos:
         language_version: python
 
   - repo: https://github.com/PyCQA/flake8
-    rev: 3.8.3
+    rev: 6.1.0
     hooks:
       - id: flake8
         language_version: python

raster_api/runtime/Dockerfile

@@ -3,7 +3,7 @@ FROM public.ecr.aws/sam/build-python3.9:latest
 WORKDIR /tmp
 
 COPY raster_api/runtime /tmp/raster
-RUN pip install "mangum>=0.14,<0.15" /tmp/raster["psycopg-binary"] -t /asset --no-binary pydantic 
+RUN pip install "mangum>=0.14,<0.15" /tmp/raster["psycopg-binary"] -t /asset --no-binary pydantic
 RUN rm -rf /tmp/raster
 
 # # Reduce package size and remove useless files
@@ -15,4 +15,4 @@ RUN rm -rdf /asset/numpy/doc/ /asset/boto3* /asset/botocore* /asset/bin /asset/g
 
 COPY raster_api/runtime/handler.py /asset/handler.py
 
-CMD ["echo", "hello world"]
+CMD ["echo", "hello world"]

raster_api/runtime/handler.py

@@ -6,12 +6,23 @@
 
 from mangum import Mangum
 from src.app import app
+from src.config import ApiSettings
 from src.monitoring import logger, metrics, tracer
 
+from titiler.pgstac.db import connect_to_db
+
+settings = ApiSettings()
+
 logging.getLogger("mangum.lifespan").setLevel(logging.ERROR)
 logging.getLogger("mangum.http").setLevel(logging.ERROR)
 
 
+@app.on_event("startup")
+async def startup_event() -> None:
+    """Connect to database on startup."""
+    await connect_to_db(app, settings=settings.load_postgres_settings())
+
+
 handler = Mangum(app, lifespan="off")
 
 if "AWS_EXECUTION_ENV" in os.environ:

raster_api/runtime/setup.py

@@ -6,27 +6,27 @@
     long_description = f.read()
 
 inst_reqs = [
-    "titiler.pgstac==0.2.3",
-    "titiler.application>=0.10,<0.11",
-    "importlib_resources>=1.1.0;python_version<='3.9'",  # https://github.com/cogeotiff/rio-tiler/pull/379
+    "boto3",
+    "titiler.pgstac==0.8.0",
+    "titiler.extensions[cogeo]>=0.15.0,<0.16",
+    "starlette-cramjam>=0.3,<0.4",
     "aws_xray_sdk>=2.6.0,<3",
     "aws-lambda-powertools>=1.18.0",
-    "pydantic<2",
 ]
 
 extra_reqs = {
     # https://www.psycopg.org/psycopg3/docs/api/pq.html#pq-module-implementations
     "psycopg": ["psycopg[pool]"],  # pure python implementation
     "psycopg-c": ["psycopg[c,pool]"],  # C implementation of the libpq wrapper
     "psycopg-binary": ["psycopg[binary,pool]"],  # pre-compiled C implementation
-    "test": ["pytest", "pytest-cov", "pytest-asyncio", "requests"],
+    "test": ["pytest", "pytest-cov", "pytest-asyncio", "requests", "brotlipy"],
 }
 
 
 setup(
     name="veda.raster_api",
     description="",
-    python_requires=">=3.7",
+    python_requires=">=3.8",
     packages=find_namespace_packages(exclude=["tests*"]),
     package_data={"src": ["templates/*.html"]},
     include_package_data=True,

raster_api/runtime/src/algorithms.py

@@ -0,0 +1,45 @@
+"""veda custom algorithms"""
+
+import math
+
+import numpy
+from rio_tiler.models import ImageData
+
+from titiler.core.algorithm import Algorithms
+from titiler.core.algorithm.base import BaseAlgorithm
+
+# https://github.com/cogeotiff/rio-tiler/blob/master/rio_tiler/reader.py#L35-L37
+
+# From eoAPI datasetparams edl_auth branch https://github.com/NASA-IMPACT/eoAPI/blob/edl_auth/src/eoapi/raster/eoapi/raster/datasetparams.py
+
+
+class SWIR(BaseAlgorithm):
+    """SWIR Custom Algorithm."""
+
+    low_value: float = math.e
+    high_value: float = 255
+    low_threshold: float = math.log(1000)
+    high_threshold: float = math.log(7500)
+
+    def __call__(self, img: ImageData) -> ImageData:
+        """Apply processing."""
+        data = numpy.log(img.array)
+        data[numpy.where(data <= self.low_threshold)] = self.low_value
+        data[numpy.where(data >= self.high_threshold)] = self.high_value
+        indices = numpy.where((data > self.low_value) & (data < self.high_value))
+        data[indices] = (
+            self.high_value
+            * (data[indices] - self.low_threshold)
+            / (self.high_threshold - self.low_threshold)
+        )
+        img.array = data.astype("uint8")
+        return img
+
+
+algorithms = Algorithms(
+    {
+        "swir": SWIR,
+    }
+)
+
+PostProcessParams = algorithms.dependency

raster_api/runtime/src/app.py

@@ -1,124 +1,121 @@
 """TiTiler+PgSTAC FastAPI application."""
 import logging
+from contextlib import asynccontextmanager
 
 from aws_lambda_powertools.metrics import MetricUnit
-from rio_cogeo.cogeo import cog_info as rio_cogeo_info
-from rio_cogeo.models import Info
+from src.algorithms import PostProcessParams
 from src.config import ApiSettings
-from src.datasetparams import DatasetParams
-from src.factory import MultiBaseTilerFactory
+from src.dependencies import ItemPathParams
+from src.extensions import stacViewerExtension
+from src.monitoring import LoggerRouteHandler, logger, metrics, tracer
 from src.version import __version__ as veda_raster_version
 
-from fastapi import APIRouter, Depends, FastAPI, Query
+from fastapi import APIRouter, FastAPI
 from starlette.middleware.cors import CORSMiddleware
 from starlette.requests import Request
-from starlette.responses import HTMLResponse
-from starlette.templating import Jinja2Templates
 from starlette_cramjam.middleware import CompressionMiddleware
-from titiler.core.dependencies import DatasetPathParams
 from titiler.core.errors import DEFAULT_STATUS_CODES, add_exception_handlers
-from titiler.core.factory import TilerFactory, TMSFactory
+from titiler.core.factory import MultiBaseTilerFactory, TilerFactory, TMSFactory
 from titiler.core.middleware import CacheControlMiddleware
 from titiler.core.resources.enums import OptionalHeader
 from titiler.core.resources.responses import JSONResponse
+from titiler.extensions import cogValidateExtension, cogViewerExtension
 from titiler.mosaic.errors import MOSAIC_STATUS_CODES
 from titiler.pgstac.db import close_db_connection, connect_to_db
-from titiler.pgstac.dependencies import ItemPathParams
 from titiler.pgstac.factory import MosaicTilerFactory
 from titiler.pgstac.reader import PgSTACReader
 
-try:
-    from importlib.resources import files as resources_files  # type: ignore
-except ImportError:
-    # Try backported to PY<39 `importlib_resources`.
-    from importlib_resources import files as resources_files  # type: ignore
-
-from .monitoring import LoggerRouteHandler, logger, metrics, tracer
-
 logging.getLogger("botocore.credentials").disabled = True
 logging.getLogger("botocore.utils").disabled = True
 logging.getLogger("rio-tiler").setLevel(logging.ERROR)
 
 settings = ApiSettings()
-templates = Jinja2Templates(directory=str(resources_files(__package__) / "templates"))  # type: ignore
+
 
 if settings.debug:
     optional_headers = [OptionalHeader.server_timing, OptionalHeader.x_assets]
 else:
     optional_headers = []
 
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """FastAPI Lifespan."""
+    # Create Connection Pool
+    await connect_to_db(app, settings=settings.load_postgres_settings())
+    yield
+    # Close the Connection Pool
+    await close_db_connection(app)
+
+
 path_prefix = settings.path_prefix
 app = FastAPI(
     title=settings.name,
     version=veda_raster_version,
     openapi_url=f"{path_prefix}/openapi.json",
     docs_url=f"{path_prefix}/docs",
+    lifespan=lifespan,
 )
 
 # router to be applied to all titiler route factories (improves logs with FastAPI context)
 router = APIRouter(route_class=LoggerRouteHandler)
 add_exception_handlers(app, DEFAULT_STATUS_CODES)
 add_exception_handlers(app, MOSAIC_STATUS_CODES)
 
-
-# Custom PgSTAC mosaic tiler
+###############################################################################
+# /mosaic - PgSTAC Mosaic titiler endpoint
+###############################################################################
 mosaic = MosaicTilerFactory(
     router_prefix=f"{path_prefix}/mosaic",
-    add_mosaic_list=settings.enable_mosaic_search,
     optional_headers=optional_headers,
     environment_dependency=settings.get_gdal_config,
-    dataset_dependency=DatasetParams,
+    process_dependency=PostProcessParams,
     router=APIRouter(route_class=LoggerRouteHandler),
+    # add /list (default to False)
+    add_mosaic_list=settings.enable_mosaic_search,
+    # add /statistics [POST] (default to False)
+    add_statistics=True,
+    # add /map viewer (default to False)
+    add_viewer=False,
+    # add /bbox [GET] and /feature  [POST] (default to False)
+    add_part=True,
 )
 app.include_router(mosaic.router, prefix=f"{path_prefix}/mosaic", tags=["Mosaic"])
+# TODO
+# prefix will be replaced by `/mosaics/{search_id}` in titiler-pgstac 0.9.0
 
-# Custom STAC titiler endpoint (not added to the openapi docs)
+###############################################################################
+# /stac - Custom STAC titiler endpoint
+###############################################################################
 stac = MultiBaseTilerFactory(
     reader=PgSTACReader,
     path_dependency=ItemPathParams,
     optional_headers=optional_headers,
     router_prefix=f"{path_prefix}/stac",
     environment_dependency=settings.get_gdal_config,
     router=APIRouter(route_class=LoggerRouteHandler),
+    extensions=[
+        stacViewerExtension(),
+    ],
 )
 app.include_router(stac.router, tags=["Items"], prefix=f"{path_prefix}/stac")
+# TODO
+# in titiler-pgstac we replaced the prefix to `/collections/{collection_id}/items/{item_id}`
 
+###############################################################################
+# /cog - External Cloud Optimized GeoTIFF endpoints
+###############################################################################
 cog = TilerFactory(
     router_prefix=f"{path_prefix}/cog",
     optional_headers=optional_headers,
     environment_dependency=settings.get_gdal_config,
     router=APIRouter(route_class=LoggerRouteHandler),
+    extensions=[
+        cogValidateExtension(),
+        cogViewerExtension(),
+    ],
 )
 
-
-@cog.router.get(
-    "/validate",
-    response_model=Info,
-    response_class=JSONResponse,
-)
-def cog_validate(
-    src_path: str = Depends(DatasetPathParams),
-    strict: bool = Query(False, description="Treat warnings as errors"),
-):
-    """Validate a COG"""
-    return rio_cogeo_info(src_path, strict=strict, config=settings.get_gdal_config())
-
-
-@cog.router.get("/viewer", response_class=HTMLResponse)
-def cog_demo(request: Request):
-    """COG Viewer."""
-    return templates.TemplateResponse(
-        name="viewer.html",
-        context={
-            "request": request,
-            "tilejson_endpoint": cog.url_for(request, "tilejson"),
-            "info_endpoint": cog.url_for(request, "info"),
-            "statistics_endpoint": cog.url_for(request, "statistics"),
-        },
-        media_type="text/html",
-    )
-
-
 app.include_router(
     cog.router, tags=["Cloud Optimized GeoTIFF"], prefix=f"{path_prefix}/cog"
 )
@@ -151,6 +148,7 @@ def ping():
 )
 app.add_middleware(
     CompressionMiddleware,
+    minimum_size=0,
     exclude_mediatype={
         "image/jpeg",
         "image/jpg",
@@ -174,8 +172,10 @@ async def add_correlation_id(request: Request, call_next):
         except KeyError:
             # If empty, use uuid
             corr_id = "local"
+
     # Add correlation id to logs
     logger.set_correlation_id(corr_id)
+
     # Add correlation id to traces
     tracer.put_annotation(key="correlation_id", value=corr_id)
 
@@ -192,15 +192,3 @@ async def validation_exception_handler(request, err):
     metrics.add_metric(name="UnhandledExceptions", unit=MetricUnit.Count, value=1)
     logger.exception("Unhandled exception")
     return JSONResponse(status_code=500, content={"detail": "Internal Server Error"})
-
-
-@app.on_event("startup")
-async def startup_event() -> None:
-    """Connect to database on startup."""
-    await connect_to_db(app, settings=settings.load_postgres_settings())
-
-
-@app.on_event("shutdown")
-async def shutdown_event() -> None:
-    """Close database connection."""
-    await close_db_connection(app)