docker-compose.yaml

version: "3.4"
x-airflow-common:
  &airflow-common
  build: ./airflow
  environment:
    &airflow-common-env
    AIRFLOW__API__AUTH_BACKEND: 'airflow.api.auth.backend.basic_auth'
    AIRFLOW__CORE__EXECUTOR: CeleryExecutor
    AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow
    AIRFLOW__CORE__FERNET_KEY: ''
    AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true'
    AIRFLOW__CORE__LOAD_EXAMPLES: 'true'
    AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://airflow:airflow@postgres/airflow
    AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0
    MLFLOW_S3_ENDPOINT_URL: http://minio:9000
    # AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
    AWS_ACCESS_KEY_ID: minioadmin
    # AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
    AWS_SECRET_ACCESS_KEY: minioadmin
  volumes:
    - ./airflow/dags:/opt/airflow/dags
    - ./airflow/logs:/opt/airflow/logs
    - ./airflow/plugins:/opt/airflow/plugins
  user: "${AIRFLOW_UID:-50000}:${AIRFLOW_GID:-50000}"
  depends_on:
    - redis
    - postgres

services:
    nginx:
        container_name: nginx
        build: ./nginx
        depends_on: 
            - python-model-service1
            - python-model-service2
        ports:
        - "80:80"
        - "443:443"
        # volumes: 
        #     - ./nginx.conf:/etc/nginx/nginx.conf
        restart: always
    
    python-model-service1:
        container_name: python-service-1
        depends_on:
            - rabbitmq
        build:
            context: ./python-model-service-1
            dockerfile: Dockerfile
        ports:
            - "8000:8000"
        volumes:
            - ./python-model-service-1:/python-model-service-1
        restart: always
    
    python-model-service2:
        container_name: python-service-2
        depends_on:
            - rabbitmq
        build: 
            context: ./python-model-service-2
            dockerfile: Dockerfile
        ports:
            - "8001:8001"
        volumes:
            - ./python-model-service-2:/python-model-service-2
        restart: always
    
    bridge-server:
        container_name: bridge_server
        build: ./bridge-server
        ports:
            - "8002:8002"
        volumes:
            - ./bridge-server/app.py:/app/app.py
            - ./bridge-server/dag_mapping.py:/app/dag_mapping.py
        restart: always
    
    postgres:
        container_name: postgres
        build: ./postgres
        environment:
            POSTGRES_USER: airflow
            POSTGRES_PASSWORD: airflow
            POSTGRES_DB: airflow
        volumes:
            - postgres-db-volume:/var/lib/postgresql/data
        healthcheck:
            test: ["CMD", "pg_isready", "-U", "airflow"]
            interval: 5s
            retries: 5
        restart: always
    
    prometheus:
        container_name: prometheus
        build: ./prometheus
        depends_on:
            - python-model-service1
            - python-model-service2
            - rabbitmq
        ports:
            - "9090:9090"
        volumes:
          - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
        restart: always

    rabbitmq:
        container_name: rabbitmq
        image: rabbitmq:3-management-alpine
        volumes:
            - ./rabbitmq/etc/definitions.json:/etc/rabbitmq/definitions.json
        # environment:
        #     RABBITMQ_ERLANG_COOKIE: "SWQOKODSQALRPCLNMEQG"
        #     RABBITMQ_DEFAULT_USER: "guest"
        #     RABBITMQ_DEFAULT_PASS: "guest"
        #     RABBITMQ_DEFAULT_VHOST: "/"
        ports:
            - "15672:15672"
            - "5672:5672"
            - "15692:15692"
        restart: always

    rabbitmq-workers:
        container_name: rabbitmq-workers
        depends_on:
            - rabbitmq
        build: ./workers
        restart: always

    mlflow:
        container_name: mlflow-server
        depends_on:
            - minio
        build: ./mlflow
        environment:
            MLFLOW_S3_ENDPOINT_URL: http://minio:9000
            # AWS_ACCESS_KEY_ID:  ${AWS_ACCESS_KEY_ID}
            # AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
            AWS_ACCESS_KEY_ID:  minioadmin
            AWS_SECRET_ACCESS_KEY: minioadmin
        ports: 
            - "5000:5000"
        command:
            mlflow server --backend-store-uri sqlite:///mlflow.db --default-artifact-root s3://mlflow --host 0.0.0.0 -p 5000
        restart: always
    
    minio:
        container_name: minio
        image: minio/minio
        ports:
          - "9000:9000"
          - "9001:9001"
        environment:
            # AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
            # AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
            AWS_ACCESS_KEY_ID: minioadmin
            AWS_SECRET_ACCESS_KEY: minioadmin
        volumes:
            - data:/data
        command:
            server /data --address ':9000' --console-address ":9001"
        restart: always

    # createbuckets:
    #     image: minio/mc
    #     depends_on:
    #         - minio
    #     entrypoint: >
    #         /bin/sh -c "
    #         # /usr/bin/mc config host add mlops http://minio:9000 ${AWS_ACCESS_KEY_ID} ${AWS_SECRET_ACCESS_KEY};
    #         /usr/bin/mc config host add mlops http://minio:9000 minioadmin minioadmin;
    #         /usr/bin/mc rm -r --force mlops/mlflow;
    #         /usr/bin/mc mb mlops/mlflow;
    #         /usr/bin/mc policy set download mlops/mlflow;
    #         exit 0;
    #         "

    influxdb:
        container_name: influxdb
        build: ./influxdb
        ports:
            - 8086:8086
            - 8083:8083
            - 2003:2003
        env_file:
            - ./influxdb/influxdb.env
        restart: always

    chronograf:
        container_name: chronograf
        image: chronograf:1.8.4-alpine
        depends_on:
            - influxdb
        user: "0"
        ports:
            - 8888:8888
        env_file:
            - ./chronograf/chronograf.env
        restart: always

    grafana:
        container_name: grafana
        image: grafana/grafana:6.3.6
        depends_on:
            - influxdb
            - prometheus
        user: "0"
        ports:
            - 3000:3000
        env_file:
            - ./grafana/grafana.env
        volumes:
            - ./grafana/provisioning/datasources/:/etc/grafana/provisioning/datasources/
            - ./grafana/provisioning/dashboards/:/etc/grafana/provisioning/dashboards/
            - ./grafana/provisioning/notifiers/:/etc/grafana/provisioning/notifiers/
        restart: always
    
    redis:
        container_name: redis
        image: redis:latest
        ports:
            - 6379:6379
        healthcheck:
            test: ["CMD", "redis-cli", "ping"]
            interval: 5s
            timeout: 30s
            retries: 50
        restart: always
    
    locust:
        container_name: locust
        build: ./locust
        ports:
          - "8089:8089"
        restart: always

    airflow-webserver:
        <<: *airflow-common
        command: webserver
        ports:
            - 8080:8080
        healthcheck:
            test: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
            interval: 10s
            timeout: 10s
            retries: 5
        restart: always

    airflow-scheduler:
        <<: *airflow-common
        depends_on:
            - airflow-webserver
        command: scheduler
        restart: always

    airflow-worker:
        <<: *airflow-common
        depends_on:
            - airflow-scheduler
        command: celery worker
        restart: always

    airflow-init:
        <<: *airflow-common
        environment:
            <<: *airflow-common-env
            _AIRFLOW_DB_UPGRADE: 'true'
            _AIRFLOW_WWW_USER_CREATE: 'true'
            _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-airflow}
            _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-airflow}
        command: version

    flower:
        <<: *airflow-common
        ports:
            - 5555:5555
        healthcheck:
            test: ["CMD", "curl", "--fail", "http://localhost:5555/"]
            interval: 10s
            timeout: 10s
            retries: 5
        command: celery flower
        restart: always
    
volumes:
  postgres-db-volume:
  data: