Use Docker Compose to run IntegrationOS locally (#3)

.env-example

@@ -0,0 +1,24 @@
+# Google KMS
+GCP_PROJECT_ID=
+GCP_LOCATION_ID=
+KMS_KEY_ID=
+KMS_KEY_RING_ID=
+
+# A secret for the gateway service. Must be 32 characters.
+GATEWAY_SECRET=replace-this-insecure-secret----  # change before deploying to production
+
+# A random MongoDB password
+MONGO_PASSWORD=replace-this-insecure-password  # change before deploying to production
+
+
+# The values below are tied to the seed data
+# Don't change without updating the data in seed-data/seed_db/
+
+BUILDABLE_SECRET=cywf.D**tGRN6yfFjA6jjEeeJKR7*e
+CORE_USER_SECRET=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzZWNyZXQiOiIyMmQ5ZmRmMC00OWI1LTQyOTgtOWFiZC0zZTdlN2I2YTc0ZjVjMzYyNzY0Yi0wMDNmLTQ5OTEtOTE5MS05Njk2NDVhMTdiN2QifQ.O8JE8t2G_YMinS2uFP26GkIdEEv89Vm5TexDqS9aiPs
+DEFAULT_LIVE_ACCESS_KEY=event_access::custom::live::default::event-inc::default-live-key
+DEFAULT_TEST_ACCESS_KEY=event_access::custom::test::default::event-inc::default-test-key
+DEVELOPER_ACCOUNT_ACCESS_KEY=sk_test_1_3pejYG_SdSxV9xkt5_GA8WoMsSnfBHvY1qpGhlX-6DKd9kyZO3ee9hWfjGWpt5dY0AzxvM51q6_45_Q6bJTWCTuax7yq4X96nhvB0uTwhhLlsxyJm02JqasmdeDVeHt08GxGPoiBc7I9u00-1EKOejw62kNO0M1EaEFqwaGXw1Y8IfFH
+DEVELOPER_ACCOUNT_ID=build-1c3cd7af757d4aebab523f5373190e1b
+EVENT_ACCESS_PASSWORD=18BBFT_i3UckCmxPwY3GGzgHdxfYs6zA
+JWT_SECRET=Qsfb9YUkdjwUULX.u96HdTCX4q7GuB

README.md

@@ -71,7 +71,7 @@ import { AuthKitToken } from "@integrationos/authkit-node";
 
 app.post("/authkit-token", async (request, response) => {
   const authKitToken = new AuthKitToken(process.env.INTEGRATIONOS_SANDBOX_API_KEY);
-  
+
 // Specifying how the token will be constructed
   const token = await authKitToken.create({
     group: "org_123", // a meaningful identifier (i.e., organizationId)
@@ -137,11 +137,11 @@ View the full guide [here](https://docs.integrationos.com/docs/quickstart).
 
 * [Docker](https://docs.docker.com/engine/) and [Docker Compose](https://docs.docker.com/compose/)
 * A [Google Cloud KMS](https://cloud.google.com/kms/docs) key ring
-* [`gcloud`](https://cloud.google.com/sdk/gcloud) installed and logged into an account with `roles/cloudkms.cryptoKeyEncrypterDecrypter` access
+* [`gcloud`](https://cloud.google.com/sdk/gcloud) installed, logged into an account that has `roles/cloudkms.cryptoKeyEncrypterDecrypter` access, and configured with [Application Default Credentials](https://cloud.google.com/docs/authentication/provide-credentials-adc)
 
 ## Setup
 
-1. Copy `.env-example` to `.env` and set the values
+1. Copy `.env-example` to `.env`. Review and update the environment variables.
 
 2. Run the containers
 
@@ -160,8 +160,8 @@ View the full guide [here](https://docs.integrationos.com/docs/quickstart).
 
     ```shell
     source .env
-    docker-compose -f docker-compose.data.yml run migrate-before
-    docker-compose -f docker-compose.data.yml run migrate-after
+    docker-compose -f docker-compose.data.yml run --rm migrate-before
+    docker-compose -f docker-compose.data.yml run --rm migrate-after
     ```
 
 ## Other actions

docker-compose.data.yml

@@ -0,0 +1,18 @@
+version: "3.8"
+
+services:
+  migrate-before:
+    platform: linux/amd64
+    image: us-docker.pkg.dev/integrationos/docker-oss/typescript-services:1.0.0
+    command: npm run run-migrations::before
+    environment:
+      - MONGO_URI=mongodb://integrationos:${MONGO_PASSWORD}@mongo:27017/events-service?authSource=admin
+      - MONGO_DB_NAME=events-service
+
+  migrate-after:
+    platform: linux/amd64
+    image: us-docker.pkg.dev/integrationos/docker-oss/typescript-services:1.0.0
+    command: npm run run-migrations::after
+    environment:
+      - MONGO_URI=mongodb://integrationos:${MONGO_PASSWORD}@mongo:27017/events-service?authSource=admin
+      - MONGO_DB_NAME=events-service

docker-compose.yml

@@ -0,0 +1,167 @@
+version: "3.8"
+
+services:
+  admin-api:
+    platform: linux/amd64
+    image: us-docker.pkg.dev/integrationos/docker-oss/api:1.0.0
+    ports:
+      - 3004:3004
+    environment:
+      - CACHE_SIZE=0
+      - CLAUDE_API_KEY=
+      - CONTROL_DATABASE_NAME=events-service
+      - CONTROL_DATABASE_URL=mongodb://integrationos:${MONGO_PASSWORD}@mongo:27017
+      - EVENT_ACCESS_PASSWORD=${EVENT_ACCESS_PASSWORD}
+      - INTERNAL_SERVER_ADDRESS=0.0.0.0:3004
+      - IS_ADMIN=true
+      - OPENAI_API_KEY=
+      - RUST_LOG=api=info,tower_http=info,udm=info
+      - SECRETS_SERVICE_BASE_URL=http://secrets-service:3006/
+      - WORKER_THREADS=4
+
+  typescript-services:
+    platform: linux/amd64
+    image: us-docker.pkg.dev/integrationos/docker-oss/typescript-services:1.0.0
+    ports:
+      - 3001:3001
+    environment:
+      - BUILDABLE_SECRET=${BUILDABLE_SECRET}
+      - CONNECTIONS_API_BASE_URL=http://connections-api:3005/
+      - CORE_USER_SECRET=${CORE_USER_SECRET}
+      - DEFAULT_LIVE_ACCESS_KEY=${DEFAULT_LIVE_ACCESS_KEY}
+      - DEFAULT_TEST_ACCESS_KEY=${DEFAULT_TEST_ACCESS_KEY}
+      - DEMO_ACCOUNT_EVENT_ACCESS_KEY=${DEVELOPER_ACCOUNT_ACCESS_KEY}
+      - ENGINEERING_ACCOUNT_BUILDABLE_ID=${DEVELOPER_ACCOUNT_ID}
+      - EVENT_ACCESS_ENCRYPTION_PASSWORD=${EVENT_ACCESS_PASSWORD}
+      - JWT_AUDIENCE=integrationos-users
+      - JWT_EXPIRES_AFTER=90d
+      - JWT_ISSUER=integrationos
+      - JWT_SECRET=${JWT_SECRET}
+      - MONGO_DB_NAME=events-service
+      - MONGO_URI=mongodb://integrationos:${MONGO_PASSWORD}@mongo:27017/events-service?authSource=admin
+      - NODE_ENV=localhost
+      - QA_ACCOUNT_BUILDABLE_ID=build-${DEVELOPER_ACCOUNT_ID}
+      - QA_ACCOUNT_EVENT_ACCESS_KEY=${DEVELOPER_ACCOUNT_ACCESS_KEY}
+      - SALT_ROUNDS=10
+      - SECRETS_SERVICE_BASE_URL=http://secrets-service:3006/
+
+  connections-api:
+    platform: linux/amd64
+    image: us-docker.pkg.dev/integrationos/docker-oss/api:1.0.0
+    ports:
+      - 3005:3005
+    environment:
+      - ACCESS_KEY_WHITELIST_REFRESH_INTERVAL_SECS=30
+      - CACHE_SIZE=1000
+      - CLAUDE_API_KEY=
+      - CONTROL_DATABASE_NAME=events-service
+      - CONTROL_DATABASE_URL=mongodb://integrationos:${MONGO_PASSWORD}@mongo:27017
+      - DEBUG_MODE=false
+      - ENGINEERING_ACCOUNT_ID=${DEVELOPER_ACCOUNT_ID}
+      - EVENT_ACCESS_PASSWORD=${EVENT_ACCESS_PASSWORD}
+      - INTERNAL_SERVER_ADDRESS=0.0.0.0:3005
+      - JWT_SECRET=${JWT_SECRET}
+      - OPENAI_API_KEY=
+      - REDIS_URL=redis://redis:6379
+      - RUST_LOG=api=info,tower_http=info,udm=info,integrationos_domain=info
+      - SECRETS_SERVICE_BASE_URL=http://secrets-service:3006/
+      - WORKER_THREADS=4
+
+  event-core:
+    platform: linux/amd64
+    image: us-docker.pkg.dev/integrationos/docker-oss/event-core:1.0.0
+    environment:
+      - CONTEXT_COLLECTION_NAME=event-transactions
+      - CONTEXT_DATABASE_NAME=events-service
+      - CONTEXT_DATABASE_URL=mongodb://integrationos:${MONGO_PASSWORD}@mongo:27017
+      - CONTROL_DATABASE_NAME=events-service
+      - CONTROL_DATABASE_URL=mongodb://integrationos:${MONGO_PASSWORD}@mongo:27017
+      - EVENT_DATABASE_NAME=events-service
+      - EVENT_DATABASE_URL-mongodb://integrationos:${MONGO_PASSWORD}@mongo:27017
+      - REDIS_URL=redis://redis:6379
+      - RUST_LOG=event_core=info
+      - SECRETS_SERVICE_BASE_URL=http://secrets-service:3006/
+
+  gateway:
+    platform: linux/amd64
+    image: us-docker.pkg.dev/integrationos/docker-oss/gateway:1.0.0
+    ports:
+      - 3002:3002
+    environment:
+      - CONTEXT_COLLECTION_NAME=event-transactions
+      - CONTEXT_DATABASE_NAME=events-service
+      - CONTEXT_DATABASE_URL=mongodb://integrationos:${MONGO_PASSWORD}@mongo:27017
+      - CONTROL_DATABASE_NAME=events-service
+      - CONTROL_DATABASE_URL=mongodb://integrationos:${MONGO_PASSWORD}@mongo:27017
+      - EVENT_DATABASE_NAME=events-service
+      - EVENT_DATABASE_URL=mongodb://integrationos:${MONGO_PASSWORD}@mongo:27017
+      - REDIS_URL=redis://redis:6379
+      - RUST_LOG=gateway=info,tower_http=info
+      - SECRET=${GATEWAY_SECRET}
+      - SERVER_ADDRESS=0.0.0.0:3002
+
+  oauth-api:
+    platform: linux/amd64
+    image: us-docker.pkg.dev/integrationos/docker-oss/oauth:1.0.0
+    ports:
+      - 3003:3003
+    environment:
+      - APP_URL=0.0.0.0:3003
+      - BURST_RATE_LIMIT=1
+      - BURST_SIZE_LIMIT=30
+      - CONTEXT_COLLECTION_NAME=event-transactions
+      - CONTEXT_DATABASE_NAME=events-service
+      - CONTEXT_DATABASE_URL=mongodb://integrationos:${MONGO_PASSWORD}@mongo:27017
+      - CONTROL_DATABASE_NAME=events-service
+      - CONTROL_DATABASE_URL=mongodb://integrationos:${MONGO_PASSWORD}@mongo:27017
+      - ENVIRONMENT=production
+      - EVENT_DATABASE_NAME=events-service
+      - EVENT_DATABASE_URL=mongodb://integrationos:${MONGO_PASSWORD}@mongo:27017
+      - HOST=0.0.0.0
+      - JWT_SECRET=${JWT_SECRET}
+      - PORT=3003
+      - REFRESH_BEFORE_IN_MINUTES=10
+      - RUST_LOG=info
+      - SECRETS_SERVICE_BASE_URL=http://secrets-service:3006/
+      - SLEEP_TIMER_IN_SECONDS=60
+
+  secrets-service:
+    platform: linux/amd64
+    image: us-docker.pkg.dev/integrationos/docker-oss/secrets-service:1.0.0
+    environment:
+      - GOOGLE_APPLICATION_CREDENTIALS=/root/.config/gcloud/application_default_credentials.json
+      - KEY_ID=${KMS_KEY_ID}
+      - KEY_RING_ID=${KMS_KEY_RING_ID}
+      - LOCATION_ID=${GCP_LOCATION_ID}
+      - MONGO_URI=mongodb://integrationos:${MONGO_PASSWORD}@mongo:27017/events-service?authSource=admin
+      - PROJECT_ID=${GCP_PROJECT_ID}
+    volumes:
+      - ${HOME}/.config/gcloud:/root/.config/gcloud:ro
+
+  watchdog:
+    platform: linux/amd64
+    image: us-docker.pkg.dev/integrationos/docker-oss/watchdog:1.0.0
+    environment:
+      - CONTEXT_COLLECTION_NAME=event-transactions
+      - CONTEXT_DATABASE_NAME=events-service
+      - CONTEXT_DATABASE_URL=mongodb://integrationos:${MONGO_PASSWORD}@mongo:27017
+      - CONTROL_DATABASE_NAME=events-service
+      - CONTROL_DATABASE_URL=mongodb://integrationos:${MONGO_PASSWORD}@mongo:27017
+      - EVENT_DATABASE_NAME=events-service
+      - EVENT_DATABASE_URL=mongodb://integrationos:${MONGO_PASSWORD}@mongo:27017
+      - POLL_DURATION=1000
+      - REDIS_URL=redis://redis:6379
+      - RUST_LOG=watchdog=info
+
+  mongo:
+    image: mongo:6.0
+    ports:
+      - 27017:27017
+    environment:
+      - MONGO_INITDB_ROOT_PASSWORD=${MONGO_PASSWORD}
+      - MONGO_INITDB_ROOT_USERNAME=integrationos
+    volumes:
+      - ./seed-data:/seed-data:ro
+
+  redis:
+    image: redis:7.0

seed-data/seed_db/clients.metadata.json

@@ -0,0 +1 @@
+{"indexes":[{"v":{"$numberInt":"2"},"key":{"_id":{"$numberInt":"1"}},"name":"_id_"}],"uuid":"a1e02023cc58458db3c4b7abd27c5a9e","collectionName":"clients","type":"collection"}

seed-data/seed_db/common-enums.metadata.json

@@ -0,0 +1 @@
+{"indexes":[{"v":{"$numberInt":"2"},"key":{"_id":{"$numberInt":"1"}},"name":"_id_"},{"v":{"$numberInt":"2"},"key":{"name":{"$numberInt":"1"}},"name":"name_1","background":true,"unique":true}],"uuid":"7d98247dc9e54ed6a6fe905c3ac3e88b","collectionName":"common-enums","type":"collection"}

seed-data/seed_db/common-models.metadata.json

@@ -0,0 +1 @@
+{"indexes":[{"v":{"$numberInt":"2"},"key":{"_id":{"$numberInt":"1"}},"name":"_id_"},{"v":{"$numberInt":"2"},"key":{"name":{"$numberInt":"1"}},"name":"name_1","background":true,"unique":true},{"v":{"$numberInt":"2"},"key":{"deleted":{"$numberInt":"1"}},"name":"deleted_1","background":true},{"v":{"$numberInt":"2"},"key":{"active":{"$numberInt":"1"}},"name":"active_1","background":true},{"v":{"$numberInt":"2"},"key":{"createdAt":{"$numberInt":"1"}},"name":"createdAt_1","background":true}],"uuid":"8897c92ff17c45a6a47933149de7a2ce","collectionName":"common-models","type":"collection"}

seed-data/seed_db/connection-definitions.metadata.json

@@ -0,0 +1 @@
+{"indexes":[{"v":{"$numberInt":"2"},"key":{"_id":{"$numberInt":"1"}},"name":"_id_"},{"v":{"$numberInt":"2"},"key":{"key":{"$numberInt":"1"}},"name":"key_1","background":true,"unique":true},{"v":{"$numberInt":"2"},"key":{"deleted":{"$numberInt":"1"}},"name":"deleted_1","background":true},{"v":{"$numberInt":"2"},"key":{"active":{"$numberInt":"1"}},"name":"active_1","background":true},{"v":{"$numberInt":"2"},"key":{"createdAt":{"$numberInt":"1"}},"name":"createdAt_1","background":true}],"uuid":"19f11b37ae934f52abb891e75af101be","collectionName":"connection-definitions","type":"collection"}

seed-data/seed_db/connection-model-definitions.metadata.json

@@ -0,0 +1 @@
+{"indexes":[{"v":{"$numberInt":"2"},"key":{"_id":{"$numberInt":"1"}},"name":"_id_"},{"v":{"$numberInt":"2"},"key":{"key":{"$numberInt":"1"}},"name":"key_1","background":true,"unique":true},{"v":{"$numberInt":"2"},"key":{"deleted":{"$numberInt":"1"}},"name":"deleted_1","background":true},{"v":{"$numberInt":"2"},"key":{"active":{"$numberInt":"1"}},"name":"active_1","background":true},{"v":{"$numberInt":"2"},"key":{"createdAt":{"$numberInt":"1"}},"name":"createdAt_1","background":true},{"v":{"$numberInt":"2"},"key":{"connectionPlatform":{"$numberInt":"1"},"actionName":{"$numberInt":"1"},"mapping.commonModelName":{"$numberInt":"1"}},"name":"connectionPlatform_1_action_1_modelName_1","background":true},{"v":{"$numberInt":"2"},"key":{"connectionPlatform":{"$numberInt":"1"}},"name":"connectionPlatform_1","background":true},{"v":{"$numberInt":"2"},"key":{"actionName":{"$numberInt":"1"}},"name":"actionName_1","background":true}],"uuid":"f2a14349988a456aadf393c9c02b8973","collectionName":"connection-model-definitions","type":"collection"}

seed-data/seed_db/connection-model-schema.metadata.json

@@ -0,0 +1 @@
+{"indexes":[{"v":{"$numberInt":"2"},"key":{"_id":{"$numberInt":"1"}},"name":"_id_"},{"v":{"$numberInt":"2"},"key":{"key":{"$numberInt":"1"}},"name":"key_1","background":true,"unique":true},{"v":{"$numberInt":"2"},"key":{"deleted":{"$numberInt":"1"}},"name":"deleted_1","background":true},{"v":{"$numberInt":"2"},"key":{"active":{"$numberInt":"1"}},"name":"active_1","background":true},{"v":{"$numberInt":"2"},"key":{"createdAt":{"$numberInt":"1"}},"name":"createdAt_1","background":true}],"uuid":"90593f4ffcd44a89bf8481741bff78e3","collectionName":"connection-model-schema","type":"collection"}

seed-data/seed_db/connection-oauth-definitions.metadata.json

@@ -0,0 +1 @@
+{"indexes":[{"v":{"$numberInt":"2"},"key":{"_id":{"$numberInt":"1"}},"name":"_id_"},{"v":{"$numberInt":"2"},"key":{"connectionPlatform":{"$numberInt":"1"}},"name":"connectionPlatform_2","background":true},{"v":{"$numberInt":"2"},"key":{"deleted":{"$numberInt":"1"}},"name":"deleted_1","background":true},{"v":{"$numberInt":"2"},"key":{"active":{"$numberInt":"1"}},"name":"active_1","background":true},{"v":{"$numberInt":"2"},"key":{"createdAt":{"$numberInt":"1"}},"name":"createdAt_1","background":true}],"uuid":"96f66f25ba5b40c88db183e90399b1c3","collectionName":"connection-oauth-definitions","type":"collection"}

seed-data/seed_db/connections.metadata.json

@@ -0,0 +1 @@
+{"indexes":[{"v":{"$numberInt":"2"},"key":{"_id":{"$numberInt":"1"}},"name":"_id_"}],"uuid":"d12c58a5b58146c78a61d3b259a46bc4","collectionName":"connections","type":"collection"}

seed-data/seed_db/embed-tokens.metadata.json

@@ -0,0 +1 @@
+{"indexes":[{"v":{"$numberInt":"2"},"key":{"_id":{"$numberInt":"1"}},"name":"_id_"}],"uuid":"4610eefe0a21419b9a1b6732e5150a57","collectionName":"embed-tokens","type":"collection"}

seed-data/seed_db/event-access.metadata.json

@@ -0,0 +1 @@
+{"indexes":[{"v":{"$numberInt":"2"},"key":{"_id":{"$numberInt":"1"}},"name":"_id_"}],"uuid":"1a1d7bb3331c4323b5d74065e76d9b8f","collectionName":"event-access","type":"collection"}

seed-data/seed_db/external-events.metadata.json

@@ -0,0 +1 @@
+{"indexes":[{"v":{"$numberInt":"2"},"key":{"_id":{"$numberInt":"1"}},"name":"_id_"}],"uuid":"7983c7719fdd4803a9f1ff512a401292","collectionName":"external-events","type":"collection"}

seed-data/seed_db/jobs.metadata.json

@@ -0,0 +1 @@
+{"indexes":[{"v":{"$numberInt":"2"},"key":{"_id":{"$numberInt":"1"}},"name":"_id_"}],"uuid":"d43a3c198b6e46c5991586eb009f11c1","collectionName":"jobs","type":"collection"}

seed-data/seed_db/platform-pages.metadata.json

@@ -0,0 +1 @@
+{"indexes":[{"v":{"$numberInt":"2"},"key":{"_id":{"$numberInt":"1"}},"name":"_id_"},{"v":{"$numberInt":"2"},"key":{"platformName":{"$numberInt":"1"},"modelName":{"$numberInt":"1"}},"name":"platformName_1_modelName_1","background":true},{"v":{"$numberInt":"2"},"key":{"deleted":{"$numberInt":"1"}},"name":"deleted_1","background":true},{"v":{"$numberInt":"2"},"key":{"active":{"$numberInt":"1"}},"name":"active_1","background":true},{"v":{"$numberInt":"2"},"key":{"createdAt":{"$numberInt":"1"}},"name":"createdAt_1","background":true}],"uuid":"53e92849455243d39ddd7618f69d75de","collectionName":"platform-pages","type":"collection"}

seed-data/seed_db/platforms.metadata.json

@@ -0,0 +1 @@
+{"indexes":[{"v":{"$numberInt":"2"},"key":{"_id":{"$numberInt":"1"}},"name":"_id_"}],"uuid":"d82a3caae9f24f05bf54b9e0063336e2","collectionName":"platforms","type":"collection"}

seed-data/seed_db/secrets.metadata.json

@@ -0,0 +1 @@
+{"indexes":[{"v":{"$numberInt":"2"},"key":{"_id":{"$numberInt":"1"}},"name":"_id_"}],"uuid":"ec7ec66bad6346fc94d1e729e6f85200","collectionName":"secrets","type":"collection"}

seed-data/seed_db/sessions.metadata.json

@@ -0,0 +1 @@
+{"indexes":[{"v":{"$numberInt":"2"},"key":{"_id":{"$numberInt":"1"}},"name":"_id_"}],"uuid":"1c1634b9deac47b98a6a5898ffdcfd0f","collectionName":"sessions","type":"collection"}

seed-data/seed_db/settings.metadata.json

@@ -0,0 +1 @@
+{"indexes":[{"v":{"$numberInt":"2"},"key":{"_id":{"$numberInt":"1"}},"name":"_id_"}],"uuid":"0c6cad137dce4d43abd77de515fcad62","collectionName":"settings","type":"collection"}

seed-data/seed_db/stages.metadata.json

@@ -0,0 +1 @@
+{"indexes":[{"v":{"$numberInt":"2"},"key":{"_id":{"$numberInt":"1"}},"name":"_id_"}],"uuid":"80a63d050164401b81d6c3497f896069","collectionName":"stages","type":"collection"}

seed-data/seed_db/system-stats.metadata.json

@@ -0,0 +1 @@
+{"indexes":[{"v":{"$numberInt":"2"},"key":{"_id":{"$numberInt":"1"}},"name":"_id_"}],"uuid":"6a50d1f51fcc48529563c14269cc3afd","collectionName":"system-stats","type":"collection"}

seed-data/seed_db/users.metadata.json

@@ -0,0 +1 @@
+{"indexes":[{"v":{"$numberInt":"2"},"key":{"_id":{"$numberInt":"1"}},"name":"_id_"}],"uuid":"731220936777496b83cf00c81e663388","collectionName":"users","type":"collection"}