feat: complete dashcam catalog ingestion demo

deploy/swarm/docker-stack.yml

@@ -1,13 +1,47 @@
 version: "3.9"
 
+x-default-deploy: &default-deploy
+  placement:
+    constraints:
+      - ${WEB_PLACEMENT_CONSTRAINT:-node.role == manager}
+  update_config:
+    order: start-first
+    parallelism: 1
+    failure_action: rollback
+    monitor: 45s
+  rollback_config:
+    order: start-first
+    parallelism: 1
+    monitor: 45s
+  restart_policy:
+    condition: on-failure
+    delay: 5s
+    max_attempts: 5
+    window: 60s
+
+x-ingestion-env: &ingestion-env
+  INGESTION_DATABASE_URL: postgres://${POSTGRES_USER:-videoreg}:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required}@postgres:5432/${POSTGRES_DB:-videoreg_ingestion}
+  INGESTION_SNAPSHOT_DIR: /app/ingestion/data/snapshots
+  INGESTION_EXPORT_DIR: /exports
+  YANDEX_MARKET_DASHCAM_CATEGORY_URL: ${YANDEX_MARKET_DASHCAM_CATEGORY_URL:-https://market.yandex.ru/catalog--avtomobilnye-videoregistratory/82798275/list?hid=82798269}
+  MVIDEO_DASHCAM_CATEGORY_URL: ${MVIDEO_DASHCAM_CATEGORY_URL:-}
+  INGESTION_CATEGORY_DAILY_BUDGET: ${INGESTION_CATEGORY_DAILY_BUDGET:-25}
+  INGESTION_PRODUCT_DAILY_BUDGET: ${INGESTION_PRODUCT_DAILY_BUDGET:-100}
+  INGESTION_SITEMAP_DAILY_BUDGET: ${INGESTION_SITEMAP_DAILY_BUDGET:-25}
+  INGESTION_FETCH_DELAY_MIN_MS: ${INGESTION_FETCH_DELAY_MIN_MS:-5000}
+  INGESTION_FETCH_DELAY_MAX_MS: ${INGESTION_FETCH_DELAY_MAX_MS:-15000}
+  INGESTION_WORKER_BATCH_SIZE: ${INGESTION_WORKER_BATCH_SIZE:-1}
+  INGESTION_WORKER_IDLE_MS: ${INGESTION_WORKER_IDLE_MS:-30000}
+
 services:
   web:
-    image: ${WEB_IMAGE}
+    image: ${WEB_IMAGE:?WEB_IMAGE is required}
     environment:
       NODE_ENV: production
       APP_ENV: ${ENVIRONMENT}
       PORT: "3000"
       VIDEOREG_DB_FILE: /data/videoreg.json
+      DASHCAM_CATALOG_EXPORT_DIR: /exports
       VIDEOREG_DB_MEMORY_TTL_MS: ${VIDEOREG_DB_MEMORY_TTL_MS:-3600000}
       VIDEOREG_NEXT_CACHE_REVALIDATE_SECONDS: ${VIDEOREG_NEXT_CACHE_REVALIDATE_SECONDS:-3600}
       COMMIT_SHA: ${COMMIT_SHA}
@@ -16,6 +50,7 @@ services:
       TEAM_ID: ${TEAM_ID:-}
     volumes:
       - videoreg_data:/data
+      - catalog_exports:/exports:ro
     networks:
       traefik: {}
     healthcheck:
@@ -25,34 +60,90 @@ services:
       retries: 3
       start_period: 20s
     deploy:
+      <<: *default-deploy
       replicas: ${WEB_REPLICAS:-1}
-      placement:
-        constraints:
-          - ${WEB_PLACEMENT_CONSTRAINT:-node.role == manager}
-      update_config:
-        order: start-first
-        parallelism: 1
-        failure_action: rollback
-        monitor: 45s
-      rollback_config:
-        order: start-first
-        parallelism: 1
-        monitor: 45s
-      restart_policy:
-        condition: on-failure
       labels:
         - "traefik.enable=true"
         - "traefik.docker.network=${TRAEFIK_NETWORK}"
         - "traefik.http.routers.${STACK_NAME}-web.rule=Host(`${PUBLIC_HOST}`) && PathPrefix(`/`)"
+        - "traefik.http.routers.${STACK_NAME}-web.priority=${TRAEFIK_WEB_PRIORITY:-1}"
         - "traefik.http.routers.${STACK_NAME}-web.entrypoints=${TRAEFIK_ENTRYPOINTS:-websecure}"
         - "traefik.http.routers.${STACK_NAME}-web.tls=${TRAEFIK_TLS:-true}"
+        - "traefik.http.routers.${STACK_NAME}-web.tls.certresolver=${TRAEFIK_CERTRESOLVER:-le}"
         - "traefik.http.services.${STACK_NAME}-web.loadbalancer.server.port=3000"
     stop_grace_period: 30s
 
+  postgres:
+    image: postgres:16-alpine
+    environment:
+      POSTGRES_DB: ${POSTGRES_DB:-videoreg_ingestion}
+      POSTGRES_USER: ${POSTGRES_USER:-videoreg}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required}
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-videoreg} -d ${POSTGRES_DB:-videoreg_ingestion}"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    volumes:
+      - ingestion_postgres:/var/lib/postgresql/data
+    networks:
+      - ingestion
+    deploy: *default-deploy
+
+  ingestion-worker:
+    image: ${INGESTION_IMAGE:?INGESTION_IMAGE is required}
+    command: sh -c "npm run ingestion:migrate && npm run ingestion:worker"
+    environment: *ingestion-env
+    volumes:
+      - ingestion_data:/app/ingestion/data
+      - catalog_exports:/exports
+    networks:
+      - ingestion
+    deploy:
+      <<: *default-deploy
+      replicas: ${INGESTION_WORKER_REPLICAS:-1}
+
+  ingestion-admin:
+    image: ${INGESTION_IMAGE:?INGESTION_IMAGE is required}
+    command: sh -c "npm run ingestion:migrate && npm run ingestion:admin"
+    environment:
+      <<: *ingestion-env
+      INGESTION_ADMIN_PORT: "4101"
+      INGESTION_ADMIN_USERNAME: ${INGESTION_ADMIN_USERNAME:-admin}
+      INGESTION_ADMIN_PASSWORD: ${INGESTION_ADMIN_PASSWORD:?INGESTION_ADMIN_PASSWORD is required}
+    volumes:
+      - ingestion_data:/app/ingestion/data
+    networks:
+      ingestion: {}
+      traefik: {}
+    healthcheck:
+      test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:4101/health >/dev/null || exit 1"]
+      interval: 15s
+      timeout: 3s
+      retries: 5
+      start_period: 20s
+    deploy:
+      <<: *default-deploy
+      replicas: 1
+      labels:
+        - "traefik.enable=true"
+        - "traefik.docker.network=${TRAEFIK_NETWORK}"
+        - "traefik.http.routers.${STACK_NAME}-admin.rule=Host(`${PUBLIC_HOST}`) && PathPrefix(`/admin`)"
+        - "traefik.http.routers.${STACK_NAME}-admin.priority=${TRAEFIK_ADMIN_PRIORITY:-100}"
+        - "traefik.http.routers.${STACK_NAME}-admin.entrypoints=${TRAEFIK_ENTRYPOINTS:-websecure}"
+        - "traefik.http.routers.${STACK_NAME}-admin.tls=${TRAEFIK_TLS:-true}"
+        - "traefik.http.routers.${STACK_NAME}-admin.tls.certresolver=${TRAEFIK_CERTRESOLVER:-le}"
+        - "traefik.http.services.${STACK_NAME}-admin.loadbalancer.server.port=4101"
+
 volumes:
   videoreg_data:
+  catalog_exports:
+  ingestion_postgres:
+  ingestion_data:
 
 networks:
+  ingestion:
+    driver: overlay
   traefik:
     external: true
     name: ${TRAEFIK_NETWORK}