feat: complete dashcam catalog ingestion demo

deploy/portainer/README.md

@@ -0,0 +1,36 @@
+# Portainer / Swarm deployment
+
+`web`, PostgreSQL, `ingestion-worker`, and `ingestion-admin` are one stack. Do
+not deploy ingestion as a separate endpoint: the approved catalog is exchanged
+through the `catalog_exports` volume mounted read-only by web and read-write by
+the worker.
+
+In Portainer, create a Swarm stack from `deploy/swarm/docker-stack.yml`.
+Configure these values as stack environment variables:
+
+- `WEB_IMAGE` - digest-pinned web image.
+- `INGESTION_IMAGE` - digest-pinned ingestion image.
+- `POSTGRES_PASSWORD` - database secret.
+- `INGESTION_ADMIN_PASSWORD` - Basic Auth password for `/admin`.
+- `PUBLIC_HOST=videoreg.ru` and `TRAEFIK_NETWORK`.
+- Optional source/crawl settings: `MVIDEO_DASHCAM_CATEGORY_URL`, daily budgets,
+  and delay bounds.
+
+Keep every service on the same manager/placement constraint. The database,
+snapshots, read model, and shared catalog volume are local Docker volumes and
+must be backed up from that node.
+
+Gitea Actions uses `.gitea/workflows/master.yml`: it tests the repository,
+builds both images, pushes immutable digests, and deploys this one stack. The
+workflow connects to the Swarm manager, pulls those digests, and runs `docker
+stack deploy --with-registry-auth`; Portainer then shows and manages the same
+updated stack. Portainer does not need registry credentials or a separate
+webhook-based image build.
+
+Verification after deploy:
+
+```bash
+curl -fsS https://videoreg.ru/health/ready
+curl -u "$INGESTION_ADMIN_USERNAME:$INGESTION_ADMIN_PASSWORD" \
+  https://videoreg.ru/admin/api/queue
+```

deploy/remote/deploy-stack.sh

@@ -19,6 +19,7 @@ require_var STACK_NAME
 require_var RELEASE_ID
 require_var COMMIT_SHA
 require_var WEB_IMAGE
+require_var INGESTION_IMAGE
 require_var PUBLIC_HOST
 require_var TRAEFIK_NETWORK
 
@@ -37,7 +38,7 @@ cleanup_release_images() {
   fi
 
   echo "Cleaning up unused release image after failed deploy." >&2
-  docker image rm "$WEB_IMAGE" >/dev/null 2>&1 || true
+  docker image rm "$WEB_IMAGE" "$INGESTION_IMAGE" >/dev/null 2>&1 || true
 
   if [ -n "${COMMIT_SHA:-}" ]; then
     docker image prune -af --filter "label=org.opencontainers.image.revision=${COMMIT_SHA}" >&2 || true
@@ -160,15 +161,32 @@ prune_docker_before_pull
 check_docker_disk_space
 
 docker image pull "$WEB_IMAGE"
+docker image pull "$INGESTION_IMAGE"
 
-export ENVIRONMENT STACK_NAME RELEASE_ID COMMIT_SHA WEB_IMAGE PUBLIC_HOST TRAEFIK_NETWORK
+export ENVIRONMENT STACK_NAME RELEASE_ID COMMIT_SHA WEB_IMAGE INGESTION_IMAGE PUBLIC_HOST TRAEFIK_NETWORK
 export REGION="${REGION:-}" TEAM_ID="${TEAM_ID:-}" WEB_REPLICAS="${WEB_REPLICAS:-1}"
 export WEB_PLACEMENT_CONSTRAINT="${WEB_PLACEMENT_CONSTRAINT:-node.role == manager}"
 export TRAEFIK_ENTRYPOINTS="${TRAEFIK_ENTRYPOINTS:-websecure}" TRAEFIK_TLS="${TRAEFIK_TLS:-true}"
+export TRAEFIK_CERTRESOLVER="${TRAEFIK_CERTRESOLVER:-le}"
+export TRAEFIK_ADMIN_PRIORITY="${TRAEFIK_ADMIN_PRIORITY:-100}"
 export VIDEOREG_DB_MEMORY_TTL_MS="${VIDEOREG_DB_MEMORY_TTL_MS:-3600000}"
 export VIDEOREG_NEXT_CACHE_REVALIDATE_SECONDS="${VIDEOREG_NEXT_CACHE_REVALIDATE_SECONDS:-3600}"
+export POSTGRES_DB="${POSTGRES_DB:-videoreg_ingestion}" POSTGRES_USER="${POSTGRES_USER:-videoreg}"
+export INGESTION_ADMIN_USERNAME="${INGESTION_ADMIN_USERNAME:-admin}"
+export YANDEX_MARKET_DASHCAM_CATEGORY_URL="${YANDEX_MARKET_DASHCAM_CATEGORY_URL:-https://market.yandex.ru/catalog--avtomobilnye-videoregistratory/82798275/list?hid=82798269}"
+export MVIDEO_DASHCAM_CATEGORY_URL="${MVIDEO_DASHCAM_CATEGORY_URL:-}"
+export INGESTION_CATEGORY_DAILY_BUDGET="${INGESTION_CATEGORY_DAILY_BUDGET:-25}"
+export INGESTION_PRODUCT_DAILY_BUDGET="${INGESTION_PRODUCT_DAILY_BUDGET:-100}"
+export INGESTION_SITEMAP_DAILY_BUDGET="${INGESTION_SITEMAP_DAILY_BUDGET:-25}"
+export INGESTION_FETCH_DELAY_MIN_MS="${INGESTION_FETCH_DELAY_MIN_MS:-5000}"
+export INGESTION_FETCH_DELAY_MAX_MS="${INGESTION_FETCH_DELAY_MAX_MS:-15000}"
+export INGESTION_WORKER_BATCH_SIZE="${INGESTION_WORKER_BATCH_SIZE:-1}"
+export INGESTION_WORKER_IDLE_MS="${INGESTION_WORKER_IDLE_MS:-30000}"
+export INGESTION_WORKER_REPLICAS="${INGESTION_WORKER_REPLICAS:-1}"
 
 docker stack deploy --with-registry-auth --prune -c "$REMOTE_DIR/docker-stack.yml" "$STACK_NAME"
 wait_service "${STACK_NAME}_web"
+wait_service "${STACK_NAME}_ingestion-worker"
+wait_service "${STACK_NAME}_ingestion-admin"
 
 log deploy_converged

deploy/scripts/deploy-over-ssh.sh

@@ -20,6 +20,7 @@ require_var STACK_NAME
 require_var RELEASE_ID
 require_var COMMIT_SHA
 require_var WEB_IMAGE
+require_var INGESTION_IMAGE
 require_var PUBLIC_HOST
 require_var TRAEFIK_NETWORK
 
@@ -32,10 +33,15 @@ scp -P "$SSH_PORT" deploy/swarm/docker-stack.yml "$SSH_TARGET:$REMOTE_DEPLOY_DIR
 scp -P "$SSH_PORT" deploy/remote/deploy-stack.sh "$SSH_TARGET:$REMOTE_DEPLOY_DIR/deploy-stack.sh"
 
 remote_vars=(
-  ENVIRONMENT STACK_NAME RELEASE_ID COMMIT_SHA WEB_IMAGE PUBLIC_HOST TRAEFIK_NETWORK
-  REGION TEAM_ID WEB_REPLICAS WEB_PLACEMENT_CONSTRAINT TRAEFIK_ENTRYPOINTS TRAEFIK_TLS REGISTRY REGISTRY_USERNAME REGISTRY_PASSWORD
+  ENVIRONMENT STACK_NAME RELEASE_ID COMMIT_SHA WEB_IMAGE INGESTION_IMAGE PUBLIC_HOST TRAEFIK_NETWORK
+  REGION TEAM_ID WEB_REPLICAS WEB_PLACEMENT_CONSTRAINT TRAEFIK_ENTRYPOINTS TRAEFIK_TLS TRAEFIK_CERTRESOLVER REGISTRY REGISTRY_USERNAME REGISTRY_PASSWORD
   RELEASE_LOCK_FILE REMOTE_DEPLOY_DIR DEPLOY_MIN_FREE_MB DEPLOY_PRUNE_BEFORE_PULL DEPLOY_PRUNE_UNTIL
   DEPLOY_CLEANUP_FAILED_IMAGES VIDEOREG_DB_MEMORY_TTL_MS VIDEOREG_NEXT_CACHE_REVALIDATE_SECONDS
+  POSTGRES_DB POSTGRES_USER POSTGRES_PASSWORD INGESTION_ADMIN_USERNAME INGESTION_ADMIN_PASSWORD
+  YANDEX_MARKET_DASHCAM_CATEGORY_URL MVIDEO_DASHCAM_CATEGORY_URL INGESTION_CATEGORY_DAILY_BUDGET
+  INGESTION_PRODUCT_DAILY_BUDGET INGESTION_SITEMAP_DAILY_BUDGET INGESTION_FETCH_DELAY_MIN_MS
+  INGESTION_FETCH_DELAY_MAX_MS INGESTION_WORKER_BATCH_SIZE INGESTION_WORKER_IDLE_MS INGESTION_WORKER_REPLICAS
+  TRAEFIK_ADMIN_PRIORITY
 )
 
 remote_command=""

deploy/scripts/predeploy-check.mjs

@@ -4,6 +4,9 @@ const required = [
   'RELEASE_ID',
   'COMMIT_SHA',
   'WEB_IMAGE',
+  'INGESTION_IMAGE',
+  'POSTGRES_PASSWORD',
+  'INGESTION_ADMIN_PASSWORD',
   'PUBLIC_HOST',
   'TRAEFIK_NETWORK',
   'SWARM_MANAGER_HOST',
@@ -16,9 +19,11 @@ if (missing.length > 0) {
   process.exit(1);
 }
 
-if (!/@sha256:[a-f0-9]{64}$/i.test(process.env.WEB_IMAGE ?? '')) {
-  console.error(`WEB_IMAGE must be pinned by immutable digest, got: ${process.env.WEB_IMAGE}`);
-  process.exit(1);
+for (const imageName of ['WEB_IMAGE', 'INGESTION_IMAGE']) {
+  if (!/@sha256:[a-f0-9]{64}$/i.test(process.env[imageName] ?? '')) {
+    console.error(`${imageName} must be pinned by immutable digest, got: ${process.env[imageName]}`);
+    process.exit(1);
+  }
 }
 
 console.log(

deploy/scripts/write-release-manifest.mjs

@@ -18,13 +18,19 @@ const manifest = {
   commitSha: requireEnv('COMMIT_SHA'),
   releaseId: requireEnv('RELEASE_ID'),
   webImageDigest: requireEnv('WEB_IMAGE_DIGEST'),
+  ingestionImageDigest: requireEnv('INGESTION_IMAGE_DIGEST'),
   deployTimestamp: new Date().toISOString(),
   smokeResult: smoke,
   workflowRunId: requireEnv('WORKFLOW_RUN_ID'),
 };
 
-if (!/@sha256:[a-f0-9]{64}$/i.test(manifest.webImageDigest)) {
-  throw new Error('webImageDigest must be digest-pinned');
+for (const [name, image] of Object.entries({
+  webImageDigest: manifest.webImageDigest,
+  ingestionImageDigest: manifest.ingestionImageDigest,
+})) {
+  if (!/@sha256:[a-f0-9]{64}$/i.test(image)) {
+    throw new Error(`${name} must be digest-pinned`);
+  }
 }
 
 console.log(JSON.stringify(manifest, null, 2));

deploy/swarm/docker-stack.yml

@@ -1,13 +1,47 @@
 version: "3.9"
 
+x-default-deploy: &default-deploy
+  placement:
+    constraints:
+      - ${WEB_PLACEMENT_CONSTRAINT:-node.role == manager}
+  update_config:
+    order: start-first
+    parallelism: 1
+    failure_action: rollback
+    monitor: 45s
+  rollback_config:
+    order: start-first
+    parallelism: 1
+    monitor: 45s
+  restart_policy:
+    condition: on-failure
+    delay: 5s
+    max_attempts: 5
+    window: 60s
+
+x-ingestion-env: &ingestion-env
+  INGESTION_DATABASE_URL: postgres://${POSTGRES_USER:-videoreg}:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required}@postgres:5432/${POSTGRES_DB:-videoreg_ingestion}
+  INGESTION_SNAPSHOT_DIR: /app/ingestion/data/snapshots
+  INGESTION_EXPORT_DIR: /exports
+  YANDEX_MARKET_DASHCAM_CATEGORY_URL: ${YANDEX_MARKET_DASHCAM_CATEGORY_URL:-https://market.yandex.ru/catalog--avtomobilnye-videoregistratory/82798275/list?hid=82798269}
+  MVIDEO_DASHCAM_CATEGORY_URL: ${MVIDEO_DASHCAM_CATEGORY_URL:-}
+  INGESTION_CATEGORY_DAILY_BUDGET: ${INGESTION_CATEGORY_DAILY_BUDGET:-25}
+  INGESTION_PRODUCT_DAILY_BUDGET: ${INGESTION_PRODUCT_DAILY_BUDGET:-100}
+  INGESTION_SITEMAP_DAILY_BUDGET: ${INGESTION_SITEMAP_DAILY_BUDGET:-25}
+  INGESTION_FETCH_DELAY_MIN_MS: ${INGESTION_FETCH_DELAY_MIN_MS:-5000}
+  INGESTION_FETCH_DELAY_MAX_MS: ${INGESTION_FETCH_DELAY_MAX_MS:-15000}
+  INGESTION_WORKER_BATCH_SIZE: ${INGESTION_WORKER_BATCH_SIZE:-1}
+  INGESTION_WORKER_IDLE_MS: ${INGESTION_WORKER_IDLE_MS:-30000}
+
 services:
   web:
-    image: ${WEB_IMAGE}
+    image: ${WEB_IMAGE:?WEB_IMAGE is required}
     environment:
       NODE_ENV: production
       APP_ENV: ${ENVIRONMENT}
       PORT: "3000"
       VIDEOREG_DB_FILE: /data/videoreg.json
+      DASHCAM_CATALOG_EXPORT_DIR: /exports
       VIDEOREG_DB_MEMORY_TTL_MS: ${VIDEOREG_DB_MEMORY_TTL_MS:-3600000}
       VIDEOREG_NEXT_CACHE_REVALIDATE_SECONDS: ${VIDEOREG_NEXT_CACHE_REVALIDATE_SECONDS:-3600}
       COMMIT_SHA: ${COMMIT_SHA}
@@ -16,6 +50,7 @@ services:
       TEAM_ID: ${TEAM_ID:-}
     volumes:
       - videoreg_data:/data
+      - catalog_exports:/exports:ro
     networks:
       traefik: {}
     healthcheck:
@@ -25,34 +60,90 @@ services:
       retries: 3
       start_period: 20s
     deploy:
+      <<: *default-deploy
       replicas: ${WEB_REPLICAS:-1}
-      placement:
-        constraints:
-          - ${WEB_PLACEMENT_CONSTRAINT:-node.role == manager}
-      update_config:
-        order: start-first
-        parallelism: 1
-        failure_action: rollback
-        monitor: 45s
-      rollback_config:
-        order: start-first
-        parallelism: 1
-        monitor: 45s
-      restart_policy:
-        condition: on-failure
       labels:
         - "traefik.enable=true"
         - "traefik.docker.network=${TRAEFIK_NETWORK}"
         - "traefik.http.routers.${STACK_NAME}-web.rule=Host(`${PUBLIC_HOST}`) && PathPrefix(`/`)"
+        - "traefik.http.routers.${STACK_NAME}-web.priority=${TRAEFIK_WEB_PRIORITY:-1}"
         - "traefik.http.routers.${STACK_NAME}-web.entrypoints=${TRAEFIK_ENTRYPOINTS:-websecure}"
         - "traefik.http.routers.${STACK_NAME}-web.tls=${TRAEFIK_TLS:-true}"
+        - "traefik.http.routers.${STACK_NAME}-web.tls.certresolver=${TRAEFIK_CERTRESOLVER:-le}"
         - "traefik.http.services.${STACK_NAME}-web.loadbalancer.server.port=3000"
     stop_grace_period: 30s
 
+  postgres:
+    image: postgres:16-alpine
+    environment:
+      POSTGRES_DB: ${POSTGRES_DB:-videoreg_ingestion}
+      POSTGRES_USER: ${POSTGRES_USER:-videoreg}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required}
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-videoreg} -d ${POSTGRES_DB:-videoreg_ingestion}"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    volumes:
+      - ingestion_postgres:/var/lib/postgresql/data
+    networks:
+      - ingestion
+    deploy: *default-deploy
+
+  ingestion-worker:
+    image: ${INGESTION_IMAGE:?INGESTION_IMAGE is required}
+    command: sh -c "npm run ingestion:migrate && npm run ingestion:worker"
+    environment: *ingestion-env
+    volumes:
+      - ingestion_data:/app/ingestion/data
+      - catalog_exports:/exports
+    networks:
+      - ingestion
+    deploy:
+      <<: *default-deploy
+      replicas: ${INGESTION_WORKER_REPLICAS:-1}
+
+  ingestion-admin:
+    image: ${INGESTION_IMAGE:?INGESTION_IMAGE is required}
+    command: sh -c "npm run ingestion:migrate && npm run ingestion:admin"
+    environment:
+      <<: *ingestion-env
+      INGESTION_ADMIN_PORT: "4101"
+      INGESTION_ADMIN_USERNAME: ${INGESTION_ADMIN_USERNAME:-admin}
+      INGESTION_ADMIN_PASSWORD: ${INGESTION_ADMIN_PASSWORD:?INGESTION_ADMIN_PASSWORD is required}
+    volumes:
+      - ingestion_data:/app/ingestion/data
+    networks:
+      ingestion: {}
+      traefik: {}
+    healthcheck:
+      test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:4101/health >/dev/null || exit 1"]
+      interval: 15s
+      timeout: 3s
+      retries: 5
+      start_period: 20s
+    deploy:
+      <<: *default-deploy
+      replicas: 1
+      labels:
+        - "traefik.enable=true"
+        - "traefik.docker.network=${TRAEFIK_NETWORK}"
+        - "traefik.http.routers.${STACK_NAME}-admin.rule=Host(`${PUBLIC_HOST}`) && PathPrefix(`/admin`)"
+        - "traefik.http.routers.${STACK_NAME}-admin.priority=${TRAEFIK_ADMIN_PRIORITY:-100}"
+        - "traefik.http.routers.${STACK_NAME}-admin.entrypoints=${TRAEFIK_ENTRYPOINTS:-websecure}"
+        - "traefik.http.routers.${STACK_NAME}-admin.tls=${TRAEFIK_TLS:-true}"
+        - "traefik.http.routers.${STACK_NAME}-admin.tls.certresolver=${TRAEFIK_CERTRESOLVER:-le}"
+        - "traefik.http.services.${STACK_NAME}-admin.loadbalancer.server.port=4101"
+
 volumes:
   videoreg_data:
+  catalog_exports:
+  ingestion_postgres:
+  ingestion_data:
 
 networks:
+  ingestion:
+    driver: overlay
   traefik:
     external: true
     name: ${TRAEFIK_NETWORK}