feat: complete dashcam catalog ingestion demo

2026-07-31 02:01:58 +03:00
parent ae6c35145f
commit 2ce58fcbc0
62 changed files with 6341 additions and 178 deletions
@@ -0,0 +1,90 @@
+services:
+  postgres:
+    image: postgres:16-alpine
+    restart: unless-stopped
+    environment:
+      POSTGRES_DB: ${POSTGRES_DB:-videoreg_ingestion}
+      POSTGRES_USER: ${POSTGRES_USER:-videoreg}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-videoreg}
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-videoreg} -d ${POSTGRES_DB:-videoreg_ingestion}"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    ports:
+      - "${INGESTION_POSTGRES_BIND:-127.0.0.1}:${INGESTION_POSTGRES_PORT:-5432}:5432"
+    volumes:
+      - postgres-data:/var/lib/postgresql/data
+
+  ingestion-migrate:
+    build:
+      context: ..
+      dockerfile: ingestion/Dockerfile
+    command: npm run ingestion:migrate
+    restart: "no"
+    depends_on:
+      postgres:
+        condition: service_healthy
+    environment: &ingestion-env
+      INGESTION_DATABASE_URL: postgres://${POSTGRES_USER:-videoreg}:${POSTGRES_PASSWORD:-videoreg}@postgres:5432/${POSTGRES_DB:-videoreg_ingestion}
+      INGESTION_SNAPSHOT_DIR: /app/ingestion/data/snapshots
+      INGESTION_EXPORT_DIR: /app/exports
+      YANDEX_MARKET_DASHCAM_CATEGORY_URL: ${YANDEX_MARKET_DASHCAM_CATEGORY_URL:-https://market.yandex.ru/catalog--avtomobilnye-videoregistratory/82798275/list?hid=82798269}
+      MVIDEO_DASHCAM_CATEGORY_URL: ${MVIDEO_DASHCAM_CATEGORY_URL:-}
+      INGESTION_CATEGORY_DAILY_BUDGET: ${INGESTION_CATEGORY_DAILY_BUDGET:-25}
+      INGESTION_PRODUCT_DAILY_BUDGET: ${INGESTION_PRODUCT_DAILY_BUDGET:-100}
+      INGESTION_SITEMAP_DAILY_BUDGET: ${INGESTION_SITEMAP_DAILY_BUDGET:-25}
+      INGESTION_FETCH_DELAY_MIN_MS: ${INGESTION_FETCH_DELAY_MIN_MS:-5000}
+      INGESTION_FETCH_DELAY_MAX_MS: ${INGESTION_FETCH_DELAY_MAX_MS:-15000}
+      INGESTION_WORKER_BATCH_SIZE: ${INGESTION_WORKER_BATCH_SIZE:-1}
+      INGESTION_WORKER_IDLE_MS: ${INGESTION_WORKER_IDLE_MS:-30000}
+    volumes:
+      - ingestion-data:/app/ingestion/data
+      - ../data/catalog-exports:/app/exports
+
+  ingestion-worker:
+    build:
+      context: ..
+      dockerfile: ingestion/Dockerfile
+    command: npm run ingestion:worker
+    restart: unless-stopped
+    depends_on:
+      postgres:
+        condition: service_healthy
+      ingestion-migrate:
+        condition: service_completed_successfully
+    environment: *ingestion-env
+    volumes:
+      - ingestion-data:/app/ingestion/data
+      - ../data/catalog-exports:/app/exports
+
+  ingestion-admin:
+    build:
+      context: ..
+      dockerfile: ingestion/Dockerfile
+    command: npm run ingestion:admin
+    restart: unless-stopped
+    depends_on:
+      postgres:
+        condition: service_healthy
+      ingestion-migrate:
+        condition: service_completed_successfully
+    environment:
+      <<: *ingestion-env
+      INGESTION_ADMIN_PORT: 4101
+      INGESTION_ADMIN_USERNAME: ${INGESTION_ADMIN_USERNAME:-admin}
+      INGESTION_ADMIN_PASSWORD: ${INGESTION_ADMIN_PASSWORD:?INGESTION_ADMIN_PASSWORD is required}
+    ports:
+      - "${INGESTION_ADMIN_BIND:-127.0.0.1}:${INGESTION_ADMIN_PORT:-4101}:4101"
+    healthcheck:
+      test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:4101/health >/dev/null || exit 1"]
+      interval: 15s
+      timeout: 3s
+      retries: 5
+      start_period: 20s
+    volumes:
+      - ingestion-data:/app/ingestion/data
+
+volumes:
+  postgres-data:
+  ingestion-data: