feat: complete dashcam catalog ingestion demo

2026-07-31 02:01:58 +03:00
parent ae6c35145f
commit 2ce58fcbc0
62 changed files with 6341 additions and 178 deletions
@@ -4,14 +4,16 @@ on:
  push:
    branches:
      - master
+  workflow_dispatch:

 concurrency:
  group: deploy-production
  cancel-in-progress: false

 env:
-  REGISTRY: ${{ vars.CONTAINER_REGISTRY }}
-  WEB_IMAGE_REPO: ${{ vars.CONTAINER_REGISTRY }}/${{ github.repository_owner }}/${{ github.event.repository.name }}-web
+  REGISTRY: ${{ vars.REGISTRY }}
+  WEB_IMAGE_REPO: ${{ vars.IMAGE_REPO }}-web
+  INGESTION_IMAGE_REPO: ${{ vars.IMAGE_REPO }}-ingestion
  RELEASE_ID: ${{ github.sha }}-${{ github.run_number }}
  NPM_CONFIG_AUDIT: "false"
  NPM_CONFIG_FUND: "false"
@@ -63,38 +65,55 @@ jobs:
            org.opencontainers.image.revision=${{ github.sha }}
            org.opencontainers.image.version=${{ env.RELEASE_ID }}

+      - name: Build and push ingestion image
+        id: build-ingestion
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ingestion/Dockerfile
+          push: true
+          tags: ${{ env.INGESTION_IMAGE_REPO }}:${{ github.sha }}
+          labels: |
+            org.opencontainers.image.revision=${{ github.sha }}
+            org.opencontainers.image.version=${{ env.RELEASE_ID }}
+
      - name: Production pre-deploy checks
        env:
          ENVIRONMENT: production
-          STACK_NAME: ${{ vars.PRODUCTION_STACK_NAME }}
+          STACK_NAME: ${{ vars.STACK_NAME }}
          RELEASE_ID: ${{ env.RELEASE_ID }}
          COMMIT_SHA: ${{ github.sha }}
          WEB_IMAGE: ${{ env.WEB_IMAGE_REPO }}@${{ steps.build-web.outputs.digest }}
-          PUBLIC_HOST: ${{ vars.PRODUCTION_PUBLIC_HOST }}
-          TRAEFIK_NETWORK: ${{ vars.PRODUCTION_TRAEFIK_NETWORK }}
-          SWARM_MANAGER_HOST: ${{ secrets.PRODUCTION_SWARM_MANAGER_HOST }}
-          SWARM_SSH_USER: ${{ secrets.PRODUCTION_SWARM_SSH_USER }}
+          INGESTION_IMAGE: ${{ env.INGESTION_IMAGE_REPO }}@${{ steps.build-ingestion.outputs.digest }}
+          POSTGRES_PASSWORD: ${{ secrets.INGESTION_POSTGRES_PASSWORD }}
+          INGESTION_ADMIN_PASSWORD: ${{ secrets.INGESTION_ADMIN_PASSWORD }}
+          PUBLIC_HOST: ${{ vars.DOMAIN }}
+          TRAEFIK_NETWORK: ${{ vars.TRAEFIK_NETWORK }}
+          SWARM_MANAGER_HOST: ${{ secrets.SWARM_MANAGER_HOST }}
+          SWARM_SSH_USER: ${{ secrets.SWARM_SSH_USER }}
        run: node deploy/scripts/predeploy-check.mjs

      - name: Deploy production
        env:
          ENVIRONMENT: production
-          STACK_NAME: ${{ vars.PRODUCTION_STACK_NAME }}
+          STACK_NAME: ${{ vars.STACK_NAME }}
          RELEASE_ID: ${{ env.RELEASE_ID }}
          COMMIT_SHA: ${{ github.sha }}
          WEB_IMAGE: ${{ env.WEB_IMAGE_REPO }}@${{ steps.build-web.outputs.digest }}
-          PUBLIC_HOST: ${{ vars.PRODUCTION_PUBLIC_HOST }}
-          TRAEFIK_NETWORK: ${{ vars.PRODUCTION_TRAEFIK_NETWORK }}
-          SWARM_MANAGER_HOST: ${{ secrets.PRODUCTION_SWARM_MANAGER_HOST }}
-          SWARM_SSH_USER: ${{ secrets.PRODUCTION_SWARM_SSH_USER }}
-          SWARM_SSH_PORT: ${{ secrets.PRODUCTION_SWARM_SSH_PORT }}
+          INGESTION_IMAGE: ${{ env.INGESTION_IMAGE_REPO }}@${{ steps.build-ingestion.outputs.digest }}
+          PUBLIC_HOST: ${{ vars.DOMAIN }}
+          TRAEFIK_NETWORK: ${{ vars.TRAEFIK_NETWORK }}
+          SWARM_MANAGER_HOST: ${{ secrets.SWARM_MANAGER_HOST }}
+          SWARM_SSH_USER: ${{ secrets.SWARM_SSH_USER }}
+          SWARM_SSH_PORT: ${{ secrets.SWARM_SSH_PORT }}
          REGISTRY: ${{ env.REGISTRY }}
          REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
          REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
-          WEB_REPLICAS: ${{ vars.PRODUCTION_WEB_REPLICAS }}
-          WEB_PLACEMENT_CONSTRAINT: ${{ vars.PRODUCTION_WEB_PLACEMENT_CONSTRAINT }}
-          TRAEFIK_ENTRYPOINTS: ${{ vars.TRAEFIK_ENTRYPOINTS }}
+          WEB_REPLICAS: ${{ vars.WEB_REPLICAS }}
+          WEB_PLACEMENT_CONSTRAINT: ${{ vars.WEB_PLACEMENT_CONSTRAINT }}
+          TRAEFIK_ENTRYPOINTS: ${{ vars.TRAEFIK_ENTRYPOINT }}
          TRAEFIK_TLS: ${{ vars.TRAEFIK_TLS }}
+          TRAEFIK_CERTRESOLVER: ${{ vars.TRAEFIK_CERTRESOLVER }}
          REGION: ${{ vars.REGION }}
          TEAM_ID: ${{ vars.TEAM_ID }}
          DEPLOY_MIN_FREE_MB: ${{ vars.DEPLOY_MIN_FREE_MB }}
@@ -103,13 +122,29 @@ jobs:
          DEPLOY_CLEANUP_FAILED_IMAGES: ${{ vars.DEPLOY_CLEANUP_FAILED_IMAGES }}
          VIDEOREG_DB_MEMORY_TTL_MS: ${{ vars.VIDEOREG_DB_MEMORY_TTL_MS }}
          VIDEOREG_NEXT_CACHE_REVALIDATE_SECONDS: ${{ vars.VIDEOREG_NEXT_CACHE_REVALIDATE_SECONDS }}
+          POSTGRES_DB: ${{ vars.INGESTION_POSTGRES_DB }}
+          POSTGRES_USER: ${{ vars.INGESTION_POSTGRES_USER }}
+          POSTGRES_PASSWORD: ${{ secrets.INGESTION_POSTGRES_PASSWORD }}
+          INGESTION_ADMIN_USERNAME: ${{ vars.INGESTION_ADMIN_USERNAME }}
+          INGESTION_ADMIN_PASSWORD: ${{ secrets.INGESTION_ADMIN_PASSWORD }}
+          YANDEX_MARKET_DASHCAM_CATEGORY_URL: ${{ vars.YANDEX_MARKET_DASHCAM_CATEGORY_URL }}
+          MVIDEO_DASHCAM_CATEGORY_URL: ${{ vars.MVIDEO_DASHCAM_CATEGORY_URL }}
+          INGESTION_CATEGORY_DAILY_BUDGET: ${{ vars.INGESTION_CATEGORY_DAILY_BUDGET }}
+          INGESTION_PRODUCT_DAILY_BUDGET: ${{ vars.INGESTION_PRODUCT_DAILY_BUDGET }}
+          INGESTION_SITEMAP_DAILY_BUDGET: ${{ vars.INGESTION_SITEMAP_DAILY_BUDGET }}
+          INGESTION_FETCH_DELAY_MIN_MS: ${{ vars.INGESTION_FETCH_DELAY_MIN_MS }}
+          INGESTION_FETCH_DELAY_MAX_MS: ${{ vars.INGESTION_FETCH_DELAY_MAX_MS }}
+          INGESTION_WORKER_BATCH_SIZE: ${{ vars.INGESTION_WORKER_BATCH_SIZE }}
+          INGESTION_WORKER_IDLE_MS: ${{ vars.INGESTION_WORKER_IDLE_MS }}
+          INGESTION_WORKER_REPLICAS: ${{ vars.INGESTION_WORKER_REPLICAS }}
+          TRAEFIK_ADMIN_PRIORITY: ${{ vars.TRAEFIK_ADMIN_PRIORITY }}
        run: deploy/scripts/deploy-over-ssh.sh

      - name: Production smoke checks
        id: smoke
        continue-on-error: true
        env:
-          BASE_URL: https://${{ vars.PRODUCTION_PUBLIC_HOST }}
+          BASE_URL: https://${{ vars.DOMAIN }}
          EXPECTED_RELEASE_ID: ${{ env.RELEASE_ID }}
          SMOKE_RESULT_PATH: smoke-result.json
        run: node deploy/scripts/smoke-check.mjs
@@ -118,10 +153,10 @@ jobs:
        if: steps.smoke.outcome == 'failure'
        env:
          ENVIRONMENT: production
-          STACK_NAME: ${{ vars.PRODUCTION_STACK_NAME }}
-          SWARM_MANAGER_HOST: ${{ secrets.PRODUCTION_SWARM_MANAGER_HOST }}
-          SWARM_SSH_USER: ${{ secrets.PRODUCTION_SWARM_SSH_USER }}
-          SWARM_SSH_PORT: ${{ secrets.PRODUCTION_SWARM_SSH_PORT }}
+          STACK_NAME: ${{ vars.STACK_NAME }}
+          SWARM_MANAGER_HOST: ${{ secrets.SWARM_MANAGER_HOST }}
+          SWARM_SSH_USER: ${{ secrets.SWARM_SSH_USER }}
+          SWARM_SSH_PORT: ${{ secrets.SWARM_SSH_PORT }}
        run: deploy/scripts/rollback-over-ssh.sh

      - name: Fail after production smoke rollback
@@ -133,6 +168,7 @@ jobs:
          COMMIT_SHA: ${{ github.sha }}
          RELEASE_ID: ${{ env.RELEASE_ID }}
          WEB_IMAGE_DIGEST: ${{ env.WEB_IMAGE_REPO }}@${{ steps.build-web.outputs.digest }}
+          INGESTION_IMAGE_DIGEST: ${{ env.INGESTION_IMAGE_REPO }}@${{ steps.build-ingestion.outputs.digest }}
          SMOKE_RESULT_PATH: smoke-result.json
          WORKFLOW_RUN_ID: ${{ github.run_id }}
        run: node deploy/scripts/write-release-manifest.mjs > release-manifest.json