feat: complete dashcam catalog ingestion demo

2026-07-31 02:01:58 +03:00
parent ae6c35145f
commit 2ce58fcbc0
62 changed files with 6341 additions and 178 deletions
@@ -0,0 +1,25 @@
+POSTGRES_DB=videoreg_ingestion
+POSTGRES_USER=videoreg
+POSTGRES_PASSWORD=change-me
+
+# Bind local operational ports to loopback.
+INGESTION_POSTGRES_BIND=127.0.0.1
+INGESTION_POSTGRES_PORT=5432
+INGESTION_ADMIN_BIND=127.0.0.1
+INGESTION_ADMIN_PORT=4101
+INGESTION_ADMIN_USERNAME=admin
+INGESTION_ADMIN_PASSWORD=replace-with-admin-password
+
+# Source seeds. MVideo is active when this URL is configured.
+YANDEX_MARKET_DASHCAM_CATEGORY_URL=https://market.yandex.ru/catalog--avtomobilnye-videoregistratory/82798275/list?hid=82798269
+MVIDEO_DASHCAM_CATEGORY_URL=https://www.mvideo.ru/product-list-page?q=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE%D1%80%D0%B5%D0%B3%D0%B8%D1%81%D1%82%D1%80%D0%B0%D1%82%D0%BE%D1%80%D1%8B
+
+# Conservative crawl limits. Local Compose maps the shared exports directory to
+# ../data/catalog-exports; the Next.js app reads that same directory directly.
+INGESTION_CATEGORY_DAILY_BUDGET=25
+INGESTION_PRODUCT_DAILY_BUDGET=100
+INGESTION_SITEMAP_DAILY_BUDGET=25
+INGESTION_FETCH_DELAY_MIN_MS=5000
+INGESTION_FETCH_DELAY_MAX_MS=15000
+INGESTION_WORKER_BATCH_SIZE=1
+INGESTION_WORKER_IDLE_MS=30000