feat: complete dashcam catalog ingestion demo

.beads/interactions.jsonl

@@ -0,0 +1,18 @@
+{"id":"int-bb1dc104","kind":"field_change","created_at":"2026-06-07T14:24:33.025471Z","actor":"E.Gavrilov","issue_id":"videoreg-kbh.2","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Implemented and verified in codebase (typecheck + 18 vitest tests pass)"}}
+{"id":"int-c9ddede0","kind":"field_change","created_at":"2026-06-07T14:24:33.344014Z","actor":"E.Gavrilov","issue_id":"videoreg-kbh.3","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Implemented and verified in codebase (typecheck + 18 vitest tests pass)"}}
+{"id":"int-2da54fc8","kind":"field_change","created_at":"2026-06-07T14:24:33.664844Z","actor":"E.Gavrilov","issue_id":"videoreg-kbh.4","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Implemented and verified in codebase (typecheck + 18 vitest tests pass)"}}
+{"id":"int-f2a63413","kind":"field_change","created_at":"2026-06-07T14:24:33.992823Z","actor":"E.Gavrilov","issue_id":"videoreg-kbh.5","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Implemented and verified in codebase (typecheck + 18 vitest tests pass)"}}
+{"id":"int-278657e9","kind":"field_change","created_at":"2026-06-07T14:24:34.319098Z","actor":"E.Gavrilov","issue_id":"videoreg-kbh.6","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Implemented and verified in codebase (typecheck + 18 vitest tests pass)"}}
+{"id":"int-f99f838f","kind":"field_change","created_at":"2026-06-07T14:24:34.647874Z","actor":"E.Gavrilov","issue_id":"videoreg-kbh.8","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Implemented and verified in codebase (typecheck + 18 vitest tests pass)"}}
+{"id":"int-2182f889","kind":"field_change","created_at":"2026-06-07T14:24:34.951525Z","actor":"E.Gavrilov","issue_id":"videoreg-seu.1","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Implemented and verified in codebase (typecheck + 18 vitest tests pass)"}}
+{"id":"int-6a49828d","kind":"field_change","created_at":"2026-06-07T14:24:35.263154Z","actor":"E.Gavrilov","issue_id":"videoreg-nnv.1","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Implemented and verified in codebase (typecheck + 18 vitest tests pass)"}}
+{"id":"int-8f64b7af","kind":"field_change","created_at":"2026-06-07T14:24:36.039743Z","actor":"E.Gavrilov","issue_id":"videoreg-kbh.1","extra":{"field":"status","new_value":"in_progress","old_value":"open"}}
+{"id":"int-9330a6c3","kind":"field_change","created_at":"2026-06-07T14:24:36.602138Z","actor":"E.Gavrilov","issue_id":"videoreg-kbh.7","extra":{"field":"status","new_value":"in_progress","old_value":"open"}}
+{"id":"int-d6554c75","kind":"field_change","created_at":"2026-06-07T14:24:37.165445Z","actor":"E.Gavrilov","issue_id":"videoreg-st9.1","extra":{"field":"status","new_value":"in_progress","old_value":"open"}}
+{"id":"int-66d79f75","kind":"field_change","created_at":"2026-07-30T23:00:42.663159Z","actor":"E.Gavrilov","issue_id":"videoreg-nnv.3","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Shared-volume bridge is implemented, documented, locally exercised, and validated by the Swarm config."}}
+{"id":"int-2599ce31","kind":"field_change","created_at":"2026-07-30T23:00:43.499008Z","actor":"E.Gavrilov","issue_id":"videoreg-nnv.4","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"The combined web, PostgreSQL, worker, and admin Swarm stack shares catalog_exports and routes /admin through Traefik."}}
+{"id":"int-c3a31f13","kind":"field_change","created_at":"2026-07-30T23:00:44.306067Z","actor":"E.Gavrilov","issue_id":"videoreg-kbh.14","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"The HTTP export server and token bridge are removed; versioned export artifacts are written atomically to the shared volume."}}
+{"id":"int-a20ed6d4","kind":"field_change","created_at":"2026-07-30T23:00:45.148032Z","actor":"E.Gavrilov","issue_id":"videoreg-kbh.15","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"The app validates manifest/artifact files from the shared export directory with SHA-256 and Zod, then updates its read model atomically."}}
+{"id":"int-b5112040","kind":"field_change","created_at":"2026-07-30T23:00:45.976151Z","actor":"E.Gavrilov","issue_id":"videoreg-kbh.9","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"MVideo discovery and product parsing are wired into the worker with smoke coverage; live source CAPTCHA is paused without bypass."}}
+{"id":"int-3528fa0c","kind":"field_change","created_at":"2026-07-30T23:00:46.850374Z","actor":"E.Gavrilov","issue_id":"videoreg-kbh.16","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Canonical model cleanup has fixtures for the SPAWNSON and iBOX cases, and canonical name/model edits are preserved in admin moderation."}}
+{"id":"int-3bda336d","kind":"field_change","created_at":"2026-07-30T23:00:47.750226Z","actor":"E.Gavrilov","issue_id":"videoreg-1v0","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Local Yandex crawl produced five real approved cards, export/import and desktop/mobile catalog rendering were verified, and quality/deploy checks pass."}}

.beads/issues.jsonl

@@ -0,0 +1,31 @@
+{"_type":"issue","id":"videoreg-1v0","title":"Demo readiness: local crawl, catalog handoff, UI verification","description":"Prepare a reproducible demo flow: co-located ingestion and web share a validated catalog export, local PostgreSQL receives real source cards, approved catalog is visible in the frontend, and the runbook plus verification artifacts are present.","acceptance_criteria":"A local documented command sequence starts the stack, crawls at least five valid cards from enabled sources within configured budgets, approves and exports catalog data, renders it in the web UI, and passes test/typecheck/build/browser checks.","status":"closed","priority":1,"issue_type":"task","assignee":"E.Gavrilov","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-07-30T21:51:59Z","created_by":"E.Gavrilov","updated_at":"2026-07-30T23:00:48Z","started_at":"2026-07-30T21:52:14Z","closed_at":"2026-07-30T23:00:48Z","close_reason":"Local Yandex crawl produced five real approved cards, export/import and desktop/mobile catalog rendering were verified, and quality/deploy checks pass.","dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-nnv.4","title":"Single co-located deploy stack with shared exports volume","description":"Merge deploy so web + ingestion (postgres, worker, admin) run in one Swarm/Portainer env. Add a shared volume carrying ingestion-data/exports that the web build/runtime can read. Remove export-server from ingestion/docker-compose.yml. Deprecate deploy/portainer/ingestion-stack.standalone.yml (separate-host). Keep ingestion Postgres/admin bound internally; only Traefik fronts web (+ optionally /admin).","status":"closed","priority":1,"issue_type":"task","assignee":"E.Gavrilov","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:53:49Z","created_by":"E.Gavrilov","updated_at":"2026-07-30T23:00:43Z","started_at":"2026-07-30T23:00:28Z","closed_at":"2026-07-30T23:00:43Z","close_reason":"The combined web, PostgreSQL, worker, and admin Swarm stack shares catalog_exports and routes /admin through Traefik.","dependencies":[{"issue_id":"videoreg-nnv.4","depends_on_id":"videoreg-nnv","type":"parent-child","created_at":"2026-06-07T17:53:49Z","created_by":"E.Gavrilov","metadata":"{}"},{"issue_id":"videoreg-nnv.4","depends_on_id":"videoreg-nnv.3","type":"blocks","created_at":"2026-06-07T17:54:02Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-kbh.15","title":"Rewrite app catalog import to read shared-volume catalog.json (drop HTTP/token)","description":"web-front/src/entities/dashcam-catalog/api/importCatalogExport.ts currently fetches manifest+artifact over HTTP with a Bearer token. Switch to reading manifest.json + catalog.json from the shared exports volume (a mounted path). Keep SHA-256 verification + exportId/createdAt/itemCount checks + Zod validation. Remove DASHCAM_CATALOG_MANIFEST_URL/EXPORT_TOKEN; add a path env. Under static-export this runs at build time.","status":"closed","priority":1,"issue_type":"task","assignee":"E.Gavrilov","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:53:48Z","created_by":"E.Gavrilov","updated_at":"2026-07-30T23:00:45Z","started_at":"2026-07-30T23:00:28Z","closed_at":"2026-07-30T23:00:45Z","close_reason":"The app validates manifest/artifact files from the shared export directory with SHA-256 and Zod, then updates its read model atomically.","dependencies":[{"issue_id":"videoreg-kbh.15","depends_on_id":"videoreg-kbh","type":"parent-child","created_at":"2026-06-07T17:53:48Z","created_by":"E.Gavrilov","metadata":"{}"},{"issue_id":"videoreg-kbh.15","depends_on_id":"videoreg-kbh.14","type":"blocks","created_at":"2026-06-07T17:54:03Z","created_by":"E.Gavrilov","metadata":"{}"},{"issue_id":"videoreg-kbh.15","depends_on_id":"videoreg-nnv.3","type":"blocks","created_at":"2026-06-07T17:54:02Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":2,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-kbh.14","title":"Drop export-server + token; write catalog export to shared volume","description":"Remove the export-server HTTP service (ingestion/src/exportServer.ts, ingestion:export script, port 4100, INGESTION_EXPORT_TOKEN/PUBLIC_BASE_URL). The ingestion-worker/export-catalog job already writes catalog.json + manifest.json to INGESTION_EXPORT_DIR (/app/ingestion/data/exports) — expose that as a shared volume instead of serving over HTTP. exportCatalog logic and Zod validation stay.","status":"closed","priority":1,"issue_type":"task","assignee":"E.Gavrilov","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:53:47Z","created_by":"E.Gavrilov","updated_at":"2026-07-30T23:00:44Z","started_at":"2026-07-30T23:00:28Z","closed_at":"2026-07-30T23:00:44Z","close_reason":"The HTTP export server and token bridge are removed; versioned export artifacts are written atomically to the shared volume.","dependencies":[{"issue_id":"videoreg-kbh.14","depends_on_id":"videoreg-kbh","type":"parent-child","created_at":"2026-06-07T17:53:47Z","created_by":"E.Gavrilov","metadata":"{}"},{"issue_id":"videoreg-kbh.14","depends_on_id":"videoreg-nnv.3","type":"blocks","created_at":"2026-06-07T17:54:01Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0}
+{"_type":"issue","id":"videoreg-nnv.3","title":"Co-locate ingestion with web; collapse catalog bridge to a shared volume","design":"Supersedes the original 'ingestion runs on a separate server/cloud' model (tasks/002-crawler.md §4, README, ingestion/README). New model: ingestion + web run in ONE cloud env (same Swarm/Portainer). The ingestion-worker writes the validated catalog.json + manifest.json to a shared exports volume; the web build reads catalog.json DIRECTLY from that volume. No export-server, no HTTP, no Bearer token. Zod + SHA-256 validation are kept at read/build time. The app still never connects to ingestion Postgres. Rationale: two separate clouds were too complex to manage.","acceptance_criteria":"export-server service removed; catalog consumed via shared volume; single co-located deploy stack; docs + memories corrected.","status":"closed","priority":1,"issue_type":"decision","assignee":"E.Gavrilov","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:53:46Z","created_by":"E.Gavrilov","updated_at":"2026-07-30T23:00:43Z","started_at":"2026-07-30T21:52:01Z","closed_at":"2026-07-30T23:00:43Z","close_reason":"Shared-volume bridge is implemented, documented, locally exercised, and validated by the Swarm config.","dependencies":[{"issue_id":"videoreg-nnv.3","depends_on_id":"videoreg-nnv","type":"parent-child","created_at":"2026-06-07T17:53:45Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":3,"comment_count":0}
+{"_type":"issue","id":"videoreg-nnv.2","title":"Commit untracked ingestion/, deploy/portainer/, ingestion CI to git","description":"The entire ingestion/ engine, deploy/portainer/, and .gitea/workflows/ingestion.yml are UNTRACKED in git (plus modified README/package.json/layout.tsx/docker-stack.yml/.dockerignore). All this project knowledge lives only on local disk. Stage, commit, and push so the work isn't stranded.","status":"in_progress","priority":1,"issue_type":"task","assignee":"E.Gavrilov","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:24:12Z","created_by":"E.Gavrilov","updated_at":"2026-07-30T21:52:02Z","started_at":"2026-07-30T21:52:02Z","dependencies":[{"issue_id":"videoreg-nnv.2","depends_on_id":"videoreg-nnv","type":"parent-child","created_at":"2026-06-07T17:24:12Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-kbh.11","title":"Deep deduplication matching (fuzzy/alias/perceptual)","description":"Implement spec §14 matching: brand + normalized model + aliases + sourceProductId + image perceptual hash + specs similarity + URL history; classify same_product/possible_duplicate/variant/oem_clone so non-trivial groups require manual moderation before export. Currently only new_product is recorded.","status":"open","priority":1,"issue_type":"feature","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:23:36Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:23:36Z","dependencies":[{"issue_id":"videoreg-kbh.11","depends_on_id":"videoreg-kbh","type":"parent-child","created_at":"2026-06-07T17:23:36Z","created_by":"E.Gavrilov","metadata":"{}"},{"issue_id":"videoreg-kbh.11","depends_on_id":"videoreg-kbh.16","type":"blocks","created_at":"2026-06-07T18:02:57Z","created_by":"E.Gavrilov","metadata":"{}"},{"issue_id":"videoreg-kbh.11","depends_on_id":"videoreg-kbh.7","type":"blocks","created_at":"2026-06-07T17:24:37Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":2,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-kbh.7","title":"Phase 7: Deduplication + moderation","description":"Canonical product grouping, moderation queue, bulk approve, conflict tracking. PARTIAL: moderation_queue + moderation_conflicts tables, products default moderation_status='needs_review', admin moderation UI/API exist. Deep dedup matching (fuzzy/alias/perceptual image hash, possible_duplicate/variant/oem_clone classification) is NOT implemented — only new_product entries are created.","notes":"Moderation queue/conflicts tables + admin UI done; deep dedup matching (kbh.11) outstanding.","status":"in_progress","priority":1,"issue_type":"feature","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:23:14Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:24:37Z","started_at":"2026-06-07T14:24:37Z","dependencies":[{"issue_id":"videoreg-kbh.7","depends_on_id":"videoreg-kbh","type":"parent-child","created_at":"2026-06-07T17:23:14Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":1,"comment_count":0}
+{"_type":"issue","id":"videoreg-kbh","title":"Dashcam Catalog Ingestion Engine","description":"Maintainable catalog ingestion engine for car dashcams (источник: Yandex Market). HTTP-only crawler, PostgreSQL source of truth, normalization, dedup, moderation, and a token-protected versioned export consumed by the Next.js app. Full spec: tasks/002-crawler.md.","status":"open","priority":1,"issue_type":"epic","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:22:41Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:22:41Z","dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-kbh.16","title":"Model/canonicalName cleaning leaks form-factor \u0026 descriptor words","description":"Verified on a live smoke run (2026 Yandex Market data). normalizer.cleanModelFromTitle/buildCanonicalModel leave non-model words in the canonical name, which will corrupt dedup/canonical grouping (kbh.7/kbh.11). Examples: title 'Видеорегистратор-зеркало iBOX Rover 2 автомобильный с базой камер' -\u003e canonicalName 'iBOX зеркало Rover 2 автомобильный' (should be ~'iBOX Rover 2'); 'iBOX F5 + WiFi с радар-детектором' -\u003e model retains 'с радар-детектором'. genericWords list lacks 'зеркало', standalone 'автомобильный', form-factor and feature tails. Fix: expand stopword/form-factor stripping and feature-tail removal; add fixtures from real titles.","status":"closed","priority":2,"issue_type":"bug","assignee":"E.Gavrilov","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T15:02:57Z","created_by":"E.Gavrilov","updated_at":"2026-07-30T23:00:47Z","started_at":"2026-07-30T21:52:03Z","closed_at":"2026-07-30T23:00:47Z","close_reason":"Canonical model cleanup has fixtures for the SPAWNSON and iBOX cases, and canonical name/model edits are preserved in admin moderation.","dependencies":[{"issue_id":"videoreg-kbh.16","depends_on_id":"videoreg-kbh","type":"parent-child","created_at":"2026-06-07T18:02:56Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":1,"comment_count":0}
+{"_type":"issue","id":"videoreg-nnv.1","title":"CI/CD pipelines + Swarm + Portainer + deploy scripts","description":"DONE: .gitea/workflows/{pr,master,ingestion}.yml, deploy/docker/Dockerfile.web, deploy/swarm/docker-stack.yml, deploy/portainer/{standalone,swarm} templates, deploy/scripts + deploy/remote ssh deploy/rollback + predeploy/smoke/release-manifest scripts.","status":"closed","priority":2,"issue_type":"feature","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:24:11Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:24:35Z","closed_at":"2026-06-07T14:24:35Z","close_reason":"Implemented and verified in codebase (typecheck + 18 vitest tests pass)","dependencies":[{"issue_id":"videoreg-nnv.1","depends_on_id":"videoreg-nnv","type":"parent-child","created_at":"2026-06-07T17:24:11Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-seu.1","title":"Video registry app: primitive JSON DB, caching, health probes","description":"Working Next.js app: dashboard + register-video flow, primitive file-backed JSON DB (shared/db/primitiveJsonDb.ts → data/videoreg.json), process+tag+HTTP read caching with revalidateTag on writes, /health/live + /health/ready, /api/videos. Tested (vitest).","status":"closed","priority":2,"issue_type":"feature","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:24:10Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:24:35Z","closed_at":"2026-06-07T14:24:35Z","close_reason":"Implemented and verified in codebase (typecheck + 18 vitest tests pass)","dependencies":[{"issue_id":"videoreg-seu.1","depends_on_id":"videoreg-seu","type":"parent-child","created_at":"2026-06-07T17:24:09Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-seu.2","title":"Replace primitive JSON DB before scaling beyond 1 replica","description":"The file-backed JSON DB forces the Swarm stack to a single replica on a manager node. Move to a real database before raising replicas or spreading across nodes. Read-model can keep being fed by the catalog import bridge.","status":"open","priority":2,"issue_type":"task","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:24:10Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:24:10Z","dependencies":[{"issue_id":"videoreg-seu.2","depends_on_id":"videoreg-seu","type":"parent-child","created_at":"2026-06-07T17:24:10Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-st9.1","title":"Phase 1: Tokens + core atomic components + 'Best Dashcams 2026' brief","description":"Typography/color/spacing tokens, light-first + dark parity, and core components for the comparison-first page. PARTIAL/mostly done in code: web-front/src/shared/ui atoms (Badge,Button,EvidenceChip,Icon,ScoreMeter), molecules (ProductCard,WinnerRail), organisms (PageHero,ProductComparisonTable), template BestDashcamsPage; globals.css tokens; dev showcase route. Trust states verified/claimed/unknown via EvidenceChip.","notes":"Core atomic components + tokens + BestDashcamsPage built; Storybook + Figma parity (st9.2) outstanding.","status":"in_progress","priority":2,"issue_type":"feature","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:24:06Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:24:37Z","started_at":"2026-06-07T14:24:37Z","dependencies":[{"issue_id":"videoreg-st9.1","depends_on_id":"videoreg-st9","type":"parent-child","created_at":"2026-06-07T17:24:05Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":1,"comment_count":0}
+{"_type":"issue","id":"videoreg-kbh.12","title":"Image storage legal/policy decision + versioned image artifacts","description":"Spec §7: legal decision on caching/serving source images must be made before production. Then cache approved images and publish versioned image artifact URLs (original URL + source metadata + optional checksum/dimensions); app must not hotlink source images. Currently product_images stores original URLs only.","status":"open","priority":2,"issue_type":"task","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:23:37Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:23:37Z","dependencies":[{"issue_id":"videoreg-kbh.12","depends_on_id":"videoreg-kbh","type":"parent-child","created_at":"2026-06-07T17:23:37Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-kbh.10","title":"Bounded sitemap-first discovery + robots.txt compliance","description":"Discovery currently parses product URLs from category HTML only (discoverYandexMarketProductsFromCategory). Spec §9 wants sitemap-first bounded discovery (diff known URLs, refresh candidates, detect near known boundaries) plus robots.txt checks. Not implemented.","status":"open","priority":2,"issue_type":"feature","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:23:35Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:23:35Z","dependencies":[{"issue_id":"videoreg-kbh.10","depends_on_id":"videoreg-kbh","type":"parent-child","created_at":"2026-06-07T17:23:35Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-kbh.9","title":"Activate MVideo source in the worker","description":"MVideo parser (ingestion/src/sources/mvideo.ts +tests) and ingestion:smoke-mvideo exist, and config.ts builds an MVIDEO seed, but worker.ts skips any seed where source!=='yandex_market' (seed.implemented guard). Wire MVideo discovery+parse into the worker crawl path.","status":"closed","priority":2,"issue_type":"feature","assignee":"E.Gavrilov","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:23:34Z","created_by":"E.Gavrilov","updated_at":"2026-07-30T23:00:46Z","started_at":"2026-07-30T21:52:03Z","closed_at":"2026-07-30T23:00:46Z","close_reason":"MVideo discovery and product parsing are wired into the worker with smoke coverage; live source CAPTCHA is paused without bypass.","dependencies":[{"issue_id":"videoreg-kbh.9","depends_on_id":"videoreg-kbh","type":"parent-child","created_at":"2026-06-07T17:23:34Z","created_by":"E.Gavrilov","metadata":"{}"},{"issue_id":"videoreg-kbh.9","depends_on_id":"videoreg-kbh.5","type":"blocks","created_at":"2026-06-07T17:24:38Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-kbh.8","title":"Phase 8: Versioned export + token-protected serving","description":"Versioned manifest + catalog artifact, Bearer-token export server, checksum verified on app import, atomic replace. DONE: ingestion/src/{exporter,exportCatalog,exportServer}.ts; excludes prices/sellers/availability/reviews/rating per spec.","notes":"Export ARTIFACT generation (exporter.ts/exportCatalog) stays valid. The token-protected HTTP SERVING is superseded by the shared-volume bridge — see the co-location decision under videoreg-nnv.","status":"closed","priority":2,"issue_type":"feature","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:23:15Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:53:50Z","closed_at":"2026-06-07T14:24:35Z","close_reason":"Implemented and verified in codebase (typecheck + 18 vitest tests pass)","dependencies":[{"issue_id":"videoreg-kbh.8","depends_on_id":"videoreg-kbh","type":"parent-child","created_at":"2026-06-07T17:23:15Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-kbh.6","title":"Phase 6: Normalizer (brand/model/specs/units/booleans)","description":"Brand dictionary, model cleaner, spec mapper, unit converter, boolean detector, alias dictionary. DONE: ingestion/src/normalizer.ts.","status":"closed","priority":2,"issue_type":"feature","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:23:13Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:24:34Z","closed_at":"2026-06-07T14:24:34Z","close_reason":"Implemented and verified in codebase (typecheck + 18 vitest tests pass)","dependencies":[{"issue_id":"videoreg-kbh.6","depends_on_id":"videoreg-kbh","type":"parent-child","created_at":"2026-06-07T17:23:13Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-kbh.5","title":"Phase 5: Parser + stableContentHash (Yandex Market)","description":"Category discovery, product/specs/image parsing, internal-only aggregate rating, source-specific stableContentHash. DONE for Yandex Market: ingestion/src/sources/yandexMarket.ts (+tests), html.ts, stableHash.ts. NOTE: bounded sitemap discovery not yet implemented (see discovery task).","status":"closed","priority":2,"issue_type":"feature","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:23:12Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:24:34Z","closed_at":"2026-06-07T14:24:34Z","close_reason":"Implemented and verified in codebase (typecheck + 18 vitest tests pass)","dependencies":[{"issue_id":"videoreg-kbh.5","depends_on_id":"videoreg-kbh","type":"parent-child","created_at":"2026-06-07T17:23:12Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":1,"comment_count":0}
+{"_type":"issue","id":"videoreg-kbh.4","title":"Phase 4: PostgreSQL-backed queue + HTTP-only fetcher","description":"Postgres queue (state machine), HTTP-only fetcher with rate limits/retry/backoff, stop on captcha/403/429, raw snapshot storage, rawSnapshotHash. DONE: ingestion/src/{queue,fetcher,worker}.ts.","status":"closed","priority":2,"issue_type":"feature","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:23:11Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:24:34Z","closed_at":"2026-06-07T14:24:34Z","close_reason":"Implemented and verified in codebase (typecheck + 18 vitest tests pass)","dependencies":[{"issue_id":"videoreg-kbh.4","depends_on_id":"videoreg-kbh","type":"parent-child","created_at":"2026-06-07T17:23:11Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-kbh.3","title":"Phase 3: Ingestion stack skeleton (Docker + Postgres + migrations)","description":"Separate docker-compose stack: postgres, ingestion-worker, ingestion-admin, export-server; migrations + schema_migrations; mounted volume for snapshots/images; healthchecks. DONE: ingestion/docker-compose.yml, migrations/001-003, ingestion/src/{db,migrate}.ts.","status":"closed","priority":2,"issue_type":"feature","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:23:10Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:24:33Z","closed_at":"2026-06-07T14:24:33Z","close_reason":"Implemented and verified in codebase (typecheck + 18 vitest tests pass)","dependencies":[{"issue_id":"videoreg-kbh.3","depends_on_id":"videoreg-kbh","type":"parent-child","created_at":"2026-06-07T17:23:10Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-kbh.2","title":"Phase 2: Shared export contract (Zod) + app import","description":"Shared Zod schema for manifest/catalog export, derived TS types, schemaVersion, operator import command, atomic replace of data/videoreg.json. DONE: web-front/src/entities/dashcam-catalog/{model/schema.ts,api/importCatalogExport.ts} + tests; package.json catalog:import.","status":"closed","priority":2,"issue_type":"feature","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:23:09Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:24:33Z","closed_at":"2026-06-07T14:24:33Z","close_reason":"Implemented and verified in codebase (typecheck + 18 vitest tests pass)","dependencies":[{"issue_id":"videoreg-kbh.2","depends_on_id":"videoreg-kbh","type":"parent-child","created_at":"2026-06-07T17:23:09Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-kbh.1","title":"Phase 1: Legal discovery \u0026 crawl policy","description":"robots.txt/sitemap/API-feed legality checks, crawl budget, allowed URL patterns, image storage policy. Crawl budget + conservative policy (1 req / 5-15s, jitter, exponential backoff, stop on captcha/403/429, daily budgets, no headless/stealth) is IMPLEMENTED in config/queue/fetcher. Still open: robots.txt + sitemap legality checks and image storage policy decision.","notes":"Crawl budget/backoff/stop-on-block implemented; robots.txt + sitemap legality checks and image storage policy still open.","status":"in_progress","priority":2,"issue_type":"task","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:23:08Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:24:36Z","started_at":"2026-06-07T14:24:36Z","dependencies":[{"issue_id":"videoreg-kbh.1","depends_on_id":"videoreg-kbh","type":"parent-child","created_at":"2026-06-07T17:23:08Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-nnv","title":"Deployment \u0026 CI/CD","description":"Gitea Actions pipelines, Docker image build/push, Swarm stack, Portainer stack templates, and SSH deploy/rollback scripts for both web and ingestion.","status":"open","priority":2,"issue_type":"epic","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:22:44Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:22:44Z","dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-seu","title":"Frontend Web App (web-front)","description":"Next.js 16 App Router frontend with FSD layering, primitive file-backed JSON DB as a replaceable read-model cache, aggressive read caching, health probes, and the operator catalog import bridge.","status":"open","priority":2,"issue_type":"epic","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:22:43Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:22:43Z","dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-st9","title":"Design System (Atomic + Tokens)","description":"Premium automotive editorial design system: tokens, atomic components, light-first + dark parity, comparison-first UX for 'Best Dashcams 2026'. Full spec: tasks/001-design-system.md.","status":"open","priority":2,"issue_type":"epic","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:22:42Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:22:42Z","dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-st9.3","title":"Phase 2: Article / review / ecommerce / filtering modules","description":"Spec Phase 2: article system, review system (pros/cons, verdict, sample footage, firmware history, heat/night tests), ecommerce modules (seller comparison, price history, CTA hierarchy), filtering system. Not implemented.","status":"open","priority":3,"issue_type":"feature","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:24:08Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:24:08Z","dependencies":[{"issue_id":"videoreg-st9.3","depends_on_id":"videoreg-st9","type":"parent-child","created_at":"2026-06-07T17:24:07Z","created_by":"E.Gavrilov","metadata":"{}"},{"issue_id":"videoreg-st9.3","depends_on_id":"videoreg-st9.1","type":"blocks","created_at":"2026-06-07T17:24:39Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0}
+{"_type":"issue","id":"videoreg-st9.2","title":"Storybook + typed component API docs + Figma parity","description":"Spec deliverables not yet present: no .storybook setup, no component API docs, no Figma↔React parity artifacts. Add Storybook, document variant/responsive/theming APIs.","status":"open","priority":3,"issue_type":"task","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:24:07Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:24:07Z","dependencies":[{"issue_id":"videoreg-st9.2","depends_on_id":"videoreg-st9","type":"parent-child","created_at":"2026-06-07T17:24:06Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-kbh.13","title":"Sync schedules: frequent light / deep / new-product detection","description":"Spec §17: frequent light sync (6-12h: first category page, bounded sitemap diff, new URLs), deep sync (7-14d: all known cards/specs/images), daily new-product detection. No scheduler implemented; worker runs an inline loop. MVP import is manual.","status":"open","priority":3,"issue_type":"feature","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:23:38Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:23:38Z","dependencies":[{"issue_id":"videoreg-kbh.13","depends_on_id":"videoreg-kbh","type":"parent-child","created_at":"2026-06-07T17:23:38Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-st9.4","title":"Phase 3: Motion, advanced comparison UX, personalization","description":"Spec Phase 3: mechanical motion system, advanced comparison UX (column pinning, snap scroll, spec highlighting), personalization, saved comparisons, dashboard-like features. Not implemented.","status":"open","priority":4,"issue_type":"feature","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:24:09Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:24:09Z","dependencies":[{"issue_id":"videoreg-st9.4","depends_on_id":"videoreg-st9","type":"parent-child","created_at":"2026-06-07T17:24:08Z","created_by":"E.Gavrilov","metadata":"{}"},{"issue_id":"videoreg-st9.4","depends_on_id":"videoreg-st9.3","type":"blocks","created_at":"2026-06-07T17:24:39Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0}

.dockerignore

@@ -1,8 +1,10 @@
 .git
 .gitea
 .next
+web-front/.next
 node_modules
 data
+ingestion/data
 coverage
 .env
 .env.local

.gitea/workflows/master.yml

@@ -4,14 +4,16 @@ on:
   push:
     branches:
       - master
+  workflow_dispatch:
 
 concurrency:
   group: deploy-production
   cancel-in-progress: false
 
 env:
-  REGISTRY: ${{ vars.CONTAINER_REGISTRY }}
-  WEB_IMAGE_REPO: ${{ vars.CONTAINER_REGISTRY }}/${{ github.repository_owner }}/${{ github.event.repository.name }}-web
+  REGISTRY: ${{ vars.REGISTRY }}
+  WEB_IMAGE_REPO: ${{ vars.IMAGE_REPO }}-web
+  INGESTION_IMAGE_REPO: ${{ vars.IMAGE_REPO }}-ingestion
   RELEASE_ID: ${{ github.sha }}-${{ github.run_number }}
   NPM_CONFIG_AUDIT: "false"
   NPM_CONFIG_FUND: "false"
@@ -63,38 +65,55 @@ jobs:
             org.opencontainers.image.revision=${{ github.sha }}
             org.opencontainers.image.version=${{ env.RELEASE_ID }}
 
+      - name: Build and push ingestion image
+        id: build-ingestion
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ingestion/Dockerfile
+          push: true
+          tags: ${{ env.INGESTION_IMAGE_REPO }}:${{ github.sha }}
+          labels: |
+            org.opencontainers.image.revision=${{ github.sha }}
+            org.opencontainers.image.version=${{ env.RELEASE_ID }}
+
       - name: Production pre-deploy checks
         env:
           ENVIRONMENT: production
-          STACK_NAME: ${{ vars.PRODUCTION_STACK_NAME }}
+          STACK_NAME: ${{ vars.STACK_NAME }}
           RELEASE_ID: ${{ env.RELEASE_ID }}
           COMMIT_SHA: ${{ github.sha }}
           WEB_IMAGE: ${{ env.WEB_IMAGE_REPO }}@${{ steps.build-web.outputs.digest }}
-          PUBLIC_HOST: ${{ vars.PRODUCTION_PUBLIC_HOST }}
-          TRAEFIK_NETWORK: ${{ vars.PRODUCTION_TRAEFIK_NETWORK }}
-          SWARM_MANAGER_HOST: ${{ secrets.PRODUCTION_SWARM_MANAGER_HOST }}
-          SWARM_SSH_USER: ${{ secrets.PRODUCTION_SWARM_SSH_USER }}
+          INGESTION_IMAGE: ${{ env.INGESTION_IMAGE_REPO }}@${{ steps.build-ingestion.outputs.digest }}
+          POSTGRES_PASSWORD: ${{ secrets.INGESTION_POSTGRES_PASSWORD }}
+          INGESTION_ADMIN_PASSWORD: ${{ secrets.INGESTION_ADMIN_PASSWORD }}
+          PUBLIC_HOST: ${{ vars.DOMAIN }}
+          TRAEFIK_NETWORK: ${{ vars.TRAEFIK_NETWORK }}
+          SWARM_MANAGER_HOST: ${{ secrets.SWARM_MANAGER_HOST }}
+          SWARM_SSH_USER: ${{ secrets.SWARM_SSH_USER }}
         run: node deploy/scripts/predeploy-check.mjs
 
       - name: Deploy production
         env:
           ENVIRONMENT: production
-          STACK_NAME: ${{ vars.PRODUCTION_STACK_NAME }}
+          STACK_NAME: ${{ vars.STACK_NAME }}
           RELEASE_ID: ${{ env.RELEASE_ID }}
           COMMIT_SHA: ${{ github.sha }}
           WEB_IMAGE: ${{ env.WEB_IMAGE_REPO }}@${{ steps.build-web.outputs.digest }}
-          PUBLIC_HOST: ${{ vars.PRODUCTION_PUBLIC_HOST }}
-          TRAEFIK_NETWORK: ${{ vars.PRODUCTION_TRAEFIK_NETWORK }}
-          SWARM_MANAGER_HOST: ${{ secrets.PRODUCTION_SWARM_MANAGER_HOST }}
-          SWARM_SSH_USER: ${{ secrets.PRODUCTION_SWARM_SSH_USER }}
-          SWARM_SSH_PORT: ${{ secrets.PRODUCTION_SWARM_SSH_PORT }}
+          INGESTION_IMAGE: ${{ env.INGESTION_IMAGE_REPO }}@${{ steps.build-ingestion.outputs.digest }}
+          PUBLIC_HOST: ${{ vars.DOMAIN }}
+          TRAEFIK_NETWORK: ${{ vars.TRAEFIK_NETWORK }}
+          SWARM_MANAGER_HOST: ${{ secrets.SWARM_MANAGER_HOST }}
+          SWARM_SSH_USER: ${{ secrets.SWARM_SSH_USER }}
+          SWARM_SSH_PORT: ${{ secrets.SWARM_SSH_PORT }}
           REGISTRY: ${{ env.REGISTRY }}
           REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
           REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
-          WEB_REPLICAS: ${{ vars.PRODUCTION_WEB_REPLICAS }}
-          WEB_PLACEMENT_CONSTRAINT: ${{ vars.PRODUCTION_WEB_PLACEMENT_CONSTRAINT }}
-          TRAEFIK_ENTRYPOINTS: ${{ vars.TRAEFIK_ENTRYPOINTS }}
+          WEB_REPLICAS: ${{ vars.WEB_REPLICAS }}
+          WEB_PLACEMENT_CONSTRAINT: ${{ vars.WEB_PLACEMENT_CONSTRAINT }}
+          TRAEFIK_ENTRYPOINTS: ${{ vars.TRAEFIK_ENTRYPOINT }}
           TRAEFIK_TLS: ${{ vars.TRAEFIK_TLS }}
+          TRAEFIK_CERTRESOLVER: ${{ vars.TRAEFIK_CERTRESOLVER }}
           REGION: ${{ vars.REGION }}
           TEAM_ID: ${{ vars.TEAM_ID }}
           DEPLOY_MIN_FREE_MB: ${{ vars.DEPLOY_MIN_FREE_MB }}
@@ -103,13 +122,29 @@ jobs:
           DEPLOY_CLEANUP_FAILED_IMAGES: ${{ vars.DEPLOY_CLEANUP_FAILED_IMAGES }}
           VIDEOREG_DB_MEMORY_TTL_MS: ${{ vars.VIDEOREG_DB_MEMORY_TTL_MS }}
           VIDEOREG_NEXT_CACHE_REVALIDATE_SECONDS: ${{ vars.VIDEOREG_NEXT_CACHE_REVALIDATE_SECONDS }}
+          POSTGRES_DB: ${{ vars.INGESTION_POSTGRES_DB }}
+          POSTGRES_USER: ${{ vars.INGESTION_POSTGRES_USER }}
+          POSTGRES_PASSWORD: ${{ secrets.INGESTION_POSTGRES_PASSWORD }}
+          INGESTION_ADMIN_USERNAME: ${{ vars.INGESTION_ADMIN_USERNAME }}
+          INGESTION_ADMIN_PASSWORD: ${{ secrets.INGESTION_ADMIN_PASSWORD }}
+          YANDEX_MARKET_DASHCAM_CATEGORY_URL: ${{ vars.YANDEX_MARKET_DASHCAM_CATEGORY_URL }}
+          MVIDEO_DASHCAM_CATEGORY_URL: ${{ vars.MVIDEO_DASHCAM_CATEGORY_URL }}
+          INGESTION_CATEGORY_DAILY_BUDGET: ${{ vars.INGESTION_CATEGORY_DAILY_BUDGET }}
+          INGESTION_PRODUCT_DAILY_BUDGET: ${{ vars.INGESTION_PRODUCT_DAILY_BUDGET }}
+          INGESTION_SITEMAP_DAILY_BUDGET: ${{ vars.INGESTION_SITEMAP_DAILY_BUDGET }}
+          INGESTION_FETCH_DELAY_MIN_MS: ${{ vars.INGESTION_FETCH_DELAY_MIN_MS }}
+          INGESTION_FETCH_DELAY_MAX_MS: ${{ vars.INGESTION_FETCH_DELAY_MAX_MS }}
+          INGESTION_WORKER_BATCH_SIZE: ${{ vars.INGESTION_WORKER_BATCH_SIZE }}
+          INGESTION_WORKER_IDLE_MS: ${{ vars.INGESTION_WORKER_IDLE_MS }}
+          INGESTION_WORKER_REPLICAS: ${{ vars.INGESTION_WORKER_REPLICAS }}
+          TRAEFIK_ADMIN_PRIORITY: ${{ vars.TRAEFIK_ADMIN_PRIORITY }}
         run: deploy/scripts/deploy-over-ssh.sh
 
       - name: Production smoke checks
         id: smoke
         continue-on-error: true
         env:
-          BASE_URL: https://${{ vars.PRODUCTION_PUBLIC_HOST }}
+          BASE_URL: https://${{ vars.DOMAIN }}
           EXPECTED_RELEASE_ID: ${{ env.RELEASE_ID }}
           SMOKE_RESULT_PATH: smoke-result.json
         run: node deploy/scripts/smoke-check.mjs
@@ -118,10 +153,10 @@ jobs:
         if: steps.smoke.outcome == 'failure'
         env:
           ENVIRONMENT: production
-          STACK_NAME: ${{ vars.PRODUCTION_STACK_NAME }}
-          SWARM_MANAGER_HOST: ${{ secrets.PRODUCTION_SWARM_MANAGER_HOST }}
-          SWARM_SSH_USER: ${{ secrets.PRODUCTION_SWARM_SSH_USER }}
-          SWARM_SSH_PORT: ${{ secrets.PRODUCTION_SWARM_SSH_PORT }}
+          STACK_NAME: ${{ vars.STACK_NAME }}
+          SWARM_MANAGER_HOST: ${{ secrets.SWARM_MANAGER_HOST }}
+          SWARM_SSH_USER: ${{ secrets.SWARM_SSH_USER }}
+          SWARM_SSH_PORT: ${{ secrets.SWARM_SSH_PORT }}
         run: deploy/scripts/rollback-over-ssh.sh
 
       - name: Fail after production smoke rollback
@@ -133,6 +168,7 @@ jobs:
           COMMIT_SHA: ${{ github.sha }}
           RELEASE_ID: ${{ env.RELEASE_ID }}
           WEB_IMAGE_DIGEST: ${{ env.WEB_IMAGE_REPO }}@${{ steps.build-web.outputs.digest }}
+          INGESTION_IMAGE_DIGEST: ${{ env.INGESTION_IMAGE_REPO }}@${{ steps.build-ingestion.outputs.digest }}
           SMOKE_RESULT_PATH: smoke-result.json
           WORKFLOW_RUN_ID: ${{ github.run_id }}
         run: node deploy/scripts/write-release-manifest.mjs > release-manifest.json

DEMO.md

@@ -0,0 +1,71 @@
+# Demo runbook
+
+The demo uses local PostgreSQL plus the same shared export contract as
+production. It never writes source prices, sellers, availability, or reviews to
+the public catalog.
+
+## 1. Start local ingestion
+
+```bash
+cp ingestion/.env.example ingestion/.env
+```
+
+Set real local passwords in `ingestion/.env`, then use five product requests per
+source for the demonstration:
+
+```bash
+INGESTION_PRODUCT_DAILY_BUDGET=5 \
+INGESTION_FETCH_DELAY_MIN_MS=5000 \
+INGESTION_FETCH_DELAY_MAX_MS=5000 \
+docker compose --env-file ingestion/.env -f ingestion/docker-compose.yml up -d --build
+```
+
+Follow the worker until the category and product jobs settle:
+
+```bash
+docker compose --env-file ingestion/.env -f ingestion/docker-compose.yml logs -f ingestion-worker
+```
+
+For a five-card demo, the worker fetches five product cards and marks the rest
+of the discovered category candidates as `rate_limited` for that budget day.
+If MVideo is configured and replies with a CAPTCHA/403/429, it remains paused
+and visible in the queue; do not retry it through a browser or proxy.
+
+## 2. Moderate and export
+
+Open `http://127.0.0.1:4101/admin`, authenticate with the values from
+`ingestion/.env`, choose **Products**, inspect the source URLs, raw technical
+characteristics, normalized values, and any conflicts. Edit the canonical model
+or canonical name where needed, approve the cards intended for the demo, then
+publish the approved export:
+
+```bash
+docker compose --env-file ingestion/.env -f ingestion/docker-compose.yml run --rm ingestion-worker \
+  npm run ingestion:export-catalog
+```
+
+The artifacts appear under `data/catalog-exports/`:
+
+- `manifest.json`
+- `artifacts/<exportId>/catalog.json`
+
+## 3. Start the web catalog
+
+```bash
+DASHCAM_CATALOG_EXPORT_DIR="$PWD/data/catalog-exports" npm run dev
+```
+
+Open `http://localhost:3000`. The root page reads and validates the shared
+catalog at request time, displays approved cards, normalized specs, and links
+back to every source card.
+
+## 4. Final checks
+
+```bash
+npm test
+npm run typecheck
+npm run build
+curl -fsS http://localhost:3000/health/ready
+curl -u "$INGESTION_ADMIN_USERNAME:$INGESTION_ADMIN_PASSWORD" \
+  http://127.0.0.1:4101/admin/api/queue
+```

README.md

@@ -8,7 +8,7 @@ The Next.js app lives under `web-front/`. Ingestion code lives under `ingestion/
 
 ```bash
 npm install
-npm run dev
+DASHCAM_CATALOG_EXPORT_DIR="$PWD/data/catalog-exports" npm run dev
 ```
 
 The app runs on [http://localhost:3000](http://localhost:3000).
@@ -29,11 +29,75 @@ Reads are intentionally cached aggressively:
 
 Writes call `revalidateTag` and update the process cache.
 
+### Dashcam catalog ingestion
+
+The catalog ingestion services run in the **same cloud** as the Next.js app
+(one Swarm/Portainer environment), not on a separate server:
+
+```bash
+docker compose -f ingestion/docker-compose.yml up --build
+```
+
+It starts PostgreSQL, an HTTP-only ingestion worker, and a minimal admin server on
+`http://localhost:4101`. The worker writes the validated `manifest.json` +
+`catalog.json` to a shared export volume; there is no HTTP export server and no
+export token — the app reads the artifacts directly from that volume.
+
+The default Yandex Market seed category is:
+
+```text
+https://market.yandex.ru/catalog--avtomobilnye-videoregistratory/82798275/list?hid=82798269
+```
+
+Seed env vars are source-specific: `YANDEX_MARKET_DASHCAM_CATEGORY_URL` is active
+today. `MVIDEO_DASHCAM_CATEGORY_URL` enables the MVideo discovery/parser path
+when configured.
+
+Crawler policy is intentionally conservative: no browser fallback in the regular
+path, no stealth/proxy behavior, PostgreSQL-backed queue and daily budgets, raw
+snapshots stored on the ingestion volume, and source pause on captcha/403/429.
+
+To import an approved export into the app read-model cache, point the import at
+the shared export volume:
+
+```bash
+DASHCAM_CATALOG_EXPORT_DIR=/path/to/ingestion/data/exports \
+npm run catalog:import
+```
+
+The import command reads `manifest.json` + `catalog.json` from that directory,
+validates them with Zod, verifies the artifact `sha256`, checks
+`exportId`/`createdAt`/`itemCount`, and atomically updates `data/videoreg.json`.
+No HTTP call and no token are involved.
+
+The ingestion admin server exposes its own operator UI at
+`http://127.0.0.1:4101/admin`. It is served by `ingestion-admin`, requires Basic
+Auth, and is not part of the Next.js app. In production Traefik should route
+`/admin` to `ingestion-admin` and `/` to Next.js, without stripping the `/admin`
+prefix.
+
+For the production setup where ingestion is co-located with the app and shares
+the export volume, see [`ingestion/README.md`](ingestion/README.md).
+For Portainer stacks, see [`deploy/portainer/README.md`](deploy/portainer/README.md).
+For the local end-to-end demonstration, see [`DEMO.md`](DEMO.md).
+
 ### Deployment
 
 Gitea Actions runs:
 
 - `.gitea/workflows/pr.yml`: quality gate for pull requests into `master`;
-- `.gitea/workflows/master.yml`: quality gate, image build, push, and direct Swarm deploy from `master`.
+- `.gitea/workflows/master.yml`: quality gate, build/push of web and ingestion images, and direct co-located Swarm deploy from `master`.
+
+The production workflow uses the runner's existing Gitea configuration names:
+
+- variables: `REGISTRY`, `IMAGE_REPO`, `DOMAIN` (`videoreg.ru`), `STACK_NAME`,
+  `TRAEFIK_NETWORK`, `TRAEFIK_ENTRYPOINT`, and `TRAEFIK_CERTRESOLVER`;
+- secrets: `REGISTRY_USERNAME`, `REGISTRY_PASSWORD`, `SWARM_MANAGER_HOST`,
+  `SWARM_SSH_USER`, `SWARM_SSH_PORT`, `INGESTION_POSTGRES_PASSWORD`, and
+  `INGESTION_ADMIN_PASSWORD`.
+
+Images are published as `${IMAGE_REPO}-web` and `${IMAGE_REPO}-ingestion`, then
+the Swarm manager pulls their immutable digests. Both `/` and `/admin` receive
+the configured TLS certificate through Traefik.
 
 The Swarm stack defaults to one replica on a manager node because the primitive database is a mounted JSON file. Use a real database before raising replicas above one or spreading the service across nodes.

deploy/portainer/README.md

@@ -0,0 +1,36 @@
+# Portainer / Swarm deployment
+
+`web`, PostgreSQL, `ingestion-worker`, and `ingestion-admin` are one stack. Do
+not deploy ingestion as a separate endpoint: the approved catalog is exchanged
+through the `catalog_exports` volume mounted read-only by web and read-write by
+the worker.
+
+In Portainer, create a Swarm stack from `deploy/swarm/docker-stack.yml`.
+Configure these values as stack environment variables:
+
+- `WEB_IMAGE` - digest-pinned web image.
+- `INGESTION_IMAGE` - digest-pinned ingestion image.
+- `POSTGRES_PASSWORD` - database secret.
+- `INGESTION_ADMIN_PASSWORD` - Basic Auth password for `/admin`.
+- `PUBLIC_HOST=videoreg.ru` and `TRAEFIK_NETWORK`.
+- Optional source/crawl settings: `MVIDEO_DASHCAM_CATEGORY_URL`, daily budgets,
+  and delay bounds.
+
+Keep every service on the same manager/placement constraint. The database,
+snapshots, read model, and shared catalog volume are local Docker volumes and
+must be backed up from that node.
+
+Gitea Actions uses `.gitea/workflows/master.yml`: it tests the repository,
+builds both images, pushes immutable digests, and deploys this one stack. The
+workflow connects to the Swarm manager, pulls those digests, and runs `docker
+stack deploy --with-registry-auth`; Portainer then shows and manages the same
+updated stack. Portainer does not need registry credentials or a separate
+webhook-based image build.
+
+Verification after deploy:
+
+```bash
+curl -fsS https://videoreg.ru/health/ready
+curl -u "$INGESTION_ADMIN_USERNAME:$INGESTION_ADMIN_PASSWORD" \
+  https://videoreg.ru/admin/api/queue
+```

deploy/remote/deploy-stack.sh

@@ -19,6 +19,7 @@ require_var STACK_NAME
 require_var RELEASE_ID
 require_var COMMIT_SHA
 require_var WEB_IMAGE
+require_var INGESTION_IMAGE
 require_var PUBLIC_HOST
 require_var TRAEFIK_NETWORK
 
@@ -37,7 +38,7 @@ cleanup_release_images() {
   fi
 
   echo "Cleaning up unused release image after failed deploy." >&2
-  docker image rm "$WEB_IMAGE" >/dev/null 2>&1 || true
+  docker image rm "$WEB_IMAGE" "$INGESTION_IMAGE" >/dev/null 2>&1 || true
 
   if [ -n "${COMMIT_SHA:-}" ]; then
     docker image prune -af --filter "label=org.opencontainers.image.revision=${COMMIT_SHA}" >&2 || true
@@ -160,15 +161,32 @@ prune_docker_before_pull
 check_docker_disk_space
 
 docker image pull "$WEB_IMAGE"
+docker image pull "$INGESTION_IMAGE"
 
-export ENVIRONMENT STACK_NAME RELEASE_ID COMMIT_SHA WEB_IMAGE PUBLIC_HOST TRAEFIK_NETWORK
+export ENVIRONMENT STACK_NAME RELEASE_ID COMMIT_SHA WEB_IMAGE INGESTION_IMAGE PUBLIC_HOST TRAEFIK_NETWORK
 export REGION="${REGION:-}" TEAM_ID="${TEAM_ID:-}" WEB_REPLICAS="${WEB_REPLICAS:-1}"
 export WEB_PLACEMENT_CONSTRAINT="${WEB_PLACEMENT_CONSTRAINT:-node.role == manager}"
 export TRAEFIK_ENTRYPOINTS="${TRAEFIK_ENTRYPOINTS:-websecure}" TRAEFIK_TLS="${TRAEFIK_TLS:-true}"
+export TRAEFIK_CERTRESOLVER="${TRAEFIK_CERTRESOLVER:-le}"
+export TRAEFIK_ADMIN_PRIORITY="${TRAEFIK_ADMIN_PRIORITY:-100}"
 export VIDEOREG_DB_MEMORY_TTL_MS="${VIDEOREG_DB_MEMORY_TTL_MS:-3600000}"
 export VIDEOREG_NEXT_CACHE_REVALIDATE_SECONDS="${VIDEOREG_NEXT_CACHE_REVALIDATE_SECONDS:-3600}"
+export POSTGRES_DB="${POSTGRES_DB:-videoreg_ingestion}" POSTGRES_USER="${POSTGRES_USER:-videoreg}"
+export INGESTION_ADMIN_USERNAME="${INGESTION_ADMIN_USERNAME:-admin}"
+export YANDEX_MARKET_DASHCAM_CATEGORY_URL="${YANDEX_MARKET_DASHCAM_CATEGORY_URL:-https://market.yandex.ru/catalog--avtomobilnye-videoregistratory/82798275/list?hid=82798269}"
+export MVIDEO_DASHCAM_CATEGORY_URL="${MVIDEO_DASHCAM_CATEGORY_URL:-}"
+export INGESTION_CATEGORY_DAILY_BUDGET="${INGESTION_CATEGORY_DAILY_BUDGET:-25}"
+export INGESTION_PRODUCT_DAILY_BUDGET="${INGESTION_PRODUCT_DAILY_BUDGET:-100}"
+export INGESTION_SITEMAP_DAILY_BUDGET="${INGESTION_SITEMAP_DAILY_BUDGET:-25}"
+export INGESTION_FETCH_DELAY_MIN_MS="${INGESTION_FETCH_DELAY_MIN_MS:-5000}"
+export INGESTION_FETCH_DELAY_MAX_MS="${INGESTION_FETCH_DELAY_MAX_MS:-15000}"
+export INGESTION_WORKER_BATCH_SIZE="${INGESTION_WORKER_BATCH_SIZE:-1}"
+export INGESTION_WORKER_IDLE_MS="${INGESTION_WORKER_IDLE_MS:-30000}"
+export INGESTION_WORKER_REPLICAS="${INGESTION_WORKER_REPLICAS:-1}"
 
 docker stack deploy --with-registry-auth --prune -c "$REMOTE_DIR/docker-stack.yml" "$STACK_NAME"
 wait_service "${STACK_NAME}_web"
+wait_service "${STACK_NAME}_ingestion-worker"
+wait_service "${STACK_NAME}_ingestion-admin"
 
 log deploy_converged

deploy/scripts/deploy-over-ssh.sh

@@ -20,6 +20,7 @@ require_var STACK_NAME
 require_var RELEASE_ID
 require_var COMMIT_SHA
 require_var WEB_IMAGE
+require_var INGESTION_IMAGE
 require_var PUBLIC_HOST
 require_var TRAEFIK_NETWORK
 
@@ -32,10 +33,15 @@ scp -P "$SSH_PORT" deploy/swarm/docker-stack.yml "$SSH_TARGET:$REMOTE_DEPLOY_DIR
 scp -P "$SSH_PORT" deploy/remote/deploy-stack.sh "$SSH_TARGET:$REMOTE_DEPLOY_DIR/deploy-stack.sh"
 
 remote_vars=(
-  ENVIRONMENT STACK_NAME RELEASE_ID COMMIT_SHA WEB_IMAGE PUBLIC_HOST TRAEFIK_NETWORK
-  REGION TEAM_ID WEB_REPLICAS WEB_PLACEMENT_CONSTRAINT TRAEFIK_ENTRYPOINTS TRAEFIK_TLS REGISTRY REGISTRY_USERNAME REGISTRY_PASSWORD
+  ENVIRONMENT STACK_NAME RELEASE_ID COMMIT_SHA WEB_IMAGE INGESTION_IMAGE PUBLIC_HOST TRAEFIK_NETWORK
+  REGION TEAM_ID WEB_REPLICAS WEB_PLACEMENT_CONSTRAINT TRAEFIK_ENTRYPOINTS TRAEFIK_TLS TRAEFIK_CERTRESOLVER REGISTRY REGISTRY_USERNAME REGISTRY_PASSWORD
   RELEASE_LOCK_FILE REMOTE_DEPLOY_DIR DEPLOY_MIN_FREE_MB DEPLOY_PRUNE_BEFORE_PULL DEPLOY_PRUNE_UNTIL
   DEPLOY_CLEANUP_FAILED_IMAGES VIDEOREG_DB_MEMORY_TTL_MS VIDEOREG_NEXT_CACHE_REVALIDATE_SECONDS
+  POSTGRES_DB POSTGRES_USER POSTGRES_PASSWORD INGESTION_ADMIN_USERNAME INGESTION_ADMIN_PASSWORD
+  YANDEX_MARKET_DASHCAM_CATEGORY_URL MVIDEO_DASHCAM_CATEGORY_URL INGESTION_CATEGORY_DAILY_BUDGET
+  INGESTION_PRODUCT_DAILY_BUDGET INGESTION_SITEMAP_DAILY_BUDGET INGESTION_FETCH_DELAY_MIN_MS
+  INGESTION_FETCH_DELAY_MAX_MS INGESTION_WORKER_BATCH_SIZE INGESTION_WORKER_IDLE_MS INGESTION_WORKER_REPLICAS
+  TRAEFIK_ADMIN_PRIORITY
 )
 
 remote_command=""

deploy/scripts/predeploy-check.mjs

@@ -4,6 +4,9 @@ const required = [
   'RELEASE_ID',
   'COMMIT_SHA',
   'WEB_IMAGE',
+  'INGESTION_IMAGE',
+  'POSTGRES_PASSWORD',
+  'INGESTION_ADMIN_PASSWORD',
   'PUBLIC_HOST',
   'TRAEFIK_NETWORK',
   'SWARM_MANAGER_HOST',
@@ -16,9 +19,11 @@ if (missing.length > 0) {
   process.exit(1);
 }
 
-if (!/@sha256:[a-f0-9]{64}$/i.test(process.env.WEB_IMAGE ?? '')) {
-  console.error(`WEB_IMAGE must be pinned by immutable digest, got: ${process.env.WEB_IMAGE}`);
-  process.exit(1);
+for (const imageName of ['WEB_IMAGE', 'INGESTION_IMAGE']) {
+  if (!/@sha256:[a-f0-9]{64}$/i.test(process.env[imageName] ?? '')) {
+    console.error(`${imageName} must be pinned by immutable digest, got: ${process.env[imageName]}`);
+    process.exit(1);
+  }
 }
 
 console.log(

deploy/scripts/write-release-manifest.mjs

@@ -18,13 +18,19 @@ const manifest = {
   commitSha: requireEnv('COMMIT_SHA'),
   releaseId: requireEnv('RELEASE_ID'),
   webImageDigest: requireEnv('WEB_IMAGE_DIGEST'),
+  ingestionImageDigest: requireEnv('INGESTION_IMAGE_DIGEST'),
   deployTimestamp: new Date().toISOString(),
   smokeResult: smoke,
   workflowRunId: requireEnv('WORKFLOW_RUN_ID'),
 };
 
-if (!/@sha256:[a-f0-9]{64}$/i.test(manifest.webImageDigest)) {
-  throw new Error('webImageDigest must be digest-pinned');
+for (const [name, image] of Object.entries({
+  webImageDigest: manifest.webImageDigest,
+  ingestionImageDigest: manifest.ingestionImageDigest,
+})) {
+  if (!/@sha256:[a-f0-9]{64}$/i.test(image)) {
+    throw new Error(`${name} must be digest-pinned`);
+  }
 }
 
 console.log(JSON.stringify(manifest, null, 2));

deploy/swarm/docker-stack.yml

@@ -1,13 +1,47 @@
 version: "3.9"
 
+x-default-deploy: &default-deploy
+  placement:
+    constraints:
+      - ${WEB_PLACEMENT_CONSTRAINT:-node.role == manager}
+  update_config:
+    order: start-first
+    parallelism: 1
+    failure_action: rollback
+    monitor: 45s
+  rollback_config:
+    order: start-first
+    parallelism: 1
+    monitor: 45s
+  restart_policy:
+    condition: on-failure
+    delay: 5s
+    max_attempts: 5
+    window: 60s
+
+x-ingestion-env: &ingestion-env
+  INGESTION_DATABASE_URL: postgres://${POSTGRES_USER:-videoreg}:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required}@postgres:5432/${POSTGRES_DB:-videoreg_ingestion}
+  INGESTION_SNAPSHOT_DIR: /app/ingestion/data/snapshots
+  INGESTION_EXPORT_DIR: /exports
+  YANDEX_MARKET_DASHCAM_CATEGORY_URL: ${YANDEX_MARKET_DASHCAM_CATEGORY_URL:-https://market.yandex.ru/catalog--avtomobilnye-videoregistratory/82798275/list?hid=82798269}
+  MVIDEO_DASHCAM_CATEGORY_URL: ${MVIDEO_DASHCAM_CATEGORY_URL:-}
+  INGESTION_CATEGORY_DAILY_BUDGET: ${INGESTION_CATEGORY_DAILY_BUDGET:-25}
+  INGESTION_PRODUCT_DAILY_BUDGET: ${INGESTION_PRODUCT_DAILY_BUDGET:-100}
+  INGESTION_SITEMAP_DAILY_BUDGET: ${INGESTION_SITEMAP_DAILY_BUDGET:-25}
+  INGESTION_FETCH_DELAY_MIN_MS: ${INGESTION_FETCH_DELAY_MIN_MS:-5000}
+  INGESTION_FETCH_DELAY_MAX_MS: ${INGESTION_FETCH_DELAY_MAX_MS:-15000}
+  INGESTION_WORKER_BATCH_SIZE: ${INGESTION_WORKER_BATCH_SIZE:-1}
+  INGESTION_WORKER_IDLE_MS: ${INGESTION_WORKER_IDLE_MS:-30000}
+
 services:
   web:
-    image: ${WEB_IMAGE}
+    image: ${WEB_IMAGE:?WEB_IMAGE is required}
     environment:
       NODE_ENV: production
       APP_ENV: ${ENVIRONMENT}
       PORT: "3000"
       VIDEOREG_DB_FILE: /data/videoreg.json
+      DASHCAM_CATALOG_EXPORT_DIR: /exports
       VIDEOREG_DB_MEMORY_TTL_MS: ${VIDEOREG_DB_MEMORY_TTL_MS:-3600000}
       VIDEOREG_NEXT_CACHE_REVALIDATE_SECONDS: ${VIDEOREG_NEXT_CACHE_REVALIDATE_SECONDS:-3600}
       COMMIT_SHA: ${COMMIT_SHA}
@@ -16,6 +50,7 @@ services:
       TEAM_ID: ${TEAM_ID:-}
     volumes:
       - videoreg_data:/data
+      - catalog_exports:/exports:ro
     networks:
       traefik: {}
     healthcheck:
@@ -25,34 +60,90 @@ services:
       retries: 3
       start_period: 20s
     deploy:
+      <<: *default-deploy
       replicas: ${WEB_REPLICAS:-1}
-      placement:
-        constraints:
-          - ${WEB_PLACEMENT_CONSTRAINT:-node.role == manager}
-      update_config:
-        order: start-first
-        parallelism: 1
-        failure_action: rollback
-        monitor: 45s
-      rollback_config:
-        order: start-first
-        parallelism: 1
-        monitor: 45s
-      restart_policy:
-        condition: on-failure
       labels:
         - "traefik.enable=true"
         - "traefik.docker.network=${TRAEFIK_NETWORK}"
         - "traefik.http.routers.${STACK_NAME}-web.rule=Host(`${PUBLIC_HOST}`) && PathPrefix(`/`)"
+        - "traefik.http.routers.${STACK_NAME}-web.priority=${TRAEFIK_WEB_PRIORITY:-1}"
         - "traefik.http.routers.${STACK_NAME}-web.entrypoints=${TRAEFIK_ENTRYPOINTS:-websecure}"
         - "traefik.http.routers.${STACK_NAME}-web.tls=${TRAEFIK_TLS:-true}"
+        - "traefik.http.routers.${STACK_NAME}-web.tls.certresolver=${TRAEFIK_CERTRESOLVER:-le}"
         - "traefik.http.services.${STACK_NAME}-web.loadbalancer.server.port=3000"
     stop_grace_period: 30s
 
+  postgres:
+    image: postgres:16-alpine
+    environment:
+      POSTGRES_DB: ${POSTGRES_DB:-videoreg_ingestion}
+      POSTGRES_USER: ${POSTGRES_USER:-videoreg}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required}
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-videoreg} -d ${POSTGRES_DB:-videoreg_ingestion}"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    volumes:
+      - ingestion_postgres:/var/lib/postgresql/data
+    networks:
+      - ingestion
+    deploy: *default-deploy
+
+  ingestion-worker:
+    image: ${INGESTION_IMAGE:?INGESTION_IMAGE is required}
+    command: sh -c "npm run ingestion:migrate && npm run ingestion:worker"
+    environment: *ingestion-env
+    volumes:
+      - ingestion_data:/app/ingestion/data
+      - catalog_exports:/exports
+    networks:
+      - ingestion
+    deploy:
+      <<: *default-deploy
+      replicas: ${INGESTION_WORKER_REPLICAS:-1}
+
+  ingestion-admin:
+    image: ${INGESTION_IMAGE:?INGESTION_IMAGE is required}
+    command: sh -c "npm run ingestion:migrate && npm run ingestion:admin"
+    environment:
+      <<: *ingestion-env
+      INGESTION_ADMIN_PORT: "4101"
+      INGESTION_ADMIN_USERNAME: ${INGESTION_ADMIN_USERNAME:-admin}
+      INGESTION_ADMIN_PASSWORD: ${INGESTION_ADMIN_PASSWORD:?INGESTION_ADMIN_PASSWORD is required}
+    volumes:
+      - ingestion_data:/app/ingestion/data
+    networks:
+      ingestion: {}
+      traefik: {}
+    healthcheck:
+      test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:4101/health >/dev/null || exit 1"]
+      interval: 15s
+      timeout: 3s
+      retries: 5
+      start_period: 20s
+    deploy:
+      <<: *default-deploy
+      replicas: 1
+      labels:
+        - "traefik.enable=true"
+        - "traefik.docker.network=${TRAEFIK_NETWORK}"
+        - "traefik.http.routers.${STACK_NAME}-admin.rule=Host(`${PUBLIC_HOST}`) && PathPrefix(`/admin`)"
+        - "traefik.http.routers.${STACK_NAME}-admin.priority=${TRAEFIK_ADMIN_PRIORITY:-100}"
+        - "traefik.http.routers.${STACK_NAME}-admin.entrypoints=${TRAEFIK_ENTRYPOINTS:-websecure}"
+        - "traefik.http.routers.${STACK_NAME}-admin.tls=${TRAEFIK_TLS:-true}"
+        - "traefik.http.routers.${STACK_NAME}-admin.tls.certresolver=${TRAEFIK_CERTRESOLVER:-le}"
+        - "traefik.http.services.${STACK_NAME}-admin.loadbalancer.server.port=4101"
+
 volumes:
   videoreg_data:
+  catalog_exports:
+  ingestion_postgres:
+  ingestion_data:
 
 networks:
+  ingestion:
+    driver: overlay
   traefik:
     external: true
     name: ${TRAEFIK_NETWORK}

ingestion/.dockerignore

@@ -0,0 +1,5 @@
+../.next
+../node_modules
+../data
+data
+coverage

ingestion/.env.example

@@ -0,0 +1,25 @@
+POSTGRES_DB=videoreg_ingestion
+POSTGRES_USER=videoreg
+POSTGRES_PASSWORD=change-me
+
+# Bind local operational ports to loopback.
+INGESTION_POSTGRES_BIND=127.0.0.1
+INGESTION_POSTGRES_PORT=5432
+INGESTION_ADMIN_BIND=127.0.0.1
+INGESTION_ADMIN_PORT=4101
+INGESTION_ADMIN_USERNAME=admin
+INGESTION_ADMIN_PASSWORD=replace-with-admin-password
+
+# Source seeds. MVideo is active when this URL is configured.
+YANDEX_MARKET_DASHCAM_CATEGORY_URL=https://market.yandex.ru/catalog--avtomobilnye-videoregistratory/82798275/list?hid=82798269
+MVIDEO_DASHCAM_CATEGORY_URL=https://www.mvideo.ru/product-list-page?q=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE%D1%80%D0%B5%D0%B3%D0%B8%D1%81%D1%82%D1%80%D0%B0%D1%82%D0%BE%D1%80%D1%8B
+
+# Conservative crawl limits. Local Compose maps the shared exports directory to
+# ../data/catalog-exports; the Next.js app reads that same directory directly.
+INGESTION_CATEGORY_DAILY_BUDGET=25
+INGESTION_PRODUCT_DAILY_BUDGET=100
+INGESTION_SITEMAP_DAILY_BUDGET=25
+INGESTION_FETCH_DELAY_MIN_MS=5000
+INGESTION_FETCH_DELAY_MAX_MS=15000
+INGESTION_WORKER_BATCH_SIZE=1
+INGESTION_WORKER_IDLE_MS=30000

ingestion/Dockerfile

@@ -0,0 +1,12 @@
+FROM node:24-alpine
+
+WORKDIR /app
+
+COPY package.json package-lock.json ./
+RUN npm ci
+
+COPY tsconfig.json ./
+COPY web-front ./web-front
+COPY ingestion ./ingestion
+
+CMD ["npm", "run", "ingestion:worker"]

ingestion/README.md

@@ -0,0 +1,228 @@
+# Dashcam catalog ingestion
+
+This stack runs in the same cloud as the Next.js application - one Swarm/Portainer environment, not a separate server. The ingestion side owns PostgreSQL, crawler state, raw snapshots, moderation state, and export artifacts. The `ingestion-worker` writes the approved catalog to a shared export volume, and the Next.js build/app reads those artifacts directly from that volume - no network call and no token. Integrity is guaranteed by the manifest `sha256` and a shared Zod schema validated at read/build time.
+
+> Note: co-locating ingestion and the web app on one shared export volume supersedes the earlier separate-server design (decision id `videoreg-nnv.3`).
+
+## Services
+
+`docker-compose.yml` starts:
+
+- `postgres` - source of truth for ingestion state.
+- `ingestion-worker` - HTTP-only crawler/parser/normalizer; also writes export artifacts to the shared export volume.
+- `ingestion-admin` - minimal operational endpoints and operator UI, internal/localhost only.
+
+The regular crawl path does not use a browser, stealth mode, proxy rotation, captcha bypass, or aggressive parallel loading.
+
+For Portainer-based deployment, deploy the same combined Swarm stack described
+in [`deploy/portainer`](../deploy/portainer/README.md).
+
+CI/CD builds and deploys both runtime images through `.gitea/workflows/master.yml`.
+
+## Server prerequisites
+
+- Docker with Compose v2.
+- Outbound HTTPS access to `market.yandex.ru` and, when enabled, `www.mvideo.ru`.
+- Persistent Docker volumes for `postgres-data` and `ingestion-data` plus the
+  shared catalog export directory.
+
+Do not expose Postgres or the admin server publicly. Keep them bound to `127.0.0.1` and use a private network for operator access.
+
+## First deploy in the application environment
+
+Deploy this stack in the same environment as the Next.js app. From the repository root:
+
+```bash
+cp ingestion/.env.example ingestion/.env
+```
+
+Edit `ingestion/.env`:
+
+```bash
+POSTGRES_PASSWORD=<strong-postgres-password>
+INGESTION_ADMIN_USERNAME=admin
+INGESTION_ADMIN_PASSWORD=<strong-admin-password>
+```
+
+The `ingestion-worker` writes the validated `manifest.json` and
+`artifacts/<exportId>/catalog.json` to `INGESTION_EXPORT_DIR` (`/app/exports`
+inside local Compose). Local Compose maps it to `data/catalog-exports`; the web
+app reads that directory directly.
+
+Start the stack:
+
+```bash
+docker compose --env-file ingestion/.env -f ingestion/docker-compose.yml up -d --build
+```
+
+Check status:
+
+```bash
+docker compose --env-file ingestion/.env -f ingestion/docker-compose.yml ps
+docker compose --env-file ingestion/.env -f ingestion/docker-compose.yml logs -f ingestion-worker
+curl http://127.0.0.1:4101/health
+```
+
+Open the ingestion admin UI:
+
+```text
+http://127.0.0.1:4101/admin
+```
+
+The UI is served by `ingestion-admin` itself and requires Basic Auth using
+`INGESTION_ADMIN_USERNAME` and `INGESTION_ADMIN_PASSWORD`. It is intentionally
+separate from the Next.js application and uses admin API endpoints under
+`/admin/api`.
+
+For a single public host/standard HTTPS port, configure Traefik to route by path:
+
+```text
+Host(`example.com`) && PathPrefix(`/admin`) -> ingestion-admin:4101
+Host(`example.com`) && PathPrefix(`/`)      -> Next.js:3000
+```
+
+Set a higher priority on the `/admin` router than the `/` router. Do not strip
+the `/admin` prefix; the admin app serves HTML/assets at `/admin/*` and its API
+at `/admin/api/*`.
+
+## Local smoke run without PostgreSQL
+
+For parser/debug checks you can run local HTTP-only smoke commands. They do not
+write to PostgreSQL and do not write snapshots.
+
+```bash
+npm run --silent ingestion:smoke-yandex
+npm run --silent ingestion:smoke-mvideo
+```
+
+Useful overrides:
+
+```bash
+INGESTION_SMOKE_LIMIT=5 \
+INGESTION_SMOKE_DELAY_MS=5000 \
+YANDEX_MARKET_DASHCAM_CATEGORY_URL="https://market.yandex.ru/catalog--avtomobilnye-videoregistratory/82798275/list?hid=82798269" \
+npm run --silent ingestion:smoke-yandex > /tmp/yandex-market-smoke.json
+
+INGESTION_SMOKE_LIMIT=5 \
+INGESTION_SMOKE_DELAY_MS=5000 \
+MVIDEO_DASHCAM_CATEGORY_URL="https://www.mvideo.ru/product-list-page?q=видеорегистраторы" \
+npm run --silent ingestion:smoke-mvideo > /tmp/mvideo-smoke.json
+```
+
+Configured category seed env vars:
+
+- `YANDEX_MARKET_DASHCAM_CATEGORY_URL` - active source, used by the worker and smoke run.
+- `MVIDEO_DASHCAM_CATEGORY_URL` - enables MVideo discovery and product parsing in
+  the worker. Leave it empty to disable that source.
+
+If a source returns a CAPTCHA, 403, or 429, the worker records the reason and
+pauses that source in PostgreSQL. It does not switch user agents, use a browser
+fallback, rotate proxies, or otherwise attempt to bypass the challenge. The
+queue and moderation UI keep the paused state visible to the operator.
+
+Progress goes to stderr, so stdout remains valid JSON. Each product includes:
+
+- `sources[]` with source name, URL, source product id, fetched time, title, and `stableContentHash`;
+- source card `title`;
+- `brand`, source `model`, `canonicalModel`, and `canonicalName`;
+- `technicalSpecs.normalized` for canonical keys used by matching/export;
+- `technicalSpecs.raw` for the fuller source characteristics with section groups, original labels, source values, and normalized key/value when known;
+- `moderation.conflicts`, currently empty in smoke mode because there is only one source per product;
+- `internalSignals.aggregateRating`, kept out of public export.
+
+## Operational endpoints
+
+Admin endpoints are intentionally small:
+
+```bash
+curl http://127.0.0.1:4101/health
+curl -u "$INGESTION_ADMIN_USERNAME:$INGESTION_ADMIN_PASSWORD" \
+  http://127.0.0.1:4101/admin/api/queue
+curl -u "$INGESTION_ADMIN_USERNAME:$INGESTION_ADMIN_PASSWORD" \
+  http://127.0.0.1:4101/admin/api/moderation
+curl -u "$INGESTION_ADMIN_USERNAME:$INGESTION_ADMIN_PASSWORD" \
+  http://127.0.0.1:4101/admin/api/moderation/products?status=needs_review
+curl -u "$INGESTION_ADMIN_USERNAME:$INGESTION_ADMIN_PASSWORD" \
+  http://127.0.0.1:4101/admin/api/manufacturers
+curl -u "$INGESTION_ADMIN_USERNAME:$INGESTION_ADMIN_PASSWORD" \
+  http://127.0.0.1:4101/admin/api/manufacturers/spawnson
+curl -u "$INGESTION_ADMIN_USERNAME:$INGESTION_ADMIN_PASSWORD" \
+  -X PATCH http://127.0.0.1:4101/admin/api/manufacturers/spawnson \
+  -H "content-type: application/json" \
+  -d '{"trustStatus":"untrusted","notes":"No stable manufacturer source found"}'
+```
+
+Manufacturer records are created automatically from parsed product brands. Use
+`trustStatus=untrusted` for brands that should not be trusted as authoritative
+manufacturer sources during moderation.
+
+When two sources produce the same canonical name, ingestion links their source
+records to one canonical product and queues `same_product` for review. Raw specs
+remain source-scoped; differing normalized values appear as source conflicts.
+This is an exact canonical-name match only, so variants are never merged by a
+fuzzy or image-based heuristic.
+
+The same manufacturer records can be edited in the ingestion admin UI at
+`http://127.0.0.1:4101/admin`.
+
+Product cards can be approved or rejected from the `Products` view in that UI.
+
+## Publishing an approved export
+
+The public export includes only approved canonical products. It excludes prices, sellers, availability, review texts, review authors, and aggregate rating.
+
+For the current MVP, moderation is represented in PostgreSQL. Approving a product means setting `products.moderation_status = 'approved'` and approving selected cached image artifacts in `product_images` once image storage policy is accepted. Products without approved image artifacts can still be exported with an empty `images` array.
+
+Generate the latest export:
+
+```bash
+docker compose --env-file ingestion/.env -f ingestion/docker-compose.yml run --rm ingestion-worker \
+  npm run ingestion:export-catalog
+```
+
+The job writes to the shared export volume:
+
+- `manifest.json`
+- `artifacts/<exportId>/catalog.json`
+
+These land directly on the shared export volume; nothing serves them over HTTP.
+
+## Importing into the Next.js/application
+
+The import reads the export artifacts directly from the shared export volume - no network call and no token. Point it at the mounted export directory and run:
+
+```bash
+DASHCAM_CATALOG_EXPORT_DIR=/path/to/exports npm run catalog:import
+```
+
+The import command:
+
+- reads `manifest.json` from the export directory;
+- validates `schemaVersion`, `exportId`, `createdAt`, `sha256`, and `itemCount`;
+- reads the catalog artifact from the same directory;
+- verifies the artifact SHA-256;
+- validates the payload through the shared Zod schema;
+- atomically updates `data/videoreg.json`.
+
+The Next.js application never connects to the ingestion PostgreSQL database; its only catalog contact is the export artifact on the shared volume.
+
+## Backups
+
+Back up both Docker volumes:
+
+- `postgres-data` - canonical ingestion database.
+- `ingestion-data` - raw snapshots.
+- `data/catalog-exports` locally, or `catalog_exports` in Swarm - approved
+  catalog artifacts shared with web.
+
+Losing `ingestion-data` removes audit/debug snapshots even if the database remains available.
+
+## Updating
+
+Pull the new repository state and rebuild:
+
+```bash
+docker compose --env-file ingestion/.env -f ingestion/docker-compose.yml up -d --build
+```
+
+Migrations run automatically before `ingestion-worker` and `ingestion-admin` start. They are tracked in the `schema_migrations` table.

ingestion/docker-compose.yml

@@ -0,0 +1,90 @@
+services:
+  postgres:
+    image: postgres:16-alpine
+    restart: unless-stopped
+    environment:
+      POSTGRES_DB: ${POSTGRES_DB:-videoreg_ingestion}
+      POSTGRES_USER: ${POSTGRES_USER:-videoreg}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-videoreg}
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-videoreg} -d ${POSTGRES_DB:-videoreg_ingestion}"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    ports:
+      - "${INGESTION_POSTGRES_BIND:-127.0.0.1}:${INGESTION_POSTGRES_PORT:-5432}:5432"
+    volumes:
+      - postgres-data:/var/lib/postgresql/data
+
+  ingestion-migrate:
+    build:
+      context: ..
+      dockerfile: ingestion/Dockerfile
+    command: npm run ingestion:migrate
+    restart: "no"
+    depends_on:
+      postgres:
+        condition: service_healthy
+    environment: &ingestion-env
+      INGESTION_DATABASE_URL: postgres://${POSTGRES_USER:-videoreg}:${POSTGRES_PASSWORD:-videoreg}@postgres:5432/${POSTGRES_DB:-videoreg_ingestion}
+      INGESTION_SNAPSHOT_DIR: /app/ingestion/data/snapshots
+      INGESTION_EXPORT_DIR: /app/exports
+      YANDEX_MARKET_DASHCAM_CATEGORY_URL: ${YANDEX_MARKET_DASHCAM_CATEGORY_URL:-https://market.yandex.ru/catalog--avtomobilnye-videoregistratory/82798275/list?hid=82798269}
+      MVIDEO_DASHCAM_CATEGORY_URL: ${MVIDEO_DASHCAM_CATEGORY_URL:-}
+      INGESTION_CATEGORY_DAILY_BUDGET: ${INGESTION_CATEGORY_DAILY_BUDGET:-25}
+      INGESTION_PRODUCT_DAILY_BUDGET: ${INGESTION_PRODUCT_DAILY_BUDGET:-100}
+      INGESTION_SITEMAP_DAILY_BUDGET: ${INGESTION_SITEMAP_DAILY_BUDGET:-25}
+      INGESTION_FETCH_DELAY_MIN_MS: ${INGESTION_FETCH_DELAY_MIN_MS:-5000}
+      INGESTION_FETCH_DELAY_MAX_MS: ${INGESTION_FETCH_DELAY_MAX_MS:-15000}
+      INGESTION_WORKER_BATCH_SIZE: ${INGESTION_WORKER_BATCH_SIZE:-1}
+      INGESTION_WORKER_IDLE_MS: ${INGESTION_WORKER_IDLE_MS:-30000}
+    volumes:
+      - ingestion-data:/app/ingestion/data
+      - ../data/catalog-exports:/app/exports
+
+  ingestion-worker:
+    build:
+      context: ..
+      dockerfile: ingestion/Dockerfile
+    command: npm run ingestion:worker
+    restart: unless-stopped
+    depends_on:
+      postgres:
+        condition: service_healthy
+      ingestion-migrate:
+        condition: service_completed_successfully
+    environment: *ingestion-env
+    volumes:
+      - ingestion-data:/app/ingestion/data
+      - ../data/catalog-exports:/app/exports
+
+  ingestion-admin:
+    build:
+      context: ..
+      dockerfile: ingestion/Dockerfile
+    command: npm run ingestion:admin
+    restart: unless-stopped
+    depends_on:
+      postgres:
+        condition: service_healthy
+      ingestion-migrate:
+        condition: service_completed_successfully
+    environment:
+      <<: *ingestion-env
+      INGESTION_ADMIN_PORT: 4101
+      INGESTION_ADMIN_USERNAME: ${INGESTION_ADMIN_USERNAME:-admin}
+      INGESTION_ADMIN_PASSWORD: ${INGESTION_ADMIN_PASSWORD:?INGESTION_ADMIN_PASSWORD is required}
+    ports:
+      - "${INGESTION_ADMIN_BIND:-127.0.0.1}:${INGESTION_ADMIN_PORT:-4101}:4101"
+    healthcheck:
+      test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:4101/health >/dev/null || exit 1"]
+      interval: 15s
+      timeout: 3s
+      retries: 5
+      start_period: 20s
+    volumes:
+      - ingestion-data:/app/ingestion/data
+
+volumes:
+  postgres-data:
+  ingestion-data:

ingestion/migrations/001_init.sql

@@ -0,0 +1,225 @@
+create extension if not exists pgcrypto;
+
+create table if not exists manufacturers (
+  id uuid primary key default gen_random_uuid(),
+  slug text not null unique,
+  name text not null,
+  trust_status text not null default 'unknown' check (trust_status in ('trusted', 'unknown', 'untrusted')),
+  notes text,
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now()
+);
+
+create table if not exists products (
+  id text primary key,
+  manufacturer_id uuid references manufacturers(id) on delete set null,
+  brand text not null,
+  model text not null,
+  canonical_name text not null,
+  title text not null,
+  moderation_status text not null default 'needs_review'
+    check (moderation_status in ('needs_review', 'approved', 'rejected', 'duplicate', 'variant', 'oem_clone')),
+  discovered_at timestamptz not null default now(),
+  updated_at timestamptz not null default now()
+);
+
+create table if not exists product_variants (
+  id uuid primary key default gen_random_uuid(),
+  product_id text references products(id) on delete cascade,
+  variant_kind text not null,
+  title text not null,
+  created_at timestamptz not null default now()
+);
+
+create table if not exists product_sources (
+  id uuid primary key default gen_random_uuid(),
+  product_id text references products(id) on delete set null,
+  source text not null check (source in ('yandex_market')),
+  url text not null,
+  source_product_id text,
+  category_url text,
+  title_preview text,
+  stable_content_hash text,
+  first_seen_at timestamptz not null default now(),
+  last_fetched_at timestamptz,
+  parsed_json jsonb not null default '{}'::jsonb,
+  state text not null default 'discovered',
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now(),
+  unique (source, url)
+);
+
+create index if not exists product_sources_product_id_idx on product_sources(product_id);
+create index if not exists product_sources_source_product_id_idx on product_sources(source, source_product_id);
+
+create table if not exists source_snapshots (
+  id uuid primary key default gen_random_uuid(),
+  source text not null check (source in ('yandex_market')),
+  url text not null,
+  fetched_at timestamptz not null,
+  http_status int not null,
+  raw_snapshot_hash text not null,
+  stable_content_hash text,
+  raw_html_path text,
+  parsed_json jsonb not null default '{}'::jsonb,
+  created_at timestamptz not null default now()
+);
+
+create index if not exists source_snapshots_url_idx on source_snapshots(source, url, fetched_at desc);
+create index if not exists source_snapshots_stable_hash_idx on source_snapshots(stable_content_hash);
+
+create table if not exists product_specs (
+  id uuid primary key default gen_random_uuid(),
+  product_id text not null references products(id) on delete cascade,
+  key text not null,
+  value jsonb not null,
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now(),
+  unique (product_id, key)
+);
+
+create table if not exists product_source_specs (
+  id uuid primary key default gen_random_uuid(),
+  product_id text not null references products(id) on delete cascade,
+  source text not null check (source in ('yandex_market')),
+  source_url text not null,
+  source_product_id text,
+  raw_group text,
+  raw_label text not null,
+  raw_value jsonb not null,
+  normalized_key text,
+  normalized_value jsonb,
+  value_hash text not null,
+  fetched_at timestamptz not null,
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now(),
+  unique (product_id, source, source_url, raw_label)
+);
+
+create index if not exists product_source_specs_product_key_idx on product_source_specs(product_id, normalized_key);
+
+create table if not exists product_images (
+  id uuid primary key default gen_random_uuid(),
+  product_id text not null references products(id) on delete cascade,
+  source text not null check (source in ('yandex_market')),
+  original_url text not null,
+  artifact_url text,
+  sha256 text,
+  width int,
+  height int,
+  sort_order int not null default 0,
+  status text not null default 'needs_review' check (status in ('needs_review', 'approved', 'rejected')),
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now(),
+  unique (product_id, original_url)
+);
+
+create table if not exists source_ratings (
+  id uuid primary key default gen_random_uuid(),
+  product_id text references products(id) on delete cascade,
+  source text not null check (source in ('yandex_market')),
+  source_product_id text,
+  rating_value numeric,
+  rating_scale numeric,
+  rating_count int,
+  fetched_at timestamptz not null,
+  created_at timestamptz not null default now()
+);
+
+create table if not exists crawl_jobs (
+  id uuid primary key default gen_random_uuid(),
+  source text not null check (source in ('yandex_market')),
+  url text not null,
+  job_type text not null check (job_type in ('category', 'product', 'sitemap')),
+  state text not null default 'pending' check (
+    state in (
+      'pending',
+      'fetching',
+      'fetched',
+      'parsed',
+      'normalized',
+      'deduplicated',
+      'needs_review',
+      'approved',
+      'exported',
+      'rate_limited',
+      'blocked',
+      'captcha',
+      'not_found',
+      'parse_failed',
+      'needs_manual_review'
+    )
+  ),
+  attempts int not null default 0,
+  next_run_at timestamptz not null default now(),
+  last_error text,
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now(),
+  unique (source, url, job_type)
+);
+
+create index if not exists crawl_jobs_pending_idx on crawl_jobs(state, next_run_at, created_at);
+
+create table if not exists crawl_errors (
+  id uuid primary key default gen_random_uuid(),
+  job_id uuid references crawl_jobs(id) on delete set null,
+  source text not null,
+  url text not null,
+  reason text not null,
+  message text not null,
+  created_at timestamptz not null default now()
+);
+
+create table if not exists source_pauses (
+  source text primary key check (source in ('yandex_market')),
+  reason text not null,
+  paused_until timestamptz not null,
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now()
+);
+
+create table if not exists source_daily_budgets (
+  source text not null check (source in ('yandex_market')),
+  budget_bucket text not null check (budget_bucket in ('category', 'product', 'sitemap')),
+  budget_day date not null,
+  daily_limit int not null,
+  used_count int not null default 0,
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now(),
+  primary key (source, budget_bucket, budget_day)
+);
+
+create table if not exists moderation_queue (
+  id uuid primary key default gen_random_uuid(),
+  product_id text not null references products(id) on delete cascade,
+  reason text not null check (
+    reason in ('new_product', 'same_product', 'possible_duplicate', 'variant', 'oem_clone', 'conflicting_specs')
+  ),
+  state text not null default 'pending' check (state in ('pending', 'resolved', 'dismissed')),
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now(),
+  unique (product_id, reason)
+);
+
+create table if not exists moderation_conflicts (
+  id uuid primary key default gen_random_uuid(),
+  product_id text not null references products(id) on delete cascade,
+  conflict_type text not null check (conflict_type in ('spec_value')),
+  key text not null,
+  values_json jsonb not null,
+  state text not null default 'pending' check (state in ('pending', 'resolved', 'dismissed')),
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now(),
+  unique (product_id, conflict_type, key)
+);
+
+create table if not exists export_runs (
+  id text primary key,
+  schema_version int not null,
+  created_at timestamptz not null,
+  artifact_path text not null,
+  manifest_path text not null,
+  sha256 text not null,
+  item_count int not null,
+  published_at timestamptz not null default now()
+);

ingestion/migrations/002_source_specs_and_conflicts.sql

@@ -0,0 +1,35 @@
+alter table products add column if not exists canonical_name text;
+update products set canonical_name = trim(brand || ' ' || model) where canonical_name is null;
+alter table products alter column canonical_name set not null;
+
+create table if not exists product_source_specs (
+  id uuid primary key default gen_random_uuid(),
+  product_id text not null references products(id) on delete cascade,
+  source text not null check (source in ('yandex_market')),
+  source_url text not null,
+  source_product_id text,
+  raw_group text,
+  raw_label text not null,
+  raw_value jsonb not null,
+  normalized_key text,
+  normalized_value jsonb,
+  value_hash text not null,
+  fetched_at timestamptz not null,
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now(),
+  unique (product_id, source, source_url, raw_label)
+);
+
+create index if not exists product_source_specs_product_key_idx on product_source_specs(product_id, normalized_key);
+
+create table if not exists moderation_conflicts (
+  id uuid primary key default gen_random_uuid(),
+  product_id text not null references products(id) on delete cascade,
+  conflict_type text not null check (conflict_type in ('spec_value')),
+  key text not null,
+  values_json jsonb not null,
+  state text not null default 'pending' check (state in ('pending', 'resolved', 'dismissed')),
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now(),
+  unique (product_id, conflict_type, key)
+);

ingestion/migrations/003_manufacturers.sql

@@ -0,0 +1,29 @@
+create table if not exists manufacturers (
+  id uuid primary key default gen_random_uuid(),
+  slug text not null unique,
+  name text not null,
+  trust_status text not null default 'unknown' check (trust_status in ('trusted', 'unknown', 'untrusted')),
+  notes text,
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now()
+);
+
+alter table products add column if not exists manufacturer_id uuid references manufacturers(id) on delete set null;
+
+insert into manufacturers (slug, name)
+select
+  lower(regexp_replace(trim(brand), '[^[:alnum:]]+', '-', 'g')) as slug,
+  brand as name
+from products
+where brand is not null
+on conflict (slug) do update
+set name = excluded.name,
+    updated_at = now();
+
+update products
+set manufacturer_id = manufacturers.id
+from manufacturers
+where products.manufacturer_id is null
+  and manufacturers.slug = lower(regexp_replace(trim(products.brand), '[^[:alnum:]]+', '-', 'g'));
+
+create index if not exists products_manufacturer_id_idx on products(manufacturer_id);

ingestion/migrations/004_mvideo_source.sql

@@ -0,0 +1,31 @@
+alter table product_sources drop constraint if exists product_sources_source_check;
+alter table product_sources add constraint product_sources_source_check
+  check (source in ('yandex_market', 'mvideo'));
+
+alter table source_snapshots drop constraint if exists source_snapshots_source_check;
+alter table source_snapshots add constraint source_snapshots_source_check
+  check (source in ('yandex_market', 'mvideo'));
+
+alter table product_source_specs drop constraint if exists product_source_specs_source_check;
+alter table product_source_specs add constraint product_source_specs_source_check
+  check (source in ('yandex_market', 'mvideo'));
+
+alter table product_images drop constraint if exists product_images_source_check;
+alter table product_images add constraint product_images_source_check
+  check (source in ('yandex_market', 'mvideo'));
+
+alter table source_ratings drop constraint if exists source_ratings_source_check;
+alter table source_ratings add constraint source_ratings_source_check
+  check (source in ('yandex_market', 'mvideo'));
+
+alter table crawl_jobs drop constraint if exists crawl_jobs_source_check;
+alter table crawl_jobs add constraint crawl_jobs_source_check
+  check (source in ('yandex_market', 'mvideo'));
+
+alter table source_pauses drop constraint if exists source_pauses_source_check;
+alter table source_pauses add constraint source_pauses_source_check
+  check (source in ('yandex_market', 'mvideo'));
+
+alter table source_daily_budgets drop constraint if exists source_daily_budgets_source_check;
+alter table source_daily_budgets add constraint source_daily_budgets_source_check
+  check (source in ('yandex_market', 'mvideo'));

ingestion/migrations/005_manual_canonical_overrides.sql

@@ -0,0 +1,3 @@
+alter table products
+  add column if not exists model_override boolean not null default false,
+  add column if not exists canonical_name_override boolean not null default false;

ingestion/migrations/006_canonical_product_grouping.sql

@@ -0,0 +1,2 @@
+create index if not exists products_canonical_name_ci_idx
+  on products (lower(canonical_name));

ingestion/src/adminAuth.test.ts

@@ -0,0 +1,28 @@
+import { describe, expect, it } from 'vitest';
+import { getAdminAuthConfig, isBasicAuthAuthorized } from './adminAuth.ts';
+
+function basic(username: string, password: string) {
+  return `Basic ${Buffer.from(`${username}:${password}`).toString('base64')}`;
+}
+
+describe('ingestion admin auth', () => {
+  it('requires an admin password', () => {
+    expect(() => getAdminAuthConfig({ INGESTION_ADMIN_USERNAME: 'admin' })).toThrow(
+      'INGESTION_ADMIN_PASSWORD is required',
+    );
+  });
+
+  it('accepts matching basic auth credentials', () => {
+    const config = { password: 'secret', username: 'operator' };
+
+    expect(isBasicAuthAuthorized(basic('operator', 'secret'), config)).toBe(true);
+  });
+
+  it('rejects missing or wrong basic auth credentials', () => {
+    const config = { password: 'secret', username: 'operator' };
+
+    expect(isBasicAuthAuthorized(undefined, config)).toBe(false);
+    expect(isBasicAuthAuthorized(basic('operator', 'wrong'), config)).toBe(false);
+    expect(isBasicAuthAuthorized(basic('wrong', 'secret'), config)).toBe(false);
+  });
+});

ingestion/src/adminAuth.ts

@@ -0,0 +1,45 @@
+import { timingSafeEqual } from 'node:crypto';
+
+export type AdminAuthConfig = {
+  password: string;
+  username: string;
+};
+
+export const adminAuthRealm = 'videoreg ingestion admin';
+
+export function getAdminAuthConfig(env: Record<string, string | undefined> = process.env): AdminAuthConfig {
+  const username = env.INGESTION_ADMIN_USERNAME ?? 'admin';
+  const password = env.INGESTION_ADMIN_PASSWORD;
+
+  if (!password) {
+    throw new Error('INGESTION_ADMIN_PASSWORD is required for ingestion-admin');
+  }
+
+  return { password, username };
+}
+
+export function isBasicAuthAuthorized(authorizationHeader: string | undefined, config: AdminAuthConfig) {
+  if (!authorizationHeader?.startsWith('Basic ')) {
+    return false;
+  }
+
+  const encodedCredentials = authorizationHeader.slice('Basic '.length).trim();
+  const decodedCredentials = Buffer.from(encodedCredentials, 'base64').toString('utf8');
+  const separatorIndex = decodedCredentials.indexOf(':');
+
+  if (separatorIndex < 0) {
+    return false;
+  }
+
+  const username = decodedCredentials.slice(0, separatorIndex);
+  const password = decodedCredentials.slice(separatorIndex + 1);
+
+  return safeEqual(username, config.username) && safeEqual(password, config.password);
+}
+
+function safeEqual(left: string, right: string) {
+  const leftBuffer = Buffer.from(left);
+  const rightBuffer = Buffer.from(right);
+
+  return leftBuffer.length === rightBuffer.length && timingSafeEqual(leftBuffer, rightBuffer);
+}

ingestion/src/adminServer.ts

@@ -0,0 +1,479 @@
+import http from 'node:http';
+import {
+  adminAuthRealm,
+  getAdminAuthConfig,
+  isBasicAuthAuthorized,
+  type AdminAuthConfig,
+} from './adminAuth.ts';
+import { adminUiCss, adminUiHtml, adminUiJs } from './adminUi.ts';
+import { createPgPool } from './db.ts';
+
+function sendJson(response: http.ServerResponse, status: number, payload: unknown) {
+  response.writeHead(status, { 'content-type': 'application/json; charset=utf-8' });
+  response.end(`${JSON.stringify(payload, null, 2)}\n`);
+}
+
+function sendText(response: http.ServerResponse, status: number, contentType: string, payload: string) {
+  response.writeHead(status, {
+    'cache-control': 'no-store',
+    'content-type': contentType,
+  });
+  response.end(payload);
+}
+
+function sendBasicAuthChallenge(response: http.ServerResponse) {
+  response.writeHead(401, {
+    'cache-control': 'no-store',
+    'content-type': 'text/plain; charset=utf-8',
+    'www-authenticate': `Basic realm="${adminAuthRealm}", charset="UTF-8"`,
+  });
+  response.end('Authentication required\n');
+}
+
+async function readJsonBody(request: http.IncomingMessage) {
+  const chunks: Buffer[] = [];
+
+  for await (const chunk of request) {
+    chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+  }
+
+  if (chunks.length === 0) return {};
+
+  return JSON.parse(Buffer.concat(chunks).toString('utf8')) as Record<string, unknown>;
+}
+
+function isTrustStatus(value: unknown): value is 'trusted' | 'unknown' | 'untrusted' {
+  return value === 'trusted' || value === 'unknown' || value === 'untrusted';
+}
+
+function isModerationStatus(value: unknown): value is 'needs_review' | 'approved' | 'rejected' | 'duplicate' | 'variant' | 'oem_clone' {
+  return ['needs_review', 'approved', 'rejected', 'duplicate', 'variant', 'oem_clone'].includes(String(value));
+}
+
+function isCanonicalText(value: unknown): value is string {
+  return typeof value === 'string' && value.trim().length > 0 && value.trim().length <= 300;
+}
+
+function isProtectedAdminPath(pathname: string) {
+  return (
+    pathname === '/admin' ||
+    pathname.startsWith('/admin/') ||
+    pathname === '/queue' ||
+    pathname === '/moderation' ||
+    pathname === '/manufacturers' ||
+    pathname.startsWith('/manufacturers/')
+  );
+}
+
+function adminApiPath(pathname: string) {
+  if (pathname === '/admin/api') return '/';
+  if (pathname.startsWith('/admin/api/')) return pathname.slice('/admin/api'.length);
+
+  return pathname;
+}
+
+type CreateAdminServerOptions = {
+  auth?: AdminAuthConfig;
+};
+
+export function createAdminServer(options: CreateAdminServerOptions = {}) {
+  const pool = createPgPool();
+  const auth = options.auth ?? getAdminAuthConfig();
+
+  return http.createServer(async (request, response) => {
+    const url = new URL(request.url ?? '/', 'http://localhost');
+
+    try {
+      if (request.method === 'GET' && url.pathname === '/') {
+        response.writeHead(302, { location: '/admin' });
+        response.end();
+        return;
+      }
+
+      if (isProtectedAdminPath(url.pathname) && !isBasicAuthAuthorized(request.headers.authorization, auth)) {
+        sendBasicAuthChallenge(response);
+        return;
+      }
+
+      if (request.method === 'GET' && url.pathname === '/admin') {
+        sendText(response, 200, 'text/html; charset=utf-8', adminUiHtml);
+        return;
+      }
+
+      if (request.method === 'GET' && url.pathname === '/admin/styles.css') {
+        sendText(response, 200, 'text/css; charset=utf-8', adminUiCss);
+        return;
+      }
+
+      if (request.method === 'GET' && url.pathname === '/admin/app.js') {
+        sendText(response, 200, 'text/javascript; charset=utf-8', adminUiJs);
+        return;
+      }
+
+      const apiPath = adminApiPath(url.pathname);
+
+      if (apiPath === '/health') {
+        await pool.query('select 1');
+        sendJson(response, 200, { status: 'ok' });
+        return;
+      }
+
+      if (apiPath === '/queue') {
+        const result = await pool.query(
+          `
+            select state, job_type as "jobType", count(*)::int as count
+            from crawl_jobs
+            group by state, job_type
+            order by state, job_type
+          `,
+        );
+
+        sendJson(response, 200, { jobs: result.rows });
+        return;
+      }
+
+      if (apiPath === '/moderation') {
+        const result = await pool.query(
+          `
+            select state, reason, count(*)::int as count
+            from moderation_queue
+            group by state, reason
+            order by state, reason
+          `,
+        );
+
+        sendJson(response, 200, { queue: result.rows });
+        return;
+      }
+
+      if (apiPath === '/moderation/products' && request.method === 'GET') {
+        const requestedStatus = url.searchParams.get('status') ?? 'needs_review';
+
+        if (!isModerationStatus(requestedStatus)) {
+          sendJson(response, 400, { error: 'invalid moderation status' });
+          return;
+        }
+
+        const result = await pool.query(
+          `
+            select
+              products.id,
+              products.brand,
+              products.model,
+              products.canonical_name as "canonicalName",
+              products.title,
+              products.moderation_status as "moderationStatus",
+              products.updated_at as "updatedAt",
+              count(distinct product_sources.id)::int as "sourceCount",
+              count(distinct product_source_specs.id)::int as "rawSpecCount",
+              count(distinct moderation_conflicts.id) filter (where moderation_conflicts.state = 'pending')::int as "pendingConflictCount"
+            from products
+            left join product_sources on product_sources.product_id = products.id
+            left join product_source_specs on product_source_specs.product_id = products.id
+            left join moderation_conflicts on moderation_conflicts.product_id = products.id
+            where products.moderation_status = $1
+            group by products.id
+            order by products.updated_at desc, products.brand asc, products.model asc
+          `,
+          [requestedStatus],
+        );
+
+        sendJson(response, 200, { products: result.rows });
+        return;
+      }
+
+      const moderationProductMatch = apiPath.match(/^\/moderation\/products\/([^/]+)$/);
+
+      if (moderationProductMatch && request.method === 'GET') {
+        const productId = decodeURIComponent(moderationProductMatch[1]);
+        const productResult = await pool.query(
+          `
+            select
+              id,
+              brand,
+              model as "canonicalModel",
+              canonical_name as "canonicalName",
+              title,
+              moderation_status as "moderationStatus",
+              model_override as "canonicalModelOverride",
+              canonical_name_override as "canonicalNameOverride",
+              updated_at as "updatedAt"
+            from products
+            where id = $1
+          `,
+          [productId],
+        );
+
+        if (!productResult.rows[0]) {
+          sendJson(response, 404, { error: 'product_not_found' });
+          return;
+        }
+
+        const [sources, specs, conflicts] = await Promise.all([
+          pool.query(
+            `
+              select
+                source,
+                url,
+                source_product_id as "sourceProductId",
+                stable_content_hash as "stableContentHash",
+                last_fetched_at as "lastFetchedAt"
+              from product_sources
+              where product_id = $1
+              order by source, url
+            `,
+            [productId],
+          ),
+          pool.query(
+            `
+              select
+                source,
+                source_url as "sourceUrl",
+                raw_group as "group",
+                raw_label as "label",
+                raw_value as "value",
+                normalized_key as "normalizedKey",
+                normalized_value as "normalizedValue"
+              from product_source_specs
+              where product_id = $1
+              order by source, raw_group nulls first, raw_label
+            `,
+            [productId],
+          ),
+          pool.query(
+            `
+              select
+                conflict_type as "conflictType",
+                key,
+                values_json as "values",
+                state
+              from moderation_conflicts
+              where product_id = $1
+              order by state, key
+            `,
+            [productId],
+          ),
+        ]);
+
+        sendJson(response, 200, {
+          product: {
+            ...productResult.rows[0],
+            conflicts: conflicts.rows,
+            rawSpecs: specs.rows,
+            sources: sources.rows,
+          },
+        });
+        return;
+      }
+
+      if (moderationProductMatch && request.method === 'PATCH') {
+        const body = await readJsonBody(request);
+        const hasModerationStatus = Object.hasOwn(body, 'moderationStatus');
+        const hasCanonicalModel = Object.hasOwn(body, 'canonicalModel');
+        const hasCanonicalName = Object.hasOwn(body, 'canonicalName');
+
+        if (hasModerationStatus && !isModerationStatus(body.moderationStatus)) {
+          sendJson(response, 400, { error: 'invalid moderationStatus' });
+          return;
+        }
+
+        if (hasCanonicalModel && !isCanonicalText(body.canonicalModel)) {
+          sendJson(response, 400, { error: 'canonicalModel must be a non-empty string up to 300 characters' });
+          return;
+        }
+
+        if (hasCanonicalName && !isCanonicalText(body.canonicalName)) {
+          sendJson(response, 400, { error: 'canonicalName must be a non-empty string up to 300 characters' });
+          return;
+        }
+
+        if (!hasModerationStatus && !hasCanonicalModel && !hasCanonicalName) {
+          sendJson(response, 400, { error: 'at least one product field is required' });
+          return;
+        }
+
+        const productId = decodeURIComponent(moderationProductMatch[1]);
+        const result = await pool.query(
+          `
+            update products
+            set moderation_status = case when $2::boolean then $3::text else moderation_status end,
+                model = case when $4::boolean then $5::text else model end,
+                canonical_name = case when $6::boolean then $7::text else canonical_name end,
+                model_override = case when $4::boolean then true else model_override end,
+                canonical_name_override = case when $6::boolean then true else canonical_name_override end,
+                updated_at = now()
+            where id = $1
+            returning
+              id,
+              model as "canonicalModel",
+              canonical_name as "canonicalName",
+              moderation_status as "moderationStatus",
+              model_override as "canonicalModelOverride",
+              canonical_name_override as "canonicalNameOverride",
+              updated_at as "updatedAt"
+          `,
+          [
+            productId,
+            hasModerationStatus,
+            hasModerationStatus ? body.moderationStatus : null,
+            hasCanonicalModel,
+            hasCanonicalModel ? String(body.canonicalModel).trim() : null,
+            hasCanonicalName,
+            hasCanonicalName ? String(body.canonicalName).trim() : null,
+          ],
+        );
+
+        if (!result.rows[0]) {
+          sendJson(response, 404, { error: 'product_not_found' });
+          return;
+        }
+
+        if (hasModerationStatus) {
+          await pool.query(
+            `
+              update moderation_queue
+              set state = case when $2 = 'needs_review' then 'pending' else 'resolved' end,
+                  updated_at = now()
+              where product_id = $1
+            `,
+            [productId, body.moderationStatus],
+          );
+
+          if (body.moderationStatus !== 'needs_review') {
+            await pool.query(
+              `
+                update crawl_jobs job
+                set state = $2,
+                    updated_at = now()
+                from product_sources source
+                where source.product_id = $1
+                  and job.source = source.source
+                  and job.url = source.url
+                  and job.job_type = 'product'
+                  and job.state = 'needs_review'
+              `,
+              [productId, body.moderationStatus === 'approved' ? 'approved' : 'needs_manual_review'],
+            );
+          }
+        }
+
+        sendJson(response, 200, { product: result.rows[0] });
+        return;
+      }
+
+      if (apiPath === '/manufacturers') {
+        const result = await pool.query(
+          `
+            select
+              manufacturers.slug,
+              manufacturers.name,
+              manufacturers.trust_status as "trustStatus",
+              manufacturers.notes,
+              count(products.id)::int as "productCount",
+              max(products.updated_at) as "lastProductUpdatedAt"
+            from manufacturers
+            left join products on products.manufacturer_id = manufacturers.id
+            group by manufacturers.id
+            order by manufacturers.trust_status desc, manufacturers.name asc
+          `,
+        );
+
+        sendJson(response, 200, { manufacturers: result.rows });
+        return;
+      }
+
+      const manufacturerMatch = apiPath.match(/^\/manufacturers\/([^/]+)$/);
+
+      if (manufacturerMatch && request.method === 'GET') {
+        const result = await pool.query(
+          `
+            select
+              manufacturers.slug,
+              manufacturers.name,
+              manufacturers.trust_status as "trustStatus",
+              manufacturers.notes,
+              coalesce(
+                jsonb_agg(
+                  jsonb_build_object(
+                    'id', products.id,
+                    'canonicalName', products.canonical_name,
+                    'title', products.title,
+                    'moderationStatus', products.moderation_status,
+                    'updatedAt', products.updated_at
+                  )
+                  order by products.updated_at desc
+                ) filter (where products.id is not null),
+                '[]'::jsonb
+              ) as products
+            from manufacturers
+            left join products on products.manufacturer_id = manufacturers.id
+            where manufacturers.slug = $1
+            group by manufacturers.id
+          `,
+          [decodeURIComponent(manufacturerMatch[1])],
+        );
+
+        if (!result.rows[0]) {
+          sendJson(response, 404, { error: 'manufacturer_not_found' });
+          return;
+        }
+
+        sendJson(response, 200, { manufacturer: result.rows[0] });
+        return;
+      }
+
+      if (manufacturerMatch && request.method === 'PATCH') {
+        const body = await readJsonBody(request);
+
+        if (!isTrustStatus(body.trustStatus)) {
+          sendJson(response, 400, { error: 'trustStatus must be trusted, unknown, or untrusted' });
+          return;
+        }
+
+        const hasNotes = Object.hasOwn(body, 'notes');
+
+        if (hasNotes && body.notes !== null && typeof body.notes !== 'string') {
+          sendJson(response, 400, { error: 'notes must be a string or null' });
+          return;
+        }
+
+        const result = await pool.query(
+          `
+            update manufacturers
+            set trust_status = $2,
+                notes = case when $3 then $4 else notes end,
+                updated_at = now()
+            where slug = $1
+            returning slug, name, trust_status as "trustStatus", notes
+          `,
+          [
+            decodeURIComponent(manufacturerMatch[1]),
+            body.trustStatus,
+            hasNotes,
+            hasNotes ? body.notes : null,
+          ],
+        );
+
+        if (!result.rows[0]) {
+          sendJson(response, 404, { error: 'manufacturer_not_found' });
+          return;
+        }
+
+        sendJson(response, 200, { manufacturer: result.rows[0] });
+        return;
+      }
+
+      sendJson(response, 404, { error: 'not_found' });
+    } catch (error) {
+      sendJson(response, 500, { error: (error as Error).message });
+    }
+  });
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  const port = Number(process.env.INGESTION_ADMIN_PORT ?? 4101);
+
+  createAdminServer().listen(port, () => {
+    console.log(`ingestion-admin listening on ${port}`);
+  });
+}

ingestion/src/adminUi.test.ts

@@ -0,0 +1,17 @@
+import { describe, expect, it } from 'vitest';
+import { adminUiHtml, adminUiJs } from './adminUi.ts';
+
+describe('ingestion admin UI', () => {
+  it('is served as a standalone ingestion app', () => {
+    expect(adminUiHtml).toContain('/admin/styles.css');
+    expect(adminUiHtml).toContain('/admin/app.js');
+    expect(adminUiHtml).not.toContain('/api/admin');
+    expect(adminUiJs).toContain("const adminApiBasePath = '/admin/api'");
+    expect(adminUiJs).toContain('data-product-form');
+    expect(adminUiJs).not.toContain("requestJson('/manufacturers");
+  });
+
+  it('ships valid browser JavaScript', () => {
+    expect(() => new Function(adminUiJs)).not.toThrow();
+  });
+});

ingestion/src/adminUi.ts

@@ -0,0 +1,790 @@
+export const adminUiHtml = `<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>Videoreg ingestion admin</title>
+    <link rel="stylesheet" href="/admin/styles.css" />
+  </head>
+  <body>
+    <main class="shell">
+      <header class="topbar">
+        <div>
+          <p class="eyebrow">catalog moderation</p>
+          <h1>Ingestion admin</h1>
+        </div>
+        <nav class="actions" aria-label="Admin navigation">
+          <a class="button ghost" href="/admin/api/health">Health</a>
+          <a class="button ghost" href="/admin/api/queue">Queue</a>
+          <a class="button ghost" href="/admin/api/moderation">Moderation</a>
+          <a class="button" href="#/moderation">Products</a>
+          <a class="button" href="#/">Manufacturers</a>
+        </nav>
+      </header>
+      <div id="app" class="app" aria-live="polite"></div>
+    </main>
+    <script src="/admin/app.js" type="module"></script>
+  </body>
+</html>
+`;
+
+export const adminUiCss = `
+:root {
+  color-scheme: light;
+  --surface-0: #f3f0e8;
+  --surface-1: #fffdf8;
+  --surface-2: #e9e4d8;
+  --text-primary: #202528;
+  --text-muted: #667076;
+  --text-subtle: #858076;
+  --border-subtle: #d7cec0;
+  --border-strong: #bdb3a4;
+  --accent: #56798a;
+  --accent-strong: #385f70;
+  --accent-soft: #dbe7e9;
+  --warning: #8f7044;
+  --warning-soft: #efe5d6;
+  --danger: #884b46;
+  --danger-soft: #eadbd8;
+  --success: #58705c;
+  --success-soft: #dde8dd;
+  --radius-sm: 4px;
+  --radius-md: 6px;
+  --radius-lg: 8px;
+  --shadow-soft: 0 14px 34px rgb(32 37 40 / 8%);
+}
+
+* {
+  box-sizing: border-box;
+}
+
+body {
+  margin: 0;
+  background: var(--surface-0);
+  color: var(--text-primary);
+  font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
+}
+
+a {
+  color: inherit;
+  text-decoration: none;
+}
+
+button,
+input,
+select,
+textarea {
+  font: inherit;
+}
+
+.shell {
+  width: min(1180px, calc(100% - 32px));
+  margin: 0 auto;
+  padding: 24px 0 40px;
+}
+
+.topbar {
+  display: flex;
+  align-items: flex-start;
+  justify-content: space-between;
+  gap: 24px;
+  border-bottom: 1px solid var(--border-subtle);
+  padding-bottom: 18px;
+}
+
+.eyebrow {
+  margin: 0 0 6px;
+  color: var(--accent-strong);
+  font-size: 0.78rem;
+  font-weight: 750;
+  text-transform: uppercase;
+}
+
+h1,
+h2,
+h3,
+p {
+  overflow-wrap: anywhere;
+}
+
+h1 {
+  margin: 0;
+  font-size: 2rem;
+  line-height: 1.1;
+}
+
+h2 {
+  margin: 0;
+  font-size: 1.1rem;
+}
+
+.actions {
+  display: flex;
+  flex-wrap: wrap;
+  gap: 8px;
+  justify-content: flex-end;
+}
+
+.button {
+  min-height: 38px;
+  display: inline-flex;
+  align-items: center;
+  justify-content: center;
+  border: 1px solid var(--border-strong);
+  border-radius: var(--radius-md);
+  background: var(--surface-1);
+  color: var(--text-primary);
+  cursor: pointer;
+  padding: 8px 12px;
+  font-size: 0.86rem;
+  font-weight: 700;
+}
+
+.button.primary {
+  border-color: var(--accent);
+  background: var(--accent);
+  color: #fff;
+}
+
+.button.ghost {
+  border-color: transparent;
+  background: transparent;
+  color: var(--text-muted);
+}
+
+.button:hover {
+  background: var(--surface-2);
+}
+
+.button.primary:hover {
+  background: var(--accent-strong);
+}
+
+.app {
+  display: grid;
+  gap: 20px;
+  padding-top: 22px;
+}
+
+.panel {
+  border: 1px solid var(--border-subtle);
+  border-radius: var(--radius-lg);
+  background: var(--surface-1);
+  box-shadow: var(--shadow-soft);
+}
+
+.panel-header {
+  display: flex;
+  align-items: flex-start;
+  justify-content: space-between;
+  gap: 16px;
+  border-bottom: 1px solid var(--border-subtle);
+  padding: 16px;
+}
+
+.panel-body {
+  padding: 16px;
+}
+
+.stats-grid {
+  display: grid;
+  grid-template-columns: repeat(4, minmax(120px, 1fr));
+  gap: 10px;
+}
+
+.metric {
+  min-height: 76px;
+  border: 1px solid var(--border-subtle);
+  border-radius: var(--radius-lg);
+  background: var(--surface-2);
+  padding: 12px;
+}
+
+.metric span {
+  display: block;
+  color: var(--text-muted);
+  font-size: 0.76rem;
+  font-weight: 750;
+  text-transform: uppercase;
+}
+
+.metric strong {
+  display: block;
+  margin-top: 7px;
+  font-size: 1.45rem;
+}
+
+.table-wrap {
+  overflow-x: auto;
+}
+
+table {
+  min-width: 760px;
+  width: 100%;
+  border-collapse: collapse;
+  text-align: left;
+  font-size: 0.9rem;
+}
+
+th,
+td {
+  border-bottom: 1px solid var(--border-subtle);
+  padding: 14px 16px;
+  vertical-align: top;
+}
+
+thead th {
+  background: var(--surface-1);
+  color: var(--text-muted);
+  font-size: 0.74rem;
+  font-weight: 750;
+  text-transform: uppercase;
+}
+
+tbody tr:hover {
+  background: color-mix(in srgb, var(--surface-2) 48%, transparent);
+}
+
+tbody tr:last-child th,
+tbody tr:last-child td {
+  border-bottom: 0;
+}
+
+.badge {
+  display: inline-flex;
+  min-height: 24px;
+  align-items: center;
+  border-radius: var(--radius-sm);
+  border: 1px solid var(--border-subtle);
+  padding: 4px 8px;
+  font-size: 0.72rem;
+  font-weight: 750;
+}
+
+.badge.trusted {
+  background: var(--success-soft);
+  color: var(--success);
+}
+
+.badge.unknown {
+  background: var(--warning-soft);
+  color: var(--warning);
+}
+
+.badge.untrusted {
+  background: var(--danger-soft);
+  color: var(--danger);
+}
+
+.muted {
+  color: var(--text-muted);
+}
+
+.mono {
+  font-family: "IBM Plex Mono", ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace;
+}
+
+.layout {
+  display: grid;
+  grid-template-columns: minmax(280px, 380px) minmax(0, 1fr);
+  gap: 20px;
+  align-items: start;
+}
+
+.form-grid {
+  display: grid;
+  gap: 14px;
+}
+
+.field {
+  display: grid;
+  gap: 6px;
+}
+
+.field span {
+  color: var(--text-muted);
+  font-size: 0.82rem;
+  font-weight: 700;
+}
+
+.field select,
+.field textarea {
+  width: 100%;
+  border: 1px solid var(--border-subtle);
+  border-radius: var(--radius-md);
+  background: var(--surface-1);
+  color: var(--text-primary);
+  padding: 9px 10px;
+}
+
+.field select {
+  min-height: 42px;
+}
+
+.field textarea {
+  min-height: 120px;
+  resize: vertical;
+}
+
+.status-line {
+  min-height: 24px;
+  color: var(--text-muted);
+  font-size: 0.88rem;
+  font-weight: 700;
+}
+
+.error,
+.empty {
+  border: 1px dashed var(--border-subtle);
+  border-radius: var(--radius-lg);
+  color: var(--text-muted);
+  padding: 22px;
+}
+
+.error strong {
+  display: block;
+  color: var(--danger);
+  margin-bottom: 6px;
+}
+
+@media (max-width: 900px) {
+  .topbar,
+  .layout {
+    display: grid;
+    grid-template-columns: 1fr;
+  }
+
+  .actions {
+    justify-content: flex-start;
+  }
+
+  .stats-grid {
+    grid-template-columns: repeat(2, minmax(0, 1fr));
+  }
+}
+
+@media (max-width: 560px) {
+  .shell {
+    width: min(100% - 20px, 1180px);
+    padding-top: 16px;
+  }
+
+  h1 {
+    font-size: 1.55rem;
+  }
+
+  .stats-grid {
+    grid-template-columns: 1fr;
+  }
+}
+`;
+
+export const adminUiJs = `
+const app = document.querySelector('#app');
+const adminApiBasePath = '/admin/api';
+const dateFormatter = new Intl.DateTimeFormat('en', {
+  dateStyle: 'medium',
+  timeStyle: 'short',
+});
+
+function escapeHtml(value) {
+  return String(value ?? '')
+    .replaceAll('&', '&amp;')
+    .replaceAll('<', '&lt;')
+    .replaceAll('>', '&gt;')
+    .replaceAll('"', '&quot;')
+    .replaceAll("'", '&#039;');
+}
+
+function formatDate(value) {
+  if (!value) return 'Never';
+  return dateFormatter.format(new Date(value));
+}
+
+function formatValue(value) {
+  if (value === undefined || value === null) return '—';
+  if (typeof value === 'string') return value;
+
+  return JSON.stringify(value);
+}
+
+function statusLabel(status) {
+  if (status === 'trusted') return 'Trusted';
+  if (status === 'untrusted') return 'Untrusted';
+  if (status === 'needs_review') return 'Needs review';
+  if (status === 'approved') return 'Approved';
+  if (status === 'rejected') return 'Rejected';
+  if (status === 'duplicate') return 'Duplicate';
+  if (status === 'variant') return 'Variant';
+  if (status === 'oem_clone') return 'OEM clone';
+  return 'Unknown';
+}
+
+async function requestJson(path, options) {
+  const response = await fetch(path, {
+    credentials: 'same-origin',
+    headers: {
+      accept: 'application/json',
+      ...(options?.headers ?? {}),
+    },
+    ...options,
+  });
+
+  const payload = await response.json().catch(() => undefined);
+
+  if (!response.ok) {
+    throw new Error(payload?.error ?? response.status + ' ' + response.statusText);
+  }
+
+  return payload;
+}
+
+function renderLoading(label) {
+  app.innerHTML = '<section class="panel"><div class="panel-body muted">' + escapeHtml(label) + '</div></section>';
+}
+
+function renderError(title, error) {
+  app.innerHTML =
+    '<section class="panel"><div class="panel-body"><div class="error">' +
+    '<strong>' + escapeHtml(title) + '</strong>' +
+    '<span>' + escapeHtml(error.message ?? error) + '</span>' +
+    '</div></div></section>';
+}
+
+function renderMetric(label, value) {
+  return '<div class="metric"><span>' + escapeHtml(label) + '</span><strong>' + escapeHtml(value) + '</strong></div>';
+}
+
+function renderBadge(status) {
+  return '<span class="badge ' + escapeHtml(status) + '">' + escapeHtml(statusLabel(status)) + '</span>';
+}
+
+function renderManufacturers(manufacturers) {
+  const stats = {
+    total: manufacturers.length,
+    trusted: manufacturers.filter((item) => item.trustStatus === 'trusted').length,
+    unknown: manufacturers.filter((item) => item.trustStatus === 'unknown').length,
+    untrusted: manufacturers.filter((item) => item.trustStatus === 'untrusted').length,
+  };
+
+  const rows = manufacturers
+    .map(
+      (manufacturer) =>
+        '<tr>' +
+        '<th scope="row"><div><strong>' + escapeHtml(manufacturer.name) + '</strong><br />' +
+        '<span class="mono muted">' + escapeHtml(manufacturer.slug) + '</span></div></th>' +
+        '<td>' + renderBadge(manufacturer.trustStatus) + '</td>' +
+        '<td class="mono">' + escapeHtml(manufacturer.productCount) + '</td>' +
+        '<td>' + escapeHtml(formatDate(manufacturer.lastProductUpdatedAt)) + '</td>' +
+        '<td class="muted">' + escapeHtml(manufacturer.notes || 'No notes') + '</td>' +
+        '<td><a class="button" href="#/manufacturers/' + encodeURIComponent(manufacturer.slug) + '">Edit</a></td>' +
+        '</tr>',
+    )
+    .join('');
+
+  app.innerHTML =
+    '<section class="stats-grid" aria-label="Manufacturer moderation stats">' +
+    renderMetric('Total', stats.total) +
+    renderMetric('Trusted', stats.trusted) +
+    renderMetric('Unknown', stats.unknown) +
+    renderMetric('Untrusted', stats.untrusted) +
+    '</section>' +
+    '<section class="panel"><div class="panel-header"><h2>Manufacturers</h2></div>' +
+    (manufacturers.length
+      ? '<div class="table-wrap"><table><thead><tr>' +
+        '<th scope="col">Brand</th><th scope="col">Trust</th><th scope="col">Products</th>' +
+        '<th scope="col">Updated</th><th scope="col">Notes</th><th scope="col">Actions</th>' +
+        '</tr></thead><tbody>' + rows + '</tbody></table></div>'
+      : '<div class="panel-body"><div class="empty">No manufacturers discovered yet</div></div>') +
+    '</section>';
+}
+
+function renderManufacturer(manufacturer) {
+  const products = manufacturer.products ?? [];
+  const productRows = products
+    .map(
+      (product) =>
+        '<tr>' +
+        '<th scope="row"><strong>' + escapeHtml(product.canonicalName || product.title) + '</strong><br />' +
+        '<span class="mono muted">' + escapeHtml(product.id) + '</span></th>' +
+        '<td>' + escapeHtml(product.moderationStatus) + '</td>' +
+        '<td>' + escapeHtml(formatDate(product.updatedAt)) + '</td>' +
+        '</tr>',
+    )
+    .join('');
+
+  app.innerHTML =
+    '<div class="layout">' +
+    '<section class="panel">' +
+    '<div class="panel-header"><div><h2>Moderation</h2><p class="muted">Mark unreliable brands so source conflicts are visible during moderation.</p></div>' +
+    renderBadge(manufacturer.trustStatus) + '</div>' +
+    '<div class="panel-body">' +
+    '<form class="form-grid" data-manufacturer-form data-slug="' + escapeHtml(manufacturer.slug) + '">' +
+    '<label class="field"><span>Trust status</span><select name="trustStatus">' +
+    '<option value="trusted"' + (manufacturer.trustStatus === 'trusted' ? ' selected' : '') + '>Trusted</option>' +
+    '<option value="unknown"' + (manufacturer.trustStatus === 'unknown' ? ' selected' : '') + '>Unknown</option>' +
+    '<option value="untrusted"' + (manufacturer.trustStatus === 'untrusted' ? ' selected' : '') + '>Untrusted</option>' +
+    '</select></label>' +
+    '<label class="field"><span>Notes</span><textarea name="notes" rows="6" placeholder="Why this manufacturer is trusted or not trusted">' +
+    escapeHtml(manufacturer.notes ?? '') + '</textarea></label>' +
+    '<div class="actions"><button class="button primary" type="submit">Save</button>' +
+    '<a class="button ghost" href="#/">Back</a></div>' +
+    '<div class="status-line" data-save-state></div>' +
+    '</form></div></section>' +
+    '<section class="panel"><div class="panel-header"><div><h2>Products</h2><p class="muted">Products currently linked to this manufacturer.</p></div>' +
+    '<span class="badge unknown">' + escapeHtml(products.length) + '</span></div>' +
+    (products.length
+      ? '<div class="table-wrap"><table><thead><tr><th scope="col">Product</th><th scope="col">Moderation</th><th scope="col">Updated</th></tr></thead><tbody>' +
+        productRows + '</tbody></table></div>'
+      : '<div class="panel-body"><div class="empty">No products linked to this manufacturer</div></div>') +
+    '</section></div>';
+}
+
+function renderModerationProducts(products) {
+  const rows = products
+    .map(
+      (product) =>
+        '<tr>' +
+        '<th scope="row"><strong>' + escapeHtml(product.canonicalName || product.title) + '</strong><br />' +
+        '<span class="muted">' + escapeHtml(product.title) + '</span></th>' +
+        '<td>' + renderBadge(product.moderationStatus) + '</td>' +
+        '<td class="mono">' + escapeHtml(product.sourceCount) + '</td>' +
+        '<td class="mono">' + escapeHtml(product.rawSpecCount) + '</td>' +
+        '<td class="mono">' + escapeHtml(product.pendingConflictCount) + '</td>' +
+        '<td>' +
+          '<div class="actions">' +
+          '<a class="button" href="#/moderation/' + encodeURIComponent(product.id) + '">Edit</a>' +
+          '<button class="button primary" type="button" data-product-status="approved" data-product-id="' + escapeHtml(product.id) + '">Approve</button>' +
+          '<button class="button" type="button" data-product-status="rejected" data-product-id="' + escapeHtml(product.id) + '">Reject</button>' +
+          '</div>' +
+        '</td>' +
+        '</tr>',
+    )
+    .join('');
+
+  app.innerHTML =
+    '<section class="panel"><div class="panel-header"><div><h2>Product moderation</h2><p class="muted">Approve source cards after checking title, canonical model and source conflicts.</p></div>' +
+    '<span class="badge unknown">' + escapeHtml(products.length) + '</span></div>' +
+    (products.length
+      ? '<div class="table-wrap"><table><thead><tr>' +
+        '<th scope="col">Product</th><th scope="col">Status</th><th scope="col">Sources</th>' +
+        '<th scope="col">Raw specs</th><th scope="col">Conflicts</th><th scope="col">Actions</th>' +
+        '</tr></thead><tbody>' + rows + '</tbody></table></div>'
+      : '<div class="panel-body"><div class="empty">No products need moderation</div></div>') +
+    '</section>';
+}
+
+function renderProductEditor(product) {
+  const sourceRows = (product.sources ?? [])
+    .map(
+      (source) =>
+        '<tr><td>' + escapeHtml(source.source) + '</td><td class="mono">' +
+        escapeHtml(source.sourceProductId ?? '—') + '</td><td><a class="source-link" href="' +
+        escapeHtml(source.url) + '" target="_blank" rel="noreferrer">Source card</a></td><td>' +
+        escapeHtml(formatDate(source.lastFetchedAt)) + '</td></tr>',
+    )
+    .join('');
+  const rawSpecRows = (product.rawSpecs ?? [])
+    .map(
+      (spec) =>
+        '<tr><td>' + escapeHtml(spec.source) + '</td><td>' + escapeHtml(spec.group ?? '—') + '</td><th scope="row">' +
+        escapeHtml(spec.label) + '</th><td>' + escapeHtml(formatValue(spec.value)) + '</td><td class="mono">' +
+        escapeHtml(spec.normalizedKey ?? '—') + '</td><td>' + escapeHtml(formatValue(spec.normalizedValue)) + '</td></tr>',
+    )
+    .join('');
+  const conflictRows = (product.conflicts ?? [])
+    .map(
+      (conflict) =>
+        '<tr><th scope="row">' + escapeHtml(conflict.key) + '</th><td>' + renderBadge(conflict.state) + '</td><td class="mono">' +
+        escapeHtml(formatValue(conflict.values)) + '</td></tr>',
+    )
+    .join('');
+
+  app.innerHTML =
+    '<div class="layout">' +
+    '<section class="panel"><div class="panel-header"><div><h2>Product editor</h2><p class="muted">' +
+    escapeHtml(product.title) + '</p></div>' + renderBadge(product.moderationStatus) + '</div><div class="panel-body">' +
+    '<form class="form-grid" data-product-form data-product-id="' + escapeHtml(product.id) + '">' +
+    '<label class="field"><span>Canonical model</span><input name="canonicalModel" value="' +
+    escapeHtml(product.canonicalModel) + '" required maxlength="300" /></label>' +
+    '<label class="field"><span>Canonical name</span><input name="canonicalName" value="' +
+    escapeHtml(product.canonicalName) + '" required maxlength="300" /></label>' +
+    '<label class="field"><span>Moderation status</span><select name="moderationStatus">' +
+    ['needs_review', 'approved', 'rejected', 'duplicate', 'variant', 'oem_clone']
+      .map((status) => '<option value="' + status + '"' + (status === product.moderationStatus ? ' selected' : '') + '>' + escapeHtml(statusLabel(status)) + '</option>')
+      .join('') +
+    '</select></label><div class="actions"><button class="button primary" type="submit">Save</button>' +
+    '<a class="button ghost" href="#/moderation">Back</a></div><div class="status-line" data-save-state></div></form></div></section>' +
+    '<section class="panel"><div class="panel-header"><div><h2>Sources</h2></div><span class="badge unknown">' +
+    escapeHtml((product.sources ?? []).length) + '</span></div>' +
+    (sourceRows ? '<div class="table-wrap"><table><thead><tr><th>Source</th><th>ID</th><th>Card</th><th>Fetched</th></tr></thead><tbody>' + sourceRows + '</tbody></table></div>' : '<div class="panel-body"><div class="empty">No source records</div></div>') +
+    '</section></div>' +
+    '<section class="panel"><div class="panel-header"><div><h2>Raw specifications</h2></div><span class="badge unknown">' +
+    escapeHtml((product.rawSpecs ?? []).length) + '</span></div>' +
+    (rawSpecRows ? '<div class="table-wrap"><table><thead><tr><th>Source</th><th>Group</th><th>Label</th><th>Raw value</th><th>Normalized key</th><th>Normalized value</th></tr></thead><tbody>' + rawSpecRows + '</tbody></table></div>' : '<div class="panel-body"><div class="empty">No raw specifications</div></div>') +
+    '</section>' +
+    '<section class="panel"><div class="panel-header"><div><h2>Source conflicts</h2></div><span class="badge unknown">' +
+    escapeHtml((product.conflicts ?? []).length) + '</span></div>' +
+    (conflictRows ? '<div class="table-wrap"><table><thead><tr><th>Key</th><th>Status</th><th>Values</th></tr></thead><tbody>' + conflictRows + '</tbody></table></div>' : '<div class="panel-body"><div class="empty">No source conflicts</div></div>') +
+    '</section>';
+}
+
+async function loadManufacturers() {
+  renderLoading('Loading manufacturers');
+  try {
+    const payload = await requestJson(adminApiBasePath + '/manufacturers');
+    renderManufacturers(payload.manufacturers ?? []);
+  } catch (error) {
+    renderError('Manufacturers could not be loaded', error);
+  }
+}
+
+async function loadManufacturer(slug) {
+  renderLoading('Loading manufacturer');
+  try {
+    const payload = await requestJson(adminApiBasePath + '/manufacturers/' + encodeURIComponent(slug));
+    renderManufacturer(payload.manufacturer);
+  } catch (error) {
+    renderError('Manufacturer could not be loaded', error);
+  }
+}
+
+async function loadModerationProducts() {
+  renderLoading('Loading product moderation');
+  try {
+    const payload = await requestJson(adminApiBasePath + '/moderation/products?status=needs_review');
+    renderModerationProducts(payload.products ?? []);
+  } catch (error) {
+    renderError('Product moderation could not be loaded', error);
+  }
+}
+
+async function loadModerationProduct(productId) {
+  renderLoading('Loading product');
+  try {
+    const payload = await requestJson(adminApiBasePath + '/moderation/products/' + encodeURIComponent(productId));
+    renderProductEditor(payload.product);
+  } catch (error) {
+    renderError('Product could not be loaded', error);
+  }
+}
+
+function route() {
+  const hash = window.location.hash || '#/';
+  const match = hash.match(/^#\\/manufacturers\\/(.+)$/);
+
+  if (match) {
+    loadManufacturer(decodeURIComponent(match[1]));
+    return;
+  }
+
+  const productMatch = hash.match(/^#\\/moderation\\/([^/]+)$/);
+
+  if (productMatch) {
+    loadModerationProduct(decodeURIComponent(productMatch[1]));
+    return;
+  }
+
+  if (hash === '#/moderation') {
+    loadModerationProducts();
+    return;
+  }
+
+  loadManufacturers();
+}
+
+app.addEventListener('submit', async (event) => {
+  const form = event.target.closest('[data-manufacturer-form]');
+
+  if (!form) return;
+
+  event.preventDefault();
+  const state = form.querySelector('[data-save-state]');
+  const slug = form.dataset.slug;
+  const formData = new FormData(form);
+  const trustStatus = formData.get('trustStatus');
+  const notesValue = String(formData.get('notes') ?? '').trim();
+
+  state.textContent = 'Saving';
+
+  try {
+    await requestJson(adminApiBasePath + '/manufacturers/' + encodeURIComponent(slug), {
+      method: 'PATCH',
+      headers: {
+        'content-type': 'application/json',
+      },
+      body: JSON.stringify({
+        trustStatus,
+        notes: notesValue || null,
+      }),
+    });
+    state.textContent = 'Saved';
+    await loadManufacturer(slug);
+  } catch (error) {
+    state.textContent = error.message ?? String(error);
+  }
+});
+
+app.addEventListener('click', async (event) => {
+  const button = event.target.closest('[data-product-status]');
+
+  if (!button) return;
+
+  const productId = button.dataset.productId;
+  const moderationStatus = button.dataset.productStatus;
+
+  if (!productId || !moderationStatus) return;
+
+  button.disabled = true;
+
+  try {
+    await requestJson(adminApiBasePath + '/moderation/products/' + encodeURIComponent(productId), {
+      method: 'PATCH',
+      headers: {
+        'content-type': 'application/json',
+      },
+      body: JSON.stringify({ moderationStatus }),
+    });
+    await loadModerationProducts();
+  } catch (error) {
+    button.disabled = false;
+    window.alert(error.message ?? String(error));
+  }
+});
+
+app.addEventListener('submit', async (event) => {
+  const form = event.target.closest('[data-product-form]');
+
+  if (!form) return;
+
+  event.preventDefault();
+  const state = form.querySelector('[data-save-state]');
+  const productId = form.dataset.productId;
+  const formData = new FormData(form);
+
+  if (!productId) return;
+
+  state.textContent = 'Saving';
+
+  try {
+    await requestJson(adminApiBasePath + '/moderation/products/' + encodeURIComponent(productId), {
+      method: 'PATCH',
+      headers: {
+        'content-type': 'application/json',
+      },
+      body: JSON.stringify({
+        canonicalModel: String(formData.get('canonicalModel') ?? '').trim(),
+        canonicalName: String(formData.get('canonicalName') ?? '').trim(),
+        moderationStatus: formData.get('moderationStatus'),
+      }),
+    });
+    state.textContent = 'Saved';
+    await loadModerationProduct(productId);
+  } catch (error) {
+    state.textContent = error.message ?? String(error);
+  }
+});
+
+window.addEventListener('hashchange', route);
+route();
+`;

ingestion/src/config.test.ts

@@ -0,0 +1,34 @@
+import { describe, expect, it } from 'vitest';
+import { buildSourceSeeds, yandexMarketDashcamCategoryUrl } from './config.ts';
+
+describe('ingestion config', () => {
+  it('always includes implemented Yandex Market source seed', () => {
+    expect(buildSourceSeeds({})).toEqual([
+      {
+        categoryUrl: yandexMarketDashcamCategoryUrl,
+        implemented: true,
+        source: 'yandex_market',
+      },
+    ]);
+  });
+
+  it('enables an explicitly configured MVideo source seed in the worker queue', () => {
+    expect(
+      buildSourceSeeds({
+        MVIDEO_DASHCAM_CATEGORY_URL: 'https://www.mvideo.ru/avtomobilnye-videoregistratory',
+        YANDEX_MARKET_DASHCAM_CATEGORY_URL: 'https://market.yandex.ru/custom',
+      }),
+    ).toEqual([
+      {
+        categoryUrl: 'https://market.yandex.ru/custom',
+        implemented: true,
+        source: 'yandex_market',
+      },
+      {
+        categoryUrl: 'https://www.mvideo.ru/avtomobilnye-videoregistratory',
+        implemented: true,
+        source: 'mvideo',
+      },
+    ]);
+  });
+});

ingestion/src/config.ts

@@ -0,0 +1,57 @@
+import path from 'node:path';
+import type { MarketSource } from './types.ts';
+
+export const yandexMarketDashcamCategoryUrl =
+  'https://market.yandex.ru/catalog--avtomobilnye-videoregistratory/82798275/list?hid=82798269';
+export const mvideoDashcamCategoryUrl =
+  'https://www.mvideo.ru/product-list-page?q=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE%D1%80%D0%B5%D0%B3%D0%B8%D1%81%D1%82%D1%80%D0%B0%D1%82%D0%BE%D1%80%D1%8B';
+
+export type SourceSeed = {
+  categoryUrl: string;
+  implemented: boolean;
+  source: MarketSource;
+};
+
+function numberEnv(name: string, fallback: number) {
+  const value = Number(process.env[name]);
+
+  return Number.isFinite(value) ? value : fallback;
+}
+
+export function buildSourceSeeds(env: Record<string, string | undefined> = process.env): SourceSeed[] {
+  const seeds: SourceSeed[] = [
+    {
+      categoryUrl: env.YANDEX_MARKET_DASHCAM_CATEGORY_URL ?? yandexMarketDashcamCategoryUrl,
+      implemented: true,
+      source: 'yandex_market',
+    },
+  ];
+
+  if (env.MVIDEO_DASHCAM_CATEGORY_URL) {
+    seeds.push({
+      categoryUrl: env.MVIDEO_DASHCAM_CATEGORY_URL,
+      implemented: true,
+      source: 'mvideo',
+    });
+  }
+
+  return seeds;
+}
+
+const sourceSeeds = buildSourceSeeds();
+
+export const ingestionConfig = {
+  databaseUrl: process.env.INGESTION_DATABASE_URL ?? 'postgres://videoreg:videoreg@localhost:5432/videoreg_ingestion',
+  snapshotDir: process.env.INGESTION_SNAPSHOT_DIR ?? path.join(process.cwd(), 'ingestion', 'data', 'snapshots'),
+  exportDir: process.env.INGESTION_EXPORT_DIR ?? path.join(process.cwd(), 'ingestion', 'data', 'exports'),
+  seedCategoryUrl: process.env.YANDEX_MARKET_DASHCAM_CATEGORY_URL ?? yandexMarketDashcamCategoryUrl,
+  sourceSeeds,
+  implementedSourceSeeds: sourceSeeds.filter((seed) => seed.implemented),
+  fetchDelayMinMs: numberEnv('INGESTION_FETCH_DELAY_MIN_MS', 5_000),
+  fetchDelayMaxMs: numberEnv('INGESTION_FETCH_DELAY_MAX_MS', 15_000),
+  categoryDailyBudget: numberEnv('INGESTION_CATEGORY_DAILY_BUDGET', 25),
+  productDailyBudget: numberEnv('INGESTION_PRODUCT_DAILY_BUDGET', 100),
+  sitemapDailyBudget: numberEnv('INGESTION_SITEMAP_DAILY_BUDGET', 25),
+  workerBatchSize: numberEnv('INGESTION_WORKER_BATCH_SIZE', 1),
+  workerIdleMs: numberEnv('INGESTION_WORKER_IDLE_MS', 30_000),
+};

ingestion/src/db.ts

@@ -0,0 +1,10 @@
+import pg from 'pg';
+import { ingestionConfig } from './config.ts';
+
+const { Pool } = pg;
+
+export function createPgPool(connectionString = ingestionConfig.databaseUrl) {
+  return new Pool({
+    connectionString,
+  });
+}

ingestion/src/exportCatalog.ts

@@ -0,0 +1,12 @@
+import { createPgPool } from './db.ts';
+import { exportApprovedCatalog } from './exporter.ts';
+
+const pool = createPgPool();
+
+try {
+  const manifest = await exportApprovedCatalog(pool);
+
+  console.log(JSON.stringify(manifest, null, 2));
+} finally {
+  await pool.end();
+}

ingestion/src/exporter.ts

@@ -0,0 +1,184 @@
+import { createHash } from 'node:crypto';
+import { mkdir, rename, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import type { Pool } from 'pg';
+import {
+  DashcamCatalogExportSchema,
+  type DashcamCatalogExport,
+  type MarketSource,
+} from '../../web-front/src/entities/dashcam-catalog/model/schema.ts';
+import { ingestionConfig } from './config.ts';
+
+type ProductRow = {
+  id: string;
+  brand: string;
+  model: string;
+  canonical_name: string;
+  title: string;
+  discovered_at: Date;
+  updated_at: Date;
+};
+
+type SpecRow = {
+  product_id: string;
+  key: string;
+  value: string | number | boolean;
+};
+
+type ImageRow = {
+  product_id: string;
+  artifact_url: string;
+  original_url: string;
+  source: MarketSource;
+  sha256: string | null;
+  width: number | null;
+  height: number | null;
+};
+
+type SourceRow = {
+  product_id: string;
+  source: MarketSource;
+  source_product_id: string | null;
+  url: string;
+  stable_content_hash: string;
+  last_fetched_at: Date;
+};
+
+function sha256(buffer: Buffer) {
+  return createHash('sha256').update(buffer).digest('hex');
+}
+
+function exportId(now: Date) {
+  return `catalog_${now.toISOString().replace(/[-:.]/g, '').replace('T', '_').replace('Z', '')}`;
+}
+
+function groupRows<T extends { product_id: string }>(rows: T[]) {
+  const grouped = new Map<string, T[]>();
+
+  for (const row of rows) {
+    grouped.set(row.product_id, [...(grouped.get(row.product_id) ?? []), row]);
+  }
+
+  return grouped;
+}
+
+async function writeFileAtomically(filePath: string, contents: Buffer) {
+  const temporaryPath = `${filePath}.${process.pid}.tmp`;
+
+  await mkdir(path.dirname(filePath), { recursive: true });
+  await writeFile(temporaryPath, contents);
+  await rename(temporaryPath, filePath);
+}
+
+export async function exportApprovedCatalog(pool: Pool, now = new Date()) {
+  const createdAt = now.toISOString();
+  const id = exportId(now);
+  const [products, specs, images, sources] = await Promise.all([
+    pool.query<ProductRow>(
+      `
+        select id, brand, model, canonical_name, title, discovered_at, updated_at
+        from products
+        where moderation_status = 'approved'
+        order by brand asc, model asc, id asc
+      `,
+    ),
+    pool.query<SpecRow>('select product_id, key, value from product_specs'),
+    pool.query<ImageRow>(
+      `
+        select product_id, artifact_url, original_url, source, sha256, width, height
+        from product_images
+        where status = 'approved'
+          and artifact_url is not null
+        order by product_id, sort_order asc, original_url asc
+      `,
+    ),
+    pool.query<SourceRow>(
+      `
+        select product_id, source, source_product_id, url, stable_content_hash, last_fetched_at
+        from product_sources
+        where product_id is not null
+          and stable_content_hash is not null
+          and last_fetched_at is not null
+        order by product_id, source, url
+      `,
+    ),
+  ]);
+
+  const specsByProduct = groupRows(specs.rows);
+  const imagesByProduct = groupRows(images.rows);
+  const sourcesByProduct = groupRows(sources.rows);
+  const catalog: DashcamCatalogExport = {
+    schemaVersion: 1,
+    exportId: id,
+    createdAt,
+    products: products.rows
+      .map((product) => ({
+        id: product.id,
+        brand: product.brand,
+        model: product.model,
+        canonicalName: product.canonical_name,
+        title: product.title,
+        specs: Object.fromEntries((specsByProduct.get(product.id) ?? []).map((spec) => [spec.key, spec.value])),
+        images: (imagesByProduct.get(product.id) ?? []).map((image) => ({
+          url: image.artifact_url,
+          originalUrl: image.original_url,
+          source: image.source,
+          ...(image.sha256 ? { sha256: image.sha256 } : {}),
+          ...(image.width ? { width: image.width } : {}),
+          ...(image.height ? { height: image.height } : {}),
+        })),
+        sources: (sourcesByProduct.get(product.id) ?? []).map((source) => ({
+          source: source.source,
+          ...(source.source_product_id ? { sourceProductId: source.source_product_id } : {}),
+          url: source.url,
+          stableContentHash: source.stable_content_hash,
+          fetchedAt: source.last_fetched_at.toISOString(),
+        })),
+        discoveredAt: product.discovered_at.toISOString(),
+        updatedAt: product.updated_at.toISOString(),
+      }))
+      .filter((product) => product.sources.length > 0),
+  };
+
+  const validatedCatalog = DashcamCatalogExportSchema.parse(catalog);
+  const artifactBuffer = Buffer.from(`${JSON.stringify(validatedCatalog, null, 2)}\n`);
+  const artifactHash = sha256(artifactBuffer);
+  const artifactRelativePath = path.posix.join('artifacts', id, 'catalog.json');
+  const artifactPath = path.join(ingestionConfig.exportDir, artifactRelativePath);
+  const manifest = {
+    schemaVersion: 1 as const,
+    exportId: id,
+    createdAt,
+    artifactPath: artifactRelativePath,
+    sha256: artifactHash,
+    itemCount: validatedCatalog.products.length,
+  };
+  const manifestBuffer = Buffer.from(`${JSON.stringify(manifest, null, 2)}\n`);
+
+  await writeFileAtomically(artifactPath, artifactBuffer);
+  await writeFileAtomically(path.join(ingestionConfig.exportDir, 'manifest.json'), manifestBuffer);
+  await pool.query(
+    `
+      insert into export_runs (id, schema_version, created_at, artifact_path, manifest_path, sha256, item_count)
+      values ($1, 1, $2, $3, $4, $5, $6)
+      on conflict (id) do nothing
+    `,
+    [id, createdAt, artifactPath, path.join(ingestionConfig.exportDir, 'manifest.json'), artifactHash, manifest.itemCount],
+  );
+  await pool.query(
+    `
+      update crawl_jobs job
+      set state = 'exported',
+          updated_at = now()
+      from product_sources source
+      join products product on product.id = source.product_id
+      where product.moderation_status = 'approved'
+        and job.source = source.source
+        and job.url = source.url
+        and job.job_type = 'product'
+        and job.state in ('needs_review', 'approved')
+    `,
+  );
+
+  return manifest;
+}

ingestion/src/fetcher.ts

@@ -0,0 +1,107 @@
+import { createHash } from 'node:crypto';
+import { mkdir, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { ingestionConfig } from './config.ts';
+import type { CrawlFailureReason, FetchClassification, MarketSource } from './types.ts';
+
+type FetchSnapshotInput = {
+  source: MarketSource;
+  url: string;
+  snapshotDir?: string;
+  fetcher?: typeof fetch;
+  headers?: HeadersInit;
+  accept?: string;
+};
+
+export type RawFetchSnapshot = {
+  source: MarketSource;
+  url: string;
+  fetchedAt: string;
+  httpStatus: number;
+  rawSnapshotHash: string;
+  rawHtmlPath: string;
+  html: string;
+  setCookieHeaders: string[];
+};
+
+function getSetCookieHeaders(response: Response) {
+  const headers = response.headers as Headers & { getSetCookie?: () => string[] };
+
+  if (typeof headers.getSetCookie === 'function') return headers.getSetCookie();
+
+  const single = response.headers.get('set-cookie');
+
+  return single ? [single] : [];
+}
+
+function classifyResponse(status: number, body: string): FetchClassification {
+  const lowered = body.slice(0, 80_000).toLowerCase();
+
+  if (status === 404) return { ok: false, reason: 'not_found', retryable: false };
+  if (status === 429) return { ok: false, reason: 'rate_limited', retryable: true };
+  if (status === 403) return { ok: false, reason: 'blocked', retryable: true };
+  if (status >= 500) return { ok: false, reason: 'rate_limited', retryable: true };
+  if (/captcha|showcaptcha|smartcaptcha|подтвердите, что вы не робот/.test(lowered)) {
+    return { ok: false, reason: 'captcha', retryable: true };
+  }
+  if (status < 200 || status >= 300) return { ok: false, reason: 'needs_manual_review', retryable: false };
+
+  return { ok: true };
+}
+
+function hash(value: string) {
+  return createHash('sha256').update(value).digest('hex');
+}
+
+function snapshotPath(snapshotDir: string, source: MarketSource, fetchedAt: string, digest: string) {
+  const day = fetchedAt.slice(0, 10);
+
+  return path.join(snapshotDir, source, day, `${digest}.html`);
+}
+
+export async function fetchRawSnapshot(input: FetchSnapshotInput): Promise<RawFetchSnapshot> {
+  const fetcher = input.fetcher ?? fetch;
+  const response = await fetcher(input.url, {
+    headers: {
+      accept: input.accept ?? 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+      'accept-language': 'ru,en;q=0.8',
+      'user-agent': 'videoreg-ingestion/1.0 (+https://videoreg.ru)',
+      ...(input.headers ?? {}),
+    },
+    redirect: 'follow',
+  });
+  const html = await response.text();
+  const classification = classifyResponse(response.status, html);
+
+  if (!classification.ok) {
+    const error = new Error(`Fetch classified as ${classification.reason} for status ${response.status}`);
+    (error as Error & { reason?: CrawlFailureReason }).reason = classification.reason;
+    throw error;
+  }
+
+  const fetchedAt = new Date().toISOString();
+  const rawSnapshotHash = hash(html);
+  const rawHtmlPath = snapshotPath(input.snapshotDir ?? ingestionConfig.snapshotDir, input.source, fetchedAt, rawSnapshotHash);
+
+  await mkdir(path.dirname(rawHtmlPath), { recursive: true });
+  await writeFile(rawHtmlPath, html, 'utf8');
+
+  return {
+    source: input.source,
+    url: response.url || input.url,
+    fetchedAt,
+    httpStatus: response.status,
+    rawSnapshotHash,
+    rawHtmlPath,
+    html,
+    setCookieHeaders: getSetCookieHeaders(response),
+  };
+}
+
+export function getFetchFailureReason(error: unknown) {
+  return (error as Error & { reason?: CrawlFailureReason }).reason;
+}
+
+export function randomDelayMs(minMs = ingestionConfig.fetchDelayMinMs, maxMs = ingestionConfig.fetchDelayMaxMs) {
+  return Math.floor(minMs + Math.random() * Math.max(0, maxMs - minMs));
+}

ingestion/src/html.ts

@@ -0,0 +1,87 @@
+const namedEntities: Record<string, string> = {
+  amp: '&',
+  gt: '>',
+  lt: '<',
+  quot: '"',
+  apos: "'",
+  nbsp: ' ',
+};
+
+export function decodeHtml(value: string) {
+  return value.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (entity, code: string) => {
+    const normalized = code.toLowerCase();
+
+    if (normalized.startsWith('#x')) {
+      return String.fromCodePoint(Number.parseInt(normalized.slice(2), 16));
+    }
+
+    if (normalized.startsWith('#')) {
+      return String.fromCodePoint(Number.parseInt(normalized.slice(1), 10));
+    }
+
+    return namedEntities[normalized] ?? entity;
+  });
+}
+
+export function stripTags(value: string) {
+  return decodeHtml(value.replace(/<[^>]*>/g, ' ')).replace(/\s+/g, ' ').trim();
+}
+
+export function normalizeWhitespace(value: string) {
+  return value.trim().replace(/\s+/g, ' ');
+}
+
+export function uniqueStable<T>(values: T[]) {
+  return [...new Set(values)];
+}
+
+function parseJsonPayload(raw: string) {
+  return JSON.parse(decodeHtml(raw).trim()) as unknown;
+}
+
+function flattenJsonLd(input: unknown): unknown[] {
+  if (Array.isArray(input)) {
+    return input.flatMap(flattenJsonLd);
+  }
+
+  if (input && typeof input === 'object') {
+    const object = input as Record<string, unknown>;
+    const graph = object['@graph'];
+
+    return graph ? [object, ...flattenJsonLd(graph)] : [object];
+  }
+
+  return [];
+}
+
+export function extractJsonLdObjects(html: string) {
+  const scripts = [...html.matchAll(/<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi)];
+  const objects: unknown[] = [];
+
+  for (const [, raw] of scripts) {
+    try {
+      objects.push(...flattenJsonLd(parseJsonPayload(raw)));
+    } catch {
+      // Ignore malformed analytics or partial JSON-LD blocks.
+    }
+  }
+
+  return objects;
+}
+
+export function getObjectString(value: unknown) {
+  if (typeof value === 'string') return normalizeWhitespace(value);
+  if (value && typeof value === 'object' && typeof (value as { name?: unknown }).name === 'string') {
+    return normalizeWhitespace((value as { name: string }).name);
+  }
+
+  return undefined;
+}
+
+export function readMetaContent(html: string, property: string) {
+  const escaped = property.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+  const pattern = new RegExp(`<meta[^>]+(?:property|name)=["']${escaped}["'][^>]+content=["']([^"']+)["'][^>]*>`, 'i');
+  const match = html.match(pattern);
+
+  return match ? decodeHtml(match[1]).trim() : undefined;
+}

ingestion/src/migrate.ts

@@ -0,0 +1,51 @@
+import { readFile, readdir } from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { createPgPool } from './db.ts';
+
+const currentFile = fileURLToPath(import.meta.url);
+const migrationsDir = path.resolve(path.dirname(currentFile), '..', 'migrations');
+
+export async function runMigrations() {
+  const pool = createPgPool();
+  const client = await pool.connect();
+
+  try {
+    await client.query("select pg_advisory_lock(hashtext('videoreg_ingestion_migrations'))");
+    await client.query(`
+      create table if not exists schema_migrations (
+        filename text primary key,
+        applied_at timestamptz not null default now()
+      )
+    `);
+
+    const files = (await readdir(migrationsDir)).filter((file) => file.endsWith('.sql')).sort();
+
+    for (const file of files) {
+      const applied = await client.query('select 1 from schema_migrations where filename = $1', [file]);
+
+      if (applied.rowCount) continue;
+
+      const sql = await readFile(path.join(migrationsDir, file), 'utf8');
+
+      await client.query('begin');
+      try {
+        await client.query(sql);
+        await client.query('insert into schema_migrations (filename) values ($1)', [file]);
+        await client.query('commit');
+        console.log(`applied migration ${file}`);
+      } catch (error) {
+        await client.query('rollback');
+        throw error;
+      }
+    }
+  } finally {
+    await client.query("select pg_advisory_unlock(hashtext('videoreg_ingestion_migrations'))").catch(() => undefined);
+    client.release();
+    await pool.end();
+  }
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  await runMigrations();
+}

ingestion/src/normalizer.ts

@@ -0,0 +1,300 @@
+import type { NormalizedMarketProduct, ParsedMarketProduct, ParsedTechnicalSpec } from './types.ts';
+
+const brandAliases = new Map<string, string>([
+  ['ibox', 'iBOX'],
+  ['sho-me', 'Sho-Me'],
+  ['trendvision', 'TrendVision'],
+  ['fujida', 'Fujida'],
+  ['intego', 'INTEGO'],
+  ['spawnson', 'SPAWNSON'],
+]);
+
+const specAliases = new Map<string, string>([
+  ['количество камер', 'channels'],
+  ['количество каналов записи видео', 'video_channels'],
+  ['количество каналов записи звука', 'audio_channels'],
+  ['качество видеосъемки/ fps', 'resolution_front'],
+  ['максимальное разрешение видеосъемки', 'resolution_max'],
+  ['макс. разрешение видеосъемки', 'resolution_max'],
+  ['максимальное разрешение камеры заднего вида', 'resolution_max'],
+  ['разрешение видеозаписи (при макс. частоте)', 'resolution_front'],
+  ['разрешение видеозаписи', 'resolution_front'],
+  ['макс. частота кадров', 'frame_rate_max_fps'],
+  ['частота кадров при макс. разрешении', 'frame_rate_max_fps'],
+  ['частота при макс. разрешении камеры заднего вида', 'frame_rate_max_fps'],
+  ['угол обзора (диагональ)', 'view_angle_degrees'],
+  ['угол обзора (ширина)', 'view_angle_width_degrees'],
+  ['угол обзора (высота)', 'view_angle_height_degrees'],
+  ['угол обзора (основная камера)', 'view_angle_degrees'],
+  ['угол обзора', 'view_angle_degrees'],
+  ['gps', 'gps'],
+  ['встроенный gps', 'gps'],
+  ['наличие gps модуля', 'gps'],
+  ['поддержка gps', 'gps'],
+  ['глонасс', 'glonass'],
+  ['встроенный глонасс', 'glonass'],
+  ['наличие глонасс', 'glonass'],
+  ['поддержка глонасс', 'glonass'],
+  ['wi-fi', 'wifi'],
+  ['wifi', 'wifi'],
+  ['поддержка wi-fi', 'wifi'],
+  ['поддержка wifi', 'wifi'],
+  ['трансляция видео через wi-fi', 'wifi'],
+  ['трансляция видео через wifi', 'wifi'],
+  ['управление с помощью смартфона', 'wifi'],
+  ['ночной режим', 'night_mode'],
+  ['режим ночной съемки', 'night_mode'],
+  ['экран', 'screen'],
+  ['запись времени и даты', 'timestamp_recording'],
+  ['запись скорости', 'speed_recording'],
+  ['запись скорости движения', 'speed_recording'],
+  ['встроенный микрофон', 'microphone'],
+  ['запись звука', 'microphone'],
+  ['встроенный динамик', 'speaker'],
+  ['с конденсатором', 'capacitor'],
+  ['максимальный размер карты памяти', 'memory_card_max_gb'],
+  ['макс. объем карты памяти', 'memory_card_max_gb'],
+  ['максимальная емкость карты памяти', 'memory_card_max_gb'],
+  ['матрица', 'sensor_type'],
+  ['размер матрицы', 'sensor_size'],
+  ['количество мегапикселей матрицы', 'sensor_megapixels'],
+  ['поддержка hd', 'hd_support'],
+  ['режим записи', 'recording_mode'],
+  ['запись роликов без разрывов', 'gapless_recording'],
+  ['циклическая запись', 'gapless_recording'],
+  ['непрерывная запись', 'gapless_recording'],
+  ['запись события в отдельный файл', 'event_file_recording'],
+  ['запись событий в отдельный файл', 'event_file_recording'],
+  ['формат записи', 'recording_format'],
+  ['используемый видеокодек', 'video_codec'],
+  ['питание от аккумулятора', 'battery_power'],
+  ['резервный источник питания', 'battery_power'],
+  ['питание от бортовой сети автомобиля', 'car_power'],
+  ['питание от прикуривателя', 'car_power'],
+  ['подключение внешних камер', 'external_cameras'],
+  ['подключение камеры заднего вида', 'external_cameras'],
+  ['дополнительная камера', 'external_cameras'],
+  ['камера заднего вида', 'external_cameras'],
+  ['поддержка карт памяти', 'memory_card_support'],
+  ['база камер', 'speedcam'],
+  ['предупреждение о камерах слежения', 'speedcam'],
+  ['speedcam', 'speedcam'],
+  ['обнаружение радаров', 'radar_detection'],
+  ['магнитный держатель', 'magnetic_mount'],
+  ['крепление на присоске', 'suction_mount'],
+  ['голосовые подсказки', 'voice_prompts'],
+  ['с радар-детектором', 'radar_detector'],
+]);
+
+function normalizeKey(value: string) {
+  return value.trim().toLowerCase().replace(/\s+/g, ' ');
+}
+
+function parseBoolean(value: unknown) {
+  if (typeof value === 'boolean') return value;
+  if (typeof value !== 'string') return undefined;
+
+  const normalized = normalizeKey(value);
+
+  if (['да', 'есть', 'true', 'yes'].includes(normalized)) return true;
+  if (['нет', 'false', 'no', 'отсутствует', 'none'].includes(normalized)) return false;
+
+  return undefined;
+}
+
+function parseFirstNumber(value: unknown) {
+  if (typeof value === 'number') return value;
+  if (typeof value !== 'string') return undefined;
+
+  const match = value.replace(',', '.').match(/\d+(?:\.\d+)?/);
+
+  return match ? Number(match[0]) : undefined;
+}
+
+function normalizeBatteryPower(value: string | number | boolean) {
+  const booleanValue = parseBoolean(value);
+
+  if (booleanValue !== undefined) return booleanValue;
+  if (typeof value === 'number') return value > 0;
+  if (typeof value !== 'string') return undefined;
+
+  const normalized = normalizeKey(value);
+  const hasBattery = /аккум|battery|li[-\s]?ion|литий/.test(normalized);
+  const hasCapacitor = /конденс|capacitor|supercap/.test(normalized);
+
+  if (hasBattery && hasCapacitor) return 'hybrid';
+  if (hasBattery) return 'battery';
+  if (hasCapacitor) return 'capacitor';
+  if (['нет', 'false', 'no', 'отсутствует', 'none'].includes(normalized)) return false;
+
+  return undefined;
+}
+
+function normalizeResolution(value: unknown) {
+  if (typeof value !== 'string') return undefined;
+
+  const match = value.replace(/[×х]/gi, 'x').match(/\d{3,5}\s*x\s*\d{3,5}/i);
+
+  return match ? match[0].replace(/\s+/g, '').toLowerCase() : undefined;
+}
+
+function normalizeSpecValue(key: string, value: string | number | boolean) {
+  if (key === 'battery_power') {
+    return normalizeBatteryPower(value);
+  }
+
+  const booleanValue = parseBoolean(value);
+
+  if (booleanValue !== undefined) {
+    return booleanValue;
+  }
+
+  if (
+    [
+      'gps',
+      'glonass',
+      'wifi',
+      'night_mode',
+      'screen',
+      'timestamp_recording',
+      'speed_recording',
+      'microphone',
+      'speaker',
+      'capacitor',
+      'speedcam',
+      'radar_detection',
+    ].includes(key)
+  ) {
+    if (typeof value === 'string' && value.trim()) {
+      return true;
+    }
+
+    return value;
+  }
+
+  if (
+    [
+      'channels',
+      'video_channels',
+      'audio_channels',
+      'frame_rate_max_fps',
+      'view_angle_degrees',
+      'view_angle_width_degrees',
+      'view_angle_height_degrees',
+      'memory_card_max_gb',
+      'sensor_megapixels',
+    ].includes(key)
+  ) {
+    return parseFirstNumber(value);
+  }
+
+  if (key === 'resolution_front' || key === 'resolution_max') {
+    return normalizeResolution(value);
+  }
+
+  return value;
+}
+
+function normalizeBrand(value: string | undefined, title: string) {
+  const raw = value ?? [...brandAliases.values()].find((brand) => title.toLowerCase().includes(brand.toLowerCase()));
+
+  if (!raw) return 'Unknown';
+
+  return brandAliases.get(raw.toLowerCase()) ?? raw.trim();
+}
+
+export function cleanModelFromTitle(title: string, brand: string) {
+  const genericWords = [
+    'видеорегистратор-зеркало',
+    'зеркало-видеорегистратор',
+    'видеорегистратор для автомобиля',
+    'видеорегистратор автомобильный',
+    'автомобильный видеорегистратор',
+    'видеорегистратор',
+    'зеркало',
+    'цвет черный',
+    'черный',
+  ];
+  let model = title;
+
+  for (const word of genericWords) {
+    model = model.replace(new RegExp(word, 'gi'), ' ');
+  }
+
+  model = model.replace(new RegExp(`["«]?${brand.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}["»]?`, 'gi'), ' ');
+  model = model.replace(new RegExp(brand.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'gi'), ' ');
+
+  return (
+    model
+      .replace(/[«»"]/g, ' ')
+      .replace(/\bwi[\s-]?fi\b/gi, 'Wi-Fi')
+      .replace(/\s*,\s*/g, ' ')
+      .replace(/^[\s,+–-]+/g, '')
+      .replace(/[,+–-]\s*$/g, '')
+      .replace(/\s+/g, ' ')
+      .trim() || title.trim()
+  );
+}
+
+function normalizeTechnicalSpec(spec: ParsedTechnicalSpec) {
+  const normalizedKey = specAliases.get(normalizeKey(spec.label));
+  const normalizedValue = normalizedKey ? normalizeSpecValue(normalizedKey, spec.value) : undefined;
+
+  return {
+    ...spec,
+    ...(normalizedKey ? { normalizedKey } : {}),
+    ...(normalizedValue !== undefined ? { normalizedValue } : {}),
+  };
+}
+
+function buildCanonicalModel(model: string) {
+  const compact = model
+    .replace(/\s+\+\s+(?:внутрисалонная|задняя)\s+камера.*$/i, '')
+    .replace(/\s+\+\s+камера.*$/i, '')
+    .replace(/\s+с\s+(?:базой камер|сигнатурным|радар-детектором|gps|глонасс).*$/i, '')
+    .replace(/\s+автомобильн(?:ый|ая|ое|ые)(?:\s+с\b.*)?$/i, '')
+    .replace(/\s+(?:зеркало|с креплением на зеркало)(?:\s+с\b.*)?$/i, '')
+    .replace(/[\s,]+(?:ночной режим|ночная съемка)$/i, '')
+    .replace(/\s+/g, ' ')
+    .trim();
+
+  return compact || model;
+}
+
+function createCanonicalName(brand: string, canonicalModel: string) {
+  if (canonicalModel.toLowerCase().startsWith(brand.toLowerCase())) return canonicalModel;
+
+  return `${brand} ${canonicalModel}`.replace(/\s+/g, ' ').trim();
+}
+
+export function normalizeMarketProduct(product: ParsedMarketProduct): NormalizedMarketProduct {
+  const brand = normalizeBrand(product.brand, product.title);
+  const model = product.model?.trim() || cleanModelFromTitle(product.title, brand);
+  const canonicalModel = buildCanonicalModel(model);
+  const rawSpecs = product.rawSpecs.map(normalizeTechnicalSpec);
+  const specs: NormalizedMarketProduct['specs'] = {};
+
+  for (const spec of rawSpecs) {
+    if (spec.normalizedKey && spec.normalizedValue !== undefined) {
+      specs[spec.normalizedKey] = spec.normalizedValue;
+    }
+  }
+
+  return {
+    source: product.source,
+    sourceProductId: product.sourceProductId,
+    sourceUrl: product.sourceUrl,
+    title: product.title,
+    brand,
+    model,
+    canonicalModel,
+    canonicalName: createCanonicalName(brand, canonicalModel),
+    specs,
+    rawSpecs,
+    images: product.images,
+    aggregateRating: product.aggregateRating,
+  };
+}
+
+// Kept as an import-compatible alias for existing Yandex parser callers.
+export const normalizeYandexMarketProduct = normalizeMarketProduct;

ingestion/src/queue.ts

@@ -0,0 +1,188 @@
+import type { Pool } from 'pg';
+import type { CrawlFailureReason, CrawlJobRecord, CrawlJobType, MarketSource } from './types.ts';
+
+type CrawlJobRow = {
+  id: string;
+  source: MarketSource;
+  url: string;
+  job_type: CrawlJobType;
+  state: CrawlJobRecord['state'];
+  attempts: number;
+  next_run_at: Date;
+  last_error: string | null;
+};
+
+type BudgetBucket = CrawlJobType;
+
+const retryableFailureReasons = new Set<CrawlFailureReason>(['rate_limited', 'blocked', 'captcha']);
+
+function toRecord(row: CrawlJobRow): CrawlJobRecord {
+  return {
+    id: row.id,
+    source: row.source,
+    url: row.url,
+    jobType: row.job_type,
+    state: row.state,
+    attempts: row.attempts,
+    nextRunAt: row.next_run_at.toISOString(),
+    ...(row.last_error ? { lastError: row.last_error } : {}),
+  };
+}
+
+export function dailyBudgetForJob(jobType: CrawlJobType) {
+  if (jobType === 'category') return Number(process.env.INGESTION_CATEGORY_DAILY_BUDGET ?? 25);
+  if (jobType === 'sitemap') return Number(process.env.INGESTION_SITEMAP_DAILY_BUDGET ?? 25);
+
+  return Number(process.env.INGESTION_PRODUCT_DAILY_BUDGET ?? 100);
+}
+
+export class CrawlQueue {
+  private readonly pool: Pool;
+
+  constructor(pool: Pool) {
+    this.pool = pool;
+  }
+
+  async enqueue(input: {
+    source: MarketSource;
+    url: string;
+    jobType: CrawlJobType;
+    nextRunAt?: Date;
+    state?: CrawlJobRecord['state'];
+  }) {
+    await this.pool.query(
+      `
+        insert into crawl_jobs (source, url, job_type, state, next_run_at)
+        values ($1, $2, $3, $4, $5)
+        on conflict (source, url, job_type) do update
+        set
+          next_run_at = least(crawl_jobs.next_run_at, excluded.next_run_at),
+          updated_at = now()
+      `,
+      [input.source, input.url, input.jobType, input.state ?? 'pending', input.nextRunAt ?? new Date()],
+    );
+  }
+
+  async leaseNext(now = new Date()) {
+    const result = await this.pool.query<CrawlJobRow>(
+      `
+        with candidate as (
+          select job.id
+          from crawl_jobs job
+          left join source_pauses pause
+            on pause.source = job.source
+           and pause.paused_until > $1
+          where job.state = 'pending'
+            and job.next_run_at <= $1
+            and pause.source is null
+          order by job.next_run_at asc, job.created_at asc
+          for update of job skip locked
+          limit 1
+        )
+        update crawl_jobs job
+        set state = 'fetching',
+            attempts = attempts + 1,
+            updated_at = now()
+        from candidate
+        where job.id = candidate.id
+        returning job.*
+      `,
+      [now],
+    );
+
+    return result.rows[0] ? toRecord(result.rows[0]) : undefined;
+  }
+
+  async reserveDailyBudget(source: MarketSource, bucket: BudgetBucket, dailyLimit: number, now = new Date()) {
+    const day = now.toISOString().slice(0, 10);
+    const result = await this.pool.query<{ reserved: boolean }>(
+      `
+        with existing as (
+          select used_count, daily_limit
+          from source_daily_budgets
+          where source = $1 and budget_bucket = $2 and budget_day = $3
+          for update
+        ),
+        upserted as (
+          insert into source_daily_budgets (source, budget_bucket, budget_day, daily_limit, used_count)
+          select $1, $2, $3, $4, 1
+          where not exists (select 1 from existing)
+          on conflict do nothing
+          returning true as reserved
+        ),
+        updated as (
+          update source_daily_budgets
+          set used_count = used_count + 1,
+              daily_limit = $4,
+              updated_at = now()
+          where source = $1
+            and budget_bucket = $2
+            and budget_day = $3
+            and exists (select 1 from existing)
+            and used_count < daily_limit
+          returning true as reserved
+        )
+        select coalesce(
+          (select reserved from upserted),
+          (select reserved from updated),
+          false
+        ) as reserved
+      `,
+      [source, bucket, day, dailyLimit],
+    );
+
+    return result.rows[0]?.reserved ?? false;
+  }
+
+  async markState(id: string, state: CrawlJobRecord['state']) {
+    await this.pool.query(
+      `
+        update crawl_jobs
+        set state = $2,
+            updated_at = now()
+        where id = $1
+      `,
+      [id, state],
+    );
+  }
+
+  async markFailure(job: CrawlJobRecord, reason: CrawlFailureReason, message: string) {
+    const nextRunAt = retryableFailureReasons.has(reason)
+      ? new Date(Date.now() + Math.min(24 * 60 * 60 * 1000, 2 ** job.attempts * 60_000))
+      : undefined;
+
+    await this.pool.query(
+      `
+        update crawl_jobs
+        set state = $2,
+            last_error = $3,
+            next_run_at = coalesce($4, next_run_at),
+            updated_at = now()
+        where id = $1
+      `,
+      [job.id, reason, message.slice(0, 2000), nextRunAt],
+    );
+
+    await this.pool.query(
+      `
+        insert into crawl_errors (job_id, source, url, reason, message)
+        values ($1, $2, $3, $4, $5)
+      `,
+      [job.id, job.source, job.url, reason, message.slice(0, 2000)],
+    );
+  }
+
+  async pauseSource(source: MarketSource, reason: CrawlFailureReason, pausedUntil: Date) {
+    await this.pool.query(
+      `
+        insert into source_pauses (source, reason, paused_until)
+        values ($1, $2, $3)
+        on conflict (source) do update
+        set reason = excluded.reason,
+            paused_until = excluded.paused_until,
+            updated_at = now()
+      `,
+      [source, reason, pausedUntil],
+    );
+  }
+}

ingestion/src/repository.test.ts

@@ -0,0 +1,69 @@
+import type { Pool } from 'pg';
+import { describe, expect, it, vi } from 'vitest';
+import { IngestionRepository } from './repository.ts';
+import type { NormalizedMarketProduct } from './types.ts';
+
+const product: NormalizedMarketProduct = {
+  source: 'mvideo',
+  sourceProductId: '400484046',
+  sourceUrl: 'https://www.mvideo.ru/products/test-400484046',
+  title: 'Verification Canonical Link 1',
+  brand: 'Verification',
+  model: 'Canonical Link 1',
+  canonicalModel: 'Canonical Link 1',
+  canonicalName: 'Verification Canonical Link 1',
+  specs: { night_mode: false },
+  rawSpecs: [
+    {
+      group: 'Verification',
+      label: 'Night mode',
+      value: 'нет',
+      normalizedKey: 'night_mode',
+      normalizedValue: false,
+    },
+  ],
+  images: [],
+};
+
+describe('IngestionRepository', () => {
+  it('links an exact canonical-name match and serializes source conflicts as JSONB', async () => {
+    const conflictValues = [
+      { source: 'mvideo', value: false },
+      { source: 'yandex_market', value: true },
+    ];
+    const query = vi.fn(async (statement: string) => {
+      if (statement.includes('select id\n        from products')) {
+        return { rows: [{ id: 'ym_5905353579' }], rowCount: 1 };
+      }
+
+      if (statement.includes('insert into manufacturers')) {
+        return { rows: [{ id: 'manufacturer-id' }], rowCount: 1 };
+      }
+
+      if (statement.includes('select\n          normalized_key')) {
+        return {
+          rows: [{ normalized_key: 'night_mode', values_json: conflictValues }],
+          rowCount: 1,
+        };
+      }
+
+      return { rows: [], rowCount: 0 };
+    });
+    const repository = new IngestionRepository({ query } as unknown as Pool);
+
+    const productId = await repository.upsertNormalizedProduct(
+      product,
+      'stable-content-hash',
+      '2026-07-30T23:00:00.000Z',
+    );
+    const calls = query.mock.calls as unknown as Array<[string, unknown[] | undefined]>;
+    const sourceCall = calls.find(([statement]) => statement.includes('insert into product_sources'));
+    const conflictCall = calls.find(([statement]) => statement.includes('insert into moderation_conflicts'));
+    const queueCalls = calls.filter(([statement]) => statement.includes('insert into moderation_queue'));
+
+    expect(productId).toBe('ym_5905353579');
+    expect(sourceCall?.[1]?.[0]).toBe('ym_5905353579');
+    expect(JSON.parse(String(conflictCall?.[1]?.[2]))).toEqual(conflictValues);
+    expect(queueCalls.at(-1)?.[1]).toEqual(['ym_5905353579', 'same_product']);
+  });
+});

ingestion/src/repository.ts

@@ -0,0 +1,317 @@
+import { createHash } from 'node:crypto';
+import type { Pool } from 'pg';
+import type { DiscoveredProduct, NormalizedMarketProduct, ProductSourceSnapshot } from './types.ts';
+
+function publicProductId(product: NormalizedMarketProduct) {
+  const sourcePrefix = product.source === 'mvideo' ? 'mv' : 'ym';
+
+  if (product.sourceProductId) return `${sourcePrefix}_${product.sourceProductId}`;
+
+  return `${sourcePrefix}_${Buffer.from(product.sourceUrl).toString('base64url').slice(0, 32)}`;
+}
+
+function valueHash(value: unknown) {
+  return createHash('sha256').update(JSON.stringify(value)).digest('hex');
+}
+
+function manufacturerSlug(name: string) {
+  const slug = name
+    .trim()
+    .toLowerCase()
+    .replace(/[^\p{Letter}\p{Number}]+/gu, '-')
+    .replace(/^-+|-+$/g, '');
+
+  return slug || 'unknown';
+}
+
+export class IngestionRepository {
+  private readonly pool: Pool;
+
+  constructor(pool: Pool) {
+    this.pool = pool;
+  }
+
+  async recordSnapshot(snapshot: ProductSourceSnapshot) {
+    await this.pool.query(
+      `
+        insert into source_snapshots (
+          source, url, fetched_at, http_status, raw_snapshot_hash,
+          stable_content_hash, raw_html_path, parsed_json
+        )
+        values ($1, $2, $3, $4, $5, $6, $7, $8)
+      `,
+      [
+        snapshot.source,
+        snapshot.url,
+        snapshot.fetchedAt,
+        snapshot.httpStatus,
+        snapshot.rawSnapshotHash,
+        snapshot.stableContentHash ?? null,
+        snapshot.rawHtmlPath ?? null,
+        snapshot.parsedJson,
+      ],
+    );
+  }
+
+  async recordDiscoveredProducts(products: DiscoveredProduct[]) {
+    for (const product of products) {
+      await this.pool.query(
+        `
+          insert into product_sources (
+            source, url, source_product_id, category_url, title_preview,
+            first_seen_at, updated_at, state
+          )
+          values ($1, $2, $3, $4, $5, $6, now(), 'discovered')
+          on conflict (source, url) do update
+          set source_product_id = coalesce(product_sources.source_product_id, excluded.source_product_id),
+              category_url = coalesce(excluded.category_url, product_sources.category_url),
+              title_preview = coalesce(excluded.title_preview, product_sources.title_preview),
+              updated_at = now()
+        `,
+        [
+          product.source,
+          product.productUrl,
+          product.sourceProductId ?? null,
+          product.categoryUrl ?? null,
+          product.titlePreview ?? null,
+          product.discoveredAt,
+        ],
+      );
+    }
+  }
+
+  async upsertNormalizedProduct(product: NormalizedMarketProduct, stableContentHash: string, fetchedAt: string) {
+    const sourceScopedProductId = publicProductId(product);
+    const matchingProduct = await this.pool.query<{ id: string }>(
+      `
+        select id
+        from products
+        where lower(canonical_name) = lower($1)
+          and id <> $2
+        order by discovered_at asc, id asc
+        limit 1
+      `,
+      [product.canonicalName, sourceScopedProductId],
+    );
+    const productId = matchingProduct.rows[0]?.id ?? sourceScopedProductId;
+    const linkedToExistingCanonicalProduct = productId !== sourceScopedProductId;
+    const manufacturerId = await this.upsertManufacturer(product.brand);
+
+    await this.pool.query(
+      `
+        insert into products (id, manufacturer_id, brand, model, canonical_name, title, moderation_status, discovered_at, updated_at)
+        values ($1, $2, $3, $4, $5, $6, 'needs_review', $7, $7)
+        on conflict (id) do update
+        set manufacturer_id = excluded.manufacturer_id,
+            brand = excluded.brand,
+            model = case when products.model_override then products.model else excluded.model end,
+            canonical_name = case when products.canonical_name_override then products.canonical_name else excluded.canonical_name end,
+            title = excluded.title,
+            updated_at = excluded.updated_at
+      `,
+      [productId, manufacturerId, product.brand, product.canonicalModel, product.canonicalName, product.title, fetchedAt],
+    );
+
+    await this.pool.query(
+      `
+        insert into product_sources (
+          product_id, source, url, source_product_id, stable_content_hash,
+          last_fetched_at, parsed_json, state, updated_at
+        )
+        values ($1, $2, $3, $4, $5, $6, $7, 'parsed', now())
+        on conflict (source, url) do update
+        set product_id = excluded.product_id,
+            source_product_id = coalesce(product_sources.source_product_id, excluded.source_product_id),
+            stable_content_hash = excluded.stable_content_hash,
+            last_fetched_at = excluded.last_fetched_at,
+            parsed_json = excluded.parsed_json,
+            state = excluded.state,
+            updated_at = now()
+      `,
+      [productId, product.source, product.sourceUrl, product.sourceProductId ?? null, stableContentHash, fetchedAt, product],
+    );
+
+    for (const [key, value] of Object.entries(product.specs)) {
+      await this.pool.query(
+        `
+          insert into product_specs (product_id, key, value, updated_at)
+          values ($1, $2, $3, now())
+          on conflict (product_id, key) do update
+          set value = excluded.value,
+              updated_at = now()
+        `,
+        [productId, key, JSON.stringify(value)],
+      );
+    }
+
+    await this.pool.query(
+      `
+        delete from product_source_specs
+        where product_id = $1
+          and source = $2
+          and source_url = $3
+      `,
+      [productId, product.source, product.sourceUrl],
+    );
+
+    for (const spec of product.rawSpecs) {
+      const comparableValue = spec.normalizedValue ?? spec.value;
+
+      await this.pool.query(
+        `
+          insert into product_source_specs (
+            product_id, source, source_url, source_product_id, raw_group, raw_label,
+            raw_value, normalized_key, normalized_value, value_hash, fetched_at
+          )
+          values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
+          on conflict (product_id, source, source_url, raw_label) do update
+          set raw_group = excluded.raw_group,
+              raw_value = excluded.raw_value,
+              normalized_key = excluded.normalized_key,
+              normalized_value = excluded.normalized_value,
+              value_hash = excluded.value_hash,
+              fetched_at = excluded.fetched_at,
+              updated_at = now()
+        `,
+        [
+          productId,
+          product.source,
+          product.sourceUrl,
+          product.sourceProductId ?? null,
+          spec.group ?? null,
+          spec.label,
+          JSON.stringify(spec.value),
+          spec.normalizedKey ?? null,
+          spec.normalizedValue === undefined ? null : JSON.stringify(spec.normalizedValue),
+          valueHash(comparableValue),
+          fetchedAt,
+        ],
+      );
+    }
+
+    await this.refreshSpecConflicts(productId);
+
+    for (const imageUrl of product.images) {
+      await this.pool.query(
+        `
+          insert into product_images (product_id, original_url, source, status)
+          values ($1, $2, $3, 'needs_review')
+          on conflict (product_id, original_url) do nothing
+        `,
+        [productId, imageUrl, product.source],
+      );
+    }
+
+    if (product.aggregateRating) {
+      await this.pool.query(
+        `
+          insert into source_ratings (product_id, source, source_product_id, rating_value, rating_scale, rating_count, fetched_at)
+          values ($1, $2, $3, $4, $5, $6, $7)
+        `,
+        [
+          productId,
+          product.source,
+          product.sourceProductId ?? null,
+          product.aggregateRating.value,
+          product.aggregateRating.scale ?? null,
+          product.aggregateRating.ratingCount ?? null,
+          fetchedAt,
+        ],
+      );
+    }
+
+    await this.pool.query(
+      `
+        insert into moderation_queue (product_id, reason, state)
+        values ($1, $2, 'pending')
+        on conflict (product_id, reason) do nothing
+      `,
+      [productId, linkedToExistingCanonicalProduct ? 'same_product' : 'new_product'],
+    );
+
+    return productId;
+  }
+
+  private async upsertManufacturer(name: string) {
+    const result = await this.pool.query<{ id: string }>(
+      `
+        insert into manufacturers (slug, name)
+        values ($1, $2)
+        on conflict (slug) do update
+        set name = excluded.name,
+            updated_at = now()
+        returning id
+      `,
+      [manufacturerSlug(name), name],
+    );
+
+    return result.rows[0].id;
+  }
+
+  private async refreshSpecConflicts(productId: string) {
+    await this.pool.query(
+      `
+        update moderation_conflicts
+        set state = 'resolved',
+            updated_at = now()
+        where product_id = $1
+          and conflict_type = 'spec_value'
+      `,
+      [productId],
+    );
+
+    const conflicts = await this.pool.query<{
+      normalized_key: string;
+      values_json: unknown;
+    }>(
+      `
+        select
+          normalized_key,
+          jsonb_agg(
+            jsonb_build_object(
+              'source', source,
+              'url', source_url,
+              'sourceProductId', source_product_id,
+              'value', coalesce(normalized_value, raw_value),
+              'rawLabel', raw_label,
+              'rawGroup', raw_group
+            )
+            order by source, source_url, raw_label
+          ) as values_json
+        from product_source_specs
+        where product_id = $1
+          and normalized_key is not null
+        group by normalized_key
+        having count(distinct value_hash) > 1
+      `,
+      [productId],
+    );
+
+    for (const conflict of conflicts.rows) {
+      await this.pool.query(
+        `
+          insert into moderation_conflicts (product_id, conflict_type, key, values_json, state)
+          values ($1, 'spec_value', $2, $3, 'pending')
+          on conflict (product_id, conflict_type, key) do update
+          set values_json = excluded.values_json,
+              state = 'pending',
+              updated_at = now()
+        `,
+        [productId, conflict.normalized_key, JSON.stringify(conflict.values_json)],
+      );
+    }
+
+    if (conflicts.rowCount) {
+      await this.pool.query(
+        `
+          insert into moderation_queue (product_id, reason, state)
+          values ($1, 'conflicting_specs', 'pending')
+          on conflict (product_id, reason) do update
+          set state = 'pending',
+              updated_at = now()
+        `,
+        [productId],
+      );
+    }
+  }
+}

ingestion/src/smokeFetchMvideo.ts

@@ -0,0 +1,284 @@
+import { ingestionConfig } from './config.ts';
+import { normalizeYandexMarketProduct } from './normalizer.ts';
+import { createStableContentHash } from './stableHash.ts';
+import {
+  buildCookieHeader,
+  discoverMvideoProductIds,
+  extractMvideoBffBasePath,
+  extractMvideoSearchQuery,
+  parseMvideoProduct,
+} from './sources/mvideo.ts';
+
+type SmokeProductResult =
+  | {
+      ok: true;
+      index: number;
+      sourceProductId: string;
+      productUrl: string;
+      productApiUrl: string;
+      httpStatus: number;
+      title: string;
+      brand: string;
+      model: string;
+      canonicalModel: string;
+      canonicalName: string;
+      sources: Array<{
+        source: 'mvideo';
+        sourceProductId?: string;
+        url: string;
+        fetchedAt: string;
+        stableContentHash: string;
+        title: string;
+        aggregateRating?: {
+          value: number;
+          scale?: number;
+          ratingCount?: number;
+        };
+      }>;
+      technicalSpecs: {
+        normalized: Record<string, string | number | boolean>;
+        raw: Array<{
+          group?: string;
+          label: string;
+          value: string | number | boolean;
+          normalizedKey?: string;
+          normalizedValue?: string | number | boolean;
+        }>;
+      };
+      images: string[];
+      stableContentHash: string;
+      moderation: {
+        conflicts: [];
+      };
+      internalSignals: {
+        aggregateRating?: {
+          value: number;
+          scale?: number;
+          ratingCount?: number;
+        };
+        rawSpecKeys: string[];
+      };
+    }
+  | {
+      ok: false;
+      index: number;
+      sourceProductId: string;
+      productApiUrl: string;
+      error: string;
+    };
+
+function numberEnv(name: string, fallback: number) {
+  const value = Number(process.env[name]);
+
+  return Number.isFinite(value) ? value : fallback;
+}
+
+function sleep(ms: number) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function setCookieHeaders(response: Response) {
+  const headers = response.headers as Headers & { getSetCookie?: () => string[] };
+
+  if (typeof headers.getSetCookie === 'function') return headers.getSetCookie();
+
+  const single = response.headers.get('set-cookie');
+
+  return single ? [single] : [];
+}
+
+async function fetchText(url: string, init?: RequestInit) {
+  const response = await fetch(url, {
+    headers: {
+      accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+      'accept-language': 'ru,en;q=0.8',
+      'user-agent': 'Mozilla/5.0 (compatible; videoreg-ingestion-smoke/1.0)',
+      ...(init?.headers ?? {}),
+    },
+    redirect: 'follow',
+    ...init,
+  });
+  const text = await response.text();
+
+  if (!response.ok) {
+    throw new Error(`HTTP ${response.status} for ${url}`);
+  }
+
+  return {
+    response,
+    url: response.url || url,
+    status: response.status,
+    text,
+  };
+}
+
+async function fetchJson(url: string, init?: RequestInit) {
+  const response = await fetch(url, {
+    headers: {
+      accept: 'application/json,text/plain,*/*',
+      'accept-language': 'ru,en;q=0.8',
+      'user-agent': 'Mozilla/5.0 (compatible; videoreg-ingestion-smoke/1.0)',
+      ...(init?.headers ?? {}),
+    },
+    redirect: 'follow',
+    ...init,
+  });
+  const text = await response.text();
+  let payload: unknown;
+
+  try {
+    payload = JSON.parse(text);
+  } catch {
+    throw new Error(`Expected JSON response from ${url}, got non-JSON body`);
+  }
+
+  if (!response.ok) {
+    throw new Error(`HTTP ${response.status} for ${url}`);
+  }
+
+  return {
+    url: response.url || url,
+    status: response.status,
+    payload,
+  };
+}
+
+async function fetchSmokeProduct(
+  productId: string,
+  index: number,
+  bffBasePath: string,
+  cookieHeader: string,
+): Promise<SmokeProductResult> {
+  const detailsBaseUrl = bffBasePath.endsWith('/') ? bffBasePath : `${bffBasePath}/`;
+  const detailsUrl = new URL('product-details', detailsBaseUrl);
+  detailsUrl.searchParams.set('productId', productId);
+
+  try {
+    const response = await fetchJson(detailsUrl.toString(), {
+      headers: {
+        cookie: cookieHeader,
+      },
+    });
+    const fetchedAt = new Date().toISOString();
+    const parsed = parseMvideoProduct(response.payload);
+    const normalized = normalizeYandexMarketProduct(parsed);
+    const stableContentHash = createStableContentHash(normalized);
+
+    return {
+      ok: true,
+      index,
+      sourceProductId: productId,
+      productUrl: normalized.sourceUrl,
+      productApiUrl: detailsUrl.toString(),
+      httpStatus: response.status,
+      title: normalized.title,
+      brand: normalized.brand,
+      model: normalized.model,
+      canonicalModel: normalized.canonicalModel,
+      canonicalName: normalized.canonicalName,
+      sources: [
+        {
+          source: 'mvideo',
+          sourceProductId: normalized.sourceProductId,
+          url: normalized.sourceUrl,
+          fetchedAt,
+          stableContentHash,
+          title: normalized.title,
+          aggregateRating: normalized.aggregateRating,
+        },
+      ],
+      technicalSpecs: {
+        normalized: normalized.specs,
+        raw: normalized.rawSpecs,
+      },
+      images: normalized.images,
+      stableContentHash,
+      moderation: {
+        conflicts: [],
+      },
+      internalSignals: {
+        aggregateRating: normalized.aggregateRating,
+        rawSpecKeys: Object.keys(parsed.specs),
+      },
+    };
+  } catch (error) {
+    return {
+      ok: false,
+      index,
+      sourceProductId: productId,
+      productApiUrl: detailsUrl.toString(),
+      error: (error as Error).message,
+    };
+  }
+}
+
+const limit = Math.max(1, numberEnv('INGESTION_SMOKE_LIMIT', 5));
+const delayMs = Math.max(0, numberEnv('INGESTION_SMOKE_DELAY_MS', ingestionConfig.fetchDelayMinMs));
+const categoryUrl =
+  process.env.MVIDEO_DASHCAM_CATEGORY_URL ?? 'https://www.mvideo.ru/product-list-page?q=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE%D1%80%D0%B5%D0%B3%D0%B8%D1%81%D1%82%D1%80%D0%B0%D1%82%D0%BE%D1%80%D1%8B';
+const startedAt = new Date().toISOString();
+
+console.error(`Fetching MVideo category page: ${categoryUrl}`);
+const categoryResponse = await fetchText(categoryUrl);
+const cookieHeader = buildCookieHeader(setCookieHeaders(categoryResponse.response));
+
+if (!cookieHeader) {
+  throw new Error('MVideo category response did not provide cookies required for BFF calls');
+}
+
+const bffBasePath = extractMvideoBffBasePath(categoryResponse.text);
+const searchQuery = extractMvideoSearchQuery(categoryResponse.url);
+const searchBaseUrl = bffBasePath.endsWith('/') ? bffBasePath : `${bffBasePath}/`;
+const searchUrl = new URL('products/v2/search', searchBaseUrl);
+searchUrl.searchParams.set('query', searchQuery);
+searchUrl.searchParams.set('offset', '0');
+searchUrl.searchParams.set('limit', String(limit));
+
+console.error(`Fetching MVideo search: ${searchUrl.toString()}`);
+const searchResponse = await fetchJson(searchUrl.toString(), {
+  headers: {
+    cookie: cookieHeader,
+  },
+});
+const discoveredProductIds = discoverMvideoProductIds(searchResponse.payload);
+const selectedProductIds = discoveredProductIds.slice(0, limit);
+const products: SmokeProductResult[] = [];
+
+for (const [index, sourceProductId] of selectedProductIds.entries()) {
+  if (index > 0 && delayMs > 0) {
+    console.error(`Waiting ${delayMs}ms before next product fetch`);
+    await sleep(delayMs);
+  }
+
+  console.error(`Fetching MVideo product ${index + 1}/${selectedProductIds.length}: ${sourceProductId}`);
+  products.push(await fetchSmokeProduct(sourceProductId, index + 1, bffBasePath, cookieHeader));
+}
+
+console.log(
+  JSON.stringify(
+    {
+      source: 'mvideo',
+      mode: 'smoke',
+      categoryUrl,
+      startedAt,
+      finishedAt: new Date().toISOString(),
+      limit,
+      delayMs,
+      category: {
+        url: categoryResponse.url,
+        httpStatus: categoryResponse.status,
+        htmlLength: categoryResponse.text.length,
+      },
+      search: {
+        query: searchQuery,
+        url: searchUrl.toString(),
+        httpStatus: searchResponse.status,
+        discoveredCount: discoveredProductIds.length,
+        selectedCount: selectedProductIds.length,
+      },
+      products,
+    },
+    null,
+    2,
+  ),
+);

ingestion/src/smokeFetchYandexMarket.ts

@@ -0,0 +1,204 @@
+import { ingestionConfig } from './config.ts';
+import { normalizeYandexMarketProduct } from './normalizer.ts';
+import { discoverYandexMarketProductsFromCategory, parseYandexMarketProduct } from './sources/yandexMarket.ts';
+import { createStableContentHash } from './stableHash.ts';
+
+type SmokeProductResult =
+  | {
+      ok: true;
+      index: number;
+      url: string;
+      httpStatus: number;
+      htmlLength: number;
+      sourceProductId?: string;
+      sources: Array<{
+        source: 'yandex_market';
+        sourceProductId?: string;
+        url: string;
+        fetchedAt: string;
+        stableContentHash: string;
+        title: string;
+        aggregateRating?: {
+          value: number;
+          scale?: number;
+          ratingCount?: number;
+        };
+      }>;
+      title: string;
+      brand: string;
+      model: string;
+      canonicalModel: string;
+      canonicalName: string;
+      technicalSpecs: {
+        normalized: Record<string, string | number | boolean>;
+        raw: Array<{
+          group?: string;
+          label: string;
+          value: string | number | boolean;
+          normalizedKey?: string;
+          normalizedValue?: string | number | boolean;
+        }>;
+      };
+      images: string[];
+      stableContentHash: string;
+      moderation: {
+        conflicts: [];
+      };
+      internalSignals: {
+        aggregateRating?: {
+          value: number;
+          scale?: number;
+          ratingCount?: number;
+        };
+        rawSpecKeys: string[];
+      };
+    }
+  | {
+      ok: false;
+      index: number;
+      url: string;
+      error: string;
+    };
+
+function numberEnv(name: string, fallback: number) {
+  const value = Number(process.env[name]);
+
+  return Number.isFinite(value) ? value : fallback;
+}
+
+function sleep(ms: number) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function classifyHtml(status: number, html: string) {
+  const lowered = html.slice(0, 80_000).toLowerCase();
+
+  if (status === 429) return 'rate_limited';
+  if (status === 403) return 'blocked';
+  if (/captcha|showcaptcha|smartcaptcha|подтвердите, что вы не робот/.test(lowered)) return 'captcha';
+  if (status === 404) return 'not_found';
+  if (status < 200 || status >= 300) return `http_${status}`;
+
+  return undefined;
+}
+
+async function fetchHtml(url: string) {
+  const response = await fetch(url, {
+    headers: {
+      accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+      'accept-language': 'ru,en;q=0.8',
+    },
+    redirect: 'follow',
+  });
+  const html = await response.text();
+  const failure = classifyHtml(response.status, html);
+
+  if (failure) {
+    throw new Error(`Fetch classified as ${failure} for status ${response.status}`);
+  }
+
+  return {
+    url: response.url || url,
+    status: response.status,
+    html,
+  };
+}
+
+async function fetchSmokeProduct(url: string, index: number): Promise<SmokeProductResult> {
+  try {
+    const response = await fetchHtml(url);
+    const fetchedAt = new Date().toISOString();
+    const parsed = parseYandexMarketProduct(response.html, url);
+    const normalized = normalizeYandexMarketProduct(parsed);
+    const stableContentHash = createStableContentHash(normalized);
+
+    return {
+      ok: true,
+      index,
+      url,
+      httpStatus: response.status,
+      htmlLength: response.html.length,
+      sourceProductId: normalized.sourceProductId,
+      sources: [
+        {
+          source: 'yandex_market',
+          sourceProductId: normalized.sourceProductId,
+          url: normalized.sourceUrl,
+          fetchedAt,
+          stableContentHash,
+          title: normalized.title,
+          aggregateRating: normalized.aggregateRating,
+        },
+      ],
+      title: normalized.title,
+      brand: normalized.brand,
+      model: normalized.model,
+      canonicalModel: normalized.canonicalModel,
+      canonicalName: normalized.canonicalName,
+      technicalSpecs: {
+        normalized: normalized.specs,
+        raw: normalized.rawSpecs,
+      },
+      images: normalized.images,
+      stableContentHash,
+      moderation: {
+        conflicts: [],
+      },
+      internalSignals: {
+        aggregateRating: normalized.aggregateRating,
+        rawSpecKeys: Object.keys(parsed.specs),
+      },
+    };
+  } catch (error) {
+    return {
+      ok: false,
+      index,
+      url,
+      error: (error as Error).message,
+    };
+  }
+}
+
+const limit = Math.max(1, numberEnv('INGESTION_SMOKE_LIMIT', 5));
+const delayMs = Math.max(0, numberEnv('INGESTION_SMOKE_DELAY_MS', ingestionConfig.fetchDelayMinMs));
+const categoryUrl = process.env.YANDEX_MARKET_DASHCAM_CATEGORY_URL ?? ingestionConfig.seedCategoryUrl;
+const startedAt = new Date().toISOString();
+
+console.error(`Fetching category: ${categoryUrl}`);
+const categoryResponse = await fetchHtml(categoryUrl);
+const discovered = discoverYandexMarketProductsFromCategory(categoryResponse.html, categoryUrl, startedAt);
+const selected = discovered.slice(0, limit);
+const products: SmokeProductResult[] = [];
+
+for (const [index, product] of selected.entries()) {
+  if (index > 0 && delayMs > 0) {
+    console.error(`Waiting ${delayMs}ms before next product fetch`);
+    await sleep(delayMs);
+  }
+
+  console.error(`Fetching product ${index + 1}/${selected.length}: ${product.productUrl}`);
+  products.push(await fetchSmokeProduct(product.productUrl, index + 1));
+}
+
+console.log(
+  JSON.stringify(
+    {
+      source: 'yandex_market',
+      mode: 'smoke',
+      categoryUrl,
+      startedAt,
+      finishedAt: new Date().toISOString(),
+      limit,
+      delayMs,
+      category: {
+        httpStatus: categoryResponse.status,
+        htmlLength: categoryResponse.html.length,
+        discoveredCount: discovered.length,
+        selectedCount: selected.length,
+      },
+      products,
+    },
+    null,
+    2,
+  ),
+);

ingestion/src/sources/mvideo.test.ts

@@ -0,0 +1,171 @@
+import { describe, expect, it } from 'vitest';
+import { normalizeYandexMarketProduct } from '../normalizer.ts';
+import {
+  buildCookieHeader,
+  discoverMvideoProductIds,
+  extractMvideoBffBasePath,
+  extractMvideoSearchQuery,
+  parseMvideoProduct,
+} from './mvideo.ts';
+
+describe('MVideo parser', () => {
+  it('extracts search query and BFF path from category HTML', () => {
+    const html = `<script>window.MVID_CONFIG={basePath:'https://www.mvideo.ru/bff'};</script>`;
+
+    expect(extractMvideoBffBasePath(html)).toBe('https://www.mvideo.ru/bff');
+    expect(
+      extractMvideoSearchQuery(
+        'https://www.mvideo.ru/product-list-page?q=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE%D1%80%D0%B5%D0%B3%D0%B8%D1%81%D1%82%D1%80%D0%B0%D1%82%D0%BE%D1%80',
+      ),
+    ).toBe('видеорегистратор');
+    expect(extractMvideoSearchQuery('https://www.mvideo.ru/category/unknown')).toBe('видеорегистраторы');
+  });
+
+  it('builds cookie header and discovers stable product ids from search response', () => {
+    const cookieHeader = buildCookieHeader([
+      'MVID_CITY_ID=CityCZ_975; Path=/; Secure',
+      'MVID_REGION_ID=1; Path=/; Secure',
+      'MVID_CITY_ID=CityCZ_975; Path=/; Secure',
+    ]);
+
+    expect(cookieHeader).toContain('MVID_CITY_ID=CityCZ_975');
+    expect(cookieHeader).toContain('MVID_REGION_ID=1');
+    expect(
+      discoverMvideoProductIds({
+        body: {
+          products: ['4254724', 4254724, '400481258', 'bad-id'],
+        },
+      }),
+    ).toEqual(['4254724', '400481258']);
+    expect(
+      discoverMvideoProductIds({
+        body: {
+          type: 'redirect',
+          url: '/products/videoregistrator-navitel-ar202-nv-400484046',
+        },
+      }),
+    ).toEqual(['400484046']);
+  });
+
+  it('maps product details to parsed and normalized shape with full raw specs', () => {
+    const parsed = parseMvideoProduct({
+      body: {
+        productId: '4254724',
+        name: 'Видеорегистратор Navitel R9 DUAL',
+        nameTranslit: 'videoregistrator-navitel-r9-dual',
+        brandName: 'Navitel',
+        modelName: 'R9 DUAL',
+        images: ['Big/4254724bb.jpg', '/Big/4254724bb1.jpg'],
+        rating: {
+          score: 4.8,
+          total: 17,
+        },
+        properties: {
+          all: [
+            {
+              name: 'Основные характеристики',
+              properties: [
+                {
+                  name: 'Поддержка Wi-Fi',
+                  value: 'Да',
+                },
+                {
+                  name: 'Качество видеосъемки/ FPS',
+                  value: 'FullHD (1920x1080 Пикс) 30 кадр/сек',
+                },
+                {
+                  name: 'Частота при макс. разрешении камеры заднего вида',
+                  value: '30 кадр/сек',
+                },
+                {
+                  name: 'Макс. разрешение видеосъемки',
+                  value: '2560x1440',
+                },
+                {
+                  name: 'База камер',
+                  value: 'Да',
+                },
+                {
+                  name: 'Запись звука',
+                  value: 'Да',
+                },
+                {
+                  name: 'Резервный источник питания',
+                  value: 'аккумулятор',
+                },
+              ],
+            },
+            {
+              name: 'Оптика',
+              properties: [
+                {
+                  name: 'Угол обзора (основная камера)',
+                  value: '170',
+                  measure: '°',
+                },
+              ],
+            },
+          ],
+        },
+      },
+    });
+    const normalized = normalizeYandexMarketProduct(parsed);
+
+    expect(parsed).toMatchObject({
+      source: 'mvideo',
+      sourceProductId: '4254724',
+      sourceUrl: 'https://www.mvideo.ru/products/videoregistrator-navitel-r9-dual-4254724',
+      brand: 'Navitel',
+      model: 'R9 DUAL',
+      title: 'Видеорегистратор Navitel R9 DUAL',
+      aggregateRating: {
+        value: 4.8,
+        scale: 5,
+        ratingCount: 17,
+      },
+    });
+    expect(parsed.images).toEqual([
+      'https://img.mvideo.ru/Big/4254724bb.jpg',
+      'https://www.mvideo.ru/Big/4254724bb1.jpg',
+    ]);
+    expect(parsed.rawSpecs).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          group: 'Основные характеристики',
+          label: 'Поддержка Wi-Fi',
+          value: 'Да',
+        }),
+        expect.objectContaining({
+          group: 'Оптика',
+          label: 'Угол обзора (основная камера)',
+          value: '170 °',
+        }),
+      ]),
+    );
+    expect(normalized.specs).toEqual(
+      expect.objectContaining({
+        wifi: true,
+        speedcam: true,
+        microphone: true,
+        battery_power: 'battery',
+        resolution_front: '1920x1080',
+        frame_rate_max_fps: 30,
+        view_angle_degrees: 170,
+      }),
+    );
+    expect(normalized.rawSpecs).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          label: 'Поддержка Wi-Fi',
+          normalizedKey: 'wifi',
+          normalizedValue: true,
+        }),
+        expect.objectContaining({
+          label: 'Макс. разрешение видеосъемки',
+          normalizedKey: 'resolution_max',
+          normalizedValue: '2560x1440',
+        }),
+      ]),
+    );
+  });
+});

ingestion/src/sources/mvideo.ts

@@ -0,0 +1,238 @@
+import { createHash } from 'node:crypto';
+import { normalizeWhitespace, uniqueStable } from '../html.ts';
+import type { ParsedMarketProduct, ParsedTechnicalSpec } from '../types.ts';
+
+export const mvideoOrigin = 'https://www.mvideo.ru';
+const productIdPattern = /^\d+$/;
+
+type ProductDetailsProperty = {
+  name?: unknown;
+  value?: unknown;
+  measure?: unknown;
+};
+
+type ProductDetailsPropertyGroup = {
+  name?: unknown;
+  properties?: unknown;
+};
+
+function stringValue(value: unknown) {
+  if (typeof value !== 'string') return undefined;
+
+  const normalized = normalizeWhitespace(value);
+
+  return normalized ? normalized : undefined;
+}
+
+function normalizeSourceImageUrl(url: string) {
+  const trimmed = url.trim();
+
+  if (trimmed.startsWith('http://') || trimmed.startsWith('https://')) return trimmed;
+  if (trimmed.startsWith('//')) return `https:${trimmed}`;
+  if (trimmed.startsWith('/')) return `${mvideoOrigin}${trimmed}`;
+
+  return `https://img.mvideo.ru/${trimmed.replace(/^\/+/, '')}`;
+}
+
+function createFallbackProductId(sourceUrl: string, title: string) {
+  return createHash('sha1').update(`${sourceUrl}:${title}`).digest('hex').slice(0, 16);
+}
+
+function parseSpecValue(value: unknown, measure: unknown): string | number | boolean | undefined {
+  if (typeof value === 'boolean' || typeof value === 'number') return value;
+
+  const normalizedValue = stringValue(value);
+
+  if (!normalizedValue) return undefined;
+
+  const normalizedMeasure = stringValue(measure);
+
+  if (!normalizedMeasure) return normalizedValue;
+  if (normalizedValue.toLowerCase().includes(normalizedMeasure.toLowerCase())) return normalizedValue;
+
+  return `${normalizedValue} ${normalizedMeasure}`;
+}
+
+function specsRecord(technicalSpecs: ParsedTechnicalSpec[]) {
+  const record: Record<string, string | number | boolean> = {};
+
+  for (const spec of technicalSpecs) {
+    record[spec.label] = spec.value;
+  }
+
+  return record;
+}
+
+export function extractMvideoBffBasePath(html: string) {
+  const directMatch = html.match(/basePath\s*:\s*['"]([^'"]+)['"]/i)?.[1];
+  const jsonMatch = html.match(/"basePath"\s*:\s*"([^"]+)"/i)?.[1];
+  const value = directMatch ?? jsonMatch;
+
+  if (!value) return `${mvideoOrigin}/bff`;
+  if (value.startsWith('http://') || value.startsWith('https://')) return value;
+  if (value.startsWith('/')) return `${mvideoOrigin}${value}`;
+
+  return `${mvideoOrigin}/${value.replace(/^\/+/, '')}`;
+}
+
+export function extractMvideoSearchQuery(categoryUrl: string) {
+  try {
+    const url = new URL(categoryUrl);
+    const query = url.searchParams.get('q') ?? url.searchParams.get('query');
+
+    if (query?.trim()) return query.trim();
+  } catch {
+    // Fall through to default query.
+  }
+
+  return 'видеорегистраторы';
+}
+
+export function buildMvideoProductUrl(productId: string) {
+  return `${mvideoOrigin}/product/${encodeURIComponent(productId)}`;
+}
+
+export function buildMvideoBffUrl(basePath: string, endpoint: string) {
+  const normalizedBasePath = basePath.endsWith('/') ? basePath : `${basePath}/`;
+
+  return new URL(endpoint.replace(/^\/+/, ''), normalizedBasePath);
+}
+
+export function buildCookieHeader(setCookieHeaders: string[]) {
+  const cookies = new Map<string, string>();
+
+  for (const header of setCookieHeaders) {
+    const cookiePart = header.split(';')[0]?.trim();
+
+    if (!cookiePart) continue;
+
+    const separatorIndex = cookiePart.indexOf('=');
+
+    if (separatorIndex <= 0) continue;
+
+    const name = cookiePart.slice(0, separatorIndex).trim();
+    const value = cookiePart.slice(separatorIndex + 1).trim();
+
+    if (!name || !value) continue;
+
+    cookies.set(name, `${name}=${value}`);
+  }
+
+  return [...cookies.values()].join('; ');
+}
+
+export function discoverMvideoProductIds(searchPayload: unknown) {
+  if (!searchPayload || typeof searchPayload !== 'object') {
+    throw new Error('MVideo search payload is not an object');
+  }
+
+  const body = (searchPayload as { body?: unknown }).body;
+  const products = body && typeof body === 'object' ? (body as { products?: unknown }).products : undefined;
+
+  const productIds = Array.isArray(products)
+    ? products
+        .map((value) => (typeof value === 'number' || typeof value === 'string' ? String(value).trim() : ''))
+        .filter((value): value is string => productIdPattern.test(value))
+    : [];
+
+  if (productIds.length > 0) return uniqueStable(productIds);
+
+  // MVideo converts exact one-result searches into a redirect instead of a
+  // one-element products array. The destination still carries the public ID.
+  const redirectUrl =
+    typeof (body as { url?: unknown }).url === 'string'
+      ? (body as { url: string }).url
+      : typeof (body as { redirectUrl?: unknown }).redirectUrl === 'string'
+        ? (body as { redirectUrl: string }).redirectUrl
+        : undefined;
+  const redirectProductId = redirectUrl?.match(/(\d{6,13})(?:[/?#]|$)/)?.[1];
+
+  if (redirectProductId) return [redirectProductId];
+
+  throw new Error('MVideo search payload does not include product identifiers');
+}
+
+export function parseMvideoProduct(detailsPayload: unknown): ParsedMarketProduct {
+  if (!detailsPayload || typeof detailsPayload !== 'object') {
+    throw new Error('MVideo product payload is not an object');
+  }
+
+  const body = (detailsPayload as { body?: unknown }).body;
+
+  if (!body || typeof body !== 'object') {
+    throw new Error('MVideo product payload has no body');
+  }
+
+  const product = body as Record<string, unknown>;
+  const title = stringValue(product.name);
+
+  if (!title) {
+    throw new Error('MVideo product title was not found');
+  }
+
+  const sourceProductIdValue = product.productId;
+  const sourceProductId =
+    (typeof sourceProductIdValue === 'number' || typeof sourceProductIdValue === 'string'
+      ? String(sourceProductIdValue).trim()
+      : undefined) ?? undefined;
+  const slug = stringValue(product.nameTranslit);
+  const sourceUrl = slug && sourceProductId
+    ? `${mvideoOrigin}/products/${slug}-${sourceProductId}`
+    : sourceProductId
+      ? `${mvideoOrigin}/product/${sourceProductId}`
+      : mvideoOrigin;
+  const brand = stringValue(product.brandName);
+  const model = stringValue(product.modelName);
+  const images = uniqueStable(
+    (Array.isArray(product.images) ? product.images : [])
+      .filter((value): value is string => typeof value === 'string' && value.trim().length > 0)
+      .map(normalizeSourceImageUrl),
+  );
+  const rawSpecs: ParsedTechnicalSpec[] = [];
+  const rawGroups = (product.properties as { all?: unknown } | undefined)?.all;
+
+  if (Array.isArray(rawGroups)) {
+    for (const rawGroup of rawGroups as ProductDetailsPropertyGroup[]) {
+      const groupName = stringValue(rawGroup.name);
+      const properties = Array.isArray(rawGroup.properties) ? rawGroup.properties : [];
+
+      for (const rawSpec of properties as ProductDetailsProperty[]) {
+        const label = stringValue(rawSpec.name);
+        const value = parseSpecValue(rawSpec.value, rawSpec.measure);
+
+        if (!label || value === undefined) continue;
+
+        rawSpecs.push({
+          ...(groupName ? { group: groupName } : {}),
+          label,
+          value,
+        });
+      }
+    }
+  }
+
+  const ratingRaw = product.rating;
+  const aggregateRating =
+    ratingRaw && typeof ratingRaw === 'object' && Number.isFinite(Number((ratingRaw as { score?: unknown }).score))
+      ? {
+          value: Number((ratingRaw as { score: number | string }).score),
+          scale: 5,
+          ...(Number.isFinite(Number((ratingRaw as { total?: unknown }).total))
+            ? { ratingCount: Number((ratingRaw as { total: number | string }).total) }
+            : {}),
+        }
+      : undefined;
+
+  return {
+    source: 'mvideo',
+    sourceProductId: sourceProductId ?? createFallbackProductId(sourceUrl, title),
+    title,
+    ...(brand ? { brand } : {}),
+    ...(model ? { model } : {}),
+    images,
+    specs: specsRecord(rawSpecs),
+    rawSpecs,
+    sourceUrl,
+    aggregateRating,
+  };
+}

ingestion/src/sources/yandexMarket.test.ts

@@ -0,0 +1,154 @@
+import { describe, expect, it } from 'vitest';
+import { normalizeYandexMarketProduct } from '../normalizer';
+import { createStableContentHash } from '../stableHash';
+import { discoverYandexMarketProductsFromCategory, parseYandexMarketProduct } from './yandexMarket';
+
+describe('Yandex Market parser', () => {
+  it('discovers and canonicalizes card and product URLs from category HTML', () => {
+    const discovered = discoverYandexMarketProductsFromCategory(
+      `
+        <a href="/card/ibox-f5-wifi-videoregistrator-s-signaturnym-radar-detektorom/5197387201?do-waremd5=x&amp;cpc=y">F5</a>
+        <script>{"url":"https://market.yandex.ru/product/312774508?offerid=secret&sku=103274083526"}</script>
+      `,
+      'https://market.yandex.ru/catalog--avtomobilnye-videoregistratory/82798275/list?hid=82798269',
+      '2026-05-16T20:00:00.000Z',
+    );
+
+    expect(discovered).toMatchObject([
+      {
+        productUrl:
+          'https://market.yandex.ru/card/ibox-f5-wifi-videoregistrator-s-signaturnym-radar-detektorom/5197387201',
+        sourceProductId: '5197387201',
+      },
+      {
+        productUrl: 'https://market.yandex.ru/product/312774508',
+        sourceProductId: '312774508',
+      },
+    ]);
+  });
+
+  it('maps product JSON-LD and DOM specs without prices, sellers or reviews', () => {
+    const html = `
+      <script type="application/ld+json">
+        {
+          "@context": "https://schema.org",
+          "@type": "Product",
+          "brand": "iBOX",
+          "image": "https://avatars.mds.yandex.net/get-mpic/18787582/2a000/orig",
+          "name": "Видеорегистратор для автомобиля iBOX F5 + WiFi с радар-детектором",
+          "offers": { "price": "17816", "availability": "https://schema.org/InStock" },
+          "aggregateRating": { "ratingValue": "4.8", "ratingCount": "17" },
+          "url": "https://market.yandex.ru/card/ibox-f5-wifi-videoregistrator-s-signaturnym-radar-detektorom/5197387201"
+        }
+      </script>
+      <label class="_3J2L5">
+        <div><span>Экран</span></div>
+        <div class="_2yz50"><span>да</span></div>
+      </label>
+      <div>
+        <span data-auto="product-spec">Количество камер</span>
+        <div class="eXP5k"><div><div class="ds-text"><span>1</span></div></div></div>
+        <span data-auto="product-spec">Разрешение видеозаписи (при макс. частоте)</span>
+        <div class="eXP5k"><div><div class="ds-text"><span>2304x1296</span></div></div></div>
+        <span data-auto="product-spec">Wi-Fi</span>
+        <div class="eXP5k"><div><div class="ds-text"><span>да</span></div></div></div>
+        <span data-auto="product-spec">С радар-детектором</span>
+        <div class="eXP5k"><div><div class="ds-text"><span>нет</span></div></div></div>
+        <label for="group-collapse-Камера"><span>Камера</span></label>
+        <span data-auto="product-spec">Угол обзора (диагональ)</span>
+        <div class="eXP5k"><div><div class="ds-text"><span>170°</span></div></div></div>
+        <span data-auto="product-spec">Матрица</span>
+        <div class="eXP5k"><div><div class="ds-text"><span>CMOS</span></div></div></div>
+      </div>
+      <span class="ds-text ds-text_weight_med">Основные характеристики</span>
+      <div data-apiary-widget-name=@card/AboutSpecV2>
+        <div><span data-auto="product-spec">Режим записи</span></div>
+        <div class="ds-flex uA8Xn"><span>циклическая</span></div>
+      </div>
+    `;
+    const parsed = parseYandexMarketProduct(
+      html,
+      'https://market.yandex.ru/card/ibox-f5-wifi-videoregistrator-s-signaturnym-radar-detektorom/5197387201',
+    );
+    const normalized = normalizeYandexMarketProduct(parsed);
+
+    expect(parsed).toMatchObject({
+      source: 'yandex_market',
+      sourceProductId: '5197387201',
+      brand: 'iBOX',
+      title: 'Видеорегистратор для автомобиля iBOX F5 + WiFi с радар-детектором',
+      aggregateRating: {
+        value: 4.8,
+        ratingCount: 17,
+      },
+    });
+    expect(normalized.specs).toEqual({
+      channels: 1,
+      resolution_front: '2304x1296',
+      wifi: true,
+      radar_detector: false,
+      view_angle_degrees: 170,
+      sensor_type: 'CMOS',
+      recording_mode: 'циклическая',
+      screen: true,
+    });
+    expect(normalized.canonicalModel).toBe('F5 + Wi-Fi');
+    expect(normalized.canonicalName).toBe('iBOX F5 + Wi-Fi');
+    expect(normalized.rawSpecs).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          group: 'Камера',
+          label: 'Матрица',
+          value: 'CMOS',
+          normalizedKey: 'sensor_type',
+          normalizedValue: 'CMOS',
+        }),
+      ]),
+    );
+    expect(parsed.rawSpecs).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          group: 'Основные характеристики',
+          label: 'Режим записи',
+          value: 'циклическая',
+        }),
+      ]),
+    );
+    expect(createStableContentHash(normalized)).toMatch(/^[a-f0-9]{64}$/);
+    expect(JSON.stringify(normalized)).not.toContain('17816');
+  });
+
+  it('keeps stable model attributes and removes feature tails from canonical model', () => {
+    const normalized = normalizeYandexMarketProduct({
+      source: 'yandex_market',
+      sourceProductId: '5247127917',
+      sourceUrl:
+        'https://market.yandex.ru/card/videoregistrator-avtomobilnyy-12-mp-1296p-s-dvumya-kamerami-2k-spawnson-full-hd-antiblikovyy-obyektiv/5247127917',
+      title: 'Видеорегистратор SPAWNSON "Spawnson", 24МП, Wi-Fi, ночной режим',
+      brand: 'SPAWNSON',
+      images: [],
+      specs: {},
+      rawSpecs: [],
+    });
+
+    expect(normalized.model).toBe('24МП Wi-Fi ночной режим');
+    expect(normalized.canonicalModel).toBe('24МП Wi-Fi');
+    expect(normalized.canonicalName).toBe('SPAWNSON 24МП Wi-Fi');
+  });
+
+  it('removes mirror form factor and automotive descriptor tails from canonical model', () => {
+    const normalized = normalizeYandexMarketProduct({
+      source: 'yandex_market',
+      sourceProductId: 'mirror-1',
+      sourceUrl: 'https://market.yandex.ru/product/mirror-1',
+      title: 'Видеорегистратор-зеркало iBOX Rover 2 автомобильный с базой камер',
+      brand: 'iBOX',
+      images: [],
+      specs: {},
+      rawSpecs: [],
+    });
+
+    expect(normalized.canonicalModel).toBe('Rover 2');
+    expect(normalized.canonicalName).toBe('iBOX Rover 2');
+  });
+});

ingestion/src/sources/yandexMarket.ts

@@ -0,0 +1,272 @@
+import { createHash } from 'node:crypto';
+import { decodeHtml, extractJsonLdObjects, getObjectString, normalizeWhitespace, readMetaContent, stripTags, uniqueStable } from '../html.ts';
+import type { DiscoveredProduct, ParsedMarketProduct, ParsedTechnicalSpec } from '../types.ts';
+
+const marketOrigin = 'https://market.yandex.ru';
+const productUrlPattern = /(?:https?:\/\/market\.yandex\.ru)?\/(?:card|product)\/[^\s"'<>\\]+/gi;
+
+function getJsonLdType(object: Record<string, unknown>) {
+  const type = object['@type'];
+
+  if (Array.isArray(type)) return type.map(String);
+  if (typeof type === 'string') return [type];
+
+  return [];
+}
+
+function isJsonLdProduct(object: unknown): object is Record<string, unknown> {
+  return !!object && typeof object === 'object' && getJsonLdType(object as Record<string, unknown>).includes('Product');
+}
+
+function normalizeSourceImageUrl(url: string) {
+  const decoded = decodeHtml(url).trim();
+
+  if (decoded.startsWith('//')) return `https:${decoded}`;
+
+  return decoded;
+}
+
+function canonicalizeMarketUrl(rawUrl: string) {
+  const decoded = decodeHtml(rawUrl).replace(/,$/, '');
+  const url = new URL(decoded, marketOrigin);
+
+  if (url.hostname !== 'market.yandex.ru') {
+    throw new Error(`Unsupported Yandex Market host: ${url.hostname}`);
+  }
+
+  const parts = url.pathname.split('/').filter(Boolean);
+  const type = parts[0];
+
+  if (type === 'product' && parts[1]) {
+    return `${marketOrigin}/product/${parts[1]}`;
+  }
+
+  if (type === 'card' && parts[1] && parts[2]) {
+    return `${marketOrigin}/card/${parts[1]}/${parts[2]}`;
+  }
+
+  throw new Error(`Unsupported Yandex Market product URL: ${rawUrl}`);
+}
+
+function sourceProductIdFromUrl(url: string) {
+  const parsed = new URL(url);
+  const parts = parsed.pathname.split('/').filter(Boolean);
+
+  if (parts[0] === 'product') return parts[1];
+  if (parts[0] === 'card') return parts[2];
+
+  return undefined;
+}
+
+function titleFromCardSlug(url: string) {
+  const parsed = new URL(url);
+  const parts = parsed.pathname.split('/').filter(Boolean);
+
+  if (parts[0] !== 'card') return undefined;
+
+  return parts[1]
+    ?.split('-')
+    .filter(Boolean)
+    .map((part) => part.charAt(0).toUpperCase() + part.slice(1))
+    .join(' ');
+}
+
+function normalizeProductUrlCandidate(rawUrl: string) {
+  try {
+    const url = canonicalizeMarketUrl(rawUrl);
+
+    return {
+      productUrl: url,
+      sourceProductId: sourceProductIdFromUrl(url),
+      titlePreview: titleFromCardSlug(url),
+    };
+  } catch {
+    return undefined;
+  }
+}
+
+function extractRating(value: unknown): ParsedMarketProduct['aggregateRating'] {
+  if (!value || typeof value !== 'object') return undefined;
+
+  const object = value as Record<string, unknown>;
+  const ratingValue = Number(object.ratingValue);
+  const ratingCount = Number(object.ratingCount ?? object.reviewCount);
+  const bestRating = Number(object.bestRating);
+
+  if (!Number.isFinite(ratingValue)) return undefined;
+
+  return {
+    value: ratingValue,
+    ...(Number.isFinite(bestRating) ? { scale: bestRating } : {}),
+    ...(Number.isFinite(ratingCount) ? { ratingCount } : {}),
+  };
+}
+
+function extractJsonLdProduct(html: string) {
+  return extractJsonLdObjects(html).find(isJsonLdProduct);
+}
+
+function extractJsonLdImages(product: Record<string, unknown> | undefined) {
+  const image = product?.image;
+  const values = Array.isArray(image) ? image : image ? [image] : [];
+
+  return values.filter((value): value is string => typeof value === 'string').map(normalizeSourceImageUrl);
+}
+
+function extractMetaImages(html: string) {
+  return [readMetaContent(html, 'og:image')]
+    .filter((value): value is string => !!value)
+    .map(normalizeSourceImageUrl);
+}
+
+function extractSpecGroup(htmlBeforeSpec: string) {
+  const matches = [
+    ...htmlBeforeSpec.matchAll(/for=["']group-collapse-([^"']+)["'][\s\S]{0,500}?<span[^>]*>([\s\S]*?)<\/span>/gi),
+    ...htmlBeforeSpec.matchAll(
+      /<span[^>]*class=["'][^"']*ds-text_weight_med[^"']*["'][^>]*>([\s\S]*?)<\/span>/gi,
+    ),
+  ];
+  const match = matches
+    .filter((candidate) => candidate.index !== undefined)
+    .sort((left, right) => (left.index ?? 0) - (right.index ?? 0))
+    .at(-1);
+
+  return match ? stripTags(match[2] || match[1]) : undefined;
+}
+
+function extractTechnicalSpecsFromDom(html: string): ParsedTechnicalSpec[] {
+  const specs: ParsedTechnicalSpec[] = [];
+  const seen = new Map<string, number>();
+  const legacySpecPattern =
+    /<span[^>]*data-auto=["']product-spec["'][^>]*>([\s\S]*?)<\/span>[\s\S]{0,1400}?<div[^>]+class=["'][^"']*ds-text[^"']*["'][^>]*>\s*<span>([\s\S]*?)<\/span>/gi;
+  const currentSpecPattern =
+    /data-apiary-widget-name=@card\/AboutSpecV2[^>]*>[\s\S]*?<span[^>]*data-auto=["']product-spec["'][^>]*>([\s\S]*?)<\/span>[\s\S]*?<div[^>]*class=["'][^"']*(?:uA8Xn|_2yz50)[^"']*["'][^>]*>\s*<span[^>]*>([\s\S]*?)<\/span>/gi;
+  const compactSpecPattern =
+    /<label[^>]*>[\s\S]*?<span[^>]*>([\s\S]*?)<\/span>[\s\S]*?<div[^>]*class=["'][^"']*_2yz50[^"']*["'][^>]*>\s*<span[^>]*>([\s\S]*?)<\/span>[\s\S]*?<\/label>/gi;
+
+  function addSpec(match: RegExpMatchArray) {
+    const rawName = match[1];
+    const rawValue = match[2];
+    const label = stripTags(rawName);
+    const value = stripTags(rawValue);
+
+    if (!label || !value || label === value) return;
+
+    const group = extractSpecGroup(html.slice(Math.max(0, (match.index ?? 0) - 8_000), match.index));
+    const dedupeKey = `${label}:${value}`;
+    const existingIndex = seen.get(dedupeKey);
+
+    if (existingIndex !== undefined) {
+      if (group && !specs[existingIndex]?.group) {
+        specs[existingIndex] = { ...specs[existingIndex], group };
+      }
+
+      return;
+    }
+
+    seen.set(dedupeKey, specs.length);
+    specs.push({
+      ...(group ? { group } : {}),
+      label,
+      value,
+    });
+  }
+
+  for (const match of html.matchAll(currentSpecPattern)) {
+    addSpec(match);
+  }
+
+  for (const match of html.matchAll(legacySpecPattern)) {
+    addSpec(match);
+  }
+
+  for (const match of html.matchAll(compactSpecPattern)) {
+    addSpec(match);
+  }
+
+  return specs;
+}
+
+function specsRecord(technicalSpecs: ParsedTechnicalSpec[]) {
+  const record: Record<string, string | number | boolean> = {};
+
+  for (const spec of technicalSpecs) {
+    record[spec.label] = spec.value;
+  }
+
+  return record;
+}
+
+function extractTitle(html: string, product?: Record<string, unknown>) {
+  const jsonTitle = getObjectString(product?.name);
+  const ogTitle = readMetaContent(html, 'og:title');
+  const titleTag = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1];
+
+  return jsonTitle ?? ogTitle ?? (titleTag ? stripTags(titleTag) : undefined);
+}
+
+function createFallbackProductId(sourceUrl: string, title: string) {
+  return createHash('sha1').update(`${sourceUrl}:${title}`).digest('hex').slice(0, 16);
+}
+
+export function discoverYandexMarketProductsFromCategory(
+  html: string,
+  categoryUrl: string,
+  discoveredAt = new Date().toISOString(),
+): DiscoveredProduct[] {
+  const products = [...html.matchAll(productUrlPattern)]
+    .map((match) => normalizeProductUrlCandidate(match[0]))
+    .filter((value): value is NonNullable<ReturnType<typeof normalizeProductUrlCandidate>> => !!value)
+    .map((value) => ({
+      source: 'yandex_market' as const,
+      categoryUrl,
+      productUrl: value.productUrl,
+      sourceProductId: value.sourceProductId,
+      titlePreview: value.titlePreview,
+      discoveredAt,
+    }));
+
+  const byUrl = new Map<string, DiscoveredProduct>();
+
+  for (const product of products) {
+    byUrl.set(product.productUrl, product);
+  }
+
+  return [...byUrl.values()];
+}
+
+export function parseYandexMarketProduct(html: string, sourceUrl: string): ParsedMarketProduct {
+  const product = extractJsonLdProduct(html);
+  const title = extractTitle(html, product);
+
+  if (!title) {
+    throw new Error('Yandex Market product title was not found');
+  }
+
+  const canonicalUrl = normalizeProductUrlCandidate(getObjectString(product?.url) ?? sourceUrl)?.productUrl ?? sourceUrl;
+  const sourceProductId =
+    sourceProductIdFromUrl(canonicalUrl) ??
+    (typeof product?.sku === 'string' ? product.sku : undefined) ??
+    createFallbackProductId(canonicalUrl, title);
+  const brand = getObjectString(product?.brand);
+  const images = uniqueStable([...extractJsonLdImages(product), ...extractMetaImages(html)]).filter((url) =>
+    url.includes('/get-mpic/'),
+  );
+  const rawSpecs = extractTechnicalSpecsFromDom(html);
+
+  return {
+    source: 'yandex_market',
+    sourceProductId,
+    title: normalizeWhitespace(title),
+    ...(brand ? { brand } : {}),
+    images,
+    specs: specsRecord(rawSpecs),
+    rawSpecs,
+    sourceUrl: canonicalUrl,
+    aggregateRating: extractRating(product?.aggregateRating),
+  };
+}
+
+export function canonicalizeYandexMarketProductUrl(rawUrl: string) {
+  return canonicalizeMarketUrl(rawUrl);
+}

ingestion/src/stableHash.ts

@@ -0,0 +1,35 @@
+import { createHash } from 'node:crypto';
+import type { NormalizedMarketProduct } from './types.ts';
+
+function stableJson(value: unknown): string {
+  if (Array.isArray(value)) {
+    return `[${value.map(stableJson).join(',')}]`;
+  }
+
+  if (value && typeof value === 'object') {
+    const object = value as Record<string, unknown>;
+
+    return `{${Object.keys(object)
+      .sort()
+      .map((key) => `${JSON.stringify(key)}:${stableJson(object[key])}`)
+      .join(',')}}`;
+  }
+
+  return JSON.stringify(value);
+}
+
+export function createStableContentHash(product: NormalizedMarketProduct) {
+  const semanticPayload = {
+    sourceProductId: product.sourceProductId,
+    canonicalProductUrl: product.sourceUrl,
+    title: product.title,
+    brand: product.brand,
+    model: product.model,
+    canonicalName: product.canonicalName,
+    specs: product.specs,
+    rawSpecs: product.rawSpecs,
+    images: product.images,
+  };
+
+  return createHash('sha256').update(stableJson(semanticPayload)).digest('hex');
+}

ingestion/src/types.ts

@@ -0,0 +1,126 @@
+import type { ExportedSpecValue } from '../../web-front/src/entities/dashcam-catalog/model/schema.ts';
+
+export type MarketSource = 'yandex_market' | 'mvideo';
+
+export type CrawlJobType = 'category' | 'product' | 'sitemap';
+
+export type CrawlJobState =
+  | 'pending'
+  | 'fetching'
+  | 'fetched'
+  | 'parsed'
+  | 'normalized'
+  | 'deduplicated'
+  | 'needs_review'
+  | 'approved'
+  | 'exported'
+  | CrawlFailureReason;
+
+export type CrawlFailureReason =
+  | 'rate_limited'
+  | 'blocked'
+  | 'captcha'
+  | 'not_found'
+  | 'parse_failed'
+  | 'needs_manual_review';
+
+export type DiscoveredProduct = {
+  source: MarketSource;
+  categoryUrl?: string;
+  productUrl: string;
+  sourceProductId?: string;
+  titlePreview?: string;
+  discoveredAt: string;
+};
+
+export type ParsedTechnicalSpec = {
+  group?: string;
+  label: string;
+  value: string | number | boolean;
+};
+
+export type NormalizedTechnicalSpec = ParsedTechnicalSpec & {
+  normalizedKey?: string;
+  normalizedValue?: ExportedSpecValue;
+};
+
+export type ParsedMarketProduct = {
+  source: MarketSource;
+  sourceProductId?: string;
+  title: string;
+  brand?: string;
+  model?: string;
+  aggregateRating?: {
+    value: number;
+    scale?: number;
+    ratingCount?: number;
+  };
+  images: string[];
+  specs: Record<string, string | number | boolean>;
+  rawSpecs: ParsedTechnicalSpec[];
+  sourceUrl: string;
+};
+
+export type NormalizedMarketProduct = {
+  source: MarketSource;
+  sourceProductId?: string;
+  sourceUrl: string;
+  title: string;
+  brand: string;
+  model: string;
+  canonicalModel: string;
+  canonicalName: string;
+  specs: Record<string, ExportedSpecValue>;
+  rawSpecs: NormalizedTechnicalSpec[];
+  images: string[];
+  aggregateRating?: ParsedMarketProduct['aggregateRating'];
+};
+
+export type ProductSourceMetadata = {
+  source: MarketSource;
+  sourceProductId?: string;
+  url: string;
+  fetchedAt?: string;
+  stableContentHash?: string;
+  title: string;
+  aggregateRating?: ParsedMarketProduct['aggregateRating'];
+};
+
+export type SpecConflict = {
+  key: string;
+  values: Array<{
+    source: MarketSource;
+    url: string;
+    value: ExportedSpecValue;
+  }>;
+};
+
+export type ProductSourceSnapshot = {
+  source: MarketSource;
+  url: string;
+  fetchedAt: string;
+  httpStatus: number;
+  rawSnapshotHash: string;
+  stableContentHash?: string;
+  rawHtmlPath?: string;
+  parsedJson: Record<string, unknown>;
+};
+
+export type FetchClassification =
+  | { ok: true }
+  | {
+      ok: false;
+      reason: CrawlFailureReason;
+      retryable: boolean;
+    };
+
+export type CrawlJobRecord = {
+  id: string;
+  source: MarketSource;
+  url: string;
+  jobType: CrawlJobType;
+  state: CrawlJobState;
+  attempts: number;
+  nextRunAt: string;
+  lastError?: string;
+};

ingestion/src/worker.ts

@@ -0,0 +1,281 @@
+import { ingestionConfig } from './config.ts';
+import { createPgPool } from './db.ts';
+import { fetchRawSnapshot, getFetchFailureReason, randomDelayMs, type RawFetchSnapshot } from './fetcher.ts';
+import { normalizeMarketProduct } from './normalizer.ts';
+import { CrawlQueue, dailyBudgetForJob } from './queue.ts';
+import { IngestionRepository } from './repository.ts';
+import {
+  buildCookieHeader,
+  buildMvideoBffUrl,
+  buildMvideoProductUrl,
+  discoverMvideoProductIds,
+  extractMvideoBffBasePath,
+  extractMvideoSearchQuery,
+  parseMvideoProduct,
+} from './sources/mvideo.ts';
+import { discoverYandexMarketProductsFromCategory, parseYandexMarketProduct } from './sources/yandexMarket.ts';
+import { createStableContentHash } from './stableHash.ts';
+import type { CrawlFailureReason, CrawlJobRecord, DiscoveredProduct } from './types.ts';
+
+function sleep(ms: number) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function isStopReason(reason: string | undefined) {
+  return reason === 'captcha' || reason === 'blocked' || reason === 'rate_limited';
+}
+
+function sourceError(reason: CrawlFailureReason, message: string) {
+  const error = new Error(message) as Error & { reason?: CrawlFailureReason };
+  error.reason = reason;
+  return error;
+}
+
+function parseJsonSnapshot(snapshot: RawFetchSnapshot, label: string) {
+  try {
+    return JSON.parse(snapshot.html) as unknown;
+  } catch (error) {
+    throw sourceError('parse_failed', `Expected JSON from ${label}: ${(error as Error).message}`);
+  }
+}
+
+async function recordSnapshot(
+  repository: IngestionRepository,
+  snapshot: RawFetchSnapshot,
+  parsedJson: Record<string, unknown>,
+  stableContentHash?: string,
+) {
+  await repository.recordSnapshot({
+    source: snapshot.source,
+    url: snapshot.url,
+    fetchedAt: snapshot.fetchedAt,
+    httpStatus: snapshot.httpStatus,
+    rawSnapshotHash: snapshot.rawSnapshotHash,
+    rawHtmlPath: snapshot.rawHtmlPath,
+    parsedJson,
+    ...(stableContentHash ? { stableContentHash } : {}),
+  });
+}
+
+async function processYandexCategory(
+  job: CrawlJobRecord,
+  repository: IngestionRepository,
+  queue: CrawlQueue,
+) {
+  const snapshot = await fetchRawSnapshot({
+    source: job.source,
+    url: job.url,
+    snapshotDir: ingestionConfig.snapshotDir,
+  });
+  const products = discoverYandexMarketProductsFromCategory(snapshot.html, job.url, snapshot.fetchedAt);
+
+  await recordSnapshot(repository, snapshot, { discoveredProducts: products });
+  await repository.recordDiscoveredProducts(products);
+
+  for (const product of products) {
+    await queue.enqueue({
+      source: product.source,
+      url: product.productUrl,
+      jobType: 'product',
+    });
+  }
+
+  await queue.markState(job.id, 'parsed');
+  console.log(`yandex_market category parsed: ${products.length} product candidates`);
+}
+
+async function processMvideoCategory(
+  job: CrawlJobRecord,
+  repository: IngestionRepository,
+  queue: CrawlQueue,
+) {
+  const categorySnapshot = await fetchRawSnapshot({
+    source: job.source,
+    url: job.url,
+    snapshotDir: ingestionConfig.snapshotDir,
+  });
+  const bffBasePath = extractMvideoBffBasePath(categorySnapshot.html);
+  const searchQuery = extractMvideoSearchQuery(categorySnapshot.url);
+  const searchUrl = buildMvideoBffUrl(bffBasePath, 'products/v2/search');
+  const cookieHeader = buildCookieHeader(categorySnapshot.setCookieHeaders);
+
+  searchUrl.searchParams.set('query', searchQuery);
+  searchUrl.searchParams.set('offset', '0');
+  searchUrl.searchParams.set('limit', String(ingestionConfig.productDailyBudget));
+
+  const searchSnapshot = await fetchRawSnapshot({
+    source: job.source,
+    url: searchUrl.toString(),
+    snapshotDir: ingestionConfig.snapshotDir,
+    accept: 'application/json,text/plain,*/*',
+    ...(cookieHeader ? { headers: { cookie: cookieHeader } } : {}),
+  });
+  const productIds = discoverMvideoProductIds(parseJsonSnapshot(searchSnapshot, 'MVideo search'));
+  const products: DiscoveredProduct[] = productIds.map((sourceProductId) => ({
+    source: 'mvideo',
+    sourceProductId,
+    productUrl: buildMvideoProductUrl(sourceProductId),
+    categoryUrl: categorySnapshot.url,
+    discoveredAt: categorySnapshot.fetchedAt,
+  }));
+
+  await recordSnapshot(repository, categorySnapshot, {
+    bffBasePath,
+    searchQuery,
+    discoveredProducts: products,
+  });
+  await recordSnapshot(repository, searchSnapshot, { productIds });
+  await repository.recordDiscoveredProducts(products);
+
+  for (const product of products) {
+    await queue.enqueue({
+      source: product.source,
+      url: product.productUrl,
+      jobType: 'product',
+    });
+  }
+
+  await queue.markState(job.id, 'parsed');
+  console.log(`mvideo category parsed: ${products.length} product candidates`);
+}
+
+async function processYandexProduct(job: CrawlJobRecord, repository: IngestionRepository, queue: CrawlQueue) {
+  const snapshot = await fetchRawSnapshot({
+    source: job.source,
+    url: job.url,
+    snapshotDir: ingestionConfig.snapshotDir,
+  });
+  const parsed = parseYandexMarketProduct(snapshot.html, job.url);
+  const normalized = normalizeMarketProduct(parsed);
+  const stableContentHash = createStableContentHash(normalized);
+
+  await recordSnapshot(repository, snapshot, { parsed, normalized }, stableContentHash);
+  await repository.upsertNormalizedProduct(normalized, stableContentHash, snapshot.fetchedAt);
+  await queue.markState(job.id, 'needs_review');
+  console.log(`yandex_market product parsed: ${normalized.brand} ${normalized.canonicalModel}`);
+}
+
+async function processMvideoProduct(job: CrawlJobRecord, repository: IngestionRepository, queue: CrawlQueue) {
+  const productPageSnapshot = await fetchRawSnapshot({
+    source: job.source,
+    url: job.url,
+    snapshotDir: ingestionConfig.snapshotDir,
+  });
+  const bffBasePath = extractMvideoBffBasePath(productPageSnapshot.html);
+  const cookieHeader = buildCookieHeader(productPageSnapshot.setCookieHeaders);
+  const sourceProductId = productPageSnapshot.url.match(/(\d+)(?:\/?(?:\?.*)?)$/)?.[1];
+
+  if (!sourceProductId) {
+    throw sourceError('parse_failed', `MVideo product id was not found in ${productPageSnapshot.url}`);
+  }
+
+  const detailsUrl = buildMvideoBffUrl(bffBasePath, 'product-details');
+  detailsUrl.searchParams.set('productId', sourceProductId);
+
+  const detailsSnapshot = await fetchRawSnapshot({
+    source: job.source,
+    url: detailsUrl.toString(),
+    snapshotDir: ingestionConfig.snapshotDir,
+    accept: 'application/json,text/plain,*/*',
+    ...(cookieHeader ? { headers: { cookie: cookieHeader } } : {}),
+  });
+  const parsed = parseMvideoProduct(parseJsonSnapshot(detailsSnapshot, 'MVideo product details'));
+  const normalized = normalizeMarketProduct(parsed);
+  const stableContentHash = createStableContentHash(normalized);
+
+  await recordSnapshot(
+    repository,
+    productPageSnapshot,
+    { productApiUrl: detailsUrl.toString(), parsed, normalized },
+    stableContentHash,
+  );
+  await recordSnapshot(repository, detailsSnapshot, { parsed, normalized }, stableContentHash);
+  await repository.upsertNormalizedProduct(normalized, stableContentHash, productPageSnapshot.fetchedAt);
+  await queue.markState(job.id, 'needs_review');
+  console.log(`mvideo product parsed: ${normalized.brand} ${normalized.canonicalModel}`);
+}
+
+async function processJob(job: CrawlJobRecord, queue: CrawlQueue, repository: IngestionRepository) {
+  const reserved = await queue.reserveDailyBudget(job.source, job.jobType, dailyBudgetForJob(job.jobType));
+
+  if (!reserved) {
+    await queue.markFailure(job, 'rate_limited', `Daily ${job.jobType} budget is exhausted`);
+    return;
+  }
+
+  try {
+    if (job.source === 'yandex_market' && job.jobType === 'category') {
+      await processYandexCategory(job, repository, queue);
+      return;
+    }
+
+    if (job.source === 'mvideo' && job.jobType === 'category') {
+      await processMvideoCategory(job, repository, queue);
+      return;
+    }
+
+    if (job.source === 'yandex_market' && job.jobType === 'product') {
+      await processYandexProduct(job, repository, queue);
+      return;
+    }
+
+    if (job.source === 'mvideo' && job.jobType === 'product') {
+      await processMvideoProduct(job, repository, queue);
+      return;
+    }
+
+    await queue.markFailure(job, 'needs_manual_review', `Unsupported job: ${job.source}/${job.jobType}`);
+  } catch (error) {
+    const reason = getFetchFailureReason(error) ?? 'parse_failed';
+
+    await queue.markFailure(job, reason, (error as Error).message);
+
+    if (isStopReason(reason)) {
+      await queue.pauseSource(job.source, reason, new Date(Date.now() + 24 * 60 * 60 * 1000));
+    }
+  }
+}
+
+export async function runWorker() {
+  const pool = createPgPool();
+  const queue = new CrawlQueue(pool);
+  const repository = new IngestionRepository(pool);
+
+  for (const seed of ingestionConfig.sourceSeeds) {
+    if (!seed.implemented) {
+      console.warn(`source seed configured but crawler is not implemented yet: ${seed.source} ${seed.categoryUrl}`);
+      continue;
+    }
+
+    await queue.enqueue({
+      source: seed.source,
+      url: seed.categoryUrl,
+      jobType: 'category',
+    });
+  }
+
+  try {
+    do {
+      let processed = 0;
+
+      for (let index = 0; index < ingestionConfig.workerBatchSize; index += 1) {
+        const job = await queue.leaseNext();
+
+        if (!job) break;
+
+        await processJob(job, queue, repository);
+        processed += 1;
+        await sleep(randomDelayMs());
+      }
+
+      if (process.env.INGESTION_ONCE === '1') break;
+      if (processed === 0) await sleep(ingestionConfig.workerIdleMs);
+    } while (true);
+  } finally {
+    await pool.end();
+  }
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  await runWorker();
+}

package.json

@@ -13,8 +13,9 @@
     "ingestion:migrate": "node --experimental-strip-types ingestion/src/migrate.ts",
     "ingestion:worker": "node --experimental-strip-types ingestion/src/worker.ts",
     "ingestion:admin": "node --experimental-strip-types ingestion/src/adminServer.ts",
-    "ingestion:export": "node --experimental-strip-types ingestion/src/exportServer.ts",
     "ingestion:export-catalog": "node --experimental-strip-types ingestion/src/exportCatalog.ts",
+    "ingestion:smoke-yandex": "node --experimental-strip-types ingestion/src/smokeFetchYandexMarket.ts",
+    "ingestion:smoke-mvideo": "node --experimental-strip-types ingestion/src/smokeFetchMvideo.ts",
     "ci:quality": "npm run typecheck && npm test && npm run build"
   },
   "dependencies": {

tasks/002-crawler.md

@@ -74,7 +74,10 @@ Headless browser допустим только как ручной diagnostic to
 
 ## 4. Deployment Model
 
-Ingestion живет отдельно от облачного runtime приложения.
+Ingestion деплоится в то же облако, что и web-приложение (один Swarm/Portainer
+environment), а не на отдельный сервер. Раньше предполагался отдельный cloud, но
+управлять двумя независимыми облаками оказалось слишком сложно. См. решение
+`videoreg-nnv.3`.
 
 Код хранится в этом же репозитории:
 
@@ -86,18 +89,20 @@ ingestion/
   Dockerfile
 ```
 
-Стек ingestion деплоится отдельным `docker-compose.yml`.
-
 Минимальные сервисы:
 
 ```text
 postgres
 ingestion-worker
 ingestion-admin
-export-server
 ```
 
-Основное Next.js-приложение не подключается напрямую к ingestion DB.
+`export-server` больше не нужен: ingestion-worker пишет валидированный
+`catalog.json` + `manifest.json` в общий (shared) export-том, а web читает их
+напрямую с этого тома. Никакого HTTP-экспорта и Bearer-токена.
+
+Основное Next.js-приложение по-прежнему не подключается напрямую к ingestion DB —
+единственный контакт с каталогом идёт через export-артефакт на shared-томе.
 
 ---
 
@@ -107,29 +112,27 @@ Source of truth для каталога: PostgreSQL ingestion stack.
 
 Текущее примитивное хранилище приложения (`data/videoreg.json`) является только read-model cache.
 
-Bridge работает со стороны приложения:
+Bridge работает со стороны приложения через общий export-том:
 
-1. ingestion публикует versioned HTTPS export;
-2. приложение вручную запускает operator import command;
-3. import command скачивает manifest;
-4. проверяет `schemaVersion`, `exportId`, `createdAt`, `sha256`, `itemCount`;
-5. скачивает catalog artifact;
+1. ingestion пишет versioned export (`manifest.json` + `catalog.json`) на shared-том;
+2. приложение/сборка читает `manifest.json` с того же тома;
+3. проверяет `schemaVersion`, `exportId`, `createdAt`, `sha256`, `itemCount`;
+4. читает `catalog.json` с тома;
+5. сверяет `sha256` артефакта;
 6. валидирует payload через shared Zod schema;
-7. атомарно заменяет `data/videoreg.json`.
+7. атомарно заменяет `data/videoreg.json` (или используется как build-time источник).
 
-В MVP import запускается вручную оператором, не по расписанию.
+В MVP import запускается вручную оператором, не по расписанию. HTTP/токен в этом
+обмене не участвуют — оба сервиса в одном облаке и делят том.
 
 ---
 
 ## 6. Export Contract
 
-Export доступен по HTTPS со статичным токеном:
-
-```http
-Authorization: Bearer <token>
-```
-
-Токен передается через env import-команды. Токен не передается в query string.
+Export публикуется как файлы на общем (shared) export-томе: `manifest.json` и
+`artifacts/<exportId>/catalog.json`. HTTP-эндпоинт и Bearer-токен не используются —
+ingestion и web живут в одном облаке и делят том. Целостность гарантируется
+`sha256` в манифесте и shared Zod-схемой при чтении.
 
 ### Manifest
 
@@ -138,7 +141,7 @@ type CatalogExportManifest = {
   schemaVersion: 1;
   exportId: string;
   createdAt: string;
-  artifactUrl: string;
+  artifactPath: string;
   sha256: string;
   itemCount: number;
 };
@@ -158,6 +161,7 @@ type ExportedDashcamProduct = {
   id: string;
   brand: string;
   model: string;
+  canonicalName?: string;
   title: string;
   specs: Record<string, string | number | boolean>;
   images: ExportedProductImage[];
@@ -169,19 +173,21 @@ type ExportedDashcamProduct = {
 type ExportedProductImage = {
   url: string;
   originalUrl: string;
-  source: "yandex_market";
+  source: MarketSource;
   sha256?: string;
   width?: number;
   height?: number;
 };
 
 type ExportedProductSource = {
-  source: "yandex_market";
+  source: MarketSource;
   sourceProductId?: string;
   url: string;
   stableContentHash: string;
   fetchedAt: string;
 };
+
+type MarketSource = "yandex_market" | "mvideo";
 ```
 
 Runtime validation: shared Zod schema в репозитории. TypeScript-типы без runtime-валидации недостаточны.
@@ -260,7 +266,7 @@ Category pages используются вторично:
 
 ```ts
 type DiscoveredProduct = {
-  source: "yandex_market";
+  source: MarketSource;
   categoryUrl?: string;
   productUrl: string;
   sourceProductId?: string;
@@ -344,7 +350,7 @@ JSON-LD > embedded state > semantic DOM > fallback DOM
 
 ```ts
 type ParsedMarketProduct = {
-  source: "yandex_market";
+  source: MarketSource;
   sourceProductId?: string;
   title: string;
   brand?: string;
@@ -372,7 +378,7 @@ type ParsedMarketProduct = {
 
 ```ts
 type ProductSourceSnapshot = {
-  source: "yandex_market";
+  source: MarketSource;
   url: string;
   fetchedAt: string;
   httpStatus: number;
@@ -623,7 +629,7 @@ export_runs
 * publish versioned manifest
 * publish versioned catalog artifact
 * publish versioned image artifacts
-* protect endpoints with Bearer token
+* write export artifacts to the shared export volume (no HTTP, no token)
 * verify checksum in app import
 * atomic replace of `data/videoreg.json`
 
@@ -633,7 +639,7 @@ export_runs
 
 Система считается готовой, если:
 
-* ingestion stack запускается отдельно через `docker-compose.yml`
+* ingestion и web деплоятся в одно облако; export идёт через shared-том, без HTTP/токена
 * PostgreSQL является source of truth
 * приложение импортирует только approved canonical catalog
 * `data/videoreg.json` заменяется только после manifest/checksum/schema validation

web-front/scripts/importCatalogExport.ts

@@ -1,17 +1,12 @@
 import { importCatalogExport } from '../src/entities/dashcam-catalog/api/importCatalogExport.ts';
 
-const manifestUrl = process.env.DASHCAM_CATALOG_MANIFEST_URL ?? process.argv[2];
-const token = process.env.DASHCAM_CATALOG_EXPORT_TOKEN;
+const exportDir = process.env.DASHCAM_CATALOG_EXPORT_DIR ?? process.argv[2];
 
-if (!manifestUrl) {
-  throw new Error('Set DASHCAM_CATALOG_MANIFEST_URL or pass manifest URL as the first argument');
+if (!exportDir) {
+  throw new Error('Set DASHCAM_CATALOG_EXPORT_DIR or pass the shared export directory as the first argument');
 }
 
-if (!token) {
-  throw new Error('Set DASHCAM_CATALOG_EXPORT_TOKEN');
-}
-
-const result = await importCatalogExport({ manifestUrl, token });
+const result = await importCatalogExport({ exportDir });
 
 console.log(
   JSON.stringify(

web-front/src/app/globals.css

@@ -365,6 +365,216 @@ th {
   text-align: center;
 }
 
+.catalog-page {
+  width: min(1180px, calc(100% - 32px));
+  margin: 0 auto;
+  padding: 28px 0 56px;
+}
+
+.catalog-header {
+  display: flex;
+  align-items: flex-end;
+  justify-content: space-between;
+  gap: 24px;
+  border-bottom: 1px solid var(--border-subtle);
+  padding-bottom: 20px;
+}
+
+.catalog-header h1 {
+  margin: 0;
+  font-size: clamp(2rem, 4vw, 3.25rem);
+  line-height: 1.05;
+}
+
+.catalog-header__eyebrow,
+.catalog-product__brand {
+  margin: 0 0 8px;
+  color: var(--accent-strong);
+  font-size: 0.75rem;
+  font-weight: 750;
+  letter-spacing: 0;
+  text-transform: uppercase;
+}
+
+.catalog-header__subtitle {
+  max-width: 46rem;
+  margin: 12px 0 0;
+  color: var(--text-muted);
+  font-size: 1rem;
+  line-height: 1.5;
+}
+
+.catalog-header__meta {
+  display: grid;
+  min-width: 126px;
+  border-left: 1px solid var(--border-subtle);
+  padding-left: 20px;
+}
+
+.catalog-header__meta strong {
+  font-family: var(--font-tech);
+  font-size: 2rem;
+  line-height: 1;
+}
+
+.catalog-header__meta span,
+.catalog-status {
+  color: var(--text-muted);
+  font-size: 0.78rem;
+}
+
+.catalog-status {
+  display: flex;
+  flex-wrap: wrap;
+  gap: 8px 16px;
+  padding: 14px 0;
+}
+
+.catalog-status span + span::before {
+  content: '•';
+  margin-right: 16px;
+  color: var(--border-strong);
+}
+
+.catalog-grid {
+  display: grid;
+  grid-template-columns: repeat(2, minmax(0, 1fr));
+  gap: 16px;
+}
+
+.catalog-product {
+  display: grid;
+  grid-template-columns: 150px minmax(0, 1fr);
+  overflow: hidden;
+  border: 1px solid var(--border-subtle);
+  border-radius: var(--radius-lg);
+  background: var(--surface-1);
+  box-shadow: var(--shadow-soft);
+}
+
+.catalog-product__media {
+  display: grid;
+  place-items: center;
+  min-height: 182px;
+  background: var(--surface-inverse);
+  padding: 18px;
+}
+
+.catalog-product__media img {
+  width: 100%;
+  max-height: 132px;
+  object-fit: contain;
+  opacity: 0.84;
+}
+
+.catalog-product__body {
+  display: grid;
+  gap: 14px;
+  padding: 18px;
+}
+
+.catalog-product__heading {
+  display: flex;
+  align-items: flex-start;
+  justify-content: space-between;
+  gap: 12px;
+}
+
+.catalog-product h2 {
+  margin: 0;
+  font-size: 1.25rem;
+  line-height: 1.2;
+}
+
+.catalog-product__source-count {
+  flex: none;
+  border: 1px solid var(--border-subtle);
+  border-radius: var(--radius-sm);
+  color: var(--text-muted);
+  padding: 4px 6px;
+  font-family: var(--font-tech);
+  font-size: 0.68rem;
+}
+
+.catalog-product__title {
+  margin: 0;
+  color: var(--text-muted);
+  font-size: 0.85rem;
+  line-height: 1.45;
+}
+
+.catalog-product__specs,
+.catalog-product__details dl {
+  display: grid;
+  grid-template-columns: repeat(2, minmax(0, 1fr));
+  gap: 8px;
+  margin: 0;
+}
+
+.catalog-product__specs div,
+.catalog-product__details dl div {
+  min-width: 0;
+  border-top: 1px solid var(--border-subtle);
+  padding-top: 7px;
+}
+
+.catalog-product__specs dt,
+.catalog-product__details dt {
+  color: var(--text-muted);
+  font-size: 0.7rem;
+}
+
+.catalog-product__specs dd,
+.catalog-product__details dd {
+  margin: 3px 0 0;
+  font-size: 0.82rem;
+  font-weight: 650;
+}
+
+.catalog-product__details summary {
+  cursor: pointer;
+  color: var(--accent-strong);
+  font-size: 0.8rem;
+  font-weight: 700;
+}
+
+.catalog-product__details dl {
+  margin-top: 10px;
+}
+
+.catalog-product__sources {
+  display: flex;
+  flex-wrap: wrap;
+  gap: 8px;
+  border-top: 1px solid var(--border-subtle);
+  padding-top: 12px;
+}
+
+.catalog-product__sources a {
+  border-bottom: 1px solid var(--accent-soft-border);
+  color: var(--accent-strong);
+  font-size: 0.78rem;
+  font-weight: 700;
+}
+
+.catalog-empty {
+  border: 1px dashed var(--border-subtle);
+  border-radius: var(--radius-lg);
+  background: var(--surface-1);
+  padding: 44px 24px;
+  text-align: center;
+}
+
+.catalog-empty h2,
+.catalog-empty p {
+  margin: 0;
+}
+
+.catalog-empty p {
+  margin-top: 8px;
+  color: var(--text-muted);
+}
+
 @media (max-width: 900px) {
   .topbar,
   .workbench {
@@ -383,6 +593,10 @@ th {
   .stats-grid {
     grid-template-columns: repeat(2, minmax(0, 1fr));
   }
+
+  .catalog-grid {
+    grid-template-columns: 1fr;
+  }
 }
 
 @media (max-width: 560px) {
@@ -400,6 +614,36 @@ th {
   .video-card {
     grid-template-columns: 1fr;
   }
+
+  .catalog-page {
+    width: min(100% - 20px, 1180px);
+    padding-top: 18px;
+  }
+
+  .catalog-header {
+    align-items: flex-start;
+    flex-direction: column;
+  }
+
+  .catalog-header__meta {
+    border-left: 0;
+    border-top: 1px solid var(--border-subtle);
+    padding: 14px 0 0;
+  }
+
+  .catalog-product {
+    grid-template-columns: 112px minmax(0, 1fr);
+  }
+
+  .catalog-product__media {
+    min-height: 0;
+    padding: 12px;
+  }
+
+  .catalog-product__specs,
+  .catalog-product__details dl {
+    grid-template-columns: 1fr;
+  }
 }
 
 @media (prefers-reduced-motion: reduce) {

web-front/src/app/layout.tsx

@@ -1,9 +1,12 @@
 import type { Metadata } from 'next';
+import Script from 'next/script';
 import './globals.css';
 
+const YANDEX_METRIKA_ID = 109268682;
+
 export const metadata: Metadata = {
-  title: 'Video Registry',
-  description: 'Operational video registry with primitive storage and aggressive caching.',
+  title: 'videoreg.ru - автомобильные видеорегистраторы',
+  description: 'Каталог автомобильных видеорегистраторов из одобренных source cards.',
 };
 
 type RootLayoutProps = {
@@ -12,8 +15,31 @@ type RootLayoutProps = {
 
 export default function RootLayout({ children }: RootLayoutProps) {
   return (
-    <html id="top" lang="en">
-      <body>{children}</body>
+    <html id="top" lang="ru">
+      <body>
+        {children}
+        <Script id="yandex-metrika" strategy="afterInteractive">
+          {`
+            (function(m,e,t,r,i,k,a){
+                m[i]=m[i]||function(){(m[i].a=m[i].a||[]).push(arguments)};
+                m[i].l=1*new Date();
+                for (var j = 0; j < document.scripts.length; j++) {if (document.scripts[j].src === r) { return; }}
+                k=e.createElement(t),a=e.getElementsByTagName(t)[0],k.async=1,k.src=r,a.parentNode.insertBefore(k,a)
+            })(window, document,'script','https://mc.yandex.ru/metrika/tag.js?id=${YANDEX_METRIKA_ID}', 'ym');
+
+            ym(${YANDEX_METRIKA_ID}, 'init', {ssr:true, webvisor:true, clickmap:true, ecommerce:"dataLayer", referrer: document.referrer, url: location.href, accurateTrackBounce:true, trackLinks:true});
+          `}
+        </Script>
+        <noscript>
+          <div>
+            <img
+              src={`https://mc.yandex.ru/watch/${YANDEX_METRIKA_ID}`}
+              style={{ position: 'absolute', left: '-9999px' }}
+              alt=""
+            />
+          </div>
+        </noscript>
+      </body>
     </html>
   );
 }

web-front/src/app/page.tsx

@@ -1,27 +1,17 @@
-import { getCachedVideoDashboard } from '@/entities/video/api/videoCache';
-import { RegisterVideoForm } from '@/features/register-video/ui/RegisterVideoForm';
-import { CachePanel } from '@/widgets/cache-panel/ui/CachePanel';
-import { VideoDashboard } from '@/widgets/video-dashboard/ui/VideoDashboard';
+import { readDashcamCatalog } from '@/entities/dashcam-catalog/api/catalogRepository';
+import { DashcamCatalog } from '@/widgets/dashcam-catalog/ui/DashcamCatalog';
 
-export const revalidate = 3600;
+export const dynamic = 'force-dynamic';
 
 export default async function Home() {
-  const dashboard = await getCachedVideoDashboard();
+  const catalog = await readDashcamCatalog();
 
   return (
-    <main className="page-shell">
-      <header className="topbar">
-        <div>
-          <p className="eyebrow">videoreg.ru</p>
-          <h1>Video Registry</h1>
-        </div>
-        <CachePanel metadata={dashboard.cache} />
-      </header>
-
-      <div className="workbench">
-        <RegisterVideoForm />
-        <VideoDashboard dashboard={dashboard} />
-      </div>
-    </main>
+    <DashcamCatalog
+      catalog={catalog.catalog}
+      exportInfo={catalog.export}
+      hasSharedExportIssue={catalog.error === 'shared_export_unavailable'}
+      source={catalog.source}
+    />
   );
 }

web-front/src/entities/dashcam-catalog/api/catalogRepository.ts

@@ -0,0 +1,62 @@
+import 'server-only';
+
+import { readPrimitiveDb } from '@/shared/db/primitiveJsonDb';
+import {
+  readCatalogExport,
+  type ReadCatalogExportResult,
+} from './importCatalogExport';
+import type { DashcamCatalogExport } from '../model/schema';
+
+export type DashcamCatalogReadModel = {
+  catalog?: DashcamCatalogExport;
+  error?: 'shared_export_unavailable';
+  source: 'shared_export' | 'read_model' | 'empty';
+  export?: Pick<ReadCatalogExportResult, 'createdAt' | 'exportId' | 'itemCount' | 'sha256'>;
+};
+
+export async function readDashcamCatalog(): Promise<DashcamCatalogReadModel> {
+  const exportDir = process.env.DASHCAM_CATALOG_EXPORT_DIR;
+
+  if (exportDir) {
+    try {
+      const result = await readCatalogExport({ exportDir });
+
+      return {
+        catalog: result.catalog,
+        source: 'shared_export',
+        export: {
+          createdAt: result.createdAt,
+          exportId: result.exportId,
+          itemCount: result.itemCount,
+          sha256: result.sha256,
+        },
+      };
+    } catch {
+      const readModel = await readPrimitiveDb();
+
+      if (readModel.dashcamCatalog) {
+        return {
+          catalog: readModel.dashcamCatalog,
+          source: 'read_model',
+          error: 'shared_export_unavailable',
+        };
+      }
+
+      return {
+        source: 'empty',
+        error: 'shared_export_unavailable',
+      };
+    }
+  }
+
+  const readModel = await readPrimitiveDb();
+
+  if (readModel.dashcamCatalog) {
+    return {
+      catalog: readModel.dashcamCatalog,
+      source: 'read_model',
+    };
+  }
+
+  return { source: 'empty' };
+}

web-front/src/entities/dashcam-catalog/api/importCatalogExport.test.ts

@@ -1,23 +1,15 @@
 import { createHash } from 'node:crypto';
-import { mkdtemp, readFile, rm } from 'node:fs/promises';
+import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
 import os from 'node:os';
 import path from 'node:path';
 import { describe, expect, it } from 'vitest';
 import { importCatalogExport } from './importCatalogExport';
 
-function jsonResponse(payload: unknown) {
-  return new Response(JSON.stringify(payload), {
-    status: 200,
-    headers: {
-      'content-type': 'application/json',
-    },
-  });
-}
-
 describe('importCatalogExport', () => {
-  it('validates manifest, checksum and atomically updates the primitive read model', async () => {
+  it('validates a shared-volume manifest, checksum and atomically updates the primitive read model', async () => {
     const tempDir = await mkdtemp(path.join(os.tmpdir(), 'videoreg-catalog-import-'));
     const dbFilePath = path.join(tempDir, 'videoreg.json');
+    const exportDir = path.join(tempDir, 'exports');
 
     try {
       const catalog = {
@@ -29,6 +21,7 @@ describe('importCatalogExport', () => {
             id: 'ibox-f5',
             brand: 'iBOX',
             model: 'F5 WiFi',
+            canonicalName: 'iBOX F5 Wi-Fi',
             title: 'Видеорегистратор для автомобиля iBOX F5 + WiFi с радар-детектором',
             specs: {
               channels: 1,
@@ -51,32 +44,25 @@ describe('importCatalogExport', () => {
       };
       const catalogBytes = Buffer.from(JSON.stringify(catalog));
       const digest = createHash('sha256').update(catalogBytes).digest('hex');
-      const fetcher = async (input: string | URL) => {
-        const url = input.toString();
+      const artifactPath = path.join(exportDir, 'artifacts', 'export_1', 'catalog.json');
 
-        if (url === 'https://exports.example.test/manifest.json') {
-          return jsonResponse({
-            schemaVersion: 1,
-            exportId: 'export_1',
-            createdAt: '2026-05-16T21:00:00.000Z',
-            artifactUrl: 'https://exports.example.test/catalog.json',
-            sha256: digest,
-            itemCount: 1,
-          });
-        }
-
-        if (url === 'https://exports.example.test/catalog.json') {
-          return jsonResponse(catalog);
-        }
-
-        return new Response('not found', { status: 404 });
-      };
+      await mkdir(path.dirname(artifactPath), { recursive: true });
+      await writeFile(artifactPath, catalogBytes);
+      await writeFile(
+        path.join(exportDir, 'manifest.json'),
+        JSON.stringify({
+          schemaVersion: 1,
+          exportId: 'export_1',
+          createdAt: '2026-05-16T21:00:00.000Z',
+          artifactPath: 'artifacts/export_1/catalog.json',
+          sha256: digest,
+          itemCount: 1,
+        }),
+      );
 
       const result = await importCatalogExport({
-        manifestUrl: 'https://exports.example.test/manifest.json',
-        token: 'secret',
+        exportDir,
         dbFilePath,
-        fetcher,
         now: new Date('2026-05-16T22:00:00.000Z'),
       });
       const stored = JSON.parse(await readFile(dbFilePath, 'utf8')) as Record<string, unknown>;

web-front/src/entities/dashcam-catalog/api/importCatalogExport.ts

@@ -7,19 +7,18 @@ import {
   type DashcamCatalogExport,
 } from '../model/schema.ts';
 
-type FetchLike = (input: string | URL, init?: RequestInit) => Promise<Response>;
-
 type PrimitiveDbCache = {
   videos?: unknown[];
   dashcamCatalog?: DashcamCatalogExport;
   dashcamCatalogImportedAt?: string;
 };
 
-export type ImportCatalogExportOptions = {
-  manifestUrl: string;
-  token: string;
+export type ReadCatalogExportOptions = {
+  exportDir?: string;
+};
+
+export type ImportCatalogExportOptions = ReadCatalogExportOptions & {
   dbFilePath?: string;
-  fetcher?: FetchLike;
   now?: Date;
 };
 
@@ -30,6 +29,15 @@ export type ImportCatalogExportResult = {
   dbFilePath: string;
 };
 
+export type ReadCatalogExportResult = {
+  catalog: DashcamCatalogExport;
+  createdAt: string;
+  exportDir: string;
+  exportId: string;
+  itemCount: number;
+  sha256: string;
+};
+
 function getDefaultDbFilePath() {
   if (process.env.VIDEOREG_DB_FILE) {
     return process.env.VIDEOREG_DB_FILE;
@@ -38,6 +46,14 @@ function getDefaultDbFilePath() {
   return path.join(process.cwd(), 'data', 'videoreg.json');
 }
 
+function getDefaultExportDir() {
+  if (process.env.DASHCAM_CATALOG_EXPORT_DIR) {
+    return process.env.DASHCAM_CATALOG_EXPORT_DIR;
+  }
+
+  return path.join(process.cwd(), 'data', 'catalog-exports');
+}
+
 function sha256(buffer: Buffer) {
   return createHash('sha256').update(buffer).digest('hex');
 }
@@ -68,20 +84,6 @@ async function writeJsonAtomically(filePath: string, payload: PrimitiveDbCache)
   await rename(temporaryPath, filePath);
 }
 
-async function fetchJsonBytes(fetcher: FetchLike, url: string, token: string) {
-  const response = await fetcher(url, {
-    headers: {
-      authorization: `Bearer ${token}`,
-    },
-  });
-
-  if (!response.ok) {
-    throw new Error(`Catalog export request failed: ${response.status} ${response.statusText}`);
-  }
-
-  return Buffer.from(await response.arrayBuffer());
-}
-
 function parseJson(buffer: Buffer, label: string) {
   try {
     return JSON.parse(buffer.toString('utf8')) as unknown;
@@ -90,14 +92,24 @@ function parseJson(buffer: Buffer, label: string) {
   }
 }
 
-export async function importCatalogExport(options: ImportCatalogExportOptions): Promise<ImportCatalogExportResult> {
-  const fetcher = options.fetcher ?? fetch;
-  const dbFilePath = options.dbFilePath ?? getDefaultDbFilePath();
-  const importedAt = (options.now ?? new Date()).toISOString();
+function resolveArtifactPath(exportDir: string, artifactPath: string) {
+  if (path.isAbsolute(artifactPath)) {
+    throw new Error('Catalog manifest artifactPath must be relative to the shared export directory');
+  }
 
-  const manifestBytes = await fetchJsonBytes(fetcher, options.manifestUrl, options.token);
+  const resolvedExportDir = path.resolve(exportDir);
+  const resolvedArtifactPath = path.resolve(resolvedExportDir, artifactPath);
+  const relativePath = path.relative(resolvedExportDir, resolvedArtifactPath);
+
+  if (relativePath === '' || relativePath === '..' || relativePath.startsWith(`..${path.sep}`)) {
+    throw new Error('Catalog manifest artifactPath escapes the shared export directory');
+  }
+
+  return resolvedArtifactPath;
+}
+
+function validateExport(manifestBytes: Buffer, artifactBytes: Buffer) {
   const manifest = parseCatalogExportManifest(parseJson(manifestBytes, 'manifest'));
-  const artifactBytes = await fetchJsonBytes(fetcher, manifest.artifactUrl, options.token);
   const artifactHash = sha256(artifactBytes);
 
   if (artifactHash.toLowerCase() !== manifest.sha256.toLowerCase()) {
@@ -118,17 +130,49 @@ export async function importCatalogExport(options: ImportCatalogExportOptions):
     throw new Error(`Catalog itemCount mismatch: manifest=${manifest.itemCount}, artifact=${catalog.products.length}`);
   }
 
+  return {
+    catalog,
+    manifest,
+    sha256: artifactHash,
+  };
+}
+
+export async function readCatalogExport(
+  options: ReadCatalogExportOptions = {},
+): Promise<ReadCatalogExportResult> {
+  const exportDir = path.resolve(options.exportDir ?? getDefaultExportDir());
+  const manifestBytes = await readFile(path.join(exportDir, 'manifest.json'));
+  const manifest = parseCatalogExportManifest(parseJson(manifestBytes, 'manifest'));
+  const artifactPath = manifest.artifactPath ?? path.posix.join('artifacts', manifest.exportId, 'catalog.json');
+  const artifactBytes = await readFile(resolveArtifactPath(exportDir, artifactPath));
+  const validated = validateExport(manifestBytes, artifactBytes);
+
+  return {
+    catalog: validated.catalog,
+    createdAt: validated.manifest.createdAt,
+    exportDir,
+    exportId: validated.manifest.exportId,
+    itemCount: validated.manifest.itemCount,
+    sha256: validated.sha256,
+  };
+}
+
+export async function importCatalogExport(options: ImportCatalogExportOptions = {}): Promise<ImportCatalogExportResult> {
+  const dbFilePath = options.dbFilePath ?? getDefaultDbFilePath();
+  const importedAt = (options.now ?? new Date()).toISOString();
+  const exportResult = await readCatalogExport(options);
+
   const current = await readExistingDb(dbFilePath);
   await writeJsonAtomically(dbFilePath, {
     ...current,
-    dashcamCatalog: catalog,
+    dashcamCatalog: exportResult.catalog,
     dashcamCatalogImportedAt: importedAt,
   });
 
   return {
-    exportId: manifest.exportId,
-    itemCount: manifest.itemCount,
-    sha256: artifactHash,
+    exportId: exportResult.exportId,
+    itemCount: exportResult.itemCount,
+    sha256: exportResult.sha256,
     dbFilePath,
   };
 }

web-front/src/entities/dashcam-catalog/model/schema.test.ts

@@ -1,10 +1,11 @@
 import { describe, expect, it } from 'vitest';
-import { DashcamCatalogExportSchema } from './schema';
+import { CatalogExportManifestSchema, DashcamCatalogExportSchema } from './schema';
 
 const baseProduct = {
   id: 'ibox-f5',
   brand: 'iBOX',
   model: 'F5 WiFi',
+  canonicalName: 'iBOX F5 Wi-Fi',
   title: 'Видеорегистратор для автомобиля iBOX F5 + WiFi с радар-детектором',
   specs: {
     channels: 1,
@@ -66,4 +67,45 @@ describe('dashcam catalog export schema', () => {
       }),
     ).toThrow();
   });
+
+  it('accepts a local shared-volume artifact path and MVideo source metadata', () => {
+    expect(
+      CatalogExportManifestSchema.parse({
+        schemaVersion: 1,
+        exportId: 'export_1',
+        createdAt: '2026-05-16T21:00:00.000Z',
+        artifactPath: 'artifacts/export_1/catalog.json',
+        sha256: 'a'.repeat(64),
+        itemCount: 1,
+      }),
+    ).toMatchObject({
+      artifactPath: 'artifacts/export_1/catalog.json',
+    });
+
+    expect(
+      DashcamCatalogExportSchema.parse({
+        schemaVersion: 1,
+        exportId: 'export_1',
+        createdAt: '2026-05-16T21:00:00.000Z',
+        products: [
+          {
+            ...baseProduct,
+            sources: [
+              {
+                ...baseProduct.sources[0],
+                source: 'mvideo',
+                url: 'https://www.mvideo.ru/products/test-4254724',
+              },
+            ],
+          },
+        ],
+      }),
+    ).toMatchObject({
+      products: [
+        {
+          sources: [{ source: 'mvideo' }],
+        },
+      ],
+    });
+  });
 });

web-front/src/entities/dashcam-catalog/model/schema.ts

@@ -5,12 +5,13 @@ const isoDateTime = z.string().refine((value) => !Number.isNaN(Date.parse(value)
 });
 
 const exportedSpecValueSchema = z.union([z.string(), z.number(), z.boolean()]);
+export const MarketSourceSchema = z.enum(['yandex_market', 'mvideo']);
 
 export const ExportedProductImageSchema = z
   .object({
     url: z.string().url(),
     originalUrl: z.string().url(),
-    source: z.literal('yandex_market'),
+    source: MarketSourceSchema,
     sha256: z.string().min(1).optional(),
     width: z.number().int().positive().optional(),
     height: z.number().int().positive().optional(),
@@ -19,7 +20,7 @@ export const ExportedProductImageSchema = z
 
 export const ExportedProductSourceSchema = z
   .object({
-    source: z.literal('yandex_market'),
+    source: MarketSourceSchema,
     sourceProductId: z.string().min(1).optional(),
     url: z.string().url(),
     stableContentHash: z.string().min(1),
@@ -32,6 +33,7 @@ export const ExportedDashcamProductSchema = z
     id: z.string().min(1),
     brand: z.string().min(1),
     model: z.string().min(1),
+    canonicalName: z.string().min(1).optional(),
     title: z.string().min(1),
     specs: z.record(z.string().min(1), exportedSpecValueSchema),
     images: z.array(ExportedProductImageSchema),
@@ -55,11 +57,15 @@ export const CatalogExportManifestSchema = z
     schemaVersion: z.literal(1),
     exportId: z.string().min(1),
     createdAt: isoDateTime,
-    artifactUrl: z.string().url(),
+    artifactPath: z.string().min(1).optional(),
+    artifactUrl: z.string().url().optional(),
     sha256: z.string().regex(/^[a-f0-9]{64}$/i),
     itemCount: z.number().int().nonnegative(),
   })
-  .strict();
+  .strict()
+  .refine((manifest) => Boolean(manifest.artifactPath || manifest.artifactUrl), {
+    message: 'Expected artifactPath or artifactUrl',
+  });
 
 export type CatalogExportManifest = z.infer<typeof CatalogExportManifestSchema>;
 export type DashcamCatalogExport = z.infer<typeof DashcamCatalogExportSchema>;
@@ -67,6 +73,7 @@ export type ExportedDashcamProduct = z.infer<typeof ExportedDashcamProductSchema
 export type ExportedProductImage = z.infer<typeof ExportedProductImageSchema>;
 export type ExportedProductSource = z.infer<typeof ExportedProductSourceSchema>;
 export type ExportedSpecValue = z.infer<typeof exportedSpecValueSchema>;
+export type MarketSource = z.infer<typeof MarketSourceSchema>;
 
 export function parseCatalogExportManifest(input: unknown) {
   return CatalogExportManifestSchema.parse(input);

web-front/src/shared/db/primitiveJsonDb.ts

@@ -49,7 +49,7 @@ function getDbFilePath() {
     return process.env.VIDEOREG_DB_FILE;
   }
 
-  return path.join(/* turbopackIgnore: true */ process.cwd(), 'data', 'videoreg.json');
+  return path.join(/*turbopackIgnore: true*/ process.cwd(), 'data', 'videoreg.json');
 }
 
 async function ensureDbFile(filePath: string) {

web-front/src/widgets/dashcam-catalog/ui/DashcamCatalog.tsx

@@ -0,0 +1,151 @@
+import type { DashcamCatalogExport, ExportedDashcamProduct, ExportedSpecValue } from '@/entities/dashcam-catalog/model/schema';
+import type { DashcamCatalogReadModel } from '@/entities/dashcam-catalog/api/catalogRepository';
+
+type DashcamCatalogProps = {
+  catalog: DashcamCatalogExport | undefined;
+  source: DashcamCatalogReadModel['source'];
+  exportInfo?: DashcamCatalogReadModel['export'];
+  hasSharedExportIssue?: boolean;
+};
+
+const preferredSpecKeys = [
+  'resolution_front',
+  'resolution_max',
+  'channels',
+  'wifi',
+  'night_mode',
+  'gps',
+  'screen',
+  'battery_power',
+  'car_power',
+] as const;
+
+const sourceLabels = {
+  mvideo: 'М.Видео',
+  yandex_market: 'Яндекс Маркет',
+};
+
+const specLabels: Record<string, string> = {
+  battery_power: 'Резервное питание',
+  car_power: 'Питание от автомобиля',
+  channels: 'Камер',
+  gps: 'GPS',
+  night_mode: 'Ночной режим',
+  resolution_front: 'Разрешение',
+  resolution_max: 'Макс. разрешение',
+  screen: 'Экран',
+  wifi: 'Wi-Fi',
+};
+
+function formatSpecValue(value: ExportedSpecValue) {
+  if (typeof value === 'boolean') return value ? 'Да' : 'Нет';
+  if (value === 'battery') return 'Аккумулятор';
+  if (value === 'capacitor') return 'Конденсатор';
+  if (value === 'hybrid') return 'Аккумулятор + конденсатор';
+
+  return String(value);
+}
+
+function formatDate(value: string) {
+  return new Intl.DateTimeFormat('ru-RU', {
+    dateStyle: 'medium',
+    timeStyle: 'short',
+  }).format(new Date(value));
+}
+
+function ProductCard({ product }: { product: ExportedDashcamProduct }) {
+  const productSpecs = preferredSpecKeys
+    .filter((key) => product.specs[key] !== undefined)
+    .map((key) => [key, product.specs[key]] as const);
+  const remainingSpecs = Object.entries(product.specs).filter(([key]) => !preferredSpecKeys.includes(key as never));
+
+  return (
+    <article className="catalog-product">
+      <div className="catalog-product__media" aria-hidden="true">
+        <img alt="" src="/dashcam-product.svg" />
+      </div>
+      <div className="catalog-product__body">
+        <div className="catalog-product__heading">
+          <div>
+            <p className="catalog-product__brand">{product.brand}</p>
+            <h2>{product.canonicalName ?? product.model}</h2>
+          </div>
+          <span className="catalog-product__source-count">{product.sources.length} источник</span>
+        </div>
+        <p className="catalog-product__title">{product.title}</p>
+
+        {productSpecs.length > 0 ? (
+          <dl className="catalog-product__specs">
+            {productSpecs.map(([key, value]) => (
+              <div key={key}>
+                <dt>{specLabels[key] ?? key}</dt>
+                <dd>{formatSpecValue(value)}</dd>
+              </div>
+            ))}
+          </dl>
+        ) : null}
+
+        {remainingSpecs.length > 0 ? (
+          <details className="catalog-product__details">
+            <summary>Все характеристики ({Object.keys(product.specs).length})</summary>
+            <dl>
+              {remainingSpecs.map(([key, value]) => (
+                <div key={key}>
+                  <dt>{specLabels[key] ?? key}</dt>
+                  <dd>{formatSpecValue(value)}</dd>
+                </div>
+              ))}
+            </dl>
+          </details>
+        ) : null}
+
+        <div className="catalog-product__sources">
+          {product.sources.map((source) => (
+            <a href={source.url} key={`${source.source}-${source.url}`} rel="noreferrer" target="_blank">
+              {sourceLabels[source.source]}
+            </a>
+          ))}
+        </div>
+      </div>
+    </article>
+  );
+}
+
+export function DashcamCatalog({ catalog, source, exportInfo, hasSharedExportIssue }: DashcamCatalogProps) {
+  const products = catalog?.products ?? [];
+
+  return (
+    <main className="catalog-page">
+      <header className="catalog-header">
+        <div>
+          <p className="catalog-header__eyebrow">videoreg.ru</p>
+          <h1>Автомобильные видеорегистраторы</h1>
+          <p className="catalog-header__subtitle">Проверенный каталог карточек из публичных источников.</p>
+        </div>
+        <div className="catalog-header__meta">
+          <strong>{products.length}</strong>
+          <span>одобренных моделей</span>
+        </div>
+      </header>
+
+      <section className="catalog-status" aria-label="Catalog status">
+        <span>Источник: {source === 'shared_export' ? 'shared export' : source === 'read_model' ? 'read model' : 'ожидается export'}</span>
+        {exportInfo ? <span>Обновлён: {formatDate(exportInfo.createdAt)}</span> : null}
+        {hasSharedExportIssue ? <span>Последний shared export недоступен, показана сохранённая версия.</span> : null}
+      </section>
+
+      {products.length > 0 ? (
+        <section className="catalog-grid" aria-label="Dashcam catalog">
+          {products.map((product) => (
+            <ProductCard key={product.id} product={product} />
+          ))}
+        </section>
+      ) : (
+        <section className="catalog-empty">
+          <h2>Каталог ожидает moderation</h2>
+          <p>Одобренные карточки появятся после первого export из ingestion.</p>
+        </section>
+      )}
+    </main>
+  );
+}