feat: complete dashcam catalog ingestion demo

.beads/.gitignore

@@ -0,0 +1,70 @@
+# Dolt database (managed by Dolt, not git)
+dolt/
+embeddeddolt/
+
+# Runtime files
+bd.sock
+bd.sock.startlock
+sync-state.json
+last-touched
+.exclusive-lock
+
+# Daemon runtime (lock, log, pid)
+daemon.*
+
+# Push state (runtime, per-machine)
+push-state.json
+
+# Lock files (various runtime locks)
+*.lock
+
+# Credential key (encryption key for federation peer auth — never commit)
+.beads-credential-key
+
+# Local version tracking (prevents upgrade notification spam after git ops)
+.local_version
+
+# Worktree redirect file (contains relative path to main repo's .beads/)
+# Must not be committed as paths would be wrong in other clones
+redirect
+
+# Sync state (local-only, per-machine)
+# These files are machine-specific and should not be shared across clones
+.sync.lock
+export-state/
+export-state.json
+
+# Ephemeral store (SQLite - wisps/molecules, intentionally not versioned)
+ephemeral.sqlite3
+ephemeral.sqlite3-journal
+ephemeral.sqlite3-wal
+ephemeral.sqlite3-shm
+
+# Dolt server management (auto-started by bd)
+dolt-server.pid
+dolt-server.log
+dolt-server.lock
+dolt-server.port
+dolt-server.activity
+
+# Corrupt backup directories (created by bd doctor --fix recovery)
+*.corrupt.backup/
+
+# Backup data (auto-exported JSONL, local-only)
+backup/
+
+# Per-project environment file (Dolt connection config, GH#2520)
+.env
+
+# Legacy files (from pre-Dolt versions)
+*.db
+*.db?*
+*.db-journal
+*.db-wal
+*.db-shm
+db.sqlite
+bd.db
+# NOTE: Do NOT add negation patterns here.
+# They would override fork protection in .git/info/exclude.
+# Config files (metadata.json, config.yaml) are tracked by git by default
+# since no pattern above ignores them.

.beads/README.md

@@ -0,0 +1,81 @@
+# Beads - AI-Native Issue Tracking
+
+Welcome to Beads! This repository uses **Beads** for issue tracking - a modern, AI-native tool designed to live directly in your codebase alongside your code.
+
+## What is Beads?
+
+Beads is issue tracking that lives in your repo, making it perfect for AI coding agents and developers who want their issues close to their code. No web UI required - everything works through the CLI and integrates seamlessly with git.
+
+**Learn more:** [github.com/steveyegge/beads](https://github.com/steveyegge/beads)
+
+## Quick Start
+
+### Essential Commands
+
+```bash
+# Create new issues
+bd create "Add user authentication"
+
+# View all issues
+bd list
+
+# View issue details
+bd show <issue-id>
+
+# Update issue status
+bd update <issue-id> --claim
+bd update <issue-id> --status done
+
+# Sync with Dolt remote
+bd dolt push
+```
+
+### Working with Issues
+
+Issues in Beads are:
+- **Git-native**: Stored in Dolt database with version control and branching
+- **AI-friendly**: CLI-first design works perfectly with AI coding agents
+- **Branch-aware**: Issues can follow your branch workflow
+- **Always in sync**: Auto-syncs with your commits
+
+## Why Beads?
+
+✨ **AI-Native Design**
+- Built specifically for AI-assisted development workflows
+- CLI-first interface works seamlessly with AI coding agents
+- No context switching to web UIs
+
+🚀 **Developer Focused**
+- Issues live in your repo, right next to your code
+- Works offline, syncs when you push
+- Fast, lightweight, and stays out of your way
+
+🔧 **Git Integration**
+- Automatic sync with git commits
+- Branch-aware issue tracking
+- Dolt-native three-way merge resolution
+
+## Get Started with Beads
+
+Try Beads in your own projects:
+
+```bash
+# Install Beads
+curl -sSL https://raw.githubusercontent.com/steveyegge/beads/main/scripts/install.sh | bash
+
+# Initialize in your repo
+bd init
+
+# Create your first issue
+bd create "Try out Beads"
+```
+
+## Learn More
+
+- **Documentation**: [github.com/steveyegge/beads/docs](https://github.com/steveyegge/beads/tree/main/docs)
+- **Quick Start Guide**: Run `bd quickstart`
+- **Examples**: [github.com/steveyegge/beads/examples](https://github.com/steveyegge/beads/tree/main/examples)
+
+---
+
+*Beads: Issue tracking that moves at the speed of thought* ⚡

.beads/config.yaml

@@ -0,0 +1,55 @@
+# Beads Configuration File
+# This file configures default behavior for all bd commands in this repository
+# All settings can also be set via environment variables (BD_* prefix)
+# or overridden with command-line flags
+
+# Issue prefix for this repository (used by bd init)
+# If not set, bd init will auto-detect from directory name
+# Example: issue-prefix: "myproject" creates issues like "myproject-1", "myproject-2", etc.
+# issue-prefix: ""
+
+# Use no-db mode: JSONL-only, no Dolt database
+# When true, bd will use .beads/issues.jsonl as the source of truth
+# no-db: false
+
+# Enable JSON output by default
+# json: false
+
+# Feedback title formatting for mutating commands (create/update/close/dep/edit)
+# 0 = hide titles, N > 0 = truncate to N characters
+# output:
+#   title-length: 255
+
+# Default actor for audit trails (overridden by BEADS_ACTOR or --actor)
+# actor: ""
+
+# Export events (audit trail) to .beads/events.jsonl on each flush/sync
+# When enabled, new events are appended incrementally using a high-water mark.
+# Use 'bd export --events' to trigger manually regardless of this setting.
+# events-export: false
+
+# Multi-repo configuration (experimental - bd-307)
+# Allows hydrating from multiple repositories and routing writes to the correct database
+# repos:
+#   primary: "."  # Primary repo (where this database lives)
+#   additional:   # Additional repos to hydrate from (read-only)
+#     - ~/beads-planning  # Personal planning repo
+#     - ~/work-planning   # Work planning repo
+
+# JSONL backup (periodic export for off-machine recovery)
+# Auto-enabled when a git remote exists. Override explicitly:
+# backup:
+#   enabled: false     # Disable auto-backup entirely
+#   interval: 15m      # Minimum time between auto-exports
+#   git-push: false    # Disable git push (export locally only)
+#   git-repo: ""       # Separate git repo for backups (default: project repo)
+
+# Integration settings (access with 'bd config get/set')
+# Non-secret keys (stored in the database):
+# - jira.url, jira.project
+# - linear.team_id
+# - github.org, github.repo
+#
+# Secret keys (stored in this file but prefer env vars to avoid git exposure):
+# - linear.api_key  → use LINEAR_API_KEY env var instead
+# - github.token    → use GITHUB_TOKEN env var instead

.beads/hooks/post-checkout

@@ -0,0 +1,24 @@
+#!/usr/bin/env sh
+# --- BEGIN BEADS INTEGRATION v1.0.4 ---
+# This section is managed by beads. Do not remove these markers.
+if command -v bd >/dev/null 2>&1; then
+  export BD_GIT_HOOK=1
+  _bd_timeout=${BEADS_HOOK_TIMEOUT:-300}
+  if command -v timeout >/dev/null 2>&1; then
+    timeout "$_bd_timeout" bd hooks run post-checkout "$@"
+    _bd_exit=$?
+    if [ $_bd_exit -eq 124 ]; then
+      echo >&2 "beads: hook 'post-checkout' timed out after ${_bd_timeout}s — continuing without beads"
+      _bd_exit=0
+    fi
+  else
+    bd hooks run post-checkout "$@"
+    _bd_exit=$?
+  fi
+  if [ $_bd_exit -eq 3 ]; then
+    echo >&2 "beads: database not initialized — skipping hook 'post-checkout'"
+    _bd_exit=0
+  fi
+  if [ $_bd_exit -ne 0 ]; then exit $_bd_exit; fi
+fi
+# --- END BEADS INTEGRATION v1.0.4 ---

.beads/hooks/post-merge

@@ -0,0 +1,24 @@
+#!/usr/bin/env sh
+# --- BEGIN BEADS INTEGRATION v1.0.4 ---
+# This section is managed by beads. Do not remove these markers.
+if command -v bd >/dev/null 2>&1; then
+  export BD_GIT_HOOK=1
+  _bd_timeout=${BEADS_HOOK_TIMEOUT:-300}
+  if command -v timeout >/dev/null 2>&1; then
+    timeout "$_bd_timeout" bd hooks run post-merge "$@"
+    _bd_exit=$?
+    if [ $_bd_exit -eq 124 ]; then
+      echo >&2 "beads: hook 'post-merge' timed out after ${_bd_timeout}s — continuing without beads"
+      _bd_exit=0
+    fi
+  else
+    bd hooks run post-merge "$@"
+    _bd_exit=$?
+  fi
+  if [ $_bd_exit -eq 3 ]; then
+    echo >&2 "beads: database not initialized — skipping hook 'post-merge'"
+    _bd_exit=0
+  fi
+  if [ $_bd_exit -ne 0 ]; then exit $_bd_exit; fi
+fi
+# --- END BEADS INTEGRATION v1.0.4 ---

.beads/hooks/pre-commit

@@ -0,0 +1,24 @@
+#!/usr/bin/env sh
+# --- BEGIN BEADS INTEGRATION v1.0.4 ---
+# This section is managed by beads. Do not remove these markers.
+if command -v bd >/dev/null 2>&1; then
+  export BD_GIT_HOOK=1
+  _bd_timeout=${BEADS_HOOK_TIMEOUT:-300}
+  if command -v timeout >/dev/null 2>&1; then
+    timeout "$_bd_timeout" bd hooks run pre-commit "$@"
+    _bd_exit=$?
+    if [ $_bd_exit -eq 124 ]; then
+      echo >&2 "beads: hook 'pre-commit' timed out after ${_bd_timeout}s — continuing without beads"
+      _bd_exit=0
+    fi
+  else
+    bd hooks run pre-commit "$@"
+    _bd_exit=$?
+  fi
+  if [ $_bd_exit -eq 3 ]; then
+    echo >&2 "beads: database not initialized — skipping hook 'pre-commit'"
+    _bd_exit=0
+  fi
+  if [ $_bd_exit -ne 0 ]; then exit $_bd_exit; fi
+fi
+# --- END BEADS INTEGRATION v1.0.4 ---

.beads/hooks/pre-push

@@ -0,0 +1,24 @@
+#!/usr/bin/env sh
+# --- BEGIN BEADS INTEGRATION v1.0.4 ---
+# This section is managed by beads. Do not remove these markers.
+if command -v bd >/dev/null 2>&1; then
+  export BD_GIT_HOOK=1
+  _bd_timeout=${BEADS_HOOK_TIMEOUT:-300}
+  if command -v timeout >/dev/null 2>&1; then
+    timeout "$_bd_timeout" bd hooks run pre-push "$@"
+    _bd_exit=$?
+    if [ $_bd_exit -eq 124 ]; then
+      echo >&2 "beads: hook 'pre-push' timed out after ${_bd_timeout}s — continuing without beads"
+      _bd_exit=0
+    fi
+  else
+    bd hooks run pre-push "$@"
+    _bd_exit=$?
+  fi
+  if [ $_bd_exit -eq 3 ]; then
+    echo >&2 "beads: database not initialized — skipping hook 'pre-push'"
+    _bd_exit=0
+  fi
+  if [ $_bd_exit -ne 0 ]; then exit $_bd_exit; fi
+fi
+# --- END BEADS INTEGRATION v1.0.4 ---

.beads/hooks/prepare-commit-msg

@@ -0,0 +1,24 @@
+#!/usr/bin/env sh
+# --- BEGIN BEADS INTEGRATION v1.0.4 ---
+# This section is managed by beads. Do not remove these markers.
+if command -v bd >/dev/null 2>&1; then
+  export BD_GIT_HOOK=1
+  _bd_timeout=${BEADS_HOOK_TIMEOUT:-300}
+  if command -v timeout >/dev/null 2>&1; then
+    timeout "$_bd_timeout" bd hooks run prepare-commit-msg "$@"
+    _bd_exit=$?
+    if [ $_bd_exit -eq 124 ]; then
+      echo >&2 "beads: hook 'prepare-commit-msg' timed out after ${_bd_timeout}s — continuing without beads"
+      _bd_exit=0
+    fi
+  else
+    bd hooks run prepare-commit-msg "$@"
+    _bd_exit=$?
+  fi
+  if [ $_bd_exit -eq 3 ]; then
+    echo >&2 "beads: database not initialized — skipping hook 'prepare-commit-msg'"
+    _bd_exit=0
+  fi
+  if [ $_bd_exit -ne 0 ]; then exit $_bd_exit; fi
+fi
+# --- END BEADS INTEGRATION v1.0.4 ---

.beads/interactions.jsonl

@@ -0,0 +1,18 @@
+{"id":"int-bb1dc104","kind":"field_change","created_at":"2026-06-07T14:24:33.025471Z","actor":"E.Gavrilov","issue_id":"videoreg-kbh.2","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Implemented and verified in codebase (typecheck + 18 vitest tests pass)"}}
+{"id":"int-c9ddede0","kind":"field_change","created_at":"2026-06-07T14:24:33.344014Z","actor":"E.Gavrilov","issue_id":"videoreg-kbh.3","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Implemented and verified in codebase (typecheck + 18 vitest tests pass)"}}
+{"id":"int-2da54fc8","kind":"field_change","created_at":"2026-06-07T14:24:33.664844Z","actor":"E.Gavrilov","issue_id":"videoreg-kbh.4","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Implemented and verified in codebase (typecheck + 18 vitest tests pass)"}}
+{"id":"int-f2a63413","kind":"field_change","created_at":"2026-06-07T14:24:33.992823Z","actor":"E.Gavrilov","issue_id":"videoreg-kbh.5","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Implemented and verified in codebase (typecheck + 18 vitest tests pass)"}}
+{"id":"int-278657e9","kind":"field_change","created_at":"2026-06-07T14:24:34.319098Z","actor":"E.Gavrilov","issue_id":"videoreg-kbh.6","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Implemented and verified in codebase (typecheck + 18 vitest tests pass)"}}
+{"id":"int-f99f838f","kind":"field_change","created_at":"2026-06-07T14:24:34.647874Z","actor":"E.Gavrilov","issue_id":"videoreg-kbh.8","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Implemented and verified in codebase (typecheck + 18 vitest tests pass)"}}
+{"id":"int-2182f889","kind":"field_change","created_at":"2026-06-07T14:24:34.951525Z","actor":"E.Gavrilov","issue_id":"videoreg-seu.1","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Implemented and verified in codebase (typecheck + 18 vitest tests pass)"}}
+{"id":"int-6a49828d","kind":"field_change","created_at":"2026-06-07T14:24:35.263154Z","actor":"E.Gavrilov","issue_id":"videoreg-nnv.1","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Implemented and verified in codebase (typecheck + 18 vitest tests pass)"}}
+{"id":"int-8f64b7af","kind":"field_change","created_at":"2026-06-07T14:24:36.039743Z","actor":"E.Gavrilov","issue_id":"videoreg-kbh.1","extra":{"field":"status","new_value":"in_progress","old_value":"open"}}
+{"id":"int-9330a6c3","kind":"field_change","created_at":"2026-06-07T14:24:36.602138Z","actor":"E.Gavrilov","issue_id":"videoreg-kbh.7","extra":{"field":"status","new_value":"in_progress","old_value":"open"}}
+{"id":"int-d6554c75","kind":"field_change","created_at":"2026-06-07T14:24:37.165445Z","actor":"E.Gavrilov","issue_id":"videoreg-st9.1","extra":{"field":"status","new_value":"in_progress","old_value":"open"}}
+{"id":"int-66d79f75","kind":"field_change","created_at":"2026-07-30T23:00:42.663159Z","actor":"E.Gavrilov","issue_id":"videoreg-nnv.3","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Shared-volume bridge is implemented, documented, locally exercised, and validated by the Swarm config."}}
+{"id":"int-2599ce31","kind":"field_change","created_at":"2026-07-30T23:00:43.499008Z","actor":"E.Gavrilov","issue_id":"videoreg-nnv.4","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"The combined web, PostgreSQL, worker, and admin Swarm stack shares catalog_exports and routes /admin through Traefik."}}
+{"id":"int-c3a31f13","kind":"field_change","created_at":"2026-07-30T23:00:44.306067Z","actor":"E.Gavrilov","issue_id":"videoreg-kbh.14","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"The HTTP export server and token bridge are removed; versioned export artifacts are written atomically to the shared volume."}}
+{"id":"int-a20ed6d4","kind":"field_change","created_at":"2026-07-30T23:00:45.148032Z","actor":"E.Gavrilov","issue_id":"videoreg-kbh.15","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"The app validates manifest/artifact files from the shared export directory with SHA-256 and Zod, then updates its read model atomically."}}
+{"id":"int-b5112040","kind":"field_change","created_at":"2026-07-30T23:00:45.976151Z","actor":"E.Gavrilov","issue_id":"videoreg-kbh.9","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"MVideo discovery and product parsing are wired into the worker with smoke coverage; live source CAPTCHA is paused without bypass."}}
+{"id":"int-3528fa0c","kind":"field_change","created_at":"2026-07-30T23:00:46.850374Z","actor":"E.Gavrilov","issue_id":"videoreg-kbh.16","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Canonical model cleanup has fixtures for the SPAWNSON and iBOX cases, and canonical name/model edits are preserved in admin moderation."}}
+{"id":"int-3bda336d","kind":"field_change","created_at":"2026-07-30T23:00:47.750226Z","actor":"E.Gavrilov","issue_id":"videoreg-1v0","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Local Yandex crawl produced five real approved cards, export/import and desktop/mobile catalog rendering were verified, and quality/deploy checks pass."}}

.beads/issues.jsonl

@@ -0,0 +1,31 @@
+{"_type":"issue","id":"videoreg-1v0","title":"Demo readiness: local crawl, catalog handoff, UI verification","description":"Prepare a reproducible demo flow: co-located ingestion and web share a validated catalog export, local PostgreSQL receives real source cards, approved catalog is visible in the frontend, and the runbook plus verification artifacts are present.","acceptance_criteria":"A local documented command sequence starts the stack, crawls at least five valid cards from enabled sources within configured budgets, approves and exports catalog data, renders it in the web UI, and passes test/typecheck/build/browser checks.","status":"closed","priority":1,"issue_type":"task","assignee":"E.Gavrilov","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-07-30T21:51:59Z","created_by":"E.Gavrilov","updated_at":"2026-07-30T23:00:48Z","started_at":"2026-07-30T21:52:14Z","closed_at":"2026-07-30T23:00:48Z","close_reason":"Local Yandex crawl produced five real approved cards, export/import and desktop/mobile catalog rendering were verified, and quality/deploy checks pass.","dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-nnv.4","title":"Single co-located deploy stack with shared exports volume","description":"Merge deploy so web + ingestion (postgres, worker, admin) run in one Swarm/Portainer env. Add a shared volume carrying ingestion-data/exports that the web build/runtime can read. Remove export-server from ingestion/docker-compose.yml. Deprecate deploy/portainer/ingestion-stack.standalone.yml (separate-host). Keep ingestion Postgres/admin bound internally; only Traefik fronts web (+ optionally /admin).","status":"closed","priority":1,"issue_type":"task","assignee":"E.Gavrilov","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:53:49Z","created_by":"E.Gavrilov","updated_at":"2026-07-30T23:00:43Z","started_at":"2026-07-30T23:00:28Z","closed_at":"2026-07-30T23:00:43Z","close_reason":"The combined web, PostgreSQL, worker, and admin Swarm stack shares catalog_exports and routes /admin through Traefik.","dependencies":[{"issue_id":"videoreg-nnv.4","depends_on_id":"videoreg-nnv","type":"parent-child","created_at":"2026-06-07T17:53:49Z","created_by":"E.Gavrilov","metadata":"{}"},{"issue_id":"videoreg-nnv.4","depends_on_id":"videoreg-nnv.3","type":"blocks","created_at":"2026-06-07T17:54:02Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-kbh.15","title":"Rewrite app catalog import to read shared-volume catalog.json (drop HTTP/token)","description":"web-front/src/entities/dashcam-catalog/api/importCatalogExport.ts currently fetches manifest+artifact over HTTP with a Bearer token. Switch to reading manifest.json + catalog.json from the shared exports volume (a mounted path). Keep SHA-256 verification + exportId/createdAt/itemCount checks + Zod validation. Remove DASHCAM_CATALOG_MANIFEST_URL/EXPORT_TOKEN; add a path env. Under static-export this runs at build time.","status":"closed","priority":1,"issue_type":"task","assignee":"E.Gavrilov","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:53:48Z","created_by":"E.Gavrilov","updated_at":"2026-07-30T23:00:45Z","started_at":"2026-07-30T23:00:28Z","closed_at":"2026-07-30T23:00:45Z","close_reason":"The app validates manifest/artifact files from the shared export directory with SHA-256 and Zod, then updates its read model atomically.","dependencies":[{"issue_id":"videoreg-kbh.15","depends_on_id":"videoreg-kbh","type":"parent-child","created_at":"2026-06-07T17:53:48Z","created_by":"E.Gavrilov","metadata":"{}"},{"issue_id":"videoreg-kbh.15","depends_on_id":"videoreg-kbh.14","type":"blocks","created_at":"2026-06-07T17:54:03Z","created_by":"E.Gavrilov","metadata":"{}"},{"issue_id":"videoreg-kbh.15","depends_on_id":"videoreg-nnv.3","type":"blocks","created_at":"2026-06-07T17:54:02Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":2,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-kbh.14","title":"Drop export-server + token; write catalog export to shared volume","description":"Remove the export-server HTTP service (ingestion/src/exportServer.ts, ingestion:export script, port 4100, INGESTION_EXPORT_TOKEN/PUBLIC_BASE_URL). The ingestion-worker/export-catalog job already writes catalog.json + manifest.json to INGESTION_EXPORT_DIR (/app/ingestion/data/exports) — expose that as a shared volume instead of serving over HTTP. exportCatalog logic and Zod validation stay.","status":"closed","priority":1,"issue_type":"task","assignee":"E.Gavrilov","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:53:47Z","created_by":"E.Gavrilov","updated_at":"2026-07-30T23:00:44Z","started_at":"2026-07-30T23:00:28Z","closed_at":"2026-07-30T23:00:44Z","close_reason":"The HTTP export server and token bridge are removed; versioned export artifacts are written atomically to the shared volume.","dependencies":[{"issue_id":"videoreg-kbh.14","depends_on_id":"videoreg-kbh","type":"parent-child","created_at":"2026-06-07T17:53:47Z","created_by":"E.Gavrilov","metadata":"{}"},{"issue_id":"videoreg-kbh.14","depends_on_id":"videoreg-nnv.3","type":"blocks","created_at":"2026-06-07T17:54:01Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0}
+{"_type":"issue","id":"videoreg-nnv.3","title":"Co-locate ingestion with web; collapse catalog bridge to a shared volume","design":"Supersedes the original 'ingestion runs on a separate server/cloud' model (tasks/002-crawler.md §4, README, ingestion/README). New model: ingestion + web run in ONE cloud env (same Swarm/Portainer). The ingestion-worker writes the validated catalog.json + manifest.json to a shared exports volume; the web build reads catalog.json DIRECTLY from that volume. No export-server, no HTTP, no Bearer token. Zod + SHA-256 validation are kept at read/build time. The app still never connects to ingestion Postgres. Rationale: two separate clouds were too complex to manage.","acceptance_criteria":"export-server service removed; catalog consumed via shared volume; single co-located deploy stack; docs + memories corrected.","status":"closed","priority":1,"issue_type":"decision","assignee":"E.Gavrilov","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:53:46Z","created_by":"E.Gavrilov","updated_at":"2026-07-30T23:00:43Z","started_at":"2026-07-30T21:52:01Z","closed_at":"2026-07-30T23:00:43Z","close_reason":"Shared-volume bridge is implemented, documented, locally exercised, and validated by the Swarm config.","dependencies":[{"issue_id":"videoreg-nnv.3","depends_on_id":"videoreg-nnv","type":"parent-child","created_at":"2026-06-07T17:53:45Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":3,"comment_count":0}
+{"_type":"issue","id":"videoreg-nnv.2","title":"Commit untracked ingestion/, deploy/portainer/, ingestion CI to git","description":"The entire ingestion/ engine, deploy/portainer/, and .gitea/workflows/ingestion.yml are UNTRACKED in git (plus modified README/package.json/layout.tsx/docker-stack.yml/.dockerignore). All this project knowledge lives only on local disk. Stage, commit, and push so the work isn't stranded.","status":"in_progress","priority":1,"issue_type":"task","assignee":"E.Gavrilov","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:24:12Z","created_by":"E.Gavrilov","updated_at":"2026-07-30T21:52:02Z","started_at":"2026-07-30T21:52:02Z","dependencies":[{"issue_id":"videoreg-nnv.2","depends_on_id":"videoreg-nnv","type":"parent-child","created_at":"2026-06-07T17:24:12Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-kbh.11","title":"Deep deduplication matching (fuzzy/alias/perceptual)","description":"Implement spec §14 matching: brand + normalized model + aliases + sourceProductId + image perceptual hash + specs similarity + URL history; classify same_product/possible_duplicate/variant/oem_clone so non-trivial groups require manual moderation before export. Currently only new_product is recorded.","status":"open","priority":1,"issue_type":"feature","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:23:36Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:23:36Z","dependencies":[{"issue_id":"videoreg-kbh.11","depends_on_id":"videoreg-kbh","type":"parent-child","created_at":"2026-06-07T17:23:36Z","created_by":"E.Gavrilov","metadata":"{}"},{"issue_id":"videoreg-kbh.11","depends_on_id":"videoreg-kbh.16","type":"blocks","created_at":"2026-06-07T18:02:57Z","created_by":"E.Gavrilov","metadata":"{}"},{"issue_id":"videoreg-kbh.11","depends_on_id":"videoreg-kbh.7","type":"blocks","created_at":"2026-06-07T17:24:37Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":2,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-kbh.7","title":"Phase 7: Deduplication + moderation","description":"Canonical product grouping, moderation queue, bulk approve, conflict tracking. PARTIAL: moderation_queue + moderation_conflicts tables, products default moderation_status='needs_review', admin moderation UI/API exist. Deep dedup matching (fuzzy/alias/perceptual image hash, possible_duplicate/variant/oem_clone classification) is NOT implemented — only new_product entries are created.","notes":"Moderation queue/conflicts tables + admin UI done; deep dedup matching (kbh.11) outstanding.","status":"in_progress","priority":1,"issue_type":"feature","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:23:14Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:24:37Z","started_at":"2026-06-07T14:24:37Z","dependencies":[{"issue_id":"videoreg-kbh.7","depends_on_id":"videoreg-kbh","type":"parent-child","created_at":"2026-06-07T17:23:14Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":1,"comment_count":0}
+{"_type":"issue","id":"videoreg-kbh","title":"Dashcam Catalog Ingestion Engine","description":"Maintainable catalog ingestion engine for car dashcams (источник: Yandex Market). HTTP-only crawler, PostgreSQL source of truth, normalization, dedup, moderation, and a token-protected versioned export consumed by the Next.js app. Full spec: tasks/002-crawler.md.","status":"open","priority":1,"issue_type":"epic","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:22:41Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:22:41Z","dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-kbh.16","title":"Model/canonicalName cleaning leaks form-factor \u0026 descriptor words","description":"Verified on a live smoke run (2026 Yandex Market data). normalizer.cleanModelFromTitle/buildCanonicalModel leave non-model words in the canonical name, which will corrupt dedup/canonical grouping (kbh.7/kbh.11). Examples: title 'Видеорегистратор-зеркало iBOX Rover 2 автомобильный с базой камер' -\u003e canonicalName 'iBOX зеркало Rover 2 автомобильный' (should be ~'iBOX Rover 2'); 'iBOX F5 + WiFi с радар-детектором' -\u003e model retains 'с радар-детектором'. genericWords list lacks 'зеркало', standalone 'автомобильный', form-factor and feature tails. Fix: expand stopword/form-factor stripping and feature-tail removal; add fixtures from real titles.","status":"closed","priority":2,"issue_type":"bug","assignee":"E.Gavrilov","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T15:02:57Z","created_by":"E.Gavrilov","updated_at":"2026-07-30T23:00:47Z","started_at":"2026-07-30T21:52:03Z","closed_at":"2026-07-30T23:00:47Z","close_reason":"Canonical model cleanup has fixtures for the SPAWNSON and iBOX cases, and canonical name/model edits are preserved in admin moderation.","dependencies":[{"issue_id":"videoreg-kbh.16","depends_on_id":"videoreg-kbh","type":"parent-child","created_at":"2026-06-07T18:02:56Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":1,"comment_count":0}
+{"_type":"issue","id":"videoreg-nnv.1","title":"CI/CD pipelines + Swarm + Portainer + deploy scripts","description":"DONE: .gitea/workflows/{pr,master,ingestion}.yml, deploy/docker/Dockerfile.web, deploy/swarm/docker-stack.yml, deploy/portainer/{standalone,swarm} templates, deploy/scripts + deploy/remote ssh deploy/rollback + predeploy/smoke/release-manifest scripts.","status":"closed","priority":2,"issue_type":"feature","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:24:11Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:24:35Z","closed_at":"2026-06-07T14:24:35Z","close_reason":"Implemented and verified in codebase (typecheck + 18 vitest tests pass)","dependencies":[{"issue_id":"videoreg-nnv.1","depends_on_id":"videoreg-nnv","type":"parent-child","created_at":"2026-06-07T17:24:11Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-seu.1","title":"Video registry app: primitive JSON DB, caching, health probes","description":"Working Next.js app: dashboard + register-video flow, primitive file-backed JSON DB (shared/db/primitiveJsonDb.ts → data/videoreg.json), process+tag+HTTP read caching with revalidateTag on writes, /health/live + /health/ready, /api/videos. Tested (vitest).","status":"closed","priority":2,"issue_type":"feature","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:24:10Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:24:35Z","closed_at":"2026-06-07T14:24:35Z","close_reason":"Implemented and verified in codebase (typecheck + 18 vitest tests pass)","dependencies":[{"issue_id":"videoreg-seu.1","depends_on_id":"videoreg-seu","type":"parent-child","created_at":"2026-06-07T17:24:09Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-seu.2","title":"Replace primitive JSON DB before scaling beyond 1 replica","description":"The file-backed JSON DB forces the Swarm stack to a single replica on a manager node. Move to a real database before raising replicas or spreading across nodes. Read-model can keep being fed by the catalog import bridge.","status":"open","priority":2,"issue_type":"task","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:24:10Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:24:10Z","dependencies":[{"issue_id":"videoreg-seu.2","depends_on_id":"videoreg-seu","type":"parent-child","created_at":"2026-06-07T17:24:10Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-st9.1","title":"Phase 1: Tokens + core atomic components + 'Best Dashcams 2026' brief","description":"Typography/color/spacing tokens, light-first + dark parity, and core components for the comparison-first page. PARTIAL/mostly done in code: web-front/src/shared/ui atoms (Badge,Button,EvidenceChip,Icon,ScoreMeter), molecules (ProductCard,WinnerRail), organisms (PageHero,ProductComparisonTable), template BestDashcamsPage; globals.css tokens; dev showcase route. Trust states verified/claimed/unknown via EvidenceChip.","notes":"Core atomic components + tokens + BestDashcamsPage built; Storybook + Figma parity (st9.2) outstanding.","status":"in_progress","priority":2,"issue_type":"feature","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:24:06Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:24:37Z","started_at":"2026-06-07T14:24:37Z","dependencies":[{"issue_id":"videoreg-st9.1","depends_on_id":"videoreg-st9","type":"parent-child","created_at":"2026-06-07T17:24:05Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":1,"comment_count":0}
+{"_type":"issue","id":"videoreg-kbh.12","title":"Image storage legal/policy decision + versioned image artifacts","description":"Spec §7: legal decision on caching/serving source images must be made before production. Then cache approved images and publish versioned image artifact URLs (original URL + source metadata + optional checksum/dimensions); app must not hotlink source images. Currently product_images stores original URLs only.","status":"open","priority":2,"issue_type":"task","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:23:37Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:23:37Z","dependencies":[{"issue_id":"videoreg-kbh.12","depends_on_id":"videoreg-kbh","type":"parent-child","created_at":"2026-06-07T17:23:37Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-kbh.10","title":"Bounded sitemap-first discovery + robots.txt compliance","description":"Discovery currently parses product URLs from category HTML only (discoverYandexMarketProductsFromCategory). Spec §9 wants sitemap-first bounded discovery (diff known URLs, refresh candidates, detect near known boundaries) plus robots.txt checks. Not implemented.","status":"open","priority":2,"issue_type":"feature","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:23:35Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:23:35Z","dependencies":[{"issue_id":"videoreg-kbh.10","depends_on_id":"videoreg-kbh","type":"parent-child","created_at":"2026-06-07T17:23:35Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-kbh.9","title":"Activate MVideo source in the worker","description":"MVideo parser (ingestion/src/sources/mvideo.ts +tests) and ingestion:smoke-mvideo exist, and config.ts builds an MVIDEO seed, but worker.ts skips any seed where source!=='yandex_market' (seed.implemented guard). Wire MVideo discovery+parse into the worker crawl path.","status":"closed","priority":2,"issue_type":"feature","assignee":"E.Gavrilov","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:23:34Z","created_by":"E.Gavrilov","updated_at":"2026-07-30T23:00:46Z","started_at":"2026-07-30T21:52:03Z","closed_at":"2026-07-30T23:00:46Z","close_reason":"MVideo discovery and product parsing are wired into the worker with smoke coverage; live source CAPTCHA is paused without bypass.","dependencies":[{"issue_id":"videoreg-kbh.9","depends_on_id":"videoreg-kbh","type":"parent-child","created_at":"2026-06-07T17:23:34Z","created_by":"E.Gavrilov","metadata":"{}"},{"issue_id":"videoreg-kbh.9","depends_on_id":"videoreg-kbh.5","type":"blocks","created_at":"2026-06-07T17:24:38Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-kbh.8","title":"Phase 8: Versioned export + token-protected serving","description":"Versioned manifest + catalog artifact, Bearer-token export server, checksum verified on app import, atomic replace. DONE: ingestion/src/{exporter,exportCatalog,exportServer}.ts; excludes prices/sellers/availability/reviews/rating per spec.","notes":"Export ARTIFACT generation (exporter.ts/exportCatalog) stays valid. The token-protected HTTP SERVING is superseded by the shared-volume bridge — see the co-location decision under videoreg-nnv.","status":"closed","priority":2,"issue_type":"feature","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:23:15Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:53:50Z","closed_at":"2026-06-07T14:24:35Z","close_reason":"Implemented and verified in codebase (typecheck + 18 vitest tests pass)","dependencies":[{"issue_id":"videoreg-kbh.8","depends_on_id":"videoreg-kbh","type":"parent-child","created_at":"2026-06-07T17:23:15Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-kbh.6","title":"Phase 6: Normalizer (brand/model/specs/units/booleans)","description":"Brand dictionary, model cleaner, spec mapper, unit converter, boolean detector, alias dictionary. DONE: ingestion/src/normalizer.ts.","status":"closed","priority":2,"issue_type":"feature","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:23:13Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:24:34Z","closed_at":"2026-06-07T14:24:34Z","close_reason":"Implemented and verified in codebase (typecheck + 18 vitest tests pass)","dependencies":[{"issue_id":"videoreg-kbh.6","depends_on_id":"videoreg-kbh","type":"parent-child","created_at":"2026-06-07T17:23:13Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-kbh.5","title":"Phase 5: Parser + stableContentHash (Yandex Market)","description":"Category discovery, product/specs/image parsing, internal-only aggregate rating, source-specific stableContentHash. DONE for Yandex Market: ingestion/src/sources/yandexMarket.ts (+tests), html.ts, stableHash.ts. NOTE: bounded sitemap discovery not yet implemented (see discovery task).","status":"closed","priority":2,"issue_type":"feature","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:23:12Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:24:34Z","closed_at":"2026-06-07T14:24:34Z","close_reason":"Implemented and verified in codebase (typecheck + 18 vitest tests pass)","dependencies":[{"issue_id":"videoreg-kbh.5","depends_on_id":"videoreg-kbh","type":"parent-child","created_at":"2026-06-07T17:23:12Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":1,"comment_count":0}
+{"_type":"issue","id":"videoreg-kbh.4","title":"Phase 4: PostgreSQL-backed queue + HTTP-only fetcher","description":"Postgres queue (state machine), HTTP-only fetcher with rate limits/retry/backoff, stop on captcha/403/429, raw snapshot storage, rawSnapshotHash. DONE: ingestion/src/{queue,fetcher,worker}.ts.","status":"closed","priority":2,"issue_type":"feature","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:23:11Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:24:34Z","closed_at":"2026-06-07T14:24:34Z","close_reason":"Implemented and verified in codebase (typecheck + 18 vitest tests pass)","dependencies":[{"issue_id":"videoreg-kbh.4","depends_on_id":"videoreg-kbh","type":"parent-child","created_at":"2026-06-07T17:23:11Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-kbh.3","title":"Phase 3: Ingestion stack skeleton (Docker + Postgres + migrations)","description":"Separate docker-compose stack: postgres, ingestion-worker, ingestion-admin, export-server; migrations + schema_migrations; mounted volume for snapshots/images; healthchecks. DONE: ingestion/docker-compose.yml, migrations/001-003, ingestion/src/{db,migrate}.ts.","status":"closed","priority":2,"issue_type":"feature","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:23:10Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:24:33Z","closed_at":"2026-06-07T14:24:33Z","close_reason":"Implemented and verified in codebase (typecheck + 18 vitest tests pass)","dependencies":[{"issue_id":"videoreg-kbh.3","depends_on_id":"videoreg-kbh","type":"parent-child","created_at":"2026-06-07T17:23:10Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-kbh.2","title":"Phase 2: Shared export contract (Zod) + app import","description":"Shared Zod schema for manifest/catalog export, derived TS types, schemaVersion, operator import command, atomic replace of data/videoreg.json. DONE: web-front/src/entities/dashcam-catalog/{model/schema.ts,api/importCatalogExport.ts} + tests; package.json catalog:import.","status":"closed","priority":2,"issue_type":"feature","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:23:09Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:24:33Z","closed_at":"2026-06-07T14:24:33Z","close_reason":"Implemented and verified in codebase (typecheck + 18 vitest tests pass)","dependencies":[{"issue_id":"videoreg-kbh.2","depends_on_id":"videoreg-kbh","type":"parent-child","created_at":"2026-06-07T17:23:09Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-kbh.1","title":"Phase 1: Legal discovery \u0026 crawl policy","description":"robots.txt/sitemap/API-feed legality checks, crawl budget, allowed URL patterns, image storage policy. Crawl budget + conservative policy (1 req / 5-15s, jitter, exponential backoff, stop on captcha/403/429, daily budgets, no headless/stealth) is IMPLEMENTED in config/queue/fetcher. Still open: robots.txt + sitemap legality checks and image storage policy decision.","notes":"Crawl budget/backoff/stop-on-block implemented; robots.txt + sitemap legality checks and image storage policy still open.","status":"in_progress","priority":2,"issue_type":"task","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:23:08Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:24:36Z","started_at":"2026-06-07T14:24:36Z","dependencies":[{"issue_id":"videoreg-kbh.1","depends_on_id":"videoreg-kbh","type":"parent-child","created_at":"2026-06-07T17:23:08Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-nnv","title":"Deployment \u0026 CI/CD","description":"Gitea Actions pipelines, Docker image build/push, Swarm stack, Portainer stack templates, and SSH deploy/rollback scripts for both web and ingestion.","status":"open","priority":2,"issue_type":"epic","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:22:44Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:22:44Z","dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-seu","title":"Frontend Web App (web-front)","description":"Next.js 16 App Router frontend with FSD layering, primitive file-backed JSON DB as a replaceable read-model cache, aggressive read caching, health probes, and the operator catalog import bridge.","status":"open","priority":2,"issue_type":"epic","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:22:43Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:22:43Z","dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-st9","title":"Design System (Atomic + Tokens)","description":"Premium automotive editorial design system: tokens, atomic components, light-first + dark parity, comparison-first UX for 'Best Dashcams 2026'. Full spec: tasks/001-design-system.md.","status":"open","priority":2,"issue_type":"epic","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:22:42Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:22:42Z","dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-st9.3","title":"Phase 2: Article / review / ecommerce / filtering modules","description":"Spec Phase 2: article system, review system (pros/cons, verdict, sample footage, firmware history, heat/night tests), ecommerce modules (seller comparison, price history, CTA hierarchy), filtering system. Not implemented.","status":"open","priority":3,"issue_type":"feature","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:24:08Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:24:08Z","dependencies":[{"issue_id":"videoreg-st9.3","depends_on_id":"videoreg-st9","type":"parent-child","created_at":"2026-06-07T17:24:07Z","created_by":"E.Gavrilov","metadata":"{}"},{"issue_id":"videoreg-st9.3","depends_on_id":"videoreg-st9.1","type":"blocks","created_at":"2026-06-07T17:24:39Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0}
+{"_type":"issue","id":"videoreg-st9.2","title":"Storybook + typed component API docs + Figma parity","description":"Spec deliverables not yet present: no .storybook setup, no component API docs, no Figma↔React parity artifacts. Add Storybook, document variant/responsive/theming APIs.","status":"open","priority":3,"issue_type":"task","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:24:07Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:24:07Z","dependencies":[{"issue_id":"videoreg-st9.2","depends_on_id":"videoreg-st9","type":"parent-child","created_at":"2026-06-07T17:24:06Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-kbh.13","title":"Sync schedules: frequent light / deep / new-product detection","description":"Spec §17: frequent light sync (6-12h: first category page, bounded sitemap diff, new URLs), deep sync (7-14d: all known cards/specs/images), daily new-product detection. No scheduler implemented; worker runs an inline loop. MVP import is manual.","status":"open","priority":3,"issue_type":"feature","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:23:38Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:23:38Z","dependencies":[{"issue_id":"videoreg-kbh.13","depends_on_id":"videoreg-kbh","type":"parent-child","created_at":"2026-06-07T17:23:38Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0}
+{"_type":"issue","id":"videoreg-st9.4","title":"Phase 3: Motion, advanced comparison UX, personalization","description":"Spec Phase 3: mechanical motion system, advanced comparison UX (column pinning, snap scroll, spec highlighting), personalization, saved comparisons, dashboard-like features. Not implemented.","status":"open","priority":4,"issue_type":"feature","owner":"gavrilovdev@yandex-team.ru","created_at":"2026-06-07T14:24:09Z","created_by":"E.Gavrilov","updated_at":"2026-06-07T14:24:09Z","dependencies":[{"issue_id":"videoreg-st9.4","depends_on_id":"videoreg-st9","type":"parent-child","created_at":"2026-06-07T17:24:08Z","created_by":"E.Gavrilov","metadata":"{}"},{"issue_id":"videoreg-st9.4","depends_on_id":"videoreg-st9.3","type":"blocks","created_at":"2026-06-07T17:24:39Z","created_by":"E.Gavrilov","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0}

.beads/metadata.json

@@ -0,0 +1,7 @@
+{
+  "database": "dolt",
+  "backend": "dolt",
+  "dolt_mode": "embedded",
+  "dolt_database": "videoreg",
+  "project_id": "3e9efadc-32e2-40ac-9833-15f223c3351d"
+}

.claude/settings.json

@@ -0,0 +1,26 @@
+{
+  "hooks": {
+    "PreCompact": [
+      {
+        "hooks": [
+          {
+            "command": "bd prime",
+            "type": "command"
+          }
+        ],
+        "matcher": ""
+      }
+    ],
+    "SessionStart": [
+      {
+        "hooks": [
+          {
+            "command": "bd prime",
+            "type": "command"
+          }
+        ],
+        "matcher": ""
+      }
+    ]
+  }
+}

.dockerignore

@@ -1,8 +1,10 @@
 .git
 .gitea
 .next
+web-front/.next
 node_modules
 data
+ingestion/data
 coverage
 .env
 .env.local

.gitea/workflows/master.yml

@@ -4,14 +4,16 @@ on:
   push:
     branches:
       - master
+  workflow_dispatch:
 
 concurrency:
   group: deploy-production
   cancel-in-progress: false
 
 env:
-  REGISTRY: ${{ vars.CONTAINER_REGISTRY }}
+  REGISTRY: ${{ vars.REGISTRY }}
-  WEB_IMAGE_REPO: ${{ vars.CONTAINER_REGISTRY }}/${{ github.repository_owner }}/${{ github.event.repository.name }}-web
+  WEB_IMAGE_REPO: ${{ vars.IMAGE_REPO }}-web
+  INGESTION_IMAGE_REPO: ${{ vars.IMAGE_REPO }}-ingestion
   RELEASE_ID: ${{ github.sha }}-${{ github.run_number }}
   NPM_CONFIG_AUDIT: "false"
   NPM_CONFIG_FUND: "false"
@@ -63,38 +65,55 @@ jobs:
             org.opencontainers.image.revision=${{ github.sha }}
             org.opencontainers.image.version=${{ env.RELEASE_ID }}
 
+      - name: Build and push ingestion image
+        id: build-ingestion
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ingestion/Dockerfile
+          push: true
+          tags: ${{ env.INGESTION_IMAGE_REPO }}:${{ github.sha }}
+          labels: |
+            org.opencontainers.image.revision=${{ github.sha }}
+            org.opencontainers.image.version=${{ env.RELEASE_ID }}
+
       - name: Production pre-deploy checks
         env:
           ENVIRONMENT: production
-          STACK_NAME: ${{ vars.PRODUCTION_STACK_NAME }}
+          STACK_NAME: ${{ vars.STACK_NAME }}
           RELEASE_ID: ${{ env.RELEASE_ID }}
           COMMIT_SHA: ${{ github.sha }}
           WEB_IMAGE: ${{ env.WEB_IMAGE_REPO }}@${{ steps.build-web.outputs.digest }}
-          PUBLIC_HOST: ${{ vars.PRODUCTION_PUBLIC_HOST }}
+          INGESTION_IMAGE: ${{ env.INGESTION_IMAGE_REPO }}@${{ steps.build-ingestion.outputs.digest }}
-          TRAEFIK_NETWORK: ${{ vars.PRODUCTION_TRAEFIK_NETWORK }}
+          POSTGRES_PASSWORD: ${{ secrets.INGESTION_POSTGRES_PASSWORD }}
-          SWARM_MANAGER_HOST: ${{ secrets.PRODUCTION_SWARM_MANAGER_HOST }}
+          INGESTION_ADMIN_PASSWORD: ${{ secrets.INGESTION_ADMIN_PASSWORD }}
-          SWARM_SSH_USER: ${{ secrets.PRODUCTION_SWARM_SSH_USER }}
+          PUBLIC_HOST: ${{ vars.DOMAIN }}
+          TRAEFIK_NETWORK: ${{ vars.TRAEFIK_NETWORK }}
+          SWARM_MANAGER_HOST: ${{ secrets.SWARM_MANAGER_HOST }}
+          SWARM_SSH_USER: ${{ secrets.SWARM_SSH_USER }}
         run: node deploy/scripts/predeploy-check.mjs
 
       - name: Deploy production
         env:
           ENVIRONMENT: production
-          STACK_NAME: ${{ vars.PRODUCTION_STACK_NAME }}
+          STACK_NAME: ${{ vars.STACK_NAME }}
           RELEASE_ID: ${{ env.RELEASE_ID }}
           COMMIT_SHA: ${{ github.sha }}
           WEB_IMAGE: ${{ env.WEB_IMAGE_REPO }}@${{ steps.build-web.outputs.digest }}
-          PUBLIC_HOST: ${{ vars.PRODUCTION_PUBLIC_HOST }}
+          INGESTION_IMAGE: ${{ env.INGESTION_IMAGE_REPO }}@${{ steps.build-ingestion.outputs.digest }}
-          TRAEFIK_NETWORK: ${{ vars.PRODUCTION_TRAEFIK_NETWORK }}
+          PUBLIC_HOST: ${{ vars.DOMAIN }}
-          SWARM_MANAGER_HOST: ${{ secrets.PRODUCTION_SWARM_MANAGER_HOST }}
+          TRAEFIK_NETWORK: ${{ vars.TRAEFIK_NETWORK }}
-          SWARM_SSH_USER: ${{ secrets.PRODUCTION_SWARM_SSH_USER }}
+          SWARM_MANAGER_HOST: ${{ secrets.SWARM_MANAGER_HOST }}
-          SWARM_SSH_PORT: ${{ secrets.PRODUCTION_SWARM_SSH_PORT }}
+          SWARM_SSH_USER: ${{ secrets.SWARM_SSH_USER }}
+          SWARM_SSH_PORT: ${{ secrets.SWARM_SSH_PORT }}
           REGISTRY: ${{ env.REGISTRY }}
           REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
           REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
-          WEB_REPLICAS: ${{ vars.PRODUCTION_WEB_REPLICAS }}
+          WEB_REPLICAS: ${{ vars.WEB_REPLICAS }}
-          WEB_PLACEMENT_CONSTRAINT: ${{ vars.PRODUCTION_WEB_PLACEMENT_CONSTRAINT }}
+          WEB_PLACEMENT_CONSTRAINT: ${{ vars.WEB_PLACEMENT_CONSTRAINT }}
-          TRAEFIK_ENTRYPOINTS: ${{ vars.TRAEFIK_ENTRYPOINTS }}
+          TRAEFIK_ENTRYPOINTS: ${{ vars.TRAEFIK_ENTRYPOINT }}
           TRAEFIK_TLS: ${{ vars.TRAEFIK_TLS }}
+          TRAEFIK_CERTRESOLVER: ${{ vars.TRAEFIK_CERTRESOLVER }}
           REGION: ${{ vars.REGION }}
           TEAM_ID: ${{ vars.TEAM_ID }}
           DEPLOY_MIN_FREE_MB: ${{ vars.DEPLOY_MIN_FREE_MB }}
@@ -103,13 +122,29 @@ jobs:
           DEPLOY_CLEANUP_FAILED_IMAGES: ${{ vars.DEPLOY_CLEANUP_FAILED_IMAGES }}
           VIDEOREG_DB_MEMORY_TTL_MS: ${{ vars.VIDEOREG_DB_MEMORY_TTL_MS }}
           VIDEOREG_NEXT_CACHE_REVALIDATE_SECONDS: ${{ vars.VIDEOREG_NEXT_CACHE_REVALIDATE_SECONDS }}
+          POSTGRES_DB: ${{ vars.INGESTION_POSTGRES_DB }}
+          POSTGRES_USER: ${{ vars.INGESTION_POSTGRES_USER }}
+          POSTGRES_PASSWORD: ${{ secrets.INGESTION_POSTGRES_PASSWORD }}
+          INGESTION_ADMIN_USERNAME: ${{ vars.INGESTION_ADMIN_USERNAME }}
+          INGESTION_ADMIN_PASSWORD: ${{ secrets.INGESTION_ADMIN_PASSWORD }}
+          YANDEX_MARKET_DASHCAM_CATEGORY_URL: ${{ vars.YANDEX_MARKET_DASHCAM_CATEGORY_URL }}
+          MVIDEO_DASHCAM_CATEGORY_URL: ${{ vars.MVIDEO_DASHCAM_CATEGORY_URL }}
+          INGESTION_CATEGORY_DAILY_BUDGET: ${{ vars.INGESTION_CATEGORY_DAILY_BUDGET }}
+          INGESTION_PRODUCT_DAILY_BUDGET: ${{ vars.INGESTION_PRODUCT_DAILY_BUDGET }}
+          INGESTION_SITEMAP_DAILY_BUDGET: ${{ vars.INGESTION_SITEMAP_DAILY_BUDGET }}
+          INGESTION_FETCH_DELAY_MIN_MS: ${{ vars.INGESTION_FETCH_DELAY_MIN_MS }}
+          INGESTION_FETCH_DELAY_MAX_MS: ${{ vars.INGESTION_FETCH_DELAY_MAX_MS }}
+          INGESTION_WORKER_BATCH_SIZE: ${{ vars.INGESTION_WORKER_BATCH_SIZE }}
+          INGESTION_WORKER_IDLE_MS: ${{ vars.INGESTION_WORKER_IDLE_MS }}
+          INGESTION_WORKER_REPLICAS: ${{ vars.INGESTION_WORKER_REPLICAS }}
+          TRAEFIK_ADMIN_PRIORITY: ${{ vars.TRAEFIK_ADMIN_PRIORITY }}
         run: deploy/scripts/deploy-over-ssh.sh
 
       - name: Production smoke checks
         id: smoke
         continue-on-error: true
         env:
-          BASE_URL: https://${{ vars.PRODUCTION_PUBLIC_HOST }}
+          BASE_URL: https://${{ vars.DOMAIN }}
           EXPECTED_RELEASE_ID: ${{ env.RELEASE_ID }}
           SMOKE_RESULT_PATH: smoke-result.json
         run: node deploy/scripts/smoke-check.mjs
@@ -118,10 +153,10 @@ jobs:
         if: steps.smoke.outcome == 'failure'
         env:
           ENVIRONMENT: production
-          STACK_NAME: ${{ vars.PRODUCTION_STACK_NAME }}
+          STACK_NAME: ${{ vars.STACK_NAME }}
-          SWARM_MANAGER_HOST: ${{ secrets.PRODUCTION_SWARM_MANAGER_HOST }}
+          SWARM_MANAGER_HOST: ${{ secrets.SWARM_MANAGER_HOST }}
-          SWARM_SSH_USER: ${{ secrets.PRODUCTION_SWARM_SSH_USER }}
+          SWARM_SSH_USER: ${{ secrets.SWARM_SSH_USER }}
-          SWARM_SSH_PORT: ${{ secrets.PRODUCTION_SWARM_SSH_PORT }}
+          SWARM_SSH_PORT: ${{ secrets.SWARM_SSH_PORT }}
         run: deploy/scripts/rollback-over-ssh.sh
 
       - name: Fail after production smoke rollback
@@ -133,6 +168,7 @@ jobs:
           COMMIT_SHA: ${{ github.sha }}
           RELEASE_ID: ${{ env.RELEASE_ID }}
           WEB_IMAGE_DIGEST: ${{ env.WEB_IMAGE_REPO }}@${{ steps.build-web.outputs.digest }}
+          INGESTION_IMAGE_DIGEST: ${{ env.INGESTION_IMAGE_REPO }}@${{ steps.build-ingestion.outputs.digest }}
           SMOKE_RESULT_PATH: smoke-result.json
           WORKFLOW_RUN_ID: ${{ github.run_id }}
         run: node deploy/scripts/write-release-manifest.mjs > release-manifest.json

.gitignore

@@ -1,8 +1,14 @@
 .next/
 node_modules/
 data/
+ingestion/data/
 coverage/
 .env
 .env.local
 npm-debug.log*
 *.tsbuildinfo
+
+# Beads / Dolt files (added by bd init)
+.dolt/
+*.db
+.beads-credential-key

.npmrc

@@ -1,3 +1,4 @@
+registry=https://registry.npmjs.org/
 audit=false
 fund=false
 update-notifier=false

AGENTS.md

@@ -0,0 +1,96 @@
+# Agent Instructions
+
+This project uses **bd** (beads) for issue tracking. Run `bd prime` for full workflow context.
+
+> **Architecture in one line:** Issues live in a local Dolt database
+> (`.beads/dolt/`); cross-machine sync uses `bd dolt push/pull` (a
+> git-compatible protocol), stored under `refs/dolt/data` on your git
+> remote — separate from `refs/heads/*` where your code lives.
+> `.beads/issues.jsonl` is a passive export, not the wire protocol.
+>
+> See [SYNC_CONCEPTS.md](https://github.com/gastownhall/beads/blob/main/docs/SYNC_CONCEPTS.md)
+> for the one-screen overview and anti-patterns (don't treat JSONL as the
+> source of truth; don't `bd import` during normal operation; don't
+> reach for third-party Dolt hosting before trying the default).
+
+## Quick Reference
+
+```bash
+bd ready              # Find available work
+bd show <id>          # View issue details
+bd update <id> --claim  # Claim work atomically
+bd close <id>         # Complete work
+bd dolt push          # Push beads data to remote
+```
+
+## Non-Interactive Shell Commands
+
+**ALWAYS use non-interactive flags** with file operations to avoid hanging on confirmation prompts.
+
+Shell commands like `cp`, `mv`, and `rm` may be aliased to include `-i` (interactive) mode on some systems, causing the agent to hang indefinitely waiting for y/n input.
+
+**Use these forms instead:**
+```bash
+# Force overwrite without prompting
+cp -f source dest           # NOT: cp source dest
+mv -f source dest           # NOT: mv source dest
+rm -f file                  # NOT: rm file
+
+# For recursive operations
+rm -rf directory            # NOT: rm -r directory
+cp -rf source dest          # NOT: cp -r source dest
+```
+
+**Other commands that may prompt:**
+- `scp` - use `-o BatchMode=yes` for non-interactive
+- `ssh` - use `-o BatchMode=yes` to fail instead of prompting
+- `apt-get` - use `-y` flag
+- `brew` - use `HOMEBREW_NO_AUTO_UPDATE=1` env var
+
+<!-- BEGIN BEADS INTEGRATION v:1 profile:minimal hash:7510c1e2 -->
+## Beads Issue Tracker
+
+This project uses **bd (beads)** for issue tracking. Run `bd prime` to see full workflow context and commands.
+
+### Quick Reference
+
+```bash
+bd ready              # Find available work
+bd show <id>          # View issue details
+bd update <id> --claim  # Claim work
+bd close <id>         # Complete work
+```
+
+### Rules
+
+- Use `bd` for ALL task tracking — do NOT use TodoWrite, TaskCreate, or markdown TODO lists
+- Run `bd prime` for detailed command reference and session close protocol
+- Use `bd remember` for persistent knowledge — do NOT use MEMORY.md files
+
+**Architecture in one line:** issues live in a local Dolt DB; sync uses `refs/dolt/data` on your git remote; `.beads/issues.jsonl` is a passive export. See https://github.com/gastownhall/beads/blob/main/docs/SYNC_CONCEPTS.md for details and anti-patterns.
+
+## Session Completion
+
+**When ending a work session**, you MUST complete ALL steps below. Work is NOT complete until `git push` succeeds.
+
+**MANDATORY WORKFLOW:**
+
+1. **File issues for remaining work** - Create issues for anything that needs follow-up
+2. **Run quality gates** (if code changed) - Tests, linters, builds
+3. **Update issue status** - Close finished work, update in-progress items
+4. **PUSH TO REMOTE** - This is MANDATORY:
+   ```bash
+   git pull --rebase
+   git push
+   git status  # MUST show "up to date with origin"
+   ```
+5. **Clean up** - Clear stashes, prune remote branches
+6. **Verify** - All changes committed AND pushed
+7. **Hand off** - Provide context for next session
+
+**CRITICAL RULES:**
+- Work is NOT complete until `git push` succeeds
+- NEVER stop before pushing - that leaves work stranded locally
+- NEVER say "ready to push when you are" - YOU must push
+- If push fails, resolve and retry until it succeeds
+<!-- END BEADS INTEGRATION -->

CLAUDE.md

@@ -0,0 +1,70 @@
+# Project Instructions for AI Agents
+
+This file provides instructions and context for AI coding agents working on this project.
+
+<!-- BEGIN BEADS INTEGRATION v:1 profile:minimal hash:7510c1e2 -->
+## Beads Issue Tracker
+
+This project uses **bd (beads)** for issue tracking. Run `bd prime` to see full workflow context and commands.
+
+### Quick Reference
+
+```bash
+bd ready              # Find available work
+bd show <id>          # View issue details
+bd update <id> --claim  # Claim work
+bd close <id>         # Complete work
+```
+
+### Rules
+
+- Use `bd` for ALL task tracking — do NOT use TodoWrite, TaskCreate, or markdown TODO lists
+- Run `bd prime` for detailed command reference and session close protocol
+- Use `bd remember` for persistent knowledge — do NOT use MEMORY.md files
+
+**Architecture in one line:** issues live in a local Dolt DB; sync uses `refs/dolt/data` on your git remote; `.beads/issues.jsonl` is a passive export. See https://github.com/gastownhall/beads/blob/main/docs/SYNC_CONCEPTS.md for details and anti-patterns.
+
+## Session Completion
+
+**When ending a work session**, you MUST complete ALL steps below. Work is NOT complete until `git push` succeeds.
+
+**MANDATORY WORKFLOW:**
+
+1. **File issues for remaining work** - Create issues for anything that needs follow-up
+2. **Run quality gates** (if code changed) - Tests, linters, builds
+3. **Update issue status** - Close finished work, update in-progress items
+4. **PUSH TO REMOTE** - This is MANDATORY:
+   ```bash
+   git pull --rebase
+   git push
+   git status  # MUST show "up to date with origin"
+   ```
+5. **Clean up** - Clear stashes, prune remote branches
+6. **Verify** - All changes committed AND pushed
+7. **Hand off** - Provide context for next session
+
+**CRITICAL RULES:**
+- Work is NOT complete until `git push` succeeds
+- NEVER stop before pushing - that leaves work stranded locally
+- NEVER say "ready to push when you are" - YOU must push
+- If push fails, resolve and retry until it succeeds
+<!-- END BEADS INTEGRATION -->
+
+
+## Build & Test
+
+_Add your build and test commands here_
+
+```bash
+# Example:
+# npm install
+# npm test
+```
+
+## Architecture Overview
+
+_Add a brief overview of your project architecture_
+
+## Conventions & Patterns
+
+_Add your project-specific conventions here_

DEMO.md

@@ -0,0 +1,71 @@
+# Demo runbook
+
+The demo uses local PostgreSQL plus the same shared export contract as
+production. It never writes source prices, sellers, availability, or reviews to
+the public catalog.
+
+## 1. Start local ingestion
+
+```bash
+cp ingestion/.env.example ingestion/.env
+```
+
+Set real local passwords in `ingestion/.env`, then use five product requests per
+source for the demonstration:
+
+```bash
+INGESTION_PRODUCT_DAILY_BUDGET=5 \
+INGESTION_FETCH_DELAY_MIN_MS=5000 \
+INGESTION_FETCH_DELAY_MAX_MS=5000 \
+docker compose --env-file ingestion/.env -f ingestion/docker-compose.yml up -d --build
+```
+
+Follow the worker until the category and product jobs settle:
+
+```bash
+docker compose --env-file ingestion/.env -f ingestion/docker-compose.yml logs -f ingestion-worker
+```
+
+For a five-card demo, the worker fetches five product cards and marks the rest
+of the discovered category candidates as `rate_limited` for that budget day.
+If MVideo is configured and replies with a CAPTCHA/403/429, it remains paused
+and visible in the queue; do not retry it through a browser or proxy.
+
+## 2. Moderate and export
+
+Open `http://127.0.0.1:4101/admin`, authenticate with the values from
+`ingestion/.env`, choose **Products**, inspect the source URLs, raw technical
+characteristics, normalized values, and any conflicts. Edit the canonical model
+or canonical name where needed, approve the cards intended for the demo, then
+publish the approved export:
+
+```bash
+docker compose --env-file ingestion/.env -f ingestion/docker-compose.yml run --rm ingestion-worker \
+  npm run ingestion:export-catalog
+```
+
+The artifacts appear under `data/catalog-exports/`:
+
+- `manifest.json`
+- `artifacts/<exportId>/catalog.json`
+
+## 3. Start the web catalog
+
+```bash
+DASHCAM_CATALOG_EXPORT_DIR="$PWD/data/catalog-exports" npm run dev
+```
+
+Open `http://localhost:3000`. The root page reads and validates the shared
+catalog at request time, displays approved cards, normalized specs, and links
+back to every source card.
+
+## 4. Final checks
+
+```bash
+npm test
+npm run typecheck
+npm run build
+curl -fsS http://localhost:3000/health/ready
+curl -u "$INGESTION_ADMIN_USERNAME:$INGESTION_ADMIN_PASSWORD" \
+  http://127.0.0.1:4101/admin/api/queue
+```

README.md

@@ -1,12 +1,14 @@
 ## videoreg.ru
 
-Small Next.js video registry using TypeScript, App Router, FSD-oriented folders, and a primitive file-backed JSON database.
+Small video registry using TypeScript, a Next.js web frontend, App Router, FSD-oriented folders, and a primitive file-backed JSON database.
+
+The Next.js app lives under `web-front/`. Ingestion code lives under `ingestion/`.
 
 ### Local development
 
 ```bash
 npm install
-npm run dev
+DASHCAM_CATALOG_EXPORT_DIR="$PWD/data/catalog-exports" npm run dev
 ```
 
 The app runs on [http://localhost:3000](http://localhost:3000).
@@ -27,11 +29,75 @@ Reads are intentionally cached aggressively:
 
 Writes call `revalidateTag` and update the process cache.
 
+### Dashcam catalog ingestion
+
+The catalog ingestion services run in the **same cloud** as the Next.js app
+(one Swarm/Portainer environment), not on a separate server:
+
+```bash
+docker compose -f ingestion/docker-compose.yml up --build
+```
+
+It starts PostgreSQL, an HTTP-only ingestion worker, and a minimal admin server on
+`http://localhost:4101`. The worker writes the validated `manifest.json` +
+`catalog.json` to a shared export volume; there is no HTTP export server and no
+export token — the app reads the artifacts directly from that volume.
+
+The default Yandex Market seed category is:
+
+```text
+https://market.yandex.ru/catalog--avtomobilnye-videoregistratory/82798275/list?hid=82798269
+```
+
+Seed env vars are source-specific: `YANDEX_MARKET_DASHCAM_CATEGORY_URL` is active
+today. `MVIDEO_DASHCAM_CATEGORY_URL` enables the MVideo discovery/parser path
+when configured.
+
+Crawler policy is intentionally conservative: no browser fallback in the regular
+path, no stealth/proxy behavior, PostgreSQL-backed queue and daily budgets, raw
+snapshots stored on the ingestion volume, and source pause on captcha/403/429.
+
+To import an approved export into the app read-model cache, point the import at
+the shared export volume:
+
+```bash
+DASHCAM_CATALOG_EXPORT_DIR=/path/to/ingestion/data/exports \
+npm run catalog:import
+```
+
+The import command reads `manifest.json` + `catalog.json` from that directory,
+validates them with Zod, verifies the artifact `sha256`, checks
+`exportId`/`createdAt`/`itemCount`, and atomically updates `data/videoreg.json`.
+No HTTP call and no token are involved.
+
+The ingestion admin server exposes its own operator UI at
+`http://127.0.0.1:4101/admin`. It is served by `ingestion-admin`, requires Basic
+Auth, and is not part of the Next.js app. In production Traefik should route
+`/admin` to `ingestion-admin` and `/` to Next.js, without stripping the `/admin`
+prefix.
+
+For the production setup where ingestion is co-located with the app and shares
+the export volume, see [`ingestion/README.md`](ingestion/README.md).
+For Portainer stacks, see [`deploy/portainer/README.md`](deploy/portainer/README.md).
+For the local end-to-end demonstration, see [`DEMO.md`](DEMO.md).
+
 ### Deployment
 
 Gitea Actions runs:
 
 - `.gitea/workflows/pr.yml`: quality gate for pull requests into `master`;
-- `.gitea/workflows/master.yml`: quality gate, image build, push, and direct Swarm deploy from `master`.
+- `.gitea/workflows/master.yml`: quality gate, build/push of web and ingestion images, and direct co-located Swarm deploy from `master`.
+
+The production workflow uses the runner's existing Gitea configuration names:
+
+- variables: `REGISTRY`, `IMAGE_REPO`, `DOMAIN` (`videoreg.ru`), `STACK_NAME`,
+  `TRAEFIK_NETWORK`, `TRAEFIK_ENTRYPOINT`, and `TRAEFIK_CERTRESOLVER`;
+- secrets: `REGISTRY_USERNAME`, `REGISTRY_PASSWORD`, `SWARM_MANAGER_HOST`,
+  `SWARM_SSH_USER`, `SWARM_SSH_PORT`, `INGESTION_POSTGRES_PASSWORD`, and
+  `INGESTION_ADMIN_PASSWORD`.
+
+Images are published as `${IMAGE_REPO}-web` and `${IMAGE_REPO}-ingestion`, then
+the Swarm manager pulls their immutable digests. Both `/` and `/admin` receive
+the configured TLS certificate through Traefik.
 
 The Swarm stack defaults to one replica on a manager node because the primitive database is a mounted JSON file. Use a real database before raising replicas above one or spreading the service across nodes.

deploy/docker/Dockerfile.web

@@ -10,9 +10,9 @@ RUN npm install -g npm@11.6.4 \
 FROM deps AS build
 WORKDIR /app
 ENV NEXT_TELEMETRY_DISABLED=1
-COPY next-env.d.ts next.config.ts tsconfig.json vitest.config.ts ./
+COPY tsconfig.json vitest.config.ts ./
-COPY src src
+COPY web-front web-front
-COPY public public
+COPY ingestion ingestion
 RUN npm run ci:quality
 
 FROM node:22-alpine AS runtime
@@ -25,9 +25,9 @@ WORKDIR /app
 
 RUN addgroup -S nodeapp && adduser -S nodeapp -G nodeapp && mkdir -p /data && chown nodeapp:nodeapp /data
 
-COPY --from=build /app/.next/standalone ./
+COPY --from=build /app/web-front/.next/standalone ./
-COPY --from=build /app/.next/static ./.next/static
+COPY --from=build /app/web-front/.next/static ./.next/static
-COPY --from=build /app/public ./public
+COPY --from=build /app/web-front/public ./public
 
 USER nodeapp
 EXPOSE 3000

deploy/portainer/README.md

@@ -0,0 +1,36 @@
+# Portainer / Swarm deployment
+
+`web`, PostgreSQL, `ingestion-worker`, and `ingestion-admin` are one stack. Do
+not deploy ingestion as a separate endpoint: the approved catalog is exchanged
+through the `catalog_exports` volume mounted read-only by web and read-write by
+the worker.
+
+In Portainer, create a Swarm stack from `deploy/swarm/docker-stack.yml`.
+Configure these values as stack environment variables:
+
+- `WEB_IMAGE` - digest-pinned web image.
+- `INGESTION_IMAGE` - digest-pinned ingestion image.
+- `POSTGRES_PASSWORD` - database secret.
+- `INGESTION_ADMIN_PASSWORD` - Basic Auth password for `/admin`.
+- `PUBLIC_HOST=videoreg.ru` and `TRAEFIK_NETWORK`.
+- Optional source/crawl settings: `MVIDEO_DASHCAM_CATEGORY_URL`, daily budgets,
+  and delay bounds.
+
+Keep every service on the same manager/placement constraint. The database,
+snapshots, read model, and shared catalog volume are local Docker volumes and
+must be backed up from that node.
+
+Gitea Actions uses `.gitea/workflows/master.yml`: it tests the repository,
+builds both images, pushes immutable digests, and deploys this one stack. The
+workflow connects to the Swarm manager, pulls those digests, and runs `docker
+stack deploy --with-registry-auth`; Portainer then shows and manages the same
+updated stack. Portainer does not need registry credentials or a separate
+webhook-based image build.
+
+Verification after deploy:
+
+```bash
+curl -fsS https://videoreg.ru/health/ready
+curl -u "$INGESTION_ADMIN_USERNAME:$INGESTION_ADMIN_PASSWORD" \
+  https://videoreg.ru/admin/api/queue
+```

deploy/remote/deploy-stack.sh

@@ -19,6 +19,7 @@ require_var STACK_NAME
 require_var RELEASE_ID
 require_var COMMIT_SHA
 require_var WEB_IMAGE
+require_var INGESTION_IMAGE
 require_var PUBLIC_HOST
 require_var TRAEFIK_NETWORK
 
@@ -37,7 +38,7 @@ cleanup_release_images() {
   fi
 
   echo "Cleaning up unused release image after failed deploy." >&2
-  docker image rm "$WEB_IMAGE" >/dev/null 2>&1 || true
+  docker image rm "$WEB_IMAGE" "$INGESTION_IMAGE" >/dev/null 2>&1 || true
 
   if [ -n "${COMMIT_SHA:-}" ]; then
     docker image prune -af --filter "label=org.opencontainers.image.revision=${COMMIT_SHA}" >&2 || true
@@ -160,15 +161,32 @@ prune_docker_before_pull
 check_docker_disk_space
 
 docker image pull "$WEB_IMAGE"
+docker image pull "$INGESTION_IMAGE"
 
-export ENVIRONMENT STACK_NAME RELEASE_ID COMMIT_SHA WEB_IMAGE PUBLIC_HOST TRAEFIK_NETWORK
+export ENVIRONMENT STACK_NAME RELEASE_ID COMMIT_SHA WEB_IMAGE INGESTION_IMAGE PUBLIC_HOST TRAEFIK_NETWORK
 export REGION="${REGION:-}" TEAM_ID="${TEAM_ID:-}" WEB_REPLICAS="${WEB_REPLICAS:-1}"
 export WEB_PLACEMENT_CONSTRAINT="${WEB_PLACEMENT_CONSTRAINT:-node.role == manager}"
 export TRAEFIK_ENTRYPOINTS="${TRAEFIK_ENTRYPOINTS:-websecure}" TRAEFIK_TLS="${TRAEFIK_TLS:-true}"
+export TRAEFIK_CERTRESOLVER="${TRAEFIK_CERTRESOLVER:-le}"
+export TRAEFIK_ADMIN_PRIORITY="${TRAEFIK_ADMIN_PRIORITY:-100}"
 export VIDEOREG_DB_MEMORY_TTL_MS="${VIDEOREG_DB_MEMORY_TTL_MS:-3600000}"
 export VIDEOREG_NEXT_CACHE_REVALIDATE_SECONDS="${VIDEOREG_NEXT_CACHE_REVALIDATE_SECONDS:-3600}"
+export POSTGRES_DB="${POSTGRES_DB:-videoreg_ingestion}" POSTGRES_USER="${POSTGRES_USER:-videoreg}"
+export INGESTION_ADMIN_USERNAME="${INGESTION_ADMIN_USERNAME:-admin}"
+export YANDEX_MARKET_DASHCAM_CATEGORY_URL="${YANDEX_MARKET_DASHCAM_CATEGORY_URL:-https://market.yandex.ru/catalog--avtomobilnye-videoregistratory/82798275/list?hid=82798269}"
+export MVIDEO_DASHCAM_CATEGORY_URL="${MVIDEO_DASHCAM_CATEGORY_URL:-}"
+export INGESTION_CATEGORY_DAILY_BUDGET="${INGESTION_CATEGORY_DAILY_BUDGET:-25}"
+export INGESTION_PRODUCT_DAILY_BUDGET="${INGESTION_PRODUCT_DAILY_BUDGET:-100}"
+export INGESTION_SITEMAP_DAILY_BUDGET="${INGESTION_SITEMAP_DAILY_BUDGET:-25}"
+export INGESTION_FETCH_DELAY_MIN_MS="${INGESTION_FETCH_DELAY_MIN_MS:-5000}"
+export INGESTION_FETCH_DELAY_MAX_MS="${INGESTION_FETCH_DELAY_MAX_MS:-15000}"
+export INGESTION_WORKER_BATCH_SIZE="${INGESTION_WORKER_BATCH_SIZE:-1}"
+export INGESTION_WORKER_IDLE_MS="${INGESTION_WORKER_IDLE_MS:-30000}"
+export INGESTION_WORKER_REPLICAS="${INGESTION_WORKER_REPLICAS:-1}"
 
 docker stack deploy --with-registry-auth --prune -c "$REMOTE_DIR/docker-stack.yml" "$STACK_NAME"
 wait_service "${STACK_NAME}_web"
+wait_service "${STACK_NAME}_ingestion-worker"
+wait_service "${STACK_NAME}_ingestion-admin"
 
 log deploy_converged

deploy/scripts/deploy-over-ssh.sh

@@ -20,6 +20,7 @@ require_var STACK_NAME
 require_var RELEASE_ID
 require_var COMMIT_SHA
 require_var WEB_IMAGE
+require_var INGESTION_IMAGE
 require_var PUBLIC_HOST
 require_var TRAEFIK_NETWORK
 
@@ -32,10 +33,15 @@ scp -P "$SSH_PORT" deploy/swarm/docker-stack.yml "$SSH_TARGET:$REMOTE_DEPLOY_DIR
 scp -P "$SSH_PORT" deploy/remote/deploy-stack.sh "$SSH_TARGET:$REMOTE_DEPLOY_DIR/deploy-stack.sh"
 
 remote_vars=(
-  ENVIRONMENT STACK_NAME RELEASE_ID COMMIT_SHA WEB_IMAGE PUBLIC_HOST TRAEFIK_NETWORK
+  ENVIRONMENT STACK_NAME RELEASE_ID COMMIT_SHA WEB_IMAGE INGESTION_IMAGE PUBLIC_HOST TRAEFIK_NETWORK
-  REGION TEAM_ID WEB_REPLICAS WEB_PLACEMENT_CONSTRAINT TRAEFIK_ENTRYPOINTS TRAEFIK_TLS REGISTRY REGISTRY_USERNAME REGISTRY_PASSWORD
+  REGION TEAM_ID WEB_REPLICAS WEB_PLACEMENT_CONSTRAINT TRAEFIK_ENTRYPOINTS TRAEFIK_TLS TRAEFIK_CERTRESOLVER REGISTRY REGISTRY_USERNAME REGISTRY_PASSWORD
   RELEASE_LOCK_FILE REMOTE_DEPLOY_DIR DEPLOY_MIN_FREE_MB DEPLOY_PRUNE_BEFORE_PULL DEPLOY_PRUNE_UNTIL
   DEPLOY_CLEANUP_FAILED_IMAGES VIDEOREG_DB_MEMORY_TTL_MS VIDEOREG_NEXT_CACHE_REVALIDATE_SECONDS
+  POSTGRES_DB POSTGRES_USER POSTGRES_PASSWORD INGESTION_ADMIN_USERNAME INGESTION_ADMIN_PASSWORD
+  YANDEX_MARKET_DASHCAM_CATEGORY_URL MVIDEO_DASHCAM_CATEGORY_URL INGESTION_CATEGORY_DAILY_BUDGET
+  INGESTION_PRODUCT_DAILY_BUDGET INGESTION_SITEMAP_DAILY_BUDGET INGESTION_FETCH_DELAY_MIN_MS
+  INGESTION_FETCH_DELAY_MAX_MS INGESTION_WORKER_BATCH_SIZE INGESTION_WORKER_IDLE_MS INGESTION_WORKER_REPLICAS
+  TRAEFIK_ADMIN_PRIORITY
 )
 
 remote_command=""

deploy/scripts/predeploy-check.mjs

@@ -4,6 +4,9 @@ const required = [
   'RELEASE_ID',
   'COMMIT_SHA',
   'WEB_IMAGE',
+  'INGESTION_IMAGE',
+  'POSTGRES_PASSWORD',
+  'INGESTION_ADMIN_PASSWORD',
   'PUBLIC_HOST',
   'TRAEFIK_NETWORK',
   'SWARM_MANAGER_HOST',
@@ -16,10 +19,12 @@ if (missing.length > 0) {
   process.exit(1);
 }
 
-if (!/@sha256:[a-f0-9]{64}$/i.test(process.env.WEB_IMAGE ?? '')) {
+for (const imageName of ['WEB_IMAGE', 'INGESTION_IMAGE']) {
-  console.error(`WEB_IMAGE must be pinned by immutable digest, got: ${process.env.WEB_IMAGE}`);
+  if (!/@sha256:[a-f0-9]{64}$/i.test(process.env[imageName] ?? '')) {
+    console.error(`${imageName} must be pinned by immutable digest, got: ${process.env[imageName]}`);
     process.exit(1);
   }
+}
 
 console.log(
   JSON.stringify({

deploy/scripts/write-release-manifest.mjs

@@ -18,13 +18,19 @@ const manifest = {
   commitSha: requireEnv('COMMIT_SHA'),
   releaseId: requireEnv('RELEASE_ID'),
   webImageDigest: requireEnv('WEB_IMAGE_DIGEST'),
+  ingestionImageDigest: requireEnv('INGESTION_IMAGE_DIGEST'),
   deployTimestamp: new Date().toISOString(),
   smokeResult: smoke,
   workflowRunId: requireEnv('WORKFLOW_RUN_ID'),
 };
 
-if (!/@sha256:[a-f0-9]{64}$/i.test(manifest.webImageDigest)) {
+for (const [name, image] of Object.entries({
-  throw new Error('webImageDigest must be digest-pinned');
+  webImageDigest: manifest.webImageDigest,
+  ingestionImageDigest: manifest.ingestionImageDigest,
+})) {
+  if (!/@sha256:[a-f0-9]{64}$/i.test(image)) {
+    throw new Error(`${name} must be digest-pinned`);
+  }
 }
 
 console.log(JSON.stringify(manifest, null, 2));

deploy/swarm/docker-stack.yml

@@ -1,31 +1,6 @@
 version: "3.9"
 
-services:
+x-default-deploy: &default-deploy
-  web:
-    image: ${WEB_IMAGE}
-    environment:
-      NODE_ENV: production
-      APP_ENV: ${ENVIRONMENT}
-      PORT: "3000"
-      VIDEOREG_DB_FILE: /data/videoreg.json
-      VIDEOREG_DB_MEMORY_TTL_MS: ${VIDEOREG_DB_MEMORY_TTL_MS:-3600000}
-      VIDEOREG_NEXT_CACHE_REVALIDATE_SECONDS: ${VIDEOREG_NEXT_CACHE_REVALIDATE_SECONDS:-3600}
-      COMMIT_SHA: ${COMMIT_SHA}
-      RELEASE_ID: ${RELEASE_ID}
-      REGION: ${REGION:-}
-      TEAM_ID: ${TEAM_ID:-}
-    volumes:
-      - videoreg_data:/data
-    networks:
-      traefik: {}
-    healthcheck:
-      test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:3000/health/ready >/dev/null || exit 1"]
-      interval: 10s
-      timeout: 3s
-      retries: 3
-      start_period: 20s
-    deploy:
-      replicas: ${WEB_REPLICAS:-1}
   placement:
     constraints:
       - ${WEB_PLACEMENT_CONSTRAINT:-node.role == manager}
@@ -40,19 +15,135 @@ services:
     monitor: 45s
   restart_policy:
     condition: on-failure
+    delay: 5s
+    max_attempts: 5
+    window: 60s
+
+x-ingestion-env: &ingestion-env
+  INGESTION_DATABASE_URL: postgres://${POSTGRES_USER:-videoreg}:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required}@postgres:5432/${POSTGRES_DB:-videoreg_ingestion}
+  INGESTION_SNAPSHOT_DIR: /app/ingestion/data/snapshots
+  INGESTION_EXPORT_DIR: /exports
+  YANDEX_MARKET_DASHCAM_CATEGORY_URL: ${YANDEX_MARKET_DASHCAM_CATEGORY_URL:-https://market.yandex.ru/catalog--avtomobilnye-videoregistratory/82798275/list?hid=82798269}
+  MVIDEO_DASHCAM_CATEGORY_URL: ${MVIDEO_DASHCAM_CATEGORY_URL:-}
+  INGESTION_CATEGORY_DAILY_BUDGET: ${INGESTION_CATEGORY_DAILY_BUDGET:-25}
+  INGESTION_PRODUCT_DAILY_BUDGET: ${INGESTION_PRODUCT_DAILY_BUDGET:-100}
+  INGESTION_SITEMAP_DAILY_BUDGET: ${INGESTION_SITEMAP_DAILY_BUDGET:-25}
+  INGESTION_FETCH_DELAY_MIN_MS: ${INGESTION_FETCH_DELAY_MIN_MS:-5000}
+  INGESTION_FETCH_DELAY_MAX_MS: ${INGESTION_FETCH_DELAY_MAX_MS:-15000}
+  INGESTION_WORKER_BATCH_SIZE: ${INGESTION_WORKER_BATCH_SIZE:-1}
+  INGESTION_WORKER_IDLE_MS: ${INGESTION_WORKER_IDLE_MS:-30000}
+
+services:
+  web:
+    image: ${WEB_IMAGE:?WEB_IMAGE is required}
+    environment:
+      NODE_ENV: production
+      APP_ENV: ${ENVIRONMENT}
+      PORT: "3000"
+      VIDEOREG_DB_FILE: /data/videoreg.json
+      DASHCAM_CATALOG_EXPORT_DIR: /exports
+      VIDEOREG_DB_MEMORY_TTL_MS: ${VIDEOREG_DB_MEMORY_TTL_MS:-3600000}
+      VIDEOREG_NEXT_CACHE_REVALIDATE_SECONDS: ${VIDEOREG_NEXT_CACHE_REVALIDATE_SECONDS:-3600}
+      COMMIT_SHA: ${COMMIT_SHA}
+      RELEASE_ID: ${RELEASE_ID}
+      REGION: ${REGION:-}
+      TEAM_ID: ${TEAM_ID:-}
+    volumes:
+      - videoreg_data:/data
+      - catalog_exports:/exports:ro
+    networks:
+      traefik: {}
+    healthcheck:
+      test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:3000/health/ready >/dev/null || exit 1"]
+      interval: 10s
+      timeout: 3s
+      retries: 3
+      start_period: 20s
+    deploy:
+      <<: *default-deploy
+      replicas: ${WEB_REPLICAS:-1}
       labels:
         - "traefik.enable=true"
         - "traefik.docker.network=${TRAEFIK_NETWORK}"
         - "traefik.http.routers.${STACK_NAME}-web.rule=Host(`${PUBLIC_HOST}`) && PathPrefix(`/`)"
+        - "traefik.http.routers.${STACK_NAME}-web.priority=${TRAEFIK_WEB_PRIORITY:-1}"
         - "traefik.http.routers.${STACK_NAME}-web.entrypoints=${TRAEFIK_ENTRYPOINTS:-websecure}"
         - "traefik.http.routers.${STACK_NAME}-web.tls=${TRAEFIK_TLS:-true}"
+        - "traefik.http.routers.${STACK_NAME}-web.tls.certresolver=${TRAEFIK_CERTRESOLVER:-le}"
         - "traefik.http.services.${STACK_NAME}-web.loadbalancer.server.port=3000"
     stop_grace_period: 30s
 
+  postgres:
+    image: postgres:16-alpine
+    environment:
+      POSTGRES_DB: ${POSTGRES_DB:-videoreg_ingestion}
+      POSTGRES_USER: ${POSTGRES_USER:-videoreg}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required}
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-videoreg} -d ${POSTGRES_DB:-videoreg_ingestion}"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    volumes:
+      - ingestion_postgres:/var/lib/postgresql/data
+    networks:
+      - ingestion
+    deploy: *default-deploy
+
+  ingestion-worker:
+    image: ${INGESTION_IMAGE:?INGESTION_IMAGE is required}
+    command: sh -c "npm run ingestion:migrate && npm run ingestion:worker"
+    environment: *ingestion-env
+    volumes:
+      - ingestion_data:/app/ingestion/data
+      - catalog_exports:/exports
+    networks:
+      - ingestion
+    deploy:
+      <<: *default-deploy
+      replicas: ${INGESTION_WORKER_REPLICAS:-1}
+
+  ingestion-admin:
+    image: ${INGESTION_IMAGE:?INGESTION_IMAGE is required}
+    command: sh -c "npm run ingestion:migrate && npm run ingestion:admin"
+    environment:
+      <<: *ingestion-env
+      INGESTION_ADMIN_PORT: "4101"
+      INGESTION_ADMIN_USERNAME: ${INGESTION_ADMIN_USERNAME:-admin}
+      INGESTION_ADMIN_PASSWORD: ${INGESTION_ADMIN_PASSWORD:?INGESTION_ADMIN_PASSWORD is required}
+    volumes:
+      - ingestion_data:/app/ingestion/data
+    networks:
+      ingestion: {}
+      traefik: {}
+    healthcheck:
+      test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:4101/health >/dev/null || exit 1"]
+      interval: 15s
+      timeout: 3s
+      retries: 5
+      start_period: 20s
+    deploy:
+      <<: *default-deploy
+      replicas: 1
+      labels:
+        - "traefik.enable=true"
+        - "traefik.docker.network=${TRAEFIK_NETWORK}"
+        - "traefik.http.routers.${STACK_NAME}-admin.rule=Host(`${PUBLIC_HOST}`) && PathPrefix(`/admin`)"
+        - "traefik.http.routers.${STACK_NAME}-admin.priority=${TRAEFIK_ADMIN_PRIORITY:-100}"
+        - "traefik.http.routers.${STACK_NAME}-admin.entrypoints=${TRAEFIK_ENTRYPOINTS:-websecure}"
+        - "traefik.http.routers.${STACK_NAME}-admin.tls=${TRAEFIK_TLS:-true}"
+        - "traefik.http.routers.${STACK_NAME}-admin.tls.certresolver=${TRAEFIK_CERTRESOLVER:-le}"
+        - "traefik.http.services.${STACK_NAME}-admin.loadbalancer.server.port=4101"
+
 volumes:
   videoreg_data:
+  catalog_exports:
+  ingestion_postgres:
+  ingestion_data:
 
 networks:
+  ingestion:
+    driver: overlay
   traefik:
     external: true
     name: ${TRAEFIK_NETWORK}

ingestion/.dockerignore

@@ -0,0 +1,5 @@
+../.next
+../node_modules
+../data
+data
+coverage

ingestion/.env.example

@@ -0,0 +1,25 @@
+POSTGRES_DB=videoreg_ingestion
+POSTGRES_USER=videoreg
+POSTGRES_PASSWORD=change-me
+
+# Bind local operational ports to loopback.
+INGESTION_POSTGRES_BIND=127.0.0.1
+INGESTION_POSTGRES_PORT=5432
+INGESTION_ADMIN_BIND=127.0.0.1
+INGESTION_ADMIN_PORT=4101
+INGESTION_ADMIN_USERNAME=admin
+INGESTION_ADMIN_PASSWORD=replace-with-admin-password
+
+# Source seeds. MVideo is active when this URL is configured.
+YANDEX_MARKET_DASHCAM_CATEGORY_URL=https://market.yandex.ru/catalog--avtomobilnye-videoregistratory/82798275/list?hid=82798269
+MVIDEO_DASHCAM_CATEGORY_URL=https://www.mvideo.ru/product-list-page?q=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE%D1%80%D0%B5%D0%B3%D0%B8%D1%81%D1%82%D1%80%D0%B0%D1%82%D0%BE%D1%80%D1%8B
+
+# Conservative crawl limits. Local Compose maps the shared exports directory to
+# ../data/catalog-exports; the Next.js app reads that same directory directly.
+INGESTION_CATEGORY_DAILY_BUDGET=25
+INGESTION_PRODUCT_DAILY_BUDGET=100
+INGESTION_SITEMAP_DAILY_BUDGET=25
+INGESTION_FETCH_DELAY_MIN_MS=5000
+INGESTION_FETCH_DELAY_MAX_MS=15000
+INGESTION_WORKER_BATCH_SIZE=1
+INGESTION_WORKER_IDLE_MS=30000

ingestion/Dockerfile

@@ -0,0 +1,12 @@
+FROM node:24-alpine
+
+WORKDIR /app
+
+COPY package.json package-lock.json ./
+RUN npm ci
+
+COPY tsconfig.json ./
+COPY web-front ./web-front
+COPY ingestion ./ingestion
+
+CMD ["npm", "run", "ingestion:worker"]

ingestion/README.md

@@ -0,0 +1,228 @@
+# Dashcam catalog ingestion
+
+This stack runs in the same cloud as the Next.js application - one Swarm/Portainer environment, not a separate server. The ingestion side owns PostgreSQL, crawler state, raw snapshots, moderation state, and export artifacts. The `ingestion-worker` writes the approved catalog to a shared export volume, and the Next.js build/app reads those artifacts directly from that volume - no network call and no token. Integrity is guaranteed by the manifest `sha256` and a shared Zod schema validated at read/build time.
+
+> Note: co-locating ingestion and the web app on one shared export volume supersedes the earlier separate-server design (decision id `videoreg-nnv.3`).
+
+## Services
+
+`docker-compose.yml` starts:
+
+- `postgres` - source of truth for ingestion state.
+- `ingestion-worker` - HTTP-only crawler/parser/normalizer; also writes export artifacts to the shared export volume.
+- `ingestion-admin` - minimal operational endpoints and operator UI, internal/localhost only.
+
+The regular crawl path does not use a browser, stealth mode, proxy rotation, captcha bypass, or aggressive parallel loading.
+
+For Portainer-based deployment, deploy the same combined Swarm stack described
+in [`deploy/portainer`](../deploy/portainer/README.md).
+
+CI/CD builds and deploys both runtime images through `.gitea/workflows/master.yml`.
+
+## Server prerequisites
+
+- Docker with Compose v2.
+- Outbound HTTPS access to `market.yandex.ru` and, when enabled, `www.mvideo.ru`.
+- Persistent Docker volumes for `postgres-data` and `ingestion-data` plus the
+  shared catalog export directory.
+
+Do not expose Postgres or the admin server publicly. Keep them bound to `127.0.0.1` and use a private network for operator access.
+
+## First deploy in the application environment
+
+Deploy this stack in the same environment as the Next.js app. From the repository root:
+
+```bash
+cp ingestion/.env.example ingestion/.env
+```
+
+Edit `ingestion/.env`:
+
+```bash
+POSTGRES_PASSWORD=<strong-postgres-password>
+INGESTION_ADMIN_USERNAME=admin
+INGESTION_ADMIN_PASSWORD=<strong-admin-password>
+```
+
+The `ingestion-worker` writes the validated `manifest.json` and
+`artifacts/<exportId>/catalog.json` to `INGESTION_EXPORT_DIR` (`/app/exports`
+inside local Compose). Local Compose maps it to `data/catalog-exports`; the web
+app reads that directory directly.
+
+Start the stack:
+
+```bash
+docker compose --env-file ingestion/.env -f ingestion/docker-compose.yml up -d --build
+```
+
+Check status:
+
+```bash
+docker compose --env-file ingestion/.env -f ingestion/docker-compose.yml ps
+docker compose --env-file ingestion/.env -f ingestion/docker-compose.yml logs -f ingestion-worker
+curl http://127.0.0.1:4101/health
+```
+
+Open the ingestion admin UI:
+
+```text
+http://127.0.0.1:4101/admin
+```
+
+The UI is served by `ingestion-admin` itself and requires Basic Auth using
+`INGESTION_ADMIN_USERNAME` and `INGESTION_ADMIN_PASSWORD`. It is intentionally
+separate from the Next.js application and uses admin API endpoints under
+`/admin/api`.
+
+For a single public host/standard HTTPS port, configure Traefik to route by path:
+
+```text
+Host(`example.com`) && PathPrefix(`/admin`) -> ingestion-admin:4101
+Host(`example.com`) && PathPrefix(`/`)      -> Next.js:3000
+```
+
+Set a higher priority on the `/admin` router than the `/` router. Do not strip
+the `/admin` prefix; the admin app serves HTML/assets at `/admin/*` and its API
+at `/admin/api/*`.
+
+## Local smoke run without PostgreSQL
+
+For parser/debug checks you can run local HTTP-only smoke commands. They do not
+write to PostgreSQL and do not write snapshots.
+
+```bash
+npm run --silent ingestion:smoke-yandex
+npm run --silent ingestion:smoke-mvideo
+```
+
+Useful overrides:
+
+```bash
+INGESTION_SMOKE_LIMIT=5 \
+INGESTION_SMOKE_DELAY_MS=5000 \
+YANDEX_MARKET_DASHCAM_CATEGORY_URL="https://market.yandex.ru/catalog--avtomobilnye-videoregistratory/82798275/list?hid=82798269" \
+npm run --silent ingestion:smoke-yandex > /tmp/yandex-market-smoke.json
+
+INGESTION_SMOKE_LIMIT=5 \
+INGESTION_SMOKE_DELAY_MS=5000 \
+MVIDEO_DASHCAM_CATEGORY_URL="https://www.mvideo.ru/product-list-page?q=видеорегистраторы" \
+npm run --silent ingestion:smoke-mvideo > /tmp/mvideo-smoke.json
+```
+
+Configured category seed env vars:
+
+- `YANDEX_MARKET_DASHCAM_CATEGORY_URL` - active source, used by the worker and smoke run.
+- `MVIDEO_DASHCAM_CATEGORY_URL` - enables MVideo discovery and product parsing in
+  the worker. Leave it empty to disable that source.
+
+If a source returns a CAPTCHA, 403, or 429, the worker records the reason and
+pauses that source in PostgreSQL. It does not switch user agents, use a browser
+fallback, rotate proxies, or otherwise attempt to bypass the challenge. The
+queue and moderation UI keep the paused state visible to the operator.
+
+Progress goes to stderr, so stdout remains valid JSON. Each product includes:
+
+- `sources[]` with source name, URL, source product id, fetched time, title, and `stableContentHash`;
+- source card `title`;
+- `brand`, source `model`, `canonicalModel`, and `canonicalName`;
+- `technicalSpecs.normalized` for canonical keys used by matching/export;
+- `technicalSpecs.raw` for the fuller source characteristics with section groups, original labels, source values, and normalized key/value when known;
+- `moderation.conflicts`, currently empty in smoke mode because there is only one source per product;
+- `internalSignals.aggregateRating`, kept out of public export.
+
+## Operational endpoints
+
+Admin endpoints are intentionally small:
+
+```bash
+curl http://127.0.0.1:4101/health
+curl -u "$INGESTION_ADMIN_USERNAME:$INGESTION_ADMIN_PASSWORD" \
+  http://127.0.0.1:4101/admin/api/queue
+curl -u "$INGESTION_ADMIN_USERNAME:$INGESTION_ADMIN_PASSWORD" \
+  http://127.0.0.1:4101/admin/api/moderation
+curl -u "$INGESTION_ADMIN_USERNAME:$INGESTION_ADMIN_PASSWORD" \
+  http://127.0.0.1:4101/admin/api/moderation/products?status=needs_review
+curl -u "$INGESTION_ADMIN_USERNAME:$INGESTION_ADMIN_PASSWORD" \
+  http://127.0.0.1:4101/admin/api/manufacturers
+curl -u "$INGESTION_ADMIN_USERNAME:$INGESTION_ADMIN_PASSWORD" \
+  http://127.0.0.1:4101/admin/api/manufacturers/spawnson
+curl -u "$INGESTION_ADMIN_USERNAME:$INGESTION_ADMIN_PASSWORD" \
+  -X PATCH http://127.0.0.1:4101/admin/api/manufacturers/spawnson \
+  -H "content-type: application/json" \
+  -d '{"trustStatus":"untrusted","notes":"No stable manufacturer source found"}'
+```
+
+Manufacturer records are created automatically from parsed product brands. Use
+`trustStatus=untrusted` for brands that should not be trusted as authoritative
+manufacturer sources during moderation.
+
+When two sources produce the same canonical name, ingestion links their source
+records to one canonical product and queues `same_product` for review. Raw specs
+remain source-scoped; differing normalized values appear as source conflicts.
+This is an exact canonical-name match only, so variants are never merged by a
+fuzzy or image-based heuristic.
+
+The same manufacturer records can be edited in the ingestion admin UI at
+`http://127.0.0.1:4101/admin`.
+
+Product cards can be approved or rejected from the `Products` view in that UI.
+
+## Publishing an approved export
+
+The public export includes only approved canonical products. It excludes prices, sellers, availability, review texts, review authors, and aggregate rating.
+
+For the current MVP, moderation is represented in PostgreSQL. Approving a product means setting `products.moderation_status = 'approved'` and approving selected cached image artifacts in `product_images` once image storage policy is accepted. Products without approved image artifacts can still be exported with an empty `images` array.
+
+Generate the latest export:
+
+```bash
+docker compose --env-file ingestion/.env -f ingestion/docker-compose.yml run --rm ingestion-worker \
+  npm run ingestion:export-catalog
+```
+
+The job writes to the shared export volume:
+
+- `manifest.json`
+- `artifacts/<exportId>/catalog.json`
+
+These land directly on the shared export volume; nothing serves them over HTTP.
+
+## Importing into the Next.js/application
+
+The import reads the export artifacts directly from the shared export volume - no network call and no token. Point it at the mounted export directory and run:
+
+```bash
+DASHCAM_CATALOG_EXPORT_DIR=/path/to/exports npm run catalog:import
+```
+
+The import command:
+
+- reads `manifest.json` from the export directory;
+- validates `schemaVersion`, `exportId`, `createdAt`, `sha256`, and `itemCount`;
+- reads the catalog artifact from the same directory;
+- verifies the artifact SHA-256;
+- validates the payload through the shared Zod schema;
+- atomically updates `data/videoreg.json`.
+
+The Next.js application never connects to the ingestion PostgreSQL database; its only catalog contact is the export artifact on the shared volume.
+
+## Backups
+
+Back up both Docker volumes:
+
+- `postgres-data` - canonical ingestion database.
+- `ingestion-data` - raw snapshots.
+- `data/catalog-exports` locally, or `catalog_exports` in Swarm - approved
+  catalog artifacts shared with web.
+
+Losing `ingestion-data` removes audit/debug snapshots even if the database remains available.
+
+## Updating
+
+Pull the new repository state and rebuild:
+
+```bash
+docker compose --env-file ingestion/.env -f ingestion/docker-compose.yml up -d --build
+```
+
+Migrations run automatically before `ingestion-worker` and `ingestion-admin` start. They are tracked in the `schema_migrations` table.

ingestion/docker-compose.yml

@@ -0,0 +1,90 @@
+services:
+  postgres:
+    image: postgres:16-alpine
+    restart: unless-stopped
+    environment:
+      POSTGRES_DB: ${POSTGRES_DB:-videoreg_ingestion}
+      POSTGRES_USER: ${POSTGRES_USER:-videoreg}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-videoreg}
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-videoreg} -d ${POSTGRES_DB:-videoreg_ingestion}"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    ports:
+      - "${INGESTION_POSTGRES_BIND:-127.0.0.1}:${INGESTION_POSTGRES_PORT:-5432}:5432"
+    volumes:
+      - postgres-data:/var/lib/postgresql/data
+
+  ingestion-migrate:
+    build:
+      context: ..
+      dockerfile: ingestion/Dockerfile
+    command: npm run ingestion:migrate
+    restart: "no"
+    depends_on:
+      postgres:
+        condition: service_healthy
+    environment: &ingestion-env
+      INGESTION_DATABASE_URL: postgres://${POSTGRES_USER:-videoreg}:${POSTGRES_PASSWORD:-videoreg}@postgres:5432/${POSTGRES_DB:-videoreg_ingestion}
+      INGESTION_SNAPSHOT_DIR: /app/ingestion/data/snapshots
+      INGESTION_EXPORT_DIR: /app/exports
+      YANDEX_MARKET_DASHCAM_CATEGORY_URL: ${YANDEX_MARKET_DASHCAM_CATEGORY_URL:-https://market.yandex.ru/catalog--avtomobilnye-videoregistratory/82798275/list?hid=82798269}
+      MVIDEO_DASHCAM_CATEGORY_URL: ${MVIDEO_DASHCAM_CATEGORY_URL:-}
+      INGESTION_CATEGORY_DAILY_BUDGET: ${INGESTION_CATEGORY_DAILY_BUDGET:-25}
+      INGESTION_PRODUCT_DAILY_BUDGET: ${INGESTION_PRODUCT_DAILY_BUDGET:-100}
+      INGESTION_SITEMAP_DAILY_BUDGET: ${INGESTION_SITEMAP_DAILY_BUDGET:-25}
+      INGESTION_FETCH_DELAY_MIN_MS: ${INGESTION_FETCH_DELAY_MIN_MS:-5000}
+      INGESTION_FETCH_DELAY_MAX_MS: ${INGESTION_FETCH_DELAY_MAX_MS:-15000}
+      INGESTION_WORKER_BATCH_SIZE: ${INGESTION_WORKER_BATCH_SIZE:-1}
+      INGESTION_WORKER_IDLE_MS: ${INGESTION_WORKER_IDLE_MS:-30000}
+    volumes:
+      - ingestion-data:/app/ingestion/data
+      - ../data/catalog-exports:/app/exports
+
+  ingestion-worker:
+    build:
+      context: ..
+      dockerfile: ingestion/Dockerfile
+    command: npm run ingestion:worker
+    restart: unless-stopped
+    depends_on:
+      postgres:
+        condition: service_healthy
+      ingestion-migrate:
+        condition: service_completed_successfully
+    environment: *ingestion-env
+    volumes:
+      - ingestion-data:/app/ingestion/data
+      - ../data/catalog-exports:/app/exports
+
+  ingestion-admin:
+    build:
+      context: ..
+      dockerfile: ingestion/Dockerfile
+    command: npm run ingestion:admin
+    restart: unless-stopped
+    depends_on:
+      postgres:
+        condition: service_healthy
+      ingestion-migrate:
+        condition: service_completed_successfully
+    environment:
+      <<: *ingestion-env
+      INGESTION_ADMIN_PORT: 4101
+      INGESTION_ADMIN_USERNAME: ${INGESTION_ADMIN_USERNAME:-admin}
+      INGESTION_ADMIN_PASSWORD: ${INGESTION_ADMIN_PASSWORD:?INGESTION_ADMIN_PASSWORD is required}
+    ports:
+      - "${INGESTION_ADMIN_BIND:-127.0.0.1}:${INGESTION_ADMIN_PORT:-4101}:4101"
+    healthcheck:
+      test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:4101/health >/dev/null || exit 1"]
+      interval: 15s
+      timeout: 3s
+      retries: 5
+      start_period: 20s
+    volumes:
+      - ingestion-data:/app/ingestion/data
+
+volumes:
+  postgres-data:
+  ingestion-data:

ingestion/migrations/001_init.sql

@@ -0,0 +1,225 @@
+create extension if not exists pgcrypto;
+
+create table if not exists manufacturers (
+  id uuid primary key default gen_random_uuid(),
+  slug text not null unique,
+  name text not null,
+  trust_status text not null default 'unknown' check (trust_status in ('trusted', 'unknown', 'untrusted')),
+  notes text,
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now()
+);
+
+create table if not exists products (
+  id text primary key,
+  manufacturer_id uuid references manufacturers(id) on delete set null,
+  brand text not null,
+  model text not null,
+  canonical_name text not null,
+  title text not null,
+  moderation_status text not null default 'needs_review'
+    check (moderation_status in ('needs_review', 'approved', 'rejected', 'duplicate', 'variant', 'oem_clone')),
+  discovered_at timestamptz not null default now(),
+  updated_at timestamptz not null default now()
+);
+
+create table if not exists product_variants (
+  id uuid primary key default gen_random_uuid(),
+  product_id text references products(id) on delete cascade,
+  variant_kind text not null,
+  title text not null,
+  created_at timestamptz not null default now()
+);
+
+create table if not exists product_sources (
+  id uuid primary key default gen_random_uuid(),
+  product_id text references products(id) on delete set null,
+  source text not null check (source in ('yandex_market')),
+  url text not null,
+  source_product_id text,
+  category_url text,
+  title_preview text,
+  stable_content_hash text,
+  first_seen_at timestamptz not null default now(),
+  last_fetched_at timestamptz,
+  parsed_json jsonb not null default '{}'::jsonb,
+  state text not null default 'discovered',
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now(),
+  unique (source, url)
+);
+
+create index if not exists product_sources_product_id_idx on product_sources(product_id);
+create index if not exists product_sources_source_product_id_idx on product_sources(source, source_product_id);
+
+create table if not exists source_snapshots (
+  id uuid primary key default gen_random_uuid(),
+  source text not null check (source in ('yandex_market')),
+  url text not null,
+  fetched_at timestamptz not null,
+  http_status int not null,
+  raw_snapshot_hash text not null,
+  stable_content_hash text,
+  raw_html_path text,
+  parsed_json jsonb not null default '{}'::jsonb,
+  created_at timestamptz not null default now()
+);
+
+create index if not exists source_snapshots_url_idx on source_snapshots(source, url, fetched_at desc);
+create index if not exists source_snapshots_stable_hash_idx on source_snapshots(stable_content_hash);
+
+create table if not exists product_specs (
+  id uuid primary key default gen_random_uuid(),
+  product_id text not null references products(id) on delete cascade,
+  key text not null,
+  value jsonb not null,
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now(),
+  unique (product_id, key)
+);
+
+create table if not exists product_source_specs (
+  id uuid primary key default gen_random_uuid(),
+  product_id text not null references products(id) on delete cascade,
+  source text not null check (source in ('yandex_market')),
+  source_url text not null,
+  source_product_id text,
+  raw_group text,
+  raw_label text not null,
+  raw_value jsonb not null,
+  normalized_key text,
+  normalized_value jsonb,
+  value_hash text not null,
+  fetched_at timestamptz not null,
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now(),
+  unique (product_id, source, source_url, raw_label)
+);
+
+create index if not exists product_source_specs_product_key_idx on product_source_specs(product_id, normalized_key);
+
+create table if not exists product_images (
+  id uuid primary key default gen_random_uuid(),
+  product_id text not null references products(id) on delete cascade,
+  source text not null check (source in ('yandex_market')),
+  original_url text not null,
+  artifact_url text,
+  sha256 text,
+  width int,
+  height int,
+  sort_order int not null default 0,
+  status text not null default 'needs_review' check (status in ('needs_review', 'approved', 'rejected')),
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now(),
+  unique (product_id, original_url)
+);
+
+create table if not exists source_ratings (
+  id uuid primary key default gen_random_uuid(),
+  product_id text references products(id) on delete cascade,
+  source text not null check (source in ('yandex_market')),
+  source_product_id text,
+  rating_value numeric,
+  rating_scale numeric,
+  rating_count int,
+  fetched_at timestamptz not null,
+  created_at timestamptz not null default now()
+);
+
+create table if not exists crawl_jobs (
+  id uuid primary key default gen_random_uuid(),
+  source text not null check (source in ('yandex_market')),
+  url text not null,
+  job_type text not null check (job_type in ('category', 'product', 'sitemap')),
+  state text not null default 'pending' check (
+    state in (
+      'pending',
+      'fetching',
+      'fetched',
+      'parsed',
+      'normalized',
+      'deduplicated',
+      'needs_review',
+      'approved',
+      'exported',
+      'rate_limited',
+      'blocked',
+      'captcha',
+      'not_found',
+      'parse_failed',
+      'needs_manual_review'
+    )
+  ),
+  attempts int not null default 0,
+  next_run_at timestamptz not null default now(),
+  last_error text,
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now(),
+  unique (source, url, job_type)
+);
+
+create index if not exists crawl_jobs_pending_idx on crawl_jobs(state, next_run_at, created_at);
+
+create table if not exists crawl_errors (
+  id uuid primary key default gen_random_uuid(),
+  job_id uuid references crawl_jobs(id) on delete set null,
+  source text not null,
+  url text not null,
+  reason text not null,
+  message text not null,
+  created_at timestamptz not null default now()
+);
+
+create table if not exists source_pauses (
+  source text primary key check (source in ('yandex_market')),
+  reason text not null,
+  paused_until timestamptz not null,
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now()
+);
+
+create table if not exists source_daily_budgets (
+  source text not null check (source in ('yandex_market')),
+  budget_bucket text not null check (budget_bucket in ('category', 'product', 'sitemap')),
+  budget_day date not null,
+  daily_limit int not null,
+  used_count int not null default 0,
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now(),
+  primary key (source, budget_bucket, budget_day)
+);
+
+create table if not exists moderation_queue (
+  id uuid primary key default gen_random_uuid(),
+  product_id text not null references products(id) on delete cascade,
+  reason text not null check (
+    reason in ('new_product', 'same_product', 'possible_duplicate', 'variant', 'oem_clone', 'conflicting_specs')
+  ),
+  state text not null default 'pending' check (state in ('pending', 'resolved', 'dismissed')),
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now(),
+  unique (product_id, reason)
+);
+
+create table if not exists moderation_conflicts (
+  id uuid primary key default gen_random_uuid(),
+  product_id text not null references products(id) on delete cascade,
+  conflict_type text not null check (conflict_type in ('spec_value')),
+  key text not null,
+  values_json jsonb not null,
+  state text not null default 'pending' check (state in ('pending', 'resolved', 'dismissed')),
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now(),
+  unique (product_id, conflict_type, key)
+);
+
+create table if not exists export_runs (
+  id text primary key,
+  schema_version int not null,
+  created_at timestamptz not null,
+  artifact_path text not null,
+  manifest_path text not null,
+  sha256 text not null,
+  item_count int not null,
+  published_at timestamptz not null default now()
+);

ingestion/migrations/002_source_specs_and_conflicts.sql

@@ -0,0 +1,35 @@
+alter table products add column if not exists canonical_name text;
+update products set canonical_name = trim(brand || ' ' || model) where canonical_name is null;
+alter table products alter column canonical_name set not null;
+
+create table if not exists product_source_specs (
+  id uuid primary key default gen_random_uuid(),
+  product_id text not null references products(id) on delete cascade,
+  source text not null check (source in ('yandex_market')),
+  source_url text not null,
+  source_product_id text,
+  raw_group text,
+  raw_label text not null,
+  raw_value jsonb not null,
+  normalized_key text,
+  normalized_value jsonb,
+  value_hash text not null,
+  fetched_at timestamptz not null,
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now(),
+  unique (product_id, source, source_url, raw_label)
+);
+
+create index if not exists product_source_specs_product_key_idx on product_source_specs(product_id, normalized_key);
+
+create table if not exists moderation_conflicts (
+  id uuid primary key default gen_random_uuid(),
+  product_id text not null references products(id) on delete cascade,
+  conflict_type text not null check (conflict_type in ('spec_value')),
+  key text not null,
+  values_json jsonb not null,
+  state text not null default 'pending' check (state in ('pending', 'resolved', 'dismissed')),
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now(),
+  unique (product_id, conflict_type, key)
+);

ingestion/migrations/003_manufacturers.sql

@@ -0,0 +1,29 @@
+create table if not exists manufacturers (
+  id uuid primary key default gen_random_uuid(),
+  slug text not null unique,
+  name text not null,
+  trust_status text not null default 'unknown' check (trust_status in ('trusted', 'unknown', 'untrusted')),
+  notes text,
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now()
+);
+
+alter table products add column if not exists manufacturer_id uuid references manufacturers(id) on delete set null;
+
+insert into manufacturers (slug, name)
+select
+  lower(regexp_replace(trim(brand), '[^[:alnum:]]+', '-', 'g')) as slug,
+  brand as name
+from products
+where brand is not null
+on conflict (slug) do update
+set name = excluded.name,
+    updated_at = now();
+
+update products
+set manufacturer_id = manufacturers.id
+from manufacturers
+where products.manufacturer_id is null
+  and manufacturers.slug = lower(regexp_replace(trim(products.brand), '[^[:alnum:]]+', '-', 'g'));
+
+create index if not exists products_manufacturer_id_idx on products(manufacturer_id);

ingestion/migrations/004_mvideo_source.sql

@@ -0,0 +1,31 @@
+alter table product_sources drop constraint if exists product_sources_source_check;
+alter table product_sources add constraint product_sources_source_check
+  check (source in ('yandex_market', 'mvideo'));
+
+alter table source_snapshots drop constraint if exists source_snapshots_source_check;
+alter table source_snapshots add constraint source_snapshots_source_check
+  check (source in ('yandex_market', 'mvideo'));
+
+alter table product_source_specs drop constraint if exists product_source_specs_source_check;
+alter table product_source_specs add constraint product_source_specs_source_check
+  check (source in ('yandex_market', 'mvideo'));
+
+alter table product_images drop constraint if exists product_images_source_check;
+alter table product_images add constraint product_images_source_check
+  check (source in ('yandex_market', 'mvideo'));
+
+alter table source_ratings drop constraint if exists source_ratings_source_check;
+alter table source_ratings add constraint source_ratings_source_check
+  check (source in ('yandex_market', 'mvideo'));
+
+alter table crawl_jobs drop constraint if exists crawl_jobs_source_check;
+alter table crawl_jobs add constraint crawl_jobs_source_check
+  check (source in ('yandex_market', 'mvideo'));
+
+alter table source_pauses drop constraint if exists source_pauses_source_check;
+alter table source_pauses add constraint source_pauses_source_check
+  check (source in ('yandex_market', 'mvideo'));
+
+alter table source_daily_budgets drop constraint if exists source_daily_budgets_source_check;
+alter table source_daily_budgets add constraint source_daily_budgets_source_check
+  check (source in ('yandex_market', 'mvideo'));

ingestion/migrations/005_manual_canonical_overrides.sql

@@ -0,0 +1,3 @@
+alter table products
+  add column if not exists model_override boolean not null default false,
+  add column if not exists canonical_name_override boolean not null default false;

ingestion/migrations/006_canonical_product_grouping.sql

@@ -0,0 +1,2 @@
+create index if not exists products_canonical_name_ci_idx
+  on products (lower(canonical_name));

ingestion/src/adminAuth.test.ts

@@ -0,0 +1,28 @@
+import { describe, expect, it } from 'vitest';
+import { getAdminAuthConfig, isBasicAuthAuthorized } from './adminAuth.ts';
+
+function basic(username: string, password: string) {
+  return `Basic ${Buffer.from(`${username}:${password}`).toString('base64')}`;
+}
+
+describe('ingestion admin auth', () => {
+  it('requires an admin password', () => {
+    expect(() => getAdminAuthConfig({ INGESTION_ADMIN_USERNAME: 'admin' })).toThrow(
+      'INGESTION_ADMIN_PASSWORD is required',
+    );
+  });
+
+  it('accepts matching basic auth credentials', () => {
+    const config = { password: 'secret', username: 'operator' };
+
+    expect(isBasicAuthAuthorized(basic('operator', 'secret'), config)).toBe(true);
+  });
+
+  it('rejects missing or wrong basic auth credentials', () => {
+    const config = { password: 'secret', username: 'operator' };
+
+    expect(isBasicAuthAuthorized(undefined, config)).toBe(false);
+    expect(isBasicAuthAuthorized(basic('operator', 'wrong'), config)).toBe(false);
+    expect(isBasicAuthAuthorized(basic('wrong', 'secret'), config)).toBe(false);
+  });
+});

ingestion/src/adminAuth.ts

@@ -0,0 +1,45 @@
+import { timingSafeEqual } from 'node:crypto';
+
+export type AdminAuthConfig = {
+  password: string;
+  username: string;
+};
+
+export const adminAuthRealm = 'videoreg ingestion admin';
+
+export function getAdminAuthConfig(env: Record<string, string | undefined> = process.env): AdminAuthConfig {
+  const username = env.INGESTION_ADMIN_USERNAME ?? 'admin';
+  const password = env.INGESTION_ADMIN_PASSWORD;
+
+  if (!password) {
+    throw new Error('INGESTION_ADMIN_PASSWORD is required for ingestion-admin');
+  }
+
+  return { password, username };
+}
+
+export function isBasicAuthAuthorized(authorizationHeader: string | undefined, config: AdminAuthConfig) {
+  if (!authorizationHeader?.startsWith('Basic ')) {
+    return false;
+  }
+
+  const encodedCredentials = authorizationHeader.slice('Basic '.length).trim();
+  const decodedCredentials = Buffer.from(encodedCredentials, 'base64').toString('utf8');
+  const separatorIndex = decodedCredentials.indexOf(':');
+
+  if (separatorIndex < 0) {
+    return false;
+  }
+
+  const username = decodedCredentials.slice(0, separatorIndex);
+  const password = decodedCredentials.slice(separatorIndex + 1);
+
+  return safeEqual(username, config.username) && safeEqual(password, config.password);
+}
+
+function safeEqual(left: string, right: string) {
+  const leftBuffer = Buffer.from(left);
+  const rightBuffer = Buffer.from(right);
+
+  return leftBuffer.length === rightBuffer.length && timingSafeEqual(leftBuffer, rightBuffer);
+}

ingestion/src/adminServer.ts

@@ -0,0 +1,479 @@
+import http from 'node:http';
+import {
+  adminAuthRealm,
+  getAdminAuthConfig,
+  isBasicAuthAuthorized,
+  type AdminAuthConfig,
+} from './adminAuth.ts';
+import { adminUiCss, adminUiHtml, adminUiJs } from './adminUi.ts';
+import { createPgPool } from './db.ts';
+
+function sendJson(response: http.ServerResponse, status: number, payload: unknown) {
+  response.writeHead(status, { 'content-type': 'application/json; charset=utf-8' });
+  response.end(`${JSON.stringify(payload, null, 2)}\n`);
+}
+
+function sendText(response: http.ServerResponse, status: number, contentType: string, payload: string) {
+  response.writeHead(status, {
+    'cache-control': 'no-store',
+    'content-type': contentType,
+  });
+  response.end(payload);
+}
+
+function sendBasicAuthChallenge(response: http.ServerResponse) {
+  response.writeHead(401, {
+    'cache-control': 'no-store',
+    'content-type': 'text/plain; charset=utf-8',
+    'www-authenticate': `Basic realm="${adminAuthRealm}", charset="UTF-8"`,
+  });
+  response.end('Authentication required\n');
+}
+
+async function readJsonBody(request: http.IncomingMessage) {
+  const chunks: Buffer[] = [];
+
+  for await (const chunk of request) {
+    chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+  }
+
+  if (chunks.length === 0) return {};
+
+  return JSON.parse(Buffer.concat(chunks).toString('utf8')) as Record<string, unknown>;
+}
+
+function isTrustStatus(value: unknown): value is 'trusted' | 'unknown' | 'untrusted' {
+  return value === 'trusted' || value === 'unknown' || value === 'untrusted';
+}
+
+function isModerationStatus(value: unknown): value is 'needs_review' | 'approved' | 'rejected' | 'duplicate' | 'variant' | 'oem_clone' {
+  return ['needs_review', 'approved', 'rejected', 'duplicate', 'variant', 'oem_clone'].includes(String(value));
+}
+
+function isCanonicalText(value: unknown): value is string {
+  return typeof value === 'string' && value.trim().length > 0 && value.trim().length <= 300;
+}
+
+function isProtectedAdminPath(pathname: string) {
+  return (
+    pathname === '/admin' ||
+    pathname.startsWith('/admin/') ||
+    pathname === '/queue' ||
+    pathname === '/moderation' ||
+    pathname === '/manufacturers' ||
+    pathname.startsWith('/manufacturers/')
+  );
+}
+
+function adminApiPath(pathname: string) {
+  if (pathname === '/admin/api') return '/';
+  if (pathname.startsWith('/admin/api/')) return pathname.slice('/admin/api'.length);
+
+  return pathname;
+}
+
+type CreateAdminServerOptions = {
+  auth?: AdminAuthConfig;
+};
+
+export function createAdminServer(options: CreateAdminServerOptions = {}) {
+  const pool = createPgPool();
+  const auth = options.auth ?? getAdminAuthConfig();
+
+  return http.createServer(async (request, response) => {
+    const url = new URL(request.url ?? '/', 'http://localhost');
+
+    try {
+      if (request.method === 'GET' && url.pathname === '/') {
+        response.writeHead(302, { location: '/admin' });
+        response.end();
+        return;
+      }
+
+      if (isProtectedAdminPath(url.pathname) && !isBasicAuthAuthorized(request.headers.authorization, auth)) {
+        sendBasicAuthChallenge(response);
+        return;
+      }
+
+      if (request.method === 'GET' && url.pathname === '/admin') {
+        sendText(response, 200, 'text/html; charset=utf-8', adminUiHtml);
+        return;
+      }
+
+      if (request.method === 'GET' && url.pathname === '/admin/styles.css') {
+        sendText(response, 200, 'text/css; charset=utf-8', adminUiCss);
+        return;
+      }
+
+      if (request.method === 'GET' && url.pathname === '/admin/app.js') {
+        sendText(response, 200, 'text/javascript; charset=utf-8', adminUiJs);
+        return;
+      }
+
+      const apiPath = adminApiPath(url.pathname);
+
+      if (apiPath === '/health') {
+        await pool.query('select 1');
+        sendJson(response, 200, { status: 'ok' });
+        return;
+      }
+
+      if (apiPath === '/queue') {
+        const result = await pool.query(
+          `
+            select state, job_type as "jobType", count(*)::int as count
+            from crawl_jobs
+            group by state, job_type
+            order by state, job_type
+          `,
+        );
+
+        sendJson(response, 200, { jobs: result.rows });
+        return;
+      }
+
+      if (apiPath === '/moderation') {
+        const result = await pool.query(
+          `
+            select state, reason, count(*)::int as count
+            from moderation_queue
+            group by state, reason
+            order by state, reason
+          `,
+        );
+
+        sendJson(response, 200, { queue: result.rows });
+        return;
+      }
+
+      if (apiPath === '/moderation/products' && request.method === 'GET') {
+        const requestedStatus = url.searchParams.get('status') ?? 'needs_review';
+
+        if (!isModerationStatus(requestedStatus)) {
+          sendJson(response, 400, { error: 'invalid moderation status' });
+          return;
+        }
+
+        const result = await pool.query(
+          `
+            select
+              products.id,
+              products.brand,
+              products.model,
+              products.canonical_name as "canonicalName",
+              products.title,
+              products.moderation_status as "moderationStatus",
+              products.updated_at as "updatedAt",
+              count(distinct product_sources.id)::int as "sourceCount",
+              count(distinct product_source_specs.id)::int as "rawSpecCount",
+              count(distinct moderation_conflicts.id) filter (where moderation_conflicts.state = 'pending')::int as "pendingConflictCount"
+            from products
+            left join product_sources on product_sources.product_id = products.id
+            left join product_source_specs on product_source_specs.product_id = products.id
+            left join moderation_conflicts on moderation_conflicts.product_id = products.id
+            where products.moderation_status = $1
+            group by products.id
+            order by products.updated_at desc, products.brand asc, products.model asc
+          `,
+          [requestedStatus],
+        );
+
+        sendJson(response, 200, { products: result.rows });
+        return;
+      }
+
+      const moderationProductMatch = apiPath.match(/^\/moderation\/products\/([^/]+)$/);
+
+      if (moderationProductMatch && request.method === 'GET') {
+        const productId = decodeURIComponent(moderationProductMatch[1]);
+        const productResult = await pool.query(
+          `
+            select
+              id,
+              brand,
+              model as "canonicalModel",
+              canonical_name as "canonicalName",
+              title,
+              moderation_status as "moderationStatus",
+              model_override as "canonicalModelOverride",
+              canonical_name_override as "canonicalNameOverride",
+              updated_at as "updatedAt"
+            from products
+            where id = $1
+          `,
+          [productId],
+        );
+
+        if (!productResult.rows[0]) {
+          sendJson(response, 404, { error: 'product_not_found' });
+          return;
+        }
+
+        const [sources, specs, conflicts] = await Promise.all([
+          pool.query(
+            `
+              select
+                source,
+                url,
+                source_product_id as "sourceProductId",
+                stable_content_hash as "stableContentHash",
+                last_fetched_at as "lastFetchedAt"
+              from product_sources
+              where product_id = $1
+              order by source, url
+            `,
+            [productId],
+          ),
+          pool.query(
+            `
+              select
+                source,
+                source_url as "sourceUrl",
+                raw_group as "group",
+                raw_label as "label",
+                raw_value as "value",
+                normalized_key as "normalizedKey",
+                normalized_value as "normalizedValue"
+              from product_source_specs
+              where product_id = $1
+              order by source, raw_group nulls first, raw_label
+            `,
+            [productId],
+          ),
+          pool.query(
+            `
+              select
+                conflict_type as "conflictType",
+                key,
+                values_json as "values",
+                state
+              from moderation_conflicts
+              where product_id = $1
+              order by state, key
+            `,
+            [productId],
+          ),
+        ]);
+
+        sendJson(response, 200, {
+          product: {
+            ...productResult.rows[0],
+            conflicts: conflicts.rows,
+            rawSpecs: specs.rows,
+            sources: sources.rows,
+          },
+        });
+        return;
+      }
+
+      if (moderationProductMatch && request.method === 'PATCH') {
+        const body = await readJsonBody(request);
+        const hasModerationStatus = Object.hasOwn(body, 'moderationStatus');
+        const hasCanonicalModel = Object.hasOwn(body, 'canonicalModel');
+        const hasCanonicalName = Object.hasOwn(body, 'canonicalName');
+
+        if (hasModerationStatus && !isModerationStatus(body.moderationStatus)) {
+          sendJson(response, 400, { error: 'invalid moderationStatus' });
+          return;
+        }
+
+        if (hasCanonicalModel && !isCanonicalText(body.canonicalModel)) {
+          sendJson(response, 400, { error: 'canonicalModel must be a non-empty string up to 300 characters' });
+          return;
+        }
+
+        if (hasCanonicalName && !isCanonicalText(body.canonicalName)) {
+          sendJson(response, 400, { error: 'canonicalName must be a non-empty string up to 300 characters' });
+          return;
+        }
+
+        if (!hasModerationStatus && !hasCanonicalModel && !hasCanonicalName) {
+          sendJson(response, 400, { error: 'at least one product field is required' });
+          return;
+        }
+
+        const productId = decodeURIComponent(moderationProductMatch[1]);
+        const result = await pool.query(
+          `
+            update products
+            set moderation_status = case when $2::boolean then $3::text else moderation_status end,
+                model = case when $4::boolean then $5::text else model end,
+                canonical_name = case when $6::boolean then $7::text else canonical_name end,
+                model_override = case when $4::boolean then true else model_override end,
+                canonical_name_override = case when $6::boolean then true else canonical_name_override end,
+                updated_at = now()
+            where id = $1
+            returning
+              id,
+              model as "canonicalModel",
+              canonical_name as "canonicalName",
+              moderation_status as "moderationStatus",
+              model_override as "canonicalModelOverride",
+              canonical_name_override as "canonicalNameOverride",
+              updated_at as "updatedAt"
+          `,
+          [
+            productId,
+            hasModerationStatus,
+            hasModerationStatus ? body.moderationStatus : null,
+            hasCanonicalModel,
+            hasCanonicalModel ? String(body.canonicalModel).trim() : null,
+            hasCanonicalName,
+            hasCanonicalName ? String(body.canonicalName).trim() : null,
+          ],
+        );
+
+        if (!result.rows[0]) {
+          sendJson(response, 404, { error: 'product_not_found' });
+          return;
+        }
+
+        if (hasModerationStatus) {
+          await pool.query(
+            `
+              update moderation_queue
+              set state = case when $2 = 'needs_review' then 'pending' else 'resolved' end,
+                  updated_at = now()
+              where product_id = $1
+            `,
+            [productId, body.moderationStatus],
+          );
+
+          if (body.moderationStatus !== 'needs_review') {
+            await pool.query(
+              `
+                update crawl_jobs job
+                set state = $2,
+                    updated_at = now()
+                from product_sources source
+                where source.product_id = $1
+                  and job.source = source.source
+                  and job.url = source.url
+                  and job.job_type = 'product'
+                  and job.state = 'needs_review'
+              `,
+              [productId, body.moderationStatus === 'approved' ? 'approved' : 'needs_manual_review'],
+            );
+          }
+        }
+
+        sendJson(response, 200, { product: result.rows[0] });
+        return;
+      }
+
+      if (apiPath === '/manufacturers') {
+        const result = await pool.query(
+          `
+            select
+              manufacturers.slug,
+              manufacturers.name,
+              manufacturers.trust_status as "trustStatus",
+              manufacturers.notes,
+              count(products.id)::int as "productCount",
+              max(products.updated_at) as "lastProductUpdatedAt"
+            from manufacturers
+            left join products on products.manufacturer_id = manufacturers.id
+            group by manufacturers.id
+            order by manufacturers.trust_status desc, manufacturers.name asc
+          `,
+        );
+
+        sendJson(response, 200, { manufacturers: result.rows });
+        return;
+      }
+
+      const manufacturerMatch = apiPath.match(/^\/manufacturers\/([^/]+)$/);
+
+      if (manufacturerMatch && request.method === 'GET') {
+        const result = await pool.query(
+          `
+            select
+              manufacturers.slug,
+              manufacturers.name,
+              manufacturers.trust_status as "trustStatus",
+              manufacturers.notes,
+              coalesce(
+                jsonb_agg(
+                  jsonb_build_object(
+                    'id', products.id,
+                    'canonicalName', products.canonical_name,
+                    'title', products.title,
+                    'moderationStatus', products.moderation_status,
+                    'updatedAt', products.updated_at
+                  )
+                  order by products.updated_at desc
+                ) filter (where products.id is not null),
+                '[]'::jsonb
+              ) as products
+            from manufacturers
+            left join products on products.manufacturer_id = manufacturers.id
+            where manufacturers.slug = $1
+            group by manufacturers.id
+          `,
+          [decodeURIComponent(manufacturerMatch[1])],
+        );
+
+        if (!result.rows[0]) {
+          sendJson(response, 404, { error: 'manufacturer_not_found' });
+          return;
+        }
+
+        sendJson(response, 200, { manufacturer: result.rows[0] });
+        return;
+      }
+
+      if (manufacturerMatch && request.method === 'PATCH') {
+        const body = await readJsonBody(request);
+
+        if (!isTrustStatus(body.trustStatus)) {
+          sendJson(response, 400, { error: 'trustStatus must be trusted, unknown, or untrusted' });
+          return;
+        }
+
+        const hasNotes = Object.hasOwn(body, 'notes');
+
+        if (hasNotes && body.notes !== null && typeof body.notes !== 'string') {
+          sendJson(response, 400, { error: 'notes must be a string or null' });
+          return;
+        }
+
+        const result = await pool.query(
+          `
+            update manufacturers
+            set trust_status = $2,
+                notes = case when $3 then $4 else notes end,
+                updated_at = now()
+            where slug = $1
+            returning slug, name, trust_status as "trustStatus", notes
+          `,
+          [
+            decodeURIComponent(manufacturerMatch[1]),
+            body.trustStatus,
+            hasNotes,
+            hasNotes ? body.notes : null,
+          ],
+        );
+
+        if (!result.rows[0]) {
+          sendJson(response, 404, { error: 'manufacturer_not_found' });
+          return;
+        }
+
+        sendJson(response, 200, { manufacturer: result.rows[0] });
+        return;
+      }
+
+      sendJson(response, 404, { error: 'not_found' });
+    } catch (error) {
+      sendJson(response, 500, { error: (error as Error).message });
+    }
+  });
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  const port = Number(process.env.INGESTION_ADMIN_PORT ?? 4101);
+
+  createAdminServer().listen(port, () => {
+    console.log(`ingestion-admin listening on ${port}`);
+  });
+}

ingestion/src/adminUi.test.ts

@@ -0,0 +1,17 @@
+import { describe, expect, it } from 'vitest';
+import { adminUiHtml, adminUiJs } from './adminUi.ts';
+
+describe('ingestion admin UI', () => {
+  it('is served as a standalone ingestion app', () => {
+    expect(adminUiHtml).toContain('/admin/styles.css');
+    expect(adminUiHtml).toContain('/admin/app.js');
+    expect(adminUiHtml).not.toContain('/api/admin');
+    expect(adminUiJs).toContain("const adminApiBasePath = '/admin/api'");
+    expect(adminUiJs).toContain('data-product-form');
+    expect(adminUiJs).not.toContain("requestJson('/manufacturers");
+  });
+
+  it('ships valid browser JavaScript', () => {
+    expect(() => new Function(adminUiJs)).not.toThrow();
+  });
+});

ingestion/src/adminUi.ts

@@ -0,0 +1,790 @@
+export const adminUiHtml = `<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>Videoreg ingestion admin</title>
+    <link rel="stylesheet" href="/admin/styles.css" />
+  </head>
+  <body>
+    <main class="shell">
+      <header class="topbar">
+        <div>
+          <p class="eyebrow">catalog moderation</p>
+          <h1>Ingestion admin</h1>
+        </div>
+        <nav class="actions" aria-label="Admin navigation">
+          <a class="button ghost" href="/admin/api/health">Health</a>
+          <a class="button ghost" href="/admin/api/queue">Queue</a>
+          <a class="button ghost" href="/admin/api/moderation">Moderation</a>
+          <a class="button" href="#/moderation">Products</a>
+          <a class="button" href="#/">Manufacturers</a>
+        </nav>
+      </header>
+      <div id="app" class="app" aria-live="polite"></div>
+    </main>
+    <script src="/admin/app.js" type="module"></script>
+  </body>
+</html>
+`;
+
+export const adminUiCss = `
+:root {
+  color-scheme: light;
+  --surface-0: #f3f0e8;
+  --surface-1: #fffdf8;
+  --surface-2: #e9e4d8;
+  --text-primary: #202528;
+  --text-muted: #667076;
+  --text-subtle: #858076;
+  --border-subtle: #d7cec0;
+  --border-strong: #bdb3a4;
+  --accent: #56798a;
+  --accent-strong: #385f70;
+  --accent-soft: #dbe7e9;
+  --warning: #8f7044;
+  --warning-soft: #efe5d6;
+  --danger: #884b46;
+  --danger-soft: #eadbd8;
+  --success: #58705c;
+  --success-soft: #dde8dd;
+  --radius-sm: 4px;
+  --radius-md: 6px;
+  --radius-lg: 8px;
+  --shadow-soft: 0 14px 34px rgb(32 37 40 / 8%);
+}
+
+* {
+  box-sizing: border-box;
+}
+
+body {
+  margin: 0;
+  background: var(--surface-0);
+  color: var(--text-primary);
+  font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
+}
+
+a {
+  color: inherit;
+  text-decoration: none;
+}
+
+button,
+input,
+select,
+textarea {
+  font: inherit;
+}
+
+.shell {
+  width: min(1180px, calc(100% - 32px));
+  margin: 0 auto;
+  padding: 24px 0 40px;
+}
+
+.topbar {
+  display: flex;
+  align-items: flex-start;
+  justify-content: space-between;
+  gap: 24px;
+  border-bottom: 1px solid var(--border-subtle);
+  padding-bottom: 18px;
+}
+
+.eyebrow {
+  margin: 0 0 6px;
+  color: var(--accent-strong);
+  font-size: 0.78rem;
+  font-weight: 750;
+  text-transform: uppercase;
+}
+
+h1,
+h2,
+h3,
+p {
+  overflow-wrap: anywhere;
+}
+
+h1 {
+  margin: 0;
+  font-size: 2rem;
+  line-height: 1.1;
+}
+
+h2 {
+  margin: 0;
+  font-size: 1.1rem;
+}
+
+.actions {
+  display: flex;
+  flex-wrap: wrap;
+  gap: 8px;
+  justify-content: flex-end;
+}
+
+.button {
+  min-height: 38px;
+  display: inline-flex;
+  align-items: center;
+  justify-content: center;
+  border: 1px solid var(--border-strong);
+  border-radius: var(--radius-md);
+  background: var(--surface-1);
+  color: var(--text-primary);
+  cursor: pointer;
+  padding: 8px 12px;
+  font-size: 0.86rem;
+  font-weight: 700;
+}
+
+.button.primary {
+  border-color: var(--accent);
+  background: var(--accent);
+  color: #fff;
+}
+
+.button.ghost {
+  border-color: transparent;
+  background: transparent;
+  color: var(--text-muted);
+}
+
+.button:hover {
+  background: var(--surface-2);
+}
+
+.button.primary:hover {
+  background: var(--accent-strong);
+}
+
+.app {
+  display: grid;
+  gap: 20px;
+  padding-top: 22px;
+}
+
+.panel {
+  border: 1px solid var(--border-subtle);
+  border-radius: var(--radius-lg);
+  background: var(--surface-1);
+  box-shadow: var(--shadow-soft);
+}
+
+.panel-header {
+  display: flex;
+  align-items: flex-start;
+  justify-content: space-between;
+  gap: 16px;
+  border-bottom: 1px solid var(--border-subtle);
+  padding: 16px;
+}
+
+.panel-body {
+  padding: 16px;
+}
+
+.stats-grid {
+  display: grid;
+  grid-template-columns: repeat(4, minmax(120px, 1fr));
+  gap: 10px;
+}
+
+.metric {
+  min-height: 76px;
+  border: 1px solid var(--border-subtle);
+  border-radius: var(--radius-lg);
+  background: var(--surface-2);
+  padding: 12px;
+}
+
+.metric span {
+  display: block;
+  color: var(--text-muted);
+  font-size: 0.76rem;
+  font-weight: 750;
+  text-transform: uppercase;
+}
+
+.metric strong {
+  display: block;
+  margin-top: 7px;
+  font-size: 1.45rem;
+}
+
+.table-wrap {
+  overflow-x: auto;
+}
+
+table {
+  min-width: 760px;
+  width: 100%;
+  border-collapse: collapse;
+  text-align: left;
+  font-size: 0.9rem;
+}
+
+th,
+td {
+  border-bottom: 1px solid var(--border-subtle);
+  padding: 14px 16px;
+  vertical-align: top;
+}
+
+thead th {
+  background: var(--surface-1);
+  color: var(--text-muted);
+  font-size: 0.74rem;
+  font-weight: 750;
+  text-transform: uppercase;
+}
+
+tbody tr:hover {
+  background: color-mix(in srgb, var(--surface-2) 48%, transparent);
+}
+
+tbody tr:last-child th,
+tbody tr:last-child td {
+  border-bottom: 0;
+}
+
+.badge {
+  display: inline-flex;
+  min-height: 24px;
+  align-items: center;
+  border-radius: var(--radius-sm);
+  border: 1px solid var(--border-subtle);
+  padding: 4px 8px;
+  font-size: 0.72rem;
+  font-weight: 750;
+}
+
+.badge.trusted {
+  background: var(--success-soft);
+  color: var(--success);
+}
+
+.badge.unknown {
+  background: var(--warning-soft);
+  color: var(--warning);
+}
+
+.badge.untrusted {
+  background: var(--danger-soft);
+  color: var(--danger);
+}
+
+.muted {
+  color: var(--text-muted);
+}
+
+.mono {
+  font-family: "IBM Plex Mono", ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace;
+}
+
+.layout {
+  display: grid;
+  grid-template-columns: minmax(280px, 380px) minmax(0, 1fr);
+  gap: 20px;
+  align-items: start;
+}
+
+.form-grid {
+  display: grid;
+  gap: 14px;
+}
+
+.field {
+  display: grid;
+  gap: 6px;
+}
+
+.field span {
+  color: var(--text-muted);
+  font-size: 0.82rem;
+  font-weight: 700;
+}
+
+.field select,
+.field textarea {
+  width: 100%;
+  border: 1px solid var(--border-subtle);
+  border-radius: var(--radius-md);
+  background: var(--surface-1);
+  color: var(--text-primary);
+  padding: 9px 10px;
+}
+
+.field select {
+  min-height: 42px;
+}
+
+.field textarea {
+  min-height: 120px;
+  resize: vertical;
+}
+
+.status-line {
+  min-height: 24px;
+  color: var(--text-muted);
+  font-size: 0.88rem;
+  font-weight: 700;
+}
+
+.error,
+.empty {
+  border: 1px dashed var(--border-subtle);
+  border-radius: var(--radius-lg);
+  color: var(--text-muted);
+  padding: 22px;
+}
+
+.error strong {
+  display: block;
+  color: var(--danger);
+  margin-bottom: 6px;
+}
+
+@media (max-width: 900px) {
+  .topbar,
+  .layout {
+    display: grid;
+    grid-template-columns: 1fr;
+  }
+
+  .actions {
+    justify-content: flex-start;
+  }
+
+  .stats-grid {
+    grid-template-columns: repeat(2, minmax(0, 1fr));
+  }
+}
+
+@media (max-width: 560px) {
+  .shell {
+    width: min(100% - 20px, 1180px);
+    padding-top: 16px;
+  }
+
+  h1 {
+    font-size: 1.55rem;
+  }
+
+  .stats-grid {
+    grid-template-columns: 1fr;
+  }
+}
+`;
+
+export const adminUiJs = `
+const app = document.querySelector('#app');
+const adminApiBasePath = '/admin/api';
+const dateFormatter = new Intl.DateTimeFormat('en', {
+  dateStyle: 'medium',
+  timeStyle: 'short',
+});
+
+function escapeHtml(value) {
+  return String(value ?? '')
+    .replaceAll('&', '&amp;')
+    .replaceAll('<', '&lt;')
+    .replaceAll('>', '&gt;')
+    .replaceAll('"', '&quot;')
+    .replaceAll("'", '&#039;');
+}
+
+function formatDate(value) {
+  if (!value) return 'Never';
+  return dateFormatter.format(new Date(value));
+}
+
+function formatValue(value) {
+  if (value === undefined || value === null) return '—';
+  if (typeof value === 'string') return value;
+
+  return JSON.stringify(value);
+}
+
+function statusLabel(status) {
+  if (status === 'trusted') return 'Trusted';
+  if (status === 'untrusted') return 'Untrusted';
+  if (status === 'needs_review') return 'Needs review';
+  if (status === 'approved') return 'Approved';
+  if (status === 'rejected') return 'Rejected';
+  if (status === 'duplicate') return 'Duplicate';
+  if (status === 'variant') return 'Variant';
+  if (status === 'oem_clone') return 'OEM clone';
+  return 'Unknown';
+}
+
+async function requestJson(path, options) {
+  const response = await fetch(path, {
+    credentials: 'same-origin',
+    headers: {
+      accept: 'application/json',
+      ...(options?.headers ?? {}),
+    },
+    ...options,
+  });
+
+  const payload = await response.json().catch(() => undefined);
+
+  if (!response.ok) {
+    throw new Error(payload?.error ?? response.status + ' ' + response.statusText);
+  }
+
+  return payload;
+}
+
+function renderLoading(label) {
+  app.innerHTML = '<section class="panel"><div class="panel-body muted">' + escapeHtml(label) + '</div></section>';
+}
+
+function renderError(title, error) {
+  app.innerHTML =
+    '<section class="panel"><div class="panel-body"><div class="error">' +
+    '<strong>' + escapeHtml(title) + '</strong>' +
+    '<span>' + escapeHtml(error.message ?? error) + '</span>' +
+    '</div></div></section>';
+}
+
+function renderMetric(label, value) {
+  return '<div class="metric"><span>' + escapeHtml(label) + '</span><strong>' + escapeHtml(value) + '</strong></div>';
+}
+
+function renderBadge(status) {
+  return '<span class="badge ' + escapeHtml(status) + '">' + escapeHtml(statusLabel(status)) + '</span>';
+}
+
+function renderManufacturers(manufacturers) {
+  const stats = {
+    total: manufacturers.length,
+    trusted: manufacturers.filter((item) => item.trustStatus === 'trusted').length,
+    unknown: manufacturers.filter((item) => item.trustStatus === 'unknown').length,
+    untrusted: manufacturers.filter((item) => item.trustStatus === 'untrusted').length,
+  };
+
+  const rows = manufacturers
+    .map(
+      (manufacturer) =>
+        '<tr>' +
+        '<th scope="row"><div><strong>' + escapeHtml(manufacturer.name) + '</strong><br />' +
+        '<span class="mono muted">' + escapeHtml(manufacturer.slug) + '</span></div></th>' +
+        '<td>' + renderBadge(manufacturer.trustStatus) + '</td>' +
+        '<td class="mono">' + escapeHtml(manufacturer.productCount) + '</td>' +
+        '<td>' + escapeHtml(formatDate(manufacturer.lastProductUpdatedAt)) + '</td>' +
+        '<td class="muted">' + escapeHtml(manufacturer.notes || 'No notes') + '</td>' +
+        '<td><a class="button" href="#/manufacturers/' + encodeURIComponent(manufacturer.slug) + '">Edit</a></td>' +
+        '</tr>',
+    )
+    .join('');
+
+  app.innerHTML =
+    '<section class="stats-grid" aria-label="Manufacturer moderation stats">' +
+    renderMetric('Total', stats.total) +
+    renderMetric('Trusted', stats.trusted) +
+    renderMetric('Unknown', stats.unknown) +
+    renderMetric('Untrusted', stats.untrusted) +
+    '</section>' +
+    '<section class="panel"><div class="panel-header"><h2>Manufacturers</h2></div>' +
+    (manufacturers.length
+      ? '<div class="table-wrap"><table><thead><tr>' +
+        '<th scope="col">Brand</th><th scope="col">Trust</th><th scope="col">Products</th>' +
+        '<th scope="col">Updated</th><th scope="col">Notes</th><th scope="col">Actions</th>' +
+        '</tr></thead><tbody>' + rows + '</tbody></table></div>'
+      : '<div class="panel-body"><div class="empty">No manufacturers discovered yet</div></div>') +
+    '</section>';
+}
+
+function renderManufacturer(manufacturer) {
+  const products = manufacturer.products ?? [];
+  const productRows = products
+    .map(
+      (product) =>
+        '<tr>' +
+        '<th scope="row"><strong>' + escapeHtml(product.canonicalName || product.title) + '</strong><br />' +
+        '<span class="mono muted">' + escapeHtml(product.id) + '</span></th>' +
+        '<td>' + escapeHtml(product.moderationStatus) + '</td>' +
+        '<td>' + escapeHtml(formatDate(product.updatedAt)) + '</td>' +
+        '</tr>',
+    )
+    .join('');
+
+  app.innerHTML =
+    '<div class="layout">' +
+    '<section class="panel">' +
+    '<div class="panel-header"><div><h2>Moderation</h2><p class="muted">Mark unreliable brands so source conflicts are visible during moderation.</p></div>' +
+    renderBadge(manufacturer.trustStatus) + '</div>' +
+    '<div class="panel-body">' +
+    '<form class="form-grid" data-manufacturer-form data-slug="' + escapeHtml(manufacturer.slug) + '">' +
+    '<label class="field"><span>Trust status</span><select name="trustStatus">' +
+    '<option value="trusted"' + (manufacturer.trustStatus === 'trusted' ? ' selected' : '') + '>Trusted</option>' +
+    '<option value="unknown"' + (manufacturer.trustStatus === 'unknown' ? ' selected' : '') + '>Unknown</option>' +
+    '<option value="untrusted"' + (manufacturer.trustStatus === 'untrusted' ? ' selected' : '') + '>Untrusted</option>' +
+    '</select></label>' +
+    '<label class="field"><span>Notes</span><textarea name="notes" rows="6" placeholder="Why this manufacturer is trusted or not trusted">' +
+    escapeHtml(manufacturer.notes ?? '') + '</textarea></label>' +
+    '<div class="actions"><button class="button primary" type="submit">Save</button>' +
+    '<a class="button ghost" href="#/">Back</a></div>' +
+    '<div class="status-line" data-save-state></div>' +
+    '</form></div></section>' +
+    '<section class="panel"><div class="panel-header"><div><h2>Products</h2><p class="muted">Products currently linked to this manufacturer.</p></div>' +
+    '<span class="badge unknown">' + escapeHtml(products.length) + '</span></div>' +
+    (products.length
+      ? '<div class="table-wrap"><table><thead><tr><th scope="col">Product</th><th scope="col">Moderation</th><th scope="col">Updated</th></tr></thead><tbody>' +
+        productRows + '</tbody></table></div>'
+      : '<div class="panel-body"><div class="empty">No products linked to this manufacturer</div></div>') +
+    '</section></div>';
+}
+
+function renderModerationProducts(products) {
+  const rows = products
+    .map(
+      (product) =>
+        '<tr>' +
+        '<th scope="row"><strong>' + escapeHtml(product.canonicalName || product.title) + '</strong><br />' +
+        '<span class="muted">' + escapeHtml(product.title) + '</span></th>' +
+        '<td>' + renderBadge(product.moderationStatus) + '</td>' +
+        '<td class="mono">' + escapeHtml(product.sourceCount) + '</td>' +
+        '<td class="mono">' + escapeHtml(product.rawSpecCount) + '</td>' +
+        '<td class="mono">' + escapeHtml(product.pendingConflictCount) + '</td>' +
+        '<td>' +
+          '<div class="actions">' +
+          '<a class="button" href="#/moderation/' + encodeURIComponent(product.id) + '">Edit</a>' +
+          '<button class="button primary" type="button" data-product-status="approved" data-product-id="' + escapeHtml(product.id) + '">Approve</button>' +
+          '<button class="button" type="button" data-product-status="rejected" data-product-id="' + escapeHtml(product.id) + '">Reject</button>' +
+          '</div>' +
+        '</td>' +
+        '</tr>',
+    )
+    .join('');
+
+  app.innerHTML =
+    '<section class="panel"><div class="panel-header"><div><h2>Product moderation</h2><p class="muted">Approve source cards after checking title, canonical model and source conflicts.</p></div>' +
+    '<span class="badge unknown">' + escapeHtml(products.length) + '</span></div>' +
+    (products.length
+      ? '<div class="table-wrap"><table><thead><tr>' +
+        '<th scope="col">Product</th><th scope="col">Status</th><th scope="col">Sources</th>' +
+        '<th scope="col">Raw specs</th><th scope="col">Conflicts</th><th scope="col">Actions</th>' +
+        '</tr></thead><tbody>' + rows + '</tbody></table></div>'
+      : '<div class="panel-body"><div class="empty">No products need moderation</div></div>') +
+    '</section>';
+}
+
+function renderProductEditor(product) {
+  const sourceRows = (product.sources ?? [])
+    .map(
+      (source) =>
+        '<tr><td>' + escapeHtml(source.source) + '</td><td class="mono">' +
+        escapeHtml(source.sourceProductId ?? '—') + '</td><td><a class="source-link" href="' +
+        escapeHtml(source.url) + '" target="_blank" rel="noreferrer">Source card</a></td><td>' +
+        escapeHtml(formatDate(source.lastFetchedAt)) + '</td></tr>',
+    )
+    .join('');
+  const rawSpecRows = (product.rawSpecs ?? [])
+    .map(
+      (spec) =>
+        '<tr><td>' + escapeHtml(spec.source) + '</td><td>' + escapeHtml(spec.group ?? '—') + '</td><th scope="row">' +
+        escapeHtml(spec.label) + '</th><td>' + escapeHtml(formatValue(spec.value)) + '</td><td class="mono">' +
+        escapeHtml(spec.normalizedKey ?? '—') + '</td><td>' + escapeHtml(formatValue(spec.normalizedValue)) + '</td></tr>',
+    )
+    .join('');
+  const conflictRows = (product.conflicts ?? [])
+    .map(
+      (conflict) =>
+        '<tr><th scope="row">' + escapeHtml(conflict.key) + '</th><td>' + renderBadge(conflict.state) + '</td><td class="mono">' +
+        escapeHtml(formatValue(conflict.values)) + '</td></tr>',
+    )
+    .join('');
+
+  app.innerHTML =
+    '<div class="layout">' +
+    '<section class="panel"><div class="panel-header"><div><h2>Product editor</h2><p class="muted">' +
+    escapeHtml(product.title) + '</p></div>' + renderBadge(product.moderationStatus) + '</div><div class="panel-body">' +
+    '<form class="form-grid" data-product-form data-product-id="' + escapeHtml(product.id) + '">' +
+    '<label class="field"><span>Canonical model</span><input name="canonicalModel" value="' +
+    escapeHtml(product.canonicalModel) + '" required maxlength="300" /></label>' +
+    '<label class="field"><span>Canonical name</span><input name="canonicalName" value="' +
+    escapeHtml(product.canonicalName) + '" required maxlength="300" /></label>' +
+    '<label class="field"><span>Moderation status</span><select name="moderationStatus">' +
+    ['needs_review', 'approved', 'rejected', 'duplicate', 'variant', 'oem_clone']
+      .map((status) => '<option value="' + status + '"' + (status === product.moderationStatus ? ' selected' : '') + '>' + escapeHtml(statusLabel(status)) + '</option>')
+      .join('') +
+    '</select></label><div class="actions"><button class="button primary" type="submit">Save</button>' +
+    '<a class="button ghost" href="#/moderation">Back</a></div><div class="status-line" data-save-state></div></form></div></section>' +
+    '<section class="panel"><div class="panel-header"><div><h2>Sources</h2></div><span class="badge unknown">' +
+    escapeHtml((product.sources ?? []).length) + '</span></div>' +
+    (sourceRows ? '<div class="table-wrap"><table><thead><tr><th>Source</th><th>ID</th><th>Card</th><th>Fetched</th></tr></thead><tbody>' + sourceRows + '</tbody></table></div>' : '<div class="panel-body"><div class="empty">No source records</div></div>') +
+    '</section></div>' +
+    '<section class="panel"><div class="panel-header"><div><h2>Raw specifications</h2></div><span class="badge unknown">' +
+    escapeHtml((product.rawSpecs ?? []).length) + '</span></div>' +
+    (rawSpecRows ? '<div class="table-wrap"><table><thead><tr><th>Source</th><th>Group</th><th>Label</th><th>Raw value</th><th>Normalized key</th><th>Normalized value</th></tr></thead><tbody>' + rawSpecRows + '</tbody></table></div>' : '<div class="panel-body"><div class="empty">No raw specifications</div></div>') +
+    '</section>' +
+    '<section class="panel"><div class="panel-header"><div><h2>Source conflicts</h2></div><span class="badge unknown">' +
+    escapeHtml((product.conflicts ?? []).length) + '</span></div>' +
+    (conflictRows ? '<div class="table-wrap"><table><thead><tr><th>Key</th><th>Status</th><th>Values</th></tr></thead><tbody>' + conflictRows + '</tbody></table></div>' : '<div class="panel-body"><div class="empty">No source conflicts</div></div>') +
+    '</section>';
+}
+
+async function loadManufacturers() {
+  renderLoading('Loading manufacturers');
+  try {
+    const payload = await requestJson(adminApiBasePath + '/manufacturers');
+    renderManufacturers(payload.manufacturers ?? []);
+  } catch (error) {
+    renderError('Manufacturers could not be loaded', error);
+  }
+}
+
+async function loadManufacturer(slug) {
+  renderLoading('Loading manufacturer');
+  try {
+    const payload = await requestJson(adminApiBasePath + '/manufacturers/' + encodeURIComponent(slug));
+    renderManufacturer(payload.manufacturer);
+  } catch (error) {
+    renderError('Manufacturer could not be loaded', error);
+  }
+}
+
+async function loadModerationProducts() {
+  renderLoading('Loading product moderation');
+  try {
+    const payload = await requestJson(adminApiBasePath + '/moderation/products?status=needs_review');
+    renderModerationProducts(payload.products ?? []);
+  } catch (error) {
+    renderError('Product moderation could not be loaded', error);
+  }
+}
+
+async function loadModerationProduct(productId) {
+  renderLoading('Loading product');
+  try {
+    const payload = await requestJson(adminApiBasePath + '/moderation/products/' + encodeURIComponent(productId));
+    renderProductEditor(payload.product);
+  } catch (error) {
+    renderError('Product could not be loaded', error);
+  }
+}
+
+function route() {
+  const hash = window.location.hash || '#/';
+  const match = hash.match(/^#\\/manufacturers\\/(.+)$/);
+
+  if (match) {
+    loadManufacturer(decodeURIComponent(match[1]));
+    return;
+  }
+
+  const productMatch = hash.match(/^#\\/moderation\\/([^/]+)$/);
+
+  if (productMatch) {
+    loadModerationProduct(decodeURIComponent(productMatch[1]));
+    return;
+  }
+
+  if (hash === '#/moderation') {
+    loadModerationProducts();
+    return;
+  }
+
+  loadManufacturers();
+}
+
+app.addEventListener('submit', async (event) => {
+  const form = event.target.closest('[data-manufacturer-form]');
+
+  if (!form) return;
+
+  event.preventDefault();
+  const state = form.querySelector('[data-save-state]');
+  const slug = form.dataset.slug;
+  const formData = new FormData(form);
+  const trustStatus = formData.get('trustStatus');
+  const notesValue = String(formData.get('notes') ?? '').trim();
+
+  state.textContent = 'Saving';
+
+  try {
+    await requestJson(adminApiBasePath + '/manufacturers/' + encodeURIComponent(slug), {
+      method: 'PATCH',
+      headers: {
+        'content-type': 'application/json',
+      },
+      body: JSON.stringify({
+        trustStatus,
+        notes: notesValue || null,
+      }),
+    });
+    state.textContent = 'Saved';
+    await loadManufacturer(slug);
+  } catch (error) {
+    state.textContent = error.message ?? String(error);
+  }
+});
+
+app.addEventListener('click', async (event) => {
+  const button = event.target.closest('[data-product-status]');
+
+  if (!button) return;
+
+  const productId = button.dataset.productId;
+  const moderationStatus = button.dataset.productStatus;
+
+  if (!productId || !moderationStatus) return;
+
+  button.disabled = true;
+
+  try {
+    await requestJson(adminApiBasePath + '/moderation/products/' + encodeURIComponent(productId), {
+      method: 'PATCH',
+      headers: {
+        'content-type': 'application/json',
+      },
+      body: JSON.stringify({ moderationStatus }),
+    });
+    await loadModerationProducts();
+  } catch (error) {
+    button.disabled = false;
+    window.alert(error.message ?? String(error));
+  }
+});
+
+app.addEventListener('submit', async (event) => {
+  const form = event.target.closest('[data-product-form]');
+
+  if (!form) return;
+
+  event.preventDefault();
+  const state = form.querySelector('[data-save-state]');
+  const productId = form.dataset.productId;
+  const formData = new FormData(form);
+
+  if (!productId) return;
+
+  state.textContent = 'Saving';
+
+  try {
+    await requestJson(adminApiBasePath + '/moderation/products/' + encodeURIComponent(productId), {
+      method: 'PATCH',
+      headers: {
+        'content-type': 'application/json',
+      },
+      body: JSON.stringify({
+        canonicalModel: String(formData.get('canonicalModel') ?? '').trim(),
+        canonicalName: String(formData.get('canonicalName') ?? '').trim(),
+        moderationStatus: formData.get('moderationStatus'),
+      }),
+    });
+    state.textContent = 'Saved';
+    await loadModerationProduct(productId);
+  } catch (error) {
+    state.textContent = error.message ?? String(error);
+  }
+});
+
+window.addEventListener('hashchange', route);
+route();
+`;

ingestion/src/config.test.ts

@@ -0,0 +1,34 @@
+import { describe, expect, it } from 'vitest';
+import { buildSourceSeeds, yandexMarketDashcamCategoryUrl } from './config.ts';
+
+describe('ingestion config', () => {
+  it('always includes implemented Yandex Market source seed', () => {
+    expect(buildSourceSeeds({})).toEqual([
+      {
+        categoryUrl: yandexMarketDashcamCategoryUrl,
+        implemented: true,
+        source: 'yandex_market',
+      },
+    ]);
+  });
+
+  it('enables an explicitly configured MVideo source seed in the worker queue', () => {
+    expect(
+      buildSourceSeeds({
+        MVIDEO_DASHCAM_CATEGORY_URL: 'https://www.mvideo.ru/avtomobilnye-videoregistratory',
+        YANDEX_MARKET_DASHCAM_CATEGORY_URL: 'https://market.yandex.ru/custom',
+      }),
+    ).toEqual([
+      {
+        categoryUrl: 'https://market.yandex.ru/custom',
+        implemented: true,
+        source: 'yandex_market',
+      },
+      {
+        categoryUrl: 'https://www.mvideo.ru/avtomobilnye-videoregistratory',
+        implemented: true,
+        source: 'mvideo',
+      },
+    ]);
+  });
+});

ingestion/src/config.ts

@@ -0,0 +1,57 @@
+import path from 'node:path';
+import type { MarketSource } from './types.ts';
+
+export const yandexMarketDashcamCategoryUrl =
+  'https://market.yandex.ru/catalog--avtomobilnye-videoregistratory/82798275/list?hid=82798269';
+export const mvideoDashcamCategoryUrl =
+  'https://www.mvideo.ru/product-list-page?q=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE%D1%80%D0%B5%D0%B3%D0%B8%D1%81%D1%82%D1%80%D0%B0%D1%82%D0%BE%D1%80%D1%8B';
+
+export type SourceSeed = {
+  categoryUrl: string;
+  implemented: boolean;
+  source: MarketSource;
+};
+
+function numberEnv(name: string, fallback: number) {
+  const value = Number(process.env[name]);
+
+  return Number.isFinite(value) ? value : fallback;
+}
+
+export function buildSourceSeeds(env: Record<string, string | undefined> = process.env): SourceSeed[] {
+  const seeds: SourceSeed[] = [
+    {
+      categoryUrl: env.YANDEX_MARKET_DASHCAM_CATEGORY_URL ?? yandexMarketDashcamCategoryUrl,
+      implemented: true,
+      source: 'yandex_market',
+    },
+  ];
+
+  if (env.MVIDEO_DASHCAM_CATEGORY_URL) {
+    seeds.push({
+      categoryUrl: env.MVIDEO_DASHCAM_CATEGORY_URL,
+      implemented: true,
+      source: 'mvideo',
+    });
+  }
+
+  return seeds;
+}
+
+const sourceSeeds = buildSourceSeeds();
+
+export const ingestionConfig = {
+  databaseUrl: process.env.INGESTION_DATABASE_URL ?? 'postgres://videoreg:videoreg@localhost:5432/videoreg_ingestion',
+  snapshotDir: process.env.INGESTION_SNAPSHOT_DIR ?? path.join(process.cwd(), 'ingestion', 'data', 'snapshots'),
+  exportDir: process.env.INGESTION_EXPORT_DIR ?? path.join(process.cwd(), 'ingestion', 'data', 'exports'),
+  seedCategoryUrl: process.env.YANDEX_MARKET_DASHCAM_CATEGORY_URL ?? yandexMarketDashcamCategoryUrl,
+  sourceSeeds,
+  implementedSourceSeeds: sourceSeeds.filter((seed) => seed.implemented),
+  fetchDelayMinMs: numberEnv('INGESTION_FETCH_DELAY_MIN_MS', 5_000),
+  fetchDelayMaxMs: numberEnv('INGESTION_FETCH_DELAY_MAX_MS', 15_000),
+  categoryDailyBudget: numberEnv('INGESTION_CATEGORY_DAILY_BUDGET', 25),
+  productDailyBudget: numberEnv('INGESTION_PRODUCT_DAILY_BUDGET', 100),
+  sitemapDailyBudget: numberEnv('INGESTION_SITEMAP_DAILY_BUDGET', 25),
+  workerBatchSize: numberEnv('INGESTION_WORKER_BATCH_SIZE', 1),
+  workerIdleMs: numberEnv('INGESTION_WORKER_IDLE_MS', 30_000),
+};

ingestion/src/db.ts

@@ -0,0 +1,10 @@
+import pg from 'pg';
+import { ingestionConfig } from './config.ts';
+
+const { Pool } = pg;
+
+export function createPgPool(connectionString = ingestionConfig.databaseUrl) {
+  return new Pool({
+    connectionString,
+  });
+}

ingestion/src/exportCatalog.ts

@@ -0,0 +1,12 @@
+import { createPgPool } from './db.ts';
+import { exportApprovedCatalog } from './exporter.ts';
+
+const pool = createPgPool();
+
+try {
+  const manifest = await exportApprovedCatalog(pool);
+
+  console.log(JSON.stringify(manifest, null, 2));
+} finally {
+  await pool.end();
+}

ingestion/src/exporter.ts

@@ -0,0 +1,184 @@
+import { createHash } from 'node:crypto';
+import { mkdir, rename, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import type { Pool } from 'pg';
+import {
+  DashcamCatalogExportSchema,
+  type DashcamCatalogExport,
+  type MarketSource,
+} from '../../web-front/src/entities/dashcam-catalog/model/schema.ts';
+import { ingestionConfig } from './config.ts';
+
+type ProductRow = {
+  id: string;
+  brand: string;
+  model: string;
+  canonical_name: string;
+  title: string;
+  discovered_at: Date;
+  updated_at: Date;
+};
+
+type SpecRow = {
+  product_id: string;
+  key: string;
+  value: string | number | boolean;
+};
+
+type ImageRow = {
+  product_id: string;
+  artifact_url: string;
+  original_url: string;
+  source: MarketSource;
+  sha256: string | null;
+  width: number | null;
+  height: number | null;
+};
+
+type SourceRow = {
+  product_id: string;
+  source: MarketSource;
+  source_product_id: string | null;
+  url: string;
+  stable_content_hash: string;
+  last_fetched_at: Date;
+};
+
+function sha256(buffer: Buffer) {
+  return createHash('sha256').update(buffer).digest('hex');
+}
+
+function exportId(now: Date) {
+  return `catalog_${now.toISOString().replace(/[-:.]/g, '').replace('T', '_').replace('Z', '')}`;
+}
+
+function groupRows<T extends { product_id: string }>(rows: T[]) {
+  const grouped = new Map<string, T[]>();
+
+  for (const row of rows) {
+    grouped.set(row.product_id, [...(grouped.get(row.product_id) ?? []), row]);
+  }
+
+  return grouped;
+}
+
+async function writeFileAtomically(filePath: string, contents: Buffer) {
+  const temporaryPath = `${filePath}.${process.pid}.tmp`;
+
+  await mkdir(path.dirname(filePath), { recursive: true });
+  await writeFile(temporaryPath, contents);
+  await rename(temporaryPath, filePath);
+}
+
+export async function exportApprovedCatalog(pool: Pool, now = new Date()) {
+  const createdAt = now.toISOString();
+  const id = exportId(now);
+  const [products, specs, images, sources] = await Promise.all([
+    pool.query<ProductRow>(
+      `
+        select id, brand, model, canonical_name, title, discovered_at, updated_at
+        from products
+        where moderation_status = 'approved'
+        order by brand asc, model asc, id asc
+      `,
+    ),
+    pool.query<SpecRow>('select product_id, key, value from product_specs'),
+    pool.query<ImageRow>(
+      `
+        select product_id, artifact_url, original_url, source, sha256, width, height
+        from product_images
+        where status = 'approved'
+          and artifact_url is not null
+        order by product_id, sort_order asc, original_url asc
+      `,
+    ),
+    pool.query<SourceRow>(
+      `
+        select product_id, source, source_product_id, url, stable_content_hash, last_fetched_at
+        from product_sources
+        where product_id is not null
+          and stable_content_hash is not null
+          and last_fetched_at is not null
+        order by product_id, source, url
+      `,
+    ),
+  ]);
+
+  const specsByProduct = groupRows(specs.rows);
+  const imagesByProduct = groupRows(images.rows);
+  const sourcesByProduct = groupRows(sources.rows);
+  const catalog: DashcamCatalogExport = {
+    schemaVersion: 1,
+    exportId: id,
+    createdAt,
+    products: products.rows
+      .map((product) => ({
+        id: product.id,
+        brand: product.brand,
+        model: product.model,
+        canonicalName: product.canonical_name,
+        title: product.title,
+        specs: Object.fromEntries((specsByProduct.get(product.id) ?? []).map((spec) => [spec.key, spec.value])),
+        images: (imagesByProduct.get(product.id) ?? []).map((image) => ({
+          url: image.artifact_url,
+          originalUrl: image.original_url,
+          source: image.source,
+          ...(image.sha256 ? { sha256: image.sha256 } : {}),
+          ...(image.width ? { width: image.width } : {}),
+          ...(image.height ? { height: image.height } : {}),
+        })),
+        sources: (sourcesByProduct.get(product.id) ?? []).map((source) => ({
+          source: source.source,
+          ...(source.source_product_id ? { sourceProductId: source.source_product_id } : {}),
+          url: source.url,
+          stableContentHash: source.stable_content_hash,
+          fetchedAt: source.last_fetched_at.toISOString(),
+        })),
+        discoveredAt: product.discovered_at.toISOString(),
+        updatedAt: product.updated_at.toISOString(),
+      }))
+      .filter((product) => product.sources.length > 0),
+  };
+
+  const validatedCatalog = DashcamCatalogExportSchema.parse(catalog);
+  const artifactBuffer = Buffer.from(`${JSON.stringify(validatedCatalog, null, 2)}\n`);
+  const artifactHash = sha256(artifactBuffer);
+  const artifactRelativePath = path.posix.join('artifacts', id, 'catalog.json');
+  const artifactPath = path.join(ingestionConfig.exportDir, artifactRelativePath);
+  const manifest = {
+    schemaVersion: 1 as const,
+    exportId: id,
+    createdAt,
+    artifactPath: artifactRelativePath,
+    sha256: artifactHash,
+    itemCount: validatedCatalog.products.length,
+  };
+  const manifestBuffer = Buffer.from(`${JSON.stringify(manifest, null, 2)}\n`);
+
+  await writeFileAtomically(artifactPath, artifactBuffer);
+  await writeFileAtomically(path.join(ingestionConfig.exportDir, 'manifest.json'), manifestBuffer);
+  await pool.query(
+    `
+      insert into export_runs (id, schema_version, created_at, artifact_path, manifest_path, sha256, item_count)
+      values ($1, 1, $2, $3, $4, $5, $6)
+      on conflict (id) do nothing
+    `,
+    [id, createdAt, artifactPath, path.join(ingestionConfig.exportDir, 'manifest.json'), artifactHash, manifest.itemCount],
+  );
+  await pool.query(
+    `
+      update crawl_jobs job
+      set state = 'exported',
+          updated_at = now()
+      from product_sources source
+      join products product on product.id = source.product_id
+      where product.moderation_status = 'approved'
+        and job.source = source.source
+        and job.url = source.url
+        and job.job_type = 'product'
+        and job.state in ('needs_review', 'approved')
+    `,
+  );
+
+  return manifest;
+}

ingestion/src/fetcher.ts

@@ -0,0 +1,107 @@
+import { createHash } from 'node:crypto';
+import { mkdir, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { ingestionConfig } from './config.ts';
+import type { CrawlFailureReason, FetchClassification, MarketSource } from './types.ts';
+
+type FetchSnapshotInput = {
+  source: MarketSource;
+  url: string;
+  snapshotDir?: string;
+  fetcher?: typeof fetch;
+  headers?: HeadersInit;
+  accept?: string;
+};
+
+export type RawFetchSnapshot = {
+  source: MarketSource;
+  url: string;
+  fetchedAt: string;
+  httpStatus: number;
+  rawSnapshotHash: string;
+  rawHtmlPath: string;
+  html: string;
+  setCookieHeaders: string[];
+};
+
+function getSetCookieHeaders(response: Response) {
+  const headers = response.headers as Headers & { getSetCookie?: () => string[] };
+
+  if (typeof headers.getSetCookie === 'function') return headers.getSetCookie();
+
+  const single = response.headers.get('set-cookie');
+
+  return single ? [single] : [];
+}
+
+function classifyResponse(status: number, body: string): FetchClassification {
+  const lowered = body.slice(0, 80_000).toLowerCase();
+
+  if (status === 404) return { ok: false, reason: 'not_found', retryable: false };
+  if (status === 429) return { ok: false, reason: 'rate_limited', retryable: true };
+  if (status === 403) return { ok: false, reason: 'blocked', retryable: true };
+  if (status >= 500) return { ok: false, reason: 'rate_limited', retryable: true };
+  if (/captcha|showcaptcha|smartcaptcha|подтвердите, что вы не робот/.test(lowered)) {
+    return { ok: false, reason: 'captcha', retryable: true };
+  }
+  if (status < 200 || status >= 300) return { ok: false, reason: 'needs_manual_review', retryable: false };
+
+  return { ok: true };
+}
+
+function hash(value: string) {
+  return createHash('sha256').update(value).digest('hex');
+}
+
+function snapshotPath(snapshotDir: string, source: MarketSource, fetchedAt: string, digest: string) {
+  const day = fetchedAt.slice(0, 10);
+
+  return path.join(snapshotDir, source, day, `${digest}.html`);
+}
+
+export async function fetchRawSnapshot(input: FetchSnapshotInput): Promise<RawFetchSnapshot> {
+  const fetcher = input.fetcher ?? fetch;
+  const response = await fetcher(input.url, {
+    headers: {
+      accept: input.accept ?? 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+      'accept-language': 'ru,en;q=0.8',
+      'user-agent': 'videoreg-ingestion/1.0 (+https://videoreg.ru)',
+      ...(input.headers ?? {}),
+    },
+    redirect: 'follow',
+  });
+  const html = await response.text();
+  const classification = classifyResponse(response.status, html);
+
+  if (!classification.ok) {
+    const error = new Error(`Fetch classified as ${classification.reason} for status ${response.status}`);
+    (error as Error & { reason?: CrawlFailureReason }).reason = classification.reason;
+    throw error;
+  }
+
+  const fetchedAt = new Date().toISOString();
+  const rawSnapshotHash = hash(html);
+  const rawHtmlPath = snapshotPath(input.snapshotDir ?? ingestionConfig.snapshotDir, input.source, fetchedAt, rawSnapshotHash);
+
+  await mkdir(path.dirname(rawHtmlPath), { recursive: true });
+  await writeFile(rawHtmlPath, html, 'utf8');
+
+  return {
+    source: input.source,
+    url: response.url || input.url,
+    fetchedAt,
+    httpStatus: response.status,
+    rawSnapshotHash,
+    rawHtmlPath,
+    html,
+    setCookieHeaders: getSetCookieHeaders(response),
+  };
+}
+
+export function getFetchFailureReason(error: unknown) {
+  return (error as Error & { reason?: CrawlFailureReason }).reason;
+}
+
+export function randomDelayMs(minMs = ingestionConfig.fetchDelayMinMs, maxMs = ingestionConfig.fetchDelayMaxMs) {
+  return Math.floor(minMs + Math.random() * Math.max(0, maxMs - minMs));
+}

ingestion/src/html.ts

@@ -0,0 +1,87 @@
+const namedEntities: Record<string, string> = {
+  amp: '&',
+  gt: '>',
+  lt: '<',
+  quot: '"',
+  apos: "'",
+  nbsp: ' ',
+};
+
+export function decodeHtml(value: string) {
+  return value.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (entity, code: string) => {
+    const normalized = code.toLowerCase();
+
+    if (normalized.startsWith('#x')) {
+      return String.fromCodePoint(Number.parseInt(normalized.slice(2), 16));
+    }
+
+    if (normalized.startsWith('#')) {
+      return String.fromCodePoint(Number.parseInt(normalized.slice(1), 10));
+    }
+
+    return namedEntities[normalized] ?? entity;
+  });
+}
+
+export function stripTags(value: string) {
+  return decodeHtml(value.replace(/<[^>]*>/g, ' ')).replace(/\s+/g, ' ').trim();
+}
+
+export function normalizeWhitespace(value: string) {
+  return value.trim().replace(/\s+/g, ' ');
+}
+
+export function uniqueStable<T>(values: T[]) {
+  return [...new Set(values)];
+}
+
+function parseJsonPayload(raw: string) {
+  return JSON.parse(decodeHtml(raw).trim()) as unknown;
+}
+
+function flattenJsonLd(input: unknown): unknown[] {
+  if (Array.isArray(input)) {
+    return input.flatMap(flattenJsonLd);
+  }
+
+  if (input && typeof input === 'object') {
+    const object = input as Record<string, unknown>;
+    const graph = object['@graph'];
+
+    return graph ? [object, ...flattenJsonLd(graph)] : [object];
+  }
+
+  return [];
+}
+
+export function extractJsonLdObjects(html: string) {
+  const scripts = [...html.matchAll(/<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi)];
+  const objects: unknown[] = [];
+
+  for (const [, raw] of scripts) {
+    try {
+      objects.push(...flattenJsonLd(parseJsonPayload(raw)));
+    } catch {
+      // Ignore malformed analytics or partial JSON-LD blocks.
+    }
+  }
+
+  return objects;
+}
+
+export function getObjectString(value: unknown) {
+  if (typeof value === 'string') return normalizeWhitespace(value);
+  if (value && typeof value === 'object' && typeof (value as { name?: unknown }).name === 'string') {
+    return normalizeWhitespace((value as { name: string }).name);
+  }
+
+  return undefined;
+}
+
+export function readMetaContent(html: string, property: string) {
+  const escaped = property.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+  const pattern = new RegExp(`<meta[^>]+(?:property|name)=["']${escaped}["'][^>]+content=["']([^"']+)["'][^>]*>`, 'i');
+  const match = html.match(pattern);
+
+  return match ? decodeHtml(match[1]).trim() : undefined;
+}

ingestion/src/migrate.ts

@@ -0,0 +1,51 @@
+import { readFile, readdir } from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { createPgPool } from './db.ts';
+
+const currentFile = fileURLToPath(import.meta.url);
+const migrationsDir = path.resolve(path.dirname(currentFile), '..', 'migrations');
+
+export async function runMigrations() {
+  const pool = createPgPool();
+  const client = await pool.connect();
+
+  try {
+    await client.query("select pg_advisory_lock(hashtext('videoreg_ingestion_migrations'))");
+    await client.query(`
+      create table if not exists schema_migrations (
+        filename text primary key,
+        applied_at timestamptz not null default now()
+      )
+    `);
+
+    const files = (await readdir(migrationsDir)).filter((file) => file.endsWith('.sql')).sort();
+
+    for (const file of files) {
+      const applied = await client.query('select 1 from schema_migrations where filename = $1', [file]);
+
+      if (applied.rowCount) continue;
+
+      const sql = await readFile(path.join(migrationsDir, file), 'utf8');
+
+      await client.query('begin');
+      try {
+        await client.query(sql);
+        await client.query('insert into schema_migrations (filename) values ($1)', [file]);
+        await client.query('commit');
+        console.log(`applied migration ${file}`);
+      } catch (error) {
+        await client.query('rollback');
+        throw error;
+      }
+    }
+  } finally {
+    await client.query("select pg_advisory_unlock(hashtext('videoreg_ingestion_migrations'))").catch(() => undefined);
+    client.release();
+    await pool.end();
+  }
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  await runMigrations();
+}

ingestion/src/normalizer.ts

@@ -0,0 +1,300 @@
+import type { NormalizedMarketProduct, ParsedMarketProduct, ParsedTechnicalSpec } from './types.ts';
+
+const brandAliases = new Map<string, string>([
+  ['ibox', 'iBOX'],
+  ['sho-me', 'Sho-Me'],
+  ['trendvision', 'TrendVision'],
+  ['fujida', 'Fujida'],
+  ['intego', 'INTEGO'],
+  ['spawnson', 'SPAWNSON'],
+]);
+
+const specAliases = new Map<string, string>([
+  ['количество камер', 'channels'],
+  ['количество каналов записи видео', 'video_channels'],
+  ['количество каналов записи звука', 'audio_channels'],
+  ['качество видеосъемки/ fps', 'resolution_front'],
+  ['максимальное разрешение видеосъемки', 'resolution_max'],
+  ['макс. разрешение видеосъемки', 'resolution_max'],
+  ['максимальное разрешение камеры заднего вида', 'resolution_max'],
+  ['разрешение видеозаписи (при макс. частоте)', 'resolution_front'],
+  ['разрешение видеозаписи', 'resolution_front'],
+  ['макс. частота кадров', 'frame_rate_max_fps'],
+  ['частота кадров при макс. разрешении', 'frame_rate_max_fps'],
+  ['частота при макс. разрешении камеры заднего вида', 'frame_rate_max_fps'],
+  ['угол обзора (диагональ)', 'view_angle_degrees'],
+  ['угол обзора (ширина)', 'view_angle_width_degrees'],
+  ['угол обзора (высота)', 'view_angle_height_degrees'],
+  ['угол обзора (основная камера)', 'view_angle_degrees'],
+  ['угол обзора', 'view_angle_degrees'],
+  ['gps', 'gps'],
+  ['встроенный gps', 'gps'],
+  ['наличие gps модуля', 'gps'],
+  ['поддержка gps', 'gps'],
+  ['глонасс', 'glonass'],
+  ['встроенный глонасс', 'glonass'],
+  ['наличие глонасс', 'glonass'],
+  ['поддержка глонасс', 'glonass'],
+  ['wi-fi', 'wifi'],
+  ['wifi', 'wifi'],
+  ['поддержка wi-fi', 'wifi'],
+  ['поддержка wifi', 'wifi'],
+  ['трансляция видео через wi-fi', 'wifi'],
+  ['трансляция видео через wifi', 'wifi'],
+  ['управление с помощью смартфона', 'wifi'],
+  ['ночной режим', 'night_mode'],
+  ['режим ночной съемки', 'night_mode'],
+  ['экран', 'screen'],
+  ['запись времени и даты', 'timestamp_recording'],
+  ['запись скорости', 'speed_recording'],
+  ['запись скорости движения', 'speed_recording'],
+  ['встроенный микрофон', 'microphone'],
+  ['запись звука', 'microphone'],
+  ['встроенный динамик', 'speaker'],
+  ['с конденсатором', 'capacitor'],
+  ['максимальный размер карты памяти', 'memory_card_max_gb'],
+  ['макс. объем карты памяти', 'memory_card_max_gb'],
+  ['максимальная емкость карты памяти', 'memory_card_max_gb'],
+  ['матрица', 'sensor_type'],
+  ['размер матрицы', 'sensor_size'],
+  ['количество мегапикселей матрицы', 'sensor_megapixels'],
+  ['поддержка hd', 'hd_support'],
+  ['режим записи', 'recording_mode'],
+  ['запись роликов без разрывов', 'gapless_recording'],
+  ['циклическая запись', 'gapless_recording'],
+  ['непрерывная запись', 'gapless_recording'],
+  ['запись события в отдельный файл', 'event_file_recording'],
+  ['запись событий в отдельный файл', 'event_file_recording'],
+  ['формат записи', 'recording_format'],
+  ['используемый видеокодек', 'video_codec'],
+  ['питание от аккумулятора', 'battery_power'],
+  ['резервный источник питания', 'battery_power'],
+  ['питание от бортовой сети автомобиля', 'car_power'],
+  ['питание от прикуривателя', 'car_power'],
+  ['подключение внешних камер', 'external_cameras'],
+  ['подключение камеры заднего вида', 'external_cameras'],
+  ['дополнительная камера', 'external_cameras'],
+  ['камера заднего вида', 'external_cameras'],
+  ['поддержка карт памяти', 'memory_card_support'],
+  ['база камер', 'speedcam'],
+  ['предупреждение о камерах слежения', 'speedcam'],
+  ['speedcam', 'speedcam'],
+  ['обнаружение радаров', 'radar_detection'],
+  ['магнитный держатель', 'magnetic_mount'],
+  ['крепление на присоске', 'suction_mount'],
+  ['голосовые подсказки', 'voice_prompts'],
+  ['с радар-детектором', 'radar_detector'],
+]);
+
+function normalizeKey(value: string) {
+  return value.trim().toLowerCase().replace(/\s+/g, ' ');
+}
+
+function parseBoolean(value: unknown) {
+  if (typeof value === 'boolean') return value;
+  if (typeof value !== 'string') return undefined;
+
+  const normalized = normalizeKey(value);
+
+  if (['да', 'есть', 'true', 'yes'].includes(normalized)) return true;
+  if (['нет', 'false', 'no', 'отсутствует', 'none'].includes(normalized)) return false;
+
+  return undefined;
+}
+
+function parseFirstNumber(value: unknown) {
+  if (typeof value === 'number') return value;
+  if (typeof value !== 'string') return undefined;
+
+  const match = value.replace(',', '.').match(/\d+(?:\.\d+)?/);
+
+  return match ? Number(match[0]) : undefined;
+}
+
+function normalizeBatteryPower(value: string | number | boolean) {
+  const booleanValue = parseBoolean(value);
+
+  if (booleanValue !== undefined) return booleanValue;
+  if (typeof value === 'number') return value > 0;
+  if (typeof value !== 'string') return undefined;
+
+  const normalized = normalizeKey(value);
+  const hasBattery = /аккум|battery|li[-\s]?ion|литий/.test(normalized);
+  const hasCapacitor = /конденс|capacitor|supercap/.test(normalized);
+
+  if (hasBattery && hasCapacitor) return 'hybrid';
+  if (hasBattery) return 'battery';
+  if (hasCapacitor) return 'capacitor';
+  if (['нет', 'false', 'no', 'отсутствует', 'none'].includes(normalized)) return false;
+
+  return undefined;
+}
+
+function normalizeResolution(value: unknown) {
+  if (typeof value !== 'string') return undefined;
+
+  const match = value.replace(/[×х]/gi, 'x').match(/\d{3,5}\s*x\s*\d{3,5}/i);
+
+  return match ? match[0].replace(/\s+/g, '').toLowerCase() : undefined;
+}
+
+function normalizeSpecValue(key: string, value: string | number | boolean) {
+  if (key === 'battery_power') {
+    return normalizeBatteryPower(value);
+  }
+
+  const booleanValue = parseBoolean(value);
+
+  if (booleanValue !== undefined) {
+    return booleanValue;
+  }
+
+  if (
+    [
+      'gps',
+      'glonass',
+      'wifi',
+      'night_mode',
+      'screen',
+      'timestamp_recording',
+      'speed_recording',
+      'microphone',
+      'speaker',
+      'capacitor',
+      'speedcam',
+      'radar_detection',
+    ].includes(key)
+  ) {
+    if (typeof value === 'string' && value.trim()) {
+      return true;
+    }
+
+    return value;
+  }
+
+  if (
+    [
+      'channels',
+      'video_channels',
+      'audio_channels',
+      'frame_rate_max_fps',
+      'view_angle_degrees',
+      'view_angle_width_degrees',
+      'view_angle_height_degrees',
+      'memory_card_max_gb',
+      'sensor_megapixels',
+    ].includes(key)
+  ) {
+    return parseFirstNumber(value);
+  }
+
+  if (key === 'resolution_front' || key === 'resolution_max') {
+    return normalizeResolution(value);
+  }
+
+  return value;
+}
+
+function normalizeBrand(value: string | undefined, title: string) {
+  const raw = value ?? [...brandAliases.values()].find((brand) => title.toLowerCase().includes(brand.toLowerCase()));
+
+  if (!raw) return 'Unknown';
+
+  return brandAliases.get(raw.toLowerCase()) ?? raw.trim();
+}
+
+export function cleanModelFromTitle(title: string, brand: string) {
+  const genericWords = [
+    'видеорегистратор-зеркало',
+    'зеркало-видеорегистратор',
+    'видеорегистратор для автомобиля',
+    'видеорегистратор автомобильный',
+    'автомобильный видеорегистратор',
+    'видеорегистратор',
+    'зеркало',
+    'цвет черный',
+    'черный',
+  ];
+  let model = title;
+
+  for (const word of genericWords) {
+    model = model.replace(new RegExp(word, 'gi'), ' ');
+  }
+
+  model = model.replace(new RegExp(`["«]?${brand.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}["»]?`, 'gi'), ' ');
+  model = model.replace(new RegExp(brand.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'gi'), ' ');
+
+  return (
+    model
+      .replace(/[«»"]/g, ' ')
+      .replace(/\bwi[\s-]?fi\b/gi, 'Wi-Fi')
+      .replace(/\s*,\s*/g, ' ')
+      .replace(/^[\s,+–-]+/g, '')
+      .replace(/[,+–-]\s*$/g, '')
+      .replace(/\s+/g, ' ')
+      .trim() || title.trim()
+  );
+}
+
+function normalizeTechnicalSpec(spec: ParsedTechnicalSpec) {
+  const normalizedKey = specAliases.get(normalizeKey(spec.label));
+  const normalizedValue = normalizedKey ? normalizeSpecValue(normalizedKey, spec.value) : undefined;
+
+  return {
+    ...spec,
+    ...(normalizedKey ? { normalizedKey } : {}),
+    ...(normalizedValue !== undefined ? { normalizedValue } : {}),
+  };
+}
+
+function buildCanonicalModel(model: string) {
+  const compact = model
+    .replace(/\s+\+\s+(?:внутрисалонная|задняя)\s+камера.*$/i, '')
+    .replace(/\s+\+\s+камера.*$/i, '')
+    .replace(/\s+с\s+(?:базой камер|сигнатурным|радар-детектором|gps|глонасс).*$/i, '')
+    .replace(/\s+автомобильн(?:ый|ая|ое|ые)(?:\s+с\b.*)?$/i, '')
+    .replace(/\s+(?:зеркало|с креплением на зеркало)(?:\s+с\b.*)?$/i, '')
+    .replace(/[\s,]+(?:ночной режим|ночная съемка)$/i, '')
+    .replace(/\s+/g, ' ')
+    .trim();
+
+  return compact || model;
+}
+
+function createCanonicalName(brand: string, canonicalModel: string) {
+  if (canonicalModel.toLowerCase().startsWith(brand.toLowerCase())) return canonicalModel;
+
+  return `${brand} ${canonicalModel}`.replace(/\s+/g, ' ').trim();
+}
+
+export function normalizeMarketProduct(product: ParsedMarketProduct): NormalizedMarketProduct {
+  const brand = normalizeBrand(product.brand, product.title);
+  const model = product.model?.trim() || cleanModelFromTitle(product.title, brand);
+  const canonicalModel = buildCanonicalModel(model);
+  const rawSpecs = product.rawSpecs.map(normalizeTechnicalSpec);
+  const specs: NormalizedMarketProduct['specs'] = {};
+
+  for (const spec of rawSpecs) {
+    if (spec.normalizedKey && spec.normalizedValue !== undefined) {
+      specs[spec.normalizedKey] = spec.normalizedValue;
+    }
+  }
+
+  return {
+    source: product.source,
+    sourceProductId: product.sourceProductId,
+    sourceUrl: product.sourceUrl,
+    title: product.title,
+    brand,
+    model,
+    canonicalModel,
+    canonicalName: createCanonicalName(brand, canonicalModel),
+    specs,
+    rawSpecs,
+    images: product.images,
+    aggregateRating: product.aggregateRating,
+  };
+}
+
+// Kept as an import-compatible alias for existing Yandex parser callers.
+export const normalizeYandexMarketProduct = normalizeMarketProduct;

ingestion/src/queue.ts

@@ -0,0 +1,188 @@
+import type { Pool } from 'pg';
+import type { CrawlFailureReason, CrawlJobRecord, CrawlJobType, MarketSource } from './types.ts';
+
+type CrawlJobRow = {
+  id: string;
+  source: MarketSource;
+  url: string;
+  job_type: CrawlJobType;
+  state: CrawlJobRecord['state'];
+  attempts: number;
+  next_run_at: Date;
+  last_error: string | null;
+};
+
+type BudgetBucket = CrawlJobType;
+
+const retryableFailureReasons = new Set<CrawlFailureReason>(['rate_limited', 'blocked', 'captcha']);
+
+function toRecord(row: CrawlJobRow): CrawlJobRecord {
+  return {
+    id: row.id,
+    source: row.source,
+    url: row.url,
+    jobType: row.job_type,
+    state: row.state,
+    attempts: row.attempts,
+    nextRunAt: row.next_run_at.toISOString(),
+    ...(row.last_error ? { lastError: row.last_error } : {}),
+  };
+}
+
+export function dailyBudgetForJob(jobType: CrawlJobType) {
+  if (jobType === 'category') return Number(process.env.INGESTION_CATEGORY_DAILY_BUDGET ?? 25);
+  if (jobType === 'sitemap') return Number(process.env.INGESTION_SITEMAP_DAILY_BUDGET ?? 25);
+
+  return Number(process.env.INGESTION_PRODUCT_DAILY_BUDGET ?? 100);
+}
+
+export class CrawlQueue {
+  private readonly pool: Pool;
+
+  constructor(pool: Pool) {
+    this.pool = pool;
+  }
+
+  async enqueue(input: {
+    source: MarketSource;
+    url: string;
+    jobType: CrawlJobType;
+    nextRunAt?: Date;
+    state?: CrawlJobRecord['state'];
+  }) {
+    await this.pool.query(
+      `
+        insert into crawl_jobs (source, url, job_type, state, next_run_at)
+        values ($1, $2, $3, $4, $5)
+        on conflict (source, url, job_type) do update
+        set
+          next_run_at = least(crawl_jobs.next_run_at, excluded.next_run_at),
+          updated_at = now()
+      `,
+      [input.source, input.url, input.jobType, input.state ?? 'pending', input.nextRunAt ?? new Date()],
+    );
+  }
+
+  async leaseNext(now = new Date()) {
+    const result = await this.pool.query<CrawlJobRow>(
+      `
+        with candidate as (
+          select job.id
+          from crawl_jobs job
+          left join source_pauses pause
+            on pause.source = job.source
+           and pause.paused_until > $1
+          where job.state = 'pending'
+            and job.next_run_at <= $1
+            and pause.source is null
+          order by job.next_run_at asc, job.created_at asc
+          for update of job skip locked
+          limit 1
+        )
+        update crawl_jobs job
+        set state = 'fetching',
+            attempts = attempts + 1,
+            updated_at = now()
+        from candidate
+        where job.id = candidate.id
+        returning job.*
+      `,
+      [now],
+    );
+
+    return result.rows[0] ? toRecord(result.rows[0]) : undefined;
+  }
+
+  async reserveDailyBudget(source: MarketSource, bucket: BudgetBucket, dailyLimit: number, now = new Date()) {
+    const day = now.toISOString().slice(0, 10);
+    const result = await this.pool.query<{ reserved: boolean }>(
+      `
+        with existing as (
+          select used_count, daily_limit
+          from source_daily_budgets
+          where source = $1 and budget_bucket = $2 and budget_day = $3
+          for update
+        ),
+        upserted as (
+          insert into source_daily_budgets (source, budget_bucket, budget_day, daily_limit, used_count)
+          select $1, $2, $3, $4, 1
+          where not exists (select 1 from existing)
+          on conflict do nothing
+          returning true as reserved
+        ),
+        updated as (
+          update source_daily_budgets
+          set used_count = used_count + 1,
+              daily_limit = $4,
+              updated_at = now()
+          where source = $1
+            and budget_bucket = $2
+            and budget_day = $3
+            and exists (select 1 from existing)
+            and used_count < daily_limit
+          returning true as reserved
+        )
+        select coalesce(
+          (select reserved from upserted),
+          (select reserved from updated),
+          false
+        ) as reserved
+      `,
+      [source, bucket, day, dailyLimit],
+    );
+
+    return result.rows[0]?.reserved ?? false;
+  }
+
+  async markState(id: string, state: CrawlJobRecord['state']) {
+    await this.pool.query(
+      `
+        update crawl_jobs
+        set state = $2,
+            updated_at = now()
+        where id = $1
+      `,
+      [id, state],
+    );
+  }
+
+  async markFailure(job: CrawlJobRecord, reason: CrawlFailureReason, message: string) {
+    const nextRunAt = retryableFailureReasons.has(reason)
+      ? new Date(Date.now() + Math.min(24 * 60 * 60 * 1000, 2 ** job.attempts * 60_000))
+      : undefined;
+
+    await this.pool.query(
+      `
+        update crawl_jobs
+        set state = $2,
+            last_error = $3,
+            next_run_at = coalesce($4, next_run_at),
+            updated_at = now()
+        where id = $1
+      `,
+      [job.id, reason, message.slice(0, 2000), nextRunAt],
+    );
+
+    await this.pool.query(
+      `
+        insert into crawl_errors (job_id, source, url, reason, message)
+        values ($1, $2, $3, $4, $5)
+      `,
+      [job.id, job.source, job.url, reason, message.slice(0, 2000)],
+    );
+  }
+
+  async pauseSource(source: MarketSource, reason: CrawlFailureReason, pausedUntil: Date) {
+    await this.pool.query(
+      `
+        insert into source_pauses (source, reason, paused_until)
+        values ($1, $2, $3)
+        on conflict (source) do update
+        set reason = excluded.reason,
+            paused_until = excluded.paused_until,
+            updated_at = now()
+      `,
+      [source, reason, pausedUntil],
+    );
+  }
+}

ingestion/src/repository.test.ts

@@ -0,0 +1,69 @@
+import type { Pool } from 'pg';
+import { describe, expect, it, vi } from 'vitest';
+import { IngestionRepository } from './repository.ts';
+import type { NormalizedMarketProduct } from './types.ts';
+
+const product: NormalizedMarketProduct = {
+  source: 'mvideo',
+  sourceProductId: '400484046',
+  sourceUrl: 'https://www.mvideo.ru/products/test-400484046',
+  title: 'Verification Canonical Link 1',
+  brand: 'Verification',
+  model: 'Canonical Link 1',
+  canonicalModel: 'Canonical Link 1',
+  canonicalName: 'Verification Canonical Link 1',
+  specs: { night_mode: false },
+  rawSpecs: [
+    {
+      group: 'Verification',
+      label: 'Night mode',
+      value: 'нет',
+      normalizedKey: 'night_mode',
+      normalizedValue: false,
+    },
+  ],
+  images: [],
+};
+
+describe('IngestionRepository', () => {
+  it('links an exact canonical-name match and serializes source conflicts as JSONB', async () => {
+    const conflictValues = [
+      { source: 'mvideo', value: false },
+      { source: 'yandex_market', value: true },
+    ];
+    const query = vi.fn(async (statement: string) => {
+      if (statement.includes('select id\n        from products')) {
+        return { rows: [{ id: 'ym_5905353579' }], rowCount: 1 };
+      }
+
+      if (statement.includes('insert into manufacturers')) {
+        return { rows: [{ id: 'manufacturer-id' }], rowCount: 1 };
+      }
+
+      if (statement.includes('select\n          normalized_key')) {
+        return {
+          rows: [{ normalized_key: 'night_mode', values_json: conflictValues }],
+          rowCount: 1,
+        };
+      }
+
+      return { rows: [], rowCount: 0 };
+    });
+    const repository = new IngestionRepository({ query } as unknown as Pool);
+
+    const productId = await repository.upsertNormalizedProduct(
+      product,
+      'stable-content-hash',
+      '2026-07-30T23:00:00.000Z',
+    );
+    const calls = query.mock.calls as unknown as Array<[string, unknown[] | undefined]>;
+    const sourceCall = calls.find(([statement]) => statement.includes('insert into product_sources'));
+    const conflictCall = calls.find(([statement]) => statement.includes('insert into moderation_conflicts'));
+    const queueCalls = calls.filter(([statement]) => statement.includes('insert into moderation_queue'));
+
+    expect(productId).toBe('ym_5905353579');
+    expect(sourceCall?.[1]?.[0]).toBe('ym_5905353579');
+    expect(JSON.parse(String(conflictCall?.[1]?.[2]))).toEqual(conflictValues);
+    expect(queueCalls.at(-1)?.[1]).toEqual(['ym_5905353579', 'same_product']);
+  });
+});

ingestion/src/repository.ts

@@ -0,0 +1,317 @@
+import { createHash } from 'node:crypto';
+import type { Pool } from 'pg';
+import type { DiscoveredProduct, NormalizedMarketProduct, ProductSourceSnapshot } from './types.ts';
+
+function publicProductId(product: NormalizedMarketProduct) {
+  const sourcePrefix = product.source === 'mvideo' ? 'mv' : 'ym';
+
+  if (product.sourceProductId) return `${sourcePrefix}_${product.sourceProductId}`;
+
+  return `${sourcePrefix}_${Buffer.from(product.sourceUrl).toString('base64url').slice(0, 32)}`;
+}
+
+function valueHash(value: unknown) {
+  return createHash('sha256').update(JSON.stringify(value)).digest('hex');
+}
+
+function manufacturerSlug(name: string) {
+  const slug = name
+    .trim()
+    .toLowerCase()
+    .replace(/[^\p{Letter}\p{Number}]+/gu, '-')
+    .replace(/^-+|-+$/g, '');
+
+  return slug || 'unknown';
+}
+
+export class IngestionRepository {
+  private readonly pool: Pool;
+
+  constructor(pool: Pool) {
+    this.pool = pool;
+  }
+
+  async recordSnapshot(snapshot: ProductSourceSnapshot) {
+    await this.pool.query(
+      `
+        insert into source_snapshots (
+          source, url, fetched_at, http_status, raw_snapshot_hash,
+          stable_content_hash, raw_html_path, parsed_json
+        )
+        values ($1, $2, $3, $4, $5, $6, $7, $8)
+      `,
+      [
+        snapshot.source,
+        snapshot.url,
+        snapshot.fetchedAt,
+        snapshot.httpStatus,
+        snapshot.rawSnapshotHash,
+        snapshot.stableContentHash ?? null,
+        snapshot.rawHtmlPath ?? null,
+        snapshot.parsedJson,
+      ],
+    );
+  }
+
+  async recordDiscoveredProducts(products: DiscoveredProduct[]) {
+    for (const product of products) {
+      await this.pool.query(
+        `
+          insert into product_sources (
+            source, url, source_product_id, category_url, title_preview,
+            first_seen_at, updated_at, state
+          )
+          values ($1, $2, $3, $4, $5, $6, now(), 'discovered')
+          on conflict (source, url) do update
+          set source_product_id = coalesce(product_sources.source_product_id, excluded.source_product_id),
+              category_url = coalesce(excluded.category_url, product_sources.category_url),
+              title_preview = coalesce(excluded.title_preview, product_sources.title_preview),
+              updated_at = now()
+        `,
+        [
+          product.source,
+          product.productUrl,
+          product.sourceProductId ?? null,
+          product.categoryUrl ?? null,
+          product.titlePreview ?? null,
+          product.discoveredAt,
+        ],
+      );
+    }
+  }
+
+  async upsertNormalizedProduct(product: NormalizedMarketProduct, stableContentHash: string, fetchedAt: string) {
+    const sourceScopedProductId = publicProductId(product);
+    const matchingProduct = await this.pool.query<{ id: string }>(
+      `
+        select id
+        from products
+        where lower(canonical_name) = lower($1)
+          and id <> $2
+        order by discovered_at asc, id asc
+        limit 1
+      `,
+      [product.canonicalName, sourceScopedProductId],
+    );
+    const productId = matchingProduct.rows[0]?.id ?? sourceScopedProductId;
+    const linkedToExistingCanonicalProduct = productId !== sourceScopedProductId;
+    const manufacturerId = await this.upsertManufacturer(product.brand);
+
+    await this.pool.query(
+      `
+        insert into products (id, manufacturer_id, brand, model, canonical_name, title, moderation_status, discovered_at, updated_at)
+        values ($1, $2, $3, $4, $5, $6, 'needs_review', $7, $7)
+        on conflict (id) do update
+        set manufacturer_id = excluded.manufacturer_id,
+            brand = excluded.brand,
+            model = case when products.model_override then products.model else excluded.model end,
+            canonical_name = case when products.canonical_name_override then products.canonical_name else excluded.canonical_name end,
+            title = excluded.title,
+            updated_at = excluded.updated_at
+      `,
+      [productId, manufacturerId, product.brand, product.canonicalModel, product.canonicalName, product.title, fetchedAt],
+    );
+
+    await this.pool.query(
+      `
+        insert into product_sources (
+          product_id, source, url, source_product_id, stable_content_hash,
+          last_fetched_at, parsed_json, state, updated_at
+        )
+        values ($1, $2, $3, $4, $5, $6, $7, 'parsed', now())
+        on conflict (source, url) do update
+        set product_id = excluded.product_id,
+            source_product_id = coalesce(product_sources.source_product_id, excluded.source_product_id),
+            stable_content_hash = excluded.stable_content_hash,
+            last_fetched_at = excluded.last_fetched_at,
+            parsed_json = excluded.parsed_json,
+            state = excluded.state,
+            updated_at = now()
+      `,
+      [productId, product.source, product.sourceUrl, product.sourceProductId ?? null, stableContentHash, fetchedAt, product],
+    );
+
+    for (const [key, value] of Object.entries(product.specs)) {
+      await this.pool.query(
+        `
+          insert into product_specs (product_id, key, value, updated_at)
+          values ($1, $2, $3, now())
+          on conflict (product_id, key) do update
+          set value = excluded.value,
+              updated_at = now()
+        `,
+        [productId, key, JSON.stringify(value)],
+      );
+    }
+
+    await this.pool.query(
+      `
+        delete from product_source_specs
+        where product_id = $1
+          and source = $2
+          and source_url = $3
+      `,
+      [productId, product.source, product.sourceUrl],
+    );
+
+    for (const spec of product.rawSpecs) {
+      const comparableValue = spec.normalizedValue ?? spec.value;
+
+      await this.pool.query(
+        `
+          insert into product_source_specs (
+            product_id, source, source_url, source_product_id, raw_group, raw_label,
+            raw_value, normalized_key, normalized_value, value_hash, fetched_at
+          )
+          values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
+          on conflict (product_id, source, source_url, raw_label) do update
+          set raw_group = excluded.raw_group,
+              raw_value = excluded.raw_value,
+              normalized_key = excluded.normalized_key,
+              normalized_value = excluded.normalized_value,
+              value_hash = excluded.value_hash,
+              fetched_at = excluded.fetched_at,
+              updated_at = now()
+        `,
+        [
+          productId,
+          product.source,
+          product.sourceUrl,
+          product.sourceProductId ?? null,
+          spec.group ?? null,
+          spec.label,
+          JSON.stringify(spec.value),
+          spec.normalizedKey ?? null,
+          spec.normalizedValue === undefined ? null : JSON.stringify(spec.normalizedValue),
+          valueHash(comparableValue),
+          fetchedAt,
+        ],
+      );
+    }
+
+    await this.refreshSpecConflicts(productId);
+
+    for (const imageUrl of product.images) {
+      await this.pool.query(
+        `
+          insert into product_images (product_id, original_url, source, status)
+          values ($1, $2, $3, 'needs_review')
+          on conflict (product_id, original_url) do nothing
+        `,
+        [productId, imageUrl, product.source],
+      );
+    }
+
+    if (product.aggregateRating) {
+      await this.pool.query(
+        `
+          insert into source_ratings (product_id, source, source_product_id, rating_value, rating_scale, rating_count, fetched_at)
+          values ($1, $2, $3, $4, $5, $6, $7)
+        `,
+        [
+          productId,
+          product.source,
+          product.sourceProductId ?? null,
+          product.aggregateRating.value,
+          product.aggregateRating.scale ?? null,
+          product.aggregateRating.ratingCount ?? null,
+          fetchedAt,
+        ],
+      );
+    }
+
+    await this.pool.query(
+      `
+        insert into moderation_queue (product_id, reason, state)
+        values ($1, $2, 'pending')
+        on conflict (product_id, reason) do nothing
+      `,
+      [productId, linkedToExistingCanonicalProduct ? 'same_product' : 'new_product'],
+    );
+
+    return productId;
+  }
+
+  private async upsertManufacturer(name: string) {
+    const result = await this.pool.query<{ id: string }>(
+      `
+        insert into manufacturers (slug, name)
+        values ($1, $2)
+        on conflict (slug) do update
+        set name = excluded.name,
+            updated_at = now()
+        returning id
+      `,
+      [manufacturerSlug(name), name],
+    );
+
+    return result.rows[0].id;
+  }
+
+  private async refreshSpecConflicts(productId: string) {
+    await this.pool.query(
+      `
+        update moderation_conflicts
+        set state = 'resolved',
+            updated_at = now()
+        where product_id = $1
+          and conflict_type = 'spec_value'
+      `,
+      [productId],
+    );
+
+    const conflicts = await this.pool.query<{
+      normalized_key: string;
+      values_json: unknown;
+    }>(
+      `
+        select
+          normalized_key,
+          jsonb_agg(
+            jsonb_build_object(
+              'source', source,
+              'url', source_url,
+              'sourceProductId', source_product_id,
+              'value', coalesce(normalized_value, raw_value),
+              'rawLabel', raw_label,
+              'rawGroup', raw_group
+            )
+            order by source, source_url, raw_label
+          ) as values_json
+        from product_source_specs
+        where product_id = $1
+          and normalized_key is not null
+        group by normalized_key
+        having count(distinct value_hash) > 1
+      `,
+      [productId],
+    );
+
+    for (const conflict of conflicts.rows) {
+      await this.pool.query(
+        `
+          insert into moderation_conflicts (product_id, conflict_type, key, values_json, state)
+          values ($1, 'spec_value', $2, $3, 'pending')
+          on conflict (product_id, conflict_type, key) do update
+          set values_json = excluded.values_json,
+              state = 'pending',
+              updated_at = now()
+        `,
+        [productId, conflict.normalized_key, JSON.stringify(conflict.values_json)],
+      );
+    }
+
+    if (conflicts.rowCount) {
+      await this.pool.query(
+        `
+          insert into moderation_queue (product_id, reason, state)
+          values ($1, 'conflicting_specs', 'pending')
+          on conflict (product_id, reason) do update
+          set state = 'pending',
+              updated_at = now()
+        `,
+        [productId],
+      );
+    }
+  }
+}

ingestion/src/smokeFetchMvideo.ts

@@ -0,0 +1,284 @@
+import { ingestionConfig } from './config.ts';
+import { normalizeYandexMarketProduct } from './normalizer.ts';
+import { createStableContentHash } from './stableHash.ts';
+import {
+  buildCookieHeader,
+  discoverMvideoProductIds,
+  extractMvideoBffBasePath,
+  extractMvideoSearchQuery,
+  parseMvideoProduct,
+} from './sources/mvideo.ts';
+
+type SmokeProductResult =
+  | {
+      ok: true;
+      index: number;
+      sourceProductId: string;
+      productUrl: string;
+      productApiUrl: string;
+      httpStatus: number;
+      title: string;
+      brand: string;
+      model: string;
+      canonicalModel: string;
+      canonicalName: string;
+      sources: Array<{
+        source: 'mvideo';
+        sourceProductId?: string;
+        url: string;
+        fetchedAt: string;
+        stableContentHash: string;
+        title: string;
+        aggregateRating?: {
+          value: number;
+          scale?: number;
+          ratingCount?: number;
+        };
+      }>;
+      technicalSpecs: {
+        normalized: Record<string, string | number | boolean>;
+        raw: Array<{
+          group?: string;
+          label: string;
+          value: string | number | boolean;
+          normalizedKey?: string;
+          normalizedValue?: string | number | boolean;
+        }>;
+      };
+      images: string[];
+      stableContentHash: string;
+      moderation: {
+        conflicts: [];
+      };
+      internalSignals: {
+        aggregateRating?: {
+          value: number;
+          scale?: number;
+          ratingCount?: number;
+        };
+        rawSpecKeys: string[];
+      };
+    }
+  | {
+      ok: false;
+      index: number;
+      sourceProductId: string;
+      productApiUrl: string;
+      error: string;
+    };
+
+function numberEnv(name: string, fallback: number) {
+  const value = Number(process.env[name]);
+
+  return Number.isFinite(value) ? value : fallback;
+}
+
+function sleep(ms: number) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function setCookieHeaders(response: Response) {
+  const headers = response.headers as Headers & { getSetCookie?: () => string[] };
+
+  if (typeof headers.getSetCookie === 'function') return headers.getSetCookie();
+
+  const single = response.headers.get('set-cookie');
+
+  return single ? [single] : [];
+}
+
+async function fetchText(url: string, init?: RequestInit) {
+  const response = await fetch(url, {
+    headers: {
+      accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+      'accept-language': 'ru,en;q=0.8',
+      'user-agent': 'Mozilla/5.0 (compatible; videoreg-ingestion-smoke/1.0)',
+      ...(init?.headers ?? {}),
+    },
+    redirect: 'follow',
+    ...init,
+  });
+  const text = await response.text();
+
+  if (!response.ok) {
+    throw new Error(`HTTP ${response.status} for ${url}`);
+  }
+
+  return {
+    response,
+    url: response.url || url,
+    status: response.status,
+    text,
+  };
+}
+
+async function fetchJson(url: string, init?: RequestInit) {
+  const response = await fetch(url, {
+    headers: {
+      accept: 'application/json,text/plain,*/*',
+      'accept-language': 'ru,en;q=0.8',
+      'user-agent': 'Mozilla/5.0 (compatible; videoreg-ingestion-smoke/1.0)',
+      ...(init?.headers ?? {}),
+    },
+    redirect: 'follow',
+    ...init,
+  });
+  const text = await response.text();
+  let payload: unknown;
+
+  try {
+    payload = JSON.parse(text);
+  } catch {
+    throw new Error(`Expected JSON response from ${url}, got non-JSON body`);
+  }
+
+  if (!response.ok) {
+    throw new Error(`HTTP ${response.status} for ${url}`);
+  }
+
+  return {
+    url: response.url || url,
+    status: response.status,
+    payload,
+  };
+}
+
+async function fetchSmokeProduct(
+  productId: string,
+  index: number,
+  bffBasePath: string,
+  cookieHeader: string,
+): Promise<SmokeProductResult> {
+  const detailsBaseUrl = bffBasePath.endsWith('/') ? bffBasePath : `${bffBasePath}/`;
+  const detailsUrl = new URL('product-details', detailsBaseUrl);
+  detailsUrl.searchParams.set('productId', productId);
+
+  try {
+    const response = await fetchJson(detailsUrl.toString(), {
+      headers: {
+        cookie: cookieHeader,
+      },
+    });
+    const fetchedAt = new Date().toISOString();
+    const parsed = parseMvideoProduct(response.payload);
+    const normalized = normalizeYandexMarketProduct(parsed);
+    const stableContentHash = createStableContentHash(normalized);
+
+    return {
+      ok: true,
+      index,
+      sourceProductId: productId,
+      productUrl: normalized.sourceUrl,
+      productApiUrl: detailsUrl.toString(),
+      httpStatus: response.status,
+      title: normalized.title,
+      brand: normalized.brand,
+      model: normalized.model,
+      canonicalModel: normalized.canonicalModel,
+      canonicalName: normalized.canonicalName,
+      sources: [
+        {
+          source: 'mvideo',
+          sourceProductId: normalized.sourceProductId,
+          url: normalized.sourceUrl,
+          fetchedAt,
+          stableContentHash,
+          title: normalized.title,
+          aggregateRating: normalized.aggregateRating,
+        },
+      ],
+      technicalSpecs: {
+        normalized: normalized.specs,
+        raw: normalized.rawSpecs,
+      },
+      images: normalized.images,
+      stableContentHash,
+      moderation: {
+        conflicts: [],
+      },
+      internalSignals: {
+        aggregateRating: normalized.aggregateRating,
+        rawSpecKeys: Object.keys(parsed.specs),
+      },
+    };
+  } catch (error) {
+    return {
+      ok: false,
+      index,
+      sourceProductId: productId,
+      productApiUrl: detailsUrl.toString(),
+      error: (error as Error).message,
+    };
+  }
+}
+
+const limit = Math.max(1, numberEnv('INGESTION_SMOKE_LIMIT', 5));
+const delayMs = Math.max(0, numberEnv('INGESTION_SMOKE_DELAY_MS', ingestionConfig.fetchDelayMinMs));
+const categoryUrl =
+  process.env.MVIDEO_DASHCAM_CATEGORY_URL ?? 'https://www.mvideo.ru/product-list-page?q=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE%D1%80%D0%B5%D0%B3%D0%B8%D1%81%D1%82%D1%80%D0%B0%D1%82%D0%BE%D1%80%D1%8B';
+const startedAt = new Date().toISOString();
+
+console.error(`Fetching MVideo category page: ${categoryUrl}`);
+const categoryResponse = await fetchText(categoryUrl);
+const cookieHeader = buildCookieHeader(setCookieHeaders(categoryResponse.response));
+
+if (!cookieHeader) {
+  throw new Error('MVideo category response did not provide cookies required for BFF calls');
+}
+
+const bffBasePath = extractMvideoBffBasePath(categoryResponse.text);
+const searchQuery = extractMvideoSearchQuery(categoryResponse.url);
+const searchBaseUrl = bffBasePath.endsWith('/') ? bffBasePath : `${bffBasePath}/`;
+const searchUrl = new URL('products/v2/search', searchBaseUrl);
+searchUrl.searchParams.set('query', searchQuery);
+searchUrl.searchParams.set('offset', '0');
+searchUrl.searchParams.set('limit', String(limit));
+
+console.error(`Fetching MVideo search: ${searchUrl.toString()}`);
+const searchResponse = await fetchJson(searchUrl.toString(), {
+  headers: {
+    cookie: cookieHeader,
+  },
+});
+const discoveredProductIds = discoverMvideoProductIds(searchResponse.payload);
+const selectedProductIds = discoveredProductIds.slice(0, limit);
+const products: SmokeProductResult[] = [];
+
+for (const [index, sourceProductId] of selectedProductIds.entries()) {
+  if (index > 0 && delayMs > 0) {
+    console.error(`Waiting ${delayMs}ms before next product fetch`);
+    await sleep(delayMs);
+  }
+
+  console.error(`Fetching MVideo product ${index + 1}/${selectedProductIds.length}: ${sourceProductId}`);
+  products.push(await fetchSmokeProduct(sourceProductId, index + 1, bffBasePath, cookieHeader));
+}
+
+console.log(
+  JSON.stringify(
+    {
+      source: 'mvideo',
+      mode: 'smoke',
+      categoryUrl,
+      startedAt,
+      finishedAt: new Date().toISOString(),
+      limit,
+      delayMs,
+      category: {
+        url: categoryResponse.url,
+        httpStatus: categoryResponse.status,
+        htmlLength: categoryResponse.text.length,
+      },
+      search: {
+        query: searchQuery,
+        url: searchUrl.toString(),
+        httpStatus: searchResponse.status,
+        discoveredCount: discoveredProductIds.length,
+        selectedCount: selectedProductIds.length,
+      },
+      products,
+    },
+    null,
+    2,
+  ),
+);

ingestion/src/smokeFetchYandexMarket.ts

@@ -0,0 +1,204 @@
+import { ingestionConfig } from './config.ts';
+import { normalizeYandexMarketProduct } from './normalizer.ts';
+import { discoverYandexMarketProductsFromCategory, parseYandexMarketProduct } from './sources/yandexMarket.ts';
+import { createStableContentHash } from './stableHash.ts';
+
+type SmokeProductResult =
+  | {
+      ok: true;
+      index: number;
+      url: string;
+      httpStatus: number;
+      htmlLength: number;
+      sourceProductId?: string;
+      sources: Array<{
+        source: 'yandex_market';
+        sourceProductId?: string;
+        url: string;
+        fetchedAt: string;
+        stableContentHash: string;
+        title: string;
+        aggregateRating?: {
+          value: number;
+          scale?: number;
+          ratingCount?: number;
+        };
+      }>;
+      title: string;
+      brand: string;
+      model: string;
+      canonicalModel: string;
+      canonicalName: string;
+      technicalSpecs: {
+        normalized: Record<string, string | number | boolean>;
+        raw: Array<{
+          group?: string;
+          label: string;
+          value: string | number | boolean;
+          normalizedKey?: string;
+          normalizedValue?: string | number | boolean;
+        }>;
+      };
+      images: string[];
+      stableContentHash: string;
+      moderation: {
+        conflicts: [];
+      };
+      internalSignals: {
+        aggregateRating?: {
+          value: number;
+          scale?: number;
+          ratingCount?: number;
+        };
+        rawSpecKeys: string[];
+      };
+    }
+  | {
+      ok: false;
+      index: number;
+      url: string;
+      error: string;
+    };
+
+function numberEnv(name: string, fallback: number) {
+  const value = Number(process.env[name]);
+
+  return Number.isFinite(value) ? value : fallback;
+}
+
+function sleep(ms: number) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function classifyHtml(status: number, html: string) {
+  const lowered = html.slice(0, 80_000).toLowerCase();
+
+  if (status === 429) return 'rate_limited';
+  if (status === 403) return 'blocked';
+  if (/captcha|showcaptcha|smartcaptcha|подтвердите, что вы не робот/.test(lowered)) return 'captcha';
+  if (status === 404) return 'not_found';
+  if (status < 200 || status >= 300) return `http_${status}`;
+
+  return undefined;
+}
+
+async function fetchHtml(url: string) {
+  const response = await fetch(url, {
+    headers: {
+      accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+      'accept-language': 'ru,en;q=0.8',
+    },
+    redirect: 'follow',
+  });
+  const html = await response.text();
+  const failure = classifyHtml(response.status, html);
+
+  if (failure) {
+    throw new Error(`Fetch classified as ${failure} for status ${response.status}`);
+  }
+
+  return {
+    url: response.url || url,
+    status: response.status,
+    html,
+  };
+}
+
+async function fetchSmokeProduct(url: string, index: number): Promise<SmokeProductResult> {
+  try {
+    const response = await fetchHtml(url);
+    const fetchedAt = new Date().toISOString();
+    const parsed = parseYandexMarketProduct(response.html, url);
+    const normalized = normalizeYandexMarketProduct(parsed);
+    const stableContentHash = createStableContentHash(normalized);
+
+    return {
+      ok: true,
+      index,
+      url,
+      httpStatus: response.status,
+      htmlLength: response.html.length,
+      sourceProductId: normalized.sourceProductId,
+      sources: [
+        {
+          source: 'yandex_market',
+          sourceProductId: normalized.sourceProductId,
+          url: normalized.sourceUrl,
+          fetchedAt,
+          stableContentHash,
+          title: normalized.title,
+          aggregateRating: normalized.aggregateRating,
+        },
+      ],
+      title: normalized.title,
+      brand: normalized.brand,
+      model: normalized.model,
+      canonicalModel: normalized.canonicalModel,
+      canonicalName: normalized.canonicalName,
+      technicalSpecs: {
+        normalized: normalized.specs,
+        raw: normalized.rawSpecs,
+      },
+      images: normalized.images,
+      stableContentHash,
+      moderation: {
+        conflicts: [],
+      },
+      internalSignals: {
+        aggregateRating: normalized.aggregateRating,
+        rawSpecKeys: Object.keys(parsed.specs),
+      },
+    };
+  } catch (error) {
+    return {
+      ok: false,
+      index,
+      url,
+      error: (error as Error).message,
+    };
+  }
+}
+
+const limit = Math.max(1, numberEnv('INGESTION_SMOKE_LIMIT', 5));
+const delayMs = Math.max(0, numberEnv('INGESTION_SMOKE_DELAY_MS', ingestionConfig.fetchDelayMinMs));
+const categoryUrl = process.env.YANDEX_MARKET_DASHCAM_CATEGORY_URL ?? ingestionConfig.seedCategoryUrl;
+const startedAt = new Date().toISOString();
+
+console.error(`Fetching category: ${categoryUrl}`);
+const categoryResponse = await fetchHtml(categoryUrl);
+const discovered = discoverYandexMarketProductsFromCategory(categoryResponse.html, categoryUrl, startedAt);
+const selected = discovered.slice(0, limit);
+const products: SmokeProductResult[] = [];
+
+for (const [index, product] of selected.entries()) {
+  if (index > 0 && delayMs > 0) {
+    console.error(`Waiting ${delayMs}ms before next product fetch`);
+    await sleep(delayMs);
+  }
+
+  console.error(`Fetching product ${index + 1}/${selected.length}: ${product.productUrl}`);
+  products.push(await fetchSmokeProduct(product.productUrl, index + 1));
+}
+
+console.log(
+  JSON.stringify(
+    {
+      source: 'yandex_market',
+      mode: 'smoke',
+      categoryUrl,
+      startedAt,
+      finishedAt: new Date().toISOString(),
+      limit,
+      delayMs,
+      category: {
+        httpStatus: categoryResponse.status,
+        htmlLength: categoryResponse.html.length,
+        discoveredCount: discovered.length,
+        selectedCount: selected.length,
+      },
+      products,
+    },
+    null,
+    2,
+  ),
+);

ingestion/src/sources/mvideo.test.ts

@@ -0,0 +1,171 @@
+import { describe, expect, it } from 'vitest';
+import { normalizeYandexMarketProduct } from '../normalizer.ts';
+import {
+  buildCookieHeader,
+  discoverMvideoProductIds,
+  extractMvideoBffBasePath,
+  extractMvideoSearchQuery,
+  parseMvideoProduct,
+} from './mvideo.ts';
+
+describe('MVideo parser', () => {
+  it('extracts search query and BFF path from category HTML', () => {
+    const html = `<script>window.MVID_CONFIG={basePath:'https://www.mvideo.ru/bff'};</script>`;
+
+    expect(extractMvideoBffBasePath(html)).toBe('https://www.mvideo.ru/bff');
+    expect(
+      extractMvideoSearchQuery(
+        'https://www.mvideo.ru/product-list-page?q=%D0%B2%D0%B8%D0%B4%D0%B5%D0%BE%D1%80%D0%B5%D0%B3%D0%B8%D1%81%D1%82%D1%80%D0%B0%D1%82%D0%BE%D1%80',
+      ),
+    ).toBe('видеорегистратор');
+    expect(extractMvideoSearchQuery('https://www.mvideo.ru/category/unknown')).toBe('видеорегистраторы');
+  });
+
+  it('builds cookie header and discovers stable product ids from search response', () => {
+    const cookieHeader = buildCookieHeader([
+      'MVID_CITY_ID=CityCZ_975; Path=/; Secure',
+      'MVID_REGION_ID=1; Path=/; Secure',
+      'MVID_CITY_ID=CityCZ_975; Path=/; Secure',
+    ]);
+
+    expect(cookieHeader).toContain('MVID_CITY_ID=CityCZ_975');
+    expect(cookieHeader).toContain('MVID_REGION_ID=1');
+    expect(
+      discoverMvideoProductIds({
+        body: {
+          products: ['4254724', 4254724, '400481258', 'bad-id'],
+        },
+      }),
+    ).toEqual(['4254724', '400481258']);
+    expect(
+      discoverMvideoProductIds({
+        body: {
+          type: 'redirect',
+          url: '/products/videoregistrator-navitel-ar202-nv-400484046',
+        },
+      }),
+    ).toEqual(['400484046']);
+  });
+
+  it('maps product details to parsed and normalized shape with full raw specs', () => {
+    const parsed = parseMvideoProduct({
+      body: {
+        productId: '4254724',
+        name: 'Видеорегистратор Navitel R9 DUAL',
+        nameTranslit: 'videoregistrator-navitel-r9-dual',
+        brandName: 'Navitel',
+        modelName: 'R9 DUAL',
+        images: ['Big/4254724bb.jpg', '/Big/4254724bb1.jpg'],
+        rating: {
+          score: 4.8,
+          total: 17,
+        },
+        properties: {
+          all: [
+            {
+              name: 'Основные характеристики',
+              properties: [
+                {
+                  name: 'Поддержка Wi-Fi',
+                  value: 'Да',
+                },
+                {
+                  name: 'Качество видеосъемки/ FPS',
+                  value: 'FullHD (1920x1080 Пикс) 30 кадр/сек',
+                },
+                {
+                  name: 'Частота при макс. разрешении камеры заднего вида',
+                  value: '30 кадр/сек',
+                },
+                {
+                  name: 'Макс. разрешение видеосъемки',
+                  value: '2560x1440',
+                },
+                {
+                  name: 'База камер',
+                  value: 'Да',
+                },
+                {
+                  name: 'Запись звука',
+                  value: 'Да',
+                },
+                {
+                  name: 'Резервный источник питания',
+                  value: 'аккумулятор',
+                },
+              ],
+            },
+            {
+              name: 'Оптика',
+              properties: [
+                {
+                  name: 'Угол обзора (основная камера)',
+                  value: '170',
+                  measure: '°',
+                },
+              ],
+            },
+          ],
+        },
+      },
+    });
+    const normalized = normalizeYandexMarketProduct(parsed);
+
+    expect(parsed).toMatchObject({
+      source: 'mvideo',
+      sourceProductId: '4254724',
+      sourceUrl: 'https://www.mvideo.ru/products/videoregistrator-navitel-r9-dual-4254724',
+      brand: 'Navitel',
+      model: 'R9 DUAL',
+      title: 'Видеорегистратор Navitel R9 DUAL',
+      aggregateRating: {
+        value: 4.8,
+        scale: 5,
+        ratingCount: 17,
+      },
+    });
+    expect(parsed.images).toEqual([
+      'https://img.mvideo.ru/Big/4254724bb.jpg',
+      'https://www.mvideo.ru/Big/4254724bb1.jpg',
+    ]);
+    expect(parsed.rawSpecs).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          group: 'Основные характеристики',
+          label: 'Поддержка Wi-Fi',
+          value: 'Да',
+        }),
+        expect.objectContaining({
+          group: 'Оптика',
+          label: 'Угол обзора (основная камера)',
+          value: '170 °',
+        }),
+      ]),
+    );
+    expect(normalized.specs).toEqual(
+      expect.objectContaining({
+        wifi: true,
+        speedcam: true,
+        microphone: true,
+        battery_power: 'battery',
+        resolution_front: '1920x1080',
+        frame_rate_max_fps: 30,
+        view_angle_degrees: 170,
+      }),
+    );
+    expect(normalized.rawSpecs).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          label: 'Поддержка Wi-Fi',
+          normalizedKey: 'wifi',
+          normalizedValue: true,
+        }),
+        expect.objectContaining({
+          label: 'Макс. разрешение видеосъемки',
+          normalizedKey: 'resolution_max',
+          normalizedValue: '2560x1440',
+        }),
+      ]),
+    );
+  });
+});

ingestion/src/sources/mvideo.ts

@@ -0,0 +1,238 @@
+import { createHash } from 'node:crypto';
+import { normalizeWhitespace, uniqueStable } from '../html.ts';
+import type { ParsedMarketProduct, ParsedTechnicalSpec } from '../types.ts';
+
+export const mvideoOrigin = 'https://www.mvideo.ru';
+const productIdPattern = /^\d+$/;
+
+type ProductDetailsProperty = {
+  name?: unknown;
+  value?: unknown;
+  measure?: unknown;
+};
+
+type ProductDetailsPropertyGroup = {
+  name?: unknown;
+  properties?: unknown;
+};
+
+function stringValue(value: unknown) {
+  if (typeof value !== 'string') return undefined;
+
+  const normalized = normalizeWhitespace(value);
+
+  return normalized ? normalized : undefined;
+}
+
+function normalizeSourceImageUrl(url: string) {
+  const trimmed = url.trim();
+
+  if (trimmed.startsWith('http://') || trimmed.startsWith('https://')) return trimmed;
+  if (trimmed.startsWith('//')) return `https:${trimmed}`;
+  if (trimmed.startsWith('/')) return `${mvideoOrigin}${trimmed}`;
+
+  return `https://img.mvideo.ru/${trimmed.replace(/^\/+/, '')}`;
+}
+
+function createFallbackProductId(sourceUrl: string, title: string) {
+  return createHash('sha1').update(`${sourceUrl}:${title}`).digest('hex').slice(0, 16);
+}
+
+function parseSpecValue(value: unknown, measure: unknown): string | number | boolean | undefined {
+  if (typeof value === 'boolean' || typeof value === 'number') return value;
+
+  const normalizedValue = stringValue(value);
+
+  if (!normalizedValue) return undefined;
+
+  const normalizedMeasure = stringValue(measure);
+
+  if (!normalizedMeasure) return normalizedValue;
+  if (normalizedValue.toLowerCase().includes(normalizedMeasure.toLowerCase())) return normalizedValue;
+
+  return `${normalizedValue} ${normalizedMeasure}`;
+}
+
+function specsRecord(technicalSpecs: ParsedTechnicalSpec[]) {
+  const record: Record<string, string | number | boolean> = {};
+
+  for (const spec of technicalSpecs) {
+    record[spec.label] = spec.value;
+  }
+
+  return record;
+}
+
+export function extractMvideoBffBasePath(html: string) {
+  const directMatch = html.match(/basePath\s*:\s*['"]([^'"]+)['"]/i)?.[1];
+  const jsonMatch = html.match(/"basePath"\s*:\s*"([^"]+)"/i)?.[1];
+  const value = directMatch ?? jsonMatch;
+
+  if (!value) return `${mvideoOrigin}/bff`;
+  if (value.startsWith('http://') || value.startsWith('https://')) return value;
+  if (value.startsWith('/')) return `${mvideoOrigin}${value}`;
+
+  return `${mvideoOrigin}/${value.replace(/^\/+/, '')}`;
+}
+
+export function extractMvideoSearchQuery(categoryUrl: string) {
+  try {
+    const url = new URL(categoryUrl);
+    const query = url.searchParams.get('q') ?? url.searchParams.get('query');
+
+    if (query?.trim()) return query.trim();
+  } catch {
+    // Fall through to default query.
+  }
+
+  return 'видеорегистраторы';
+}
+
+export function buildMvideoProductUrl(productId: string) {
+  return `${mvideoOrigin}/product/${encodeURIComponent(productId)}`;
+}
+
+export function buildMvideoBffUrl(basePath: string, endpoint: string) {
+  const normalizedBasePath = basePath.endsWith('/') ? basePath : `${basePath}/`;
+
+  return new URL(endpoint.replace(/^\/+/, ''), normalizedBasePath);
+}
+
+export function buildCookieHeader(setCookieHeaders: string[]) {
+  const cookies = new Map<string, string>();
+
+  for (const header of setCookieHeaders) {
+    const cookiePart = header.split(';')[0]?.trim();
+
+    if (!cookiePart) continue;
+
+    const separatorIndex = cookiePart.indexOf('=');
+
+    if (separatorIndex <= 0) continue;
+
+    const name = cookiePart.slice(0, separatorIndex).trim();
+    const value = cookiePart.slice(separatorIndex + 1).trim();
+
+    if (!name || !value) continue;
+
+    cookies.set(name, `${name}=${value}`);
+  }
+
+  return [...cookies.values()].join('; ');
+}
+
+export function discoverMvideoProductIds(searchPayload: unknown) {
+  if (!searchPayload || typeof searchPayload !== 'object') {
+    throw new Error('MVideo search payload is not an object');
+  }
+
+  const body = (searchPayload as { body?: unknown }).body;
+  const products = body && typeof body === 'object' ? (body as { products?: unknown }).products : undefined;
+
+  const productIds = Array.isArray(products)
+    ? products
+        .map((value) => (typeof value === 'number' || typeof value === 'string' ? String(value).trim() : ''))
+        .filter((value): value is string => productIdPattern.test(value))
+    : [];
+
+  if (productIds.length > 0) return uniqueStable(productIds);
+
+  // MVideo converts exact one-result searches into a redirect instead of a
+  // one-element products array. The destination still carries the public ID.
+  const redirectUrl =
+    typeof (body as { url?: unknown }).url === 'string'
+      ? (body as { url: string }).url
+      : typeof (body as { redirectUrl?: unknown }).redirectUrl === 'string'
+        ? (body as { redirectUrl: string }).redirectUrl
+        : undefined;
+  const redirectProductId = redirectUrl?.match(/(\d{6,13})(?:[/?#]|$)/)?.[1];
+
+  if (redirectProductId) return [redirectProductId];
+
+  throw new Error('MVideo search payload does not include product identifiers');
+}
+
+export function parseMvideoProduct(detailsPayload: unknown): ParsedMarketProduct {
+  if (!detailsPayload || typeof detailsPayload !== 'object') {
+    throw new Error('MVideo product payload is not an object');
+  }
+
+  const body = (detailsPayload as { body?: unknown }).body;
+
+  if (!body || typeof body !== 'object') {
+    throw new Error('MVideo product payload has no body');
+  }
+
+  const product = body as Record<string, unknown>;
+  const title = stringValue(product.name);
+
+  if (!title) {
+    throw new Error('MVideo product title was not found');
+  }
+
+  const sourceProductIdValue = product.productId;
+  const sourceProductId =
+    (typeof sourceProductIdValue === 'number' || typeof sourceProductIdValue === 'string'
+      ? String(sourceProductIdValue).trim()
+      : undefined) ?? undefined;
+  const slug = stringValue(product.nameTranslit);
+  const sourceUrl = slug && sourceProductId
+    ? `${mvideoOrigin}/products/${slug}-${sourceProductId}`
+    : sourceProductId
+      ? `${mvideoOrigin}/product/${sourceProductId}`
+      : mvideoOrigin;
+  const brand = stringValue(product.brandName);
+  const model = stringValue(product.modelName);
+  const images = uniqueStable(
+    (Array.isArray(product.images) ? product.images : [])
+      .filter((value): value is string => typeof value === 'string' && value.trim().length > 0)
+      .map(normalizeSourceImageUrl),
+  );
+  const rawSpecs: ParsedTechnicalSpec[] = [];
+  const rawGroups = (product.properties as { all?: unknown } | undefined)?.all;
+
+  if (Array.isArray(rawGroups)) {
+    for (const rawGroup of rawGroups as ProductDetailsPropertyGroup[]) {
+      const groupName = stringValue(rawGroup.name);
+      const properties = Array.isArray(rawGroup.properties) ? rawGroup.properties : [];
+
+      for (const rawSpec of properties as ProductDetailsProperty[]) {
+        const label = stringValue(rawSpec.name);
+        const value = parseSpecValue(rawSpec.value, rawSpec.measure);
+
+        if (!label || value === undefined) continue;
+
+        rawSpecs.push({
+          ...(groupName ? { group: groupName } : {}),
+          label,
+          value,
+        });
+      }
+    }
+  }
+
+  const ratingRaw = product.rating;
+  const aggregateRating =
+    ratingRaw && typeof ratingRaw === 'object' && Number.isFinite(Number((ratingRaw as { score?: unknown }).score))
+      ? {
+          value: Number((ratingRaw as { score: number | string }).score),
+          scale: 5,
+          ...(Number.isFinite(Number((ratingRaw as { total?: unknown }).total))
+            ? { ratingCount: Number((ratingRaw as { total: number | string }).total) }
+            : {}),
+        }
+      : undefined;
+
+  return {
+    source: 'mvideo',
+    sourceProductId: sourceProductId ?? createFallbackProductId(sourceUrl, title),
+    title,
+    ...(brand ? { brand } : {}),
+    ...(model ? { model } : {}),
+    images,
+    specs: specsRecord(rawSpecs),
+    rawSpecs,
+    sourceUrl,
+    aggregateRating,
+  };
+}

ingestion/src/sources/yandexMarket.test.ts

@@ -0,0 +1,154 @@
+import { describe, expect, it } from 'vitest';
+import { normalizeYandexMarketProduct } from '../normalizer';
+import { createStableContentHash } from '../stableHash';
+import { discoverYandexMarketProductsFromCategory, parseYandexMarketProduct } from './yandexMarket';
+
+describe('Yandex Market parser', () => {
+  it('discovers and canonicalizes card and product URLs from category HTML', () => {
+    const discovered = discoverYandexMarketProductsFromCategory(
+      `
+        <a href="/card/ibox-f5-wifi-videoregistrator-s-signaturnym-radar-detektorom/5197387201?do-waremd5=x&amp;cpc=y">F5</a>
+        <script>{"url":"https://market.yandex.ru/product/312774508?offerid=secret&sku=103274083526"}</script>
+      `,
+      'https://market.yandex.ru/catalog--avtomobilnye-videoregistratory/82798275/list?hid=82798269',
+      '2026-05-16T20:00:00.000Z',
+    );
+
+    expect(discovered).toMatchObject([
+      {
+        productUrl:
+          'https://market.yandex.ru/card/ibox-f5-wifi-videoregistrator-s-signaturnym-radar-detektorom/5197387201',
+        sourceProductId: '5197387201',
+      },
+      {
+        productUrl: 'https://market.yandex.ru/product/312774508',
+        sourceProductId: '312774508',
+      },
+    ]);
+  });
+
+  it('maps product JSON-LD and DOM specs without prices, sellers or reviews', () => {
+    const html = `
+      <script type="application/ld+json">
+        {
+          "@context": "https://schema.org",
+          "@type": "Product",
+          "brand": "iBOX",
+          "image": "https://avatars.mds.yandex.net/get-mpic/18787582/2a000/orig",
+          "name": "Видеорегистратор для автомобиля iBOX F5 + WiFi с радар-детектором",
+          "offers": { "price": "17816", "availability": "https://schema.org/InStock" },
+          "aggregateRating": { "ratingValue": "4.8", "ratingCount": "17" },
+          "url": "https://market.yandex.ru/card/ibox-f5-wifi-videoregistrator-s-signaturnym-radar-detektorom/5197387201"
+        }
+      </script>
+      <label class="_3J2L5">
+        <div><span>Экран</span></div>
+        <div class="_2yz50"><span>да</span></div>
+      </label>
+      <div>
+        <span data-auto="product-spec">Количество камер</span>
+        <div class="eXP5k"><div><div class="ds-text"><span>1</span></div></div></div>
+        <span data-auto="product-spec">Разрешение видеозаписи (при макс. частоте)</span>
+        <div class="eXP5k"><div><div class="ds-text"><span>2304x1296</span></div></div></div>
+        <span data-auto="product-spec">Wi-Fi</span>
+        <div class="eXP5k"><div><div class="ds-text"><span>да</span></div></div></div>
+        <span data-auto="product-spec">С радар-детектором</span>
+        <div class="eXP5k"><div><div class="ds-text"><span>нет</span></div></div></div>
+        <label for="group-collapse-Камера"><span>Камера</span></label>
+        <span data-auto="product-spec">Угол обзора (диагональ)</span>
+        <div class="eXP5k"><div><div class="ds-text"><span>170°</span></div></div></div>
+        <span data-auto="product-spec">Матрица</span>
+        <div class="eXP5k"><div><div class="ds-text"><span>CMOS</span></div></div></div>
+      </div>
+      <span class="ds-text ds-text_weight_med">Основные характеристики</span>
+      <div data-apiary-widget-name=@card/AboutSpecV2>
+        <div><span data-auto="product-spec">Режим записи</span></div>
+        <div class="ds-flex uA8Xn"><span>циклическая</span></div>
+      </div>
+    `;
+    const parsed = parseYandexMarketProduct(
+      html,
+      'https://market.yandex.ru/card/ibox-f5-wifi-videoregistrator-s-signaturnym-radar-detektorom/5197387201',
+    );
+    const normalized = normalizeYandexMarketProduct(parsed);
+
+    expect(parsed).toMatchObject({
+      source: 'yandex_market',
+      sourceProductId: '5197387201',
+      brand: 'iBOX',
+      title: 'Видеорегистратор для автомобиля iBOX F5 + WiFi с радар-детектором',
+      aggregateRating: {
+        value: 4.8,
+        ratingCount: 17,
+      },
+    });
+    expect(normalized.specs).toEqual({
+      channels: 1,
+      resolution_front: '2304x1296',
+      wifi: true,
+      radar_detector: false,
+      view_angle_degrees: 170,
+      sensor_type: 'CMOS',
+      recording_mode: 'циклическая',
+      screen: true,
+    });
+    expect(normalized.canonicalModel).toBe('F5 + Wi-Fi');
+    expect(normalized.canonicalName).toBe('iBOX F5 + Wi-Fi');
+    expect(normalized.rawSpecs).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          group: 'Камера',
+          label: 'Матрица',
+          value: 'CMOS',
+          normalizedKey: 'sensor_type',
+          normalizedValue: 'CMOS',
+        }),
+      ]),
+    );
+    expect(parsed.rawSpecs).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          group: 'Основные характеристики',
+          label: 'Режим записи',
+          value: 'циклическая',
+        }),
+      ]),
+    );
+    expect(createStableContentHash(normalized)).toMatch(/^[a-f0-9]{64}$/);
+    expect(JSON.stringify(normalized)).not.toContain('17816');
+  });
+
+  it('keeps stable model attributes and removes feature tails from canonical model', () => {
+    const normalized = normalizeYandexMarketProduct({
+      source: 'yandex_market',
+      sourceProductId: '5247127917',
+      sourceUrl:
+        'https://market.yandex.ru/card/videoregistrator-avtomobilnyy-12-mp-1296p-s-dvumya-kamerami-2k-spawnson-full-hd-antiblikovyy-obyektiv/5247127917',
+      title: 'Видеорегистратор SPAWNSON "Spawnson", 24МП, Wi-Fi, ночной режим',
+      brand: 'SPAWNSON',
+      images: [],
+      specs: {},
+      rawSpecs: [],
+    });
+
+    expect(normalized.model).toBe('24МП Wi-Fi ночной режим');
+    expect(normalized.canonicalModel).toBe('24МП Wi-Fi');
+    expect(normalized.canonicalName).toBe('SPAWNSON 24МП Wi-Fi');
+  });
+
+  it('removes mirror form factor and automotive descriptor tails from canonical model', () => {
+    const normalized = normalizeYandexMarketProduct({
+      source: 'yandex_market',
+      sourceProductId: 'mirror-1',
+      sourceUrl: 'https://market.yandex.ru/product/mirror-1',
+      title: 'Видеорегистратор-зеркало iBOX Rover 2 автомобильный с базой камер',
+      brand: 'iBOX',
+      images: [],
+      specs: {},
+      rawSpecs: [],
+    });
+
+    expect(normalized.canonicalModel).toBe('Rover 2');
+    expect(normalized.canonicalName).toBe('iBOX Rover 2');
+  });
+});

ingestion/src/sources/yandexMarket.ts

@@ -0,0 +1,272 @@
+import { createHash } from 'node:crypto';
+import { decodeHtml, extractJsonLdObjects, getObjectString, normalizeWhitespace, readMetaContent, stripTags, uniqueStable } from '../html.ts';
+import type { DiscoveredProduct, ParsedMarketProduct, ParsedTechnicalSpec } from '../types.ts';
+
+const marketOrigin = 'https://market.yandex.ru';
+const productUrlPattern = /(?:https?:\/\/market\.yandex\.ru)?\/(?:card|product)\/[^\s"'<>\\]+/gi;
+
+function getJsonLdType(object: Record<string, unknown>) {
+  const type = object['@type'];
+
+  if (Array.isArray(type)) return type.map(String);
+  if (typeof type === 'string') return [type];
+
+  return [];
+}
+
+function isJsonLdProduct(object: unknown): object is Record<string, unknown> {
+  return !!object && typeof object === 'object' && getJsonLdType(object as Record<string, unknown>).includes('Product');
+}
+
+function normalizeSourceImageUrl(url: string) {
+  const decoded = decodeHtml(url).trim();
+
+  if (decoded.startsWith('//')) return `https:${decoded}`;
+
+  return decoded;
+}
+
+function canonicalizeMarketUrl(rawUrl: string) {
+  const decoded = decodeHtml(rawUrl).replace(/,$/, '');
+  const url = new URL(decoded, marketOrigin);
+
+  if (url.hostname !== 'market.yandex.ru') {
+    throw new Error(`Unsupported Yandex Market host: ${url.hostname}`);
+  }
+
+  const parts = url.pathname.split('/').filter(Boolean);
+  const type = parts[0];
+
+  if (type === 'product' && parts[1]) {
+    return `${marketOrigin}/product/${parts[1]}`;
+  }
+
+  if (type === 'card' && parts[1] && parts[2]) {
+    return `${marketOrigin}/card/${parts[1]}/${parts[2]}`;
+  }
+
+  throw new Error(`Unsupported Yandex Market product URL: ${rawUrl}`);
+}
+
+function sourceProductIdFromUrl(url: string) {
+  const parsed = new URL(url);
+  const parts = parsed.pathname.split('/').filter(Boolean);
+
+  if (parts[0] === 'product') return parts[1];
+  if (parts[0] === 'card') return parts[2];
+
+  return undefined;
+}
+
+function titleFromCardSlug(url: string) {
+  const parsed = new URL(url);
+  const parts = parsed.pathname.split('/').filter(Boolean);
+
+  if (parts[0] !== 'card') return undefined;
+
+  return parts[1]
+    ?.split('-')
+    .filter(Boolean)
+    .map((part) => part.charAt(0).toUpperCase() + part.slice(1))
+    .join(' ');
+}
+
+function normalizeProductUrlCandidate(rawUrl: string) {
+  try {
+    const url = canonicalizeMarketUrl(rawUrl);
+
+    return {
+      productUrl: url,
+      sourceProductId: sourceProductIdFromUrl(url),
+      titlePreview: titleFromCardSlug(url),
+    };
+  } catch {
+    return undefined;
+  }
+}
+
+function extractRating(value: unknown): ParsedMarketProduct['aggregateRating'] {
+  if (!value || typeof value !== 'object') return undefined;
+
+  const object = value as Record<string, unknown>;
+  const ratingValue = Number(object.ratingValue);
+  const ratingCount = Number(object.ratingCount ?? object.reviewCount);
+  const bestRating = Number(object.bestRating);
+
+  if (!Number.isFinite(ratingValue)) return undefined;
+
+  return {
+    value: ratingValue,
+    ...(Number.isFinite(bestRating) ? { scale: bestRating } : {}),
+    ...(Number.isFinite(ratingCount) ? { ratingCount } : {}),
+  };
+}
+
+function extractJsonLdProduct(html: string) {
+  return extractJsonLdObjects(html).find(isJsonLdProduct);
+}
+
+function extractJsonLdImages(product: Record<string, unknown> | undefined) {
+  const image = product?.image;
+  const values = Array.isArray(image) ? image : image ? [image] : [];
+
+  return values.filter((value): value is string => typeof value === 'string').map(normalizeSourceImageUrl);
+}
+
+function extractMetaImages(html: string) {
+  return [readMetaContent(html, 'og:image')]
+    .filter((value): value is string => !!value)
+    .map(normalizeSourceImageUrl);
+}
+
+function extractSpecGroup(htmlBeforeSpec: string) {
+  const matches = [
+    ...htmlBeforeSpec.matchAll(/for=["']group-collapse-([^"']+)["'][\s\S]{0,500}?<span[^>]*>([\s\S]*?)<\/span>/gi),
+    ...htmlBeforeSpec.matchAll(
+      /<span[^>]*class=["'][^"']*ds-text_weight_med[^"']*["'][^>]*>([\s\S]*?)<\/span>/gi,
+    ),
+  ];
+  const match = matches
+    .filter((candidate) => candidate.index !== undefined)
+    .sort((left, right) => (left.index ?? 0) - (right.index ?? 0))
+    .at(-1);
+
+  return match ? stripTags(match[2] || match[1]) : undefined;
+}
+
+function extractTechnicalSpecsFromDom(html: string): ParsedTechnicalSpec[] {
+  const specs: ParsedTechnicalSpec[] = [];
+  const seen = new Map<string, number>();
+  const legacySpecPattern =
+    /<span[^>]*data-auto=["']product-spec["'][^>]*>([\s\S]*?)<\/span>[\s\S]{0,1400}?<div[^>]+class=["'][^"']*ds-text[^"']*["'][^>]*>\s*<span>([\s\S]*?)<\/span>/gi;
+  const currentSpecPattern =
+    /data-apiary-widget-name=@card\/AboutSpecV2[^>]*>[\s\S]*?<span[^>]*data-auto=["']product-spec["'][^>]*>([\s\S]*?)<\/span>[\s\S]*?<div[^>]*class=["'][^"']*(?:uA8Xn|_2yz50)[^"']*["'][^>]*>\s*<span[^>]*>([\s\S]*?)<\/span>/gi;
+  const compactSpecPattern =
+    /<label[^>]*>[\s\S]*?<span[^>]*>([\s\S]*?)<\/span>[\s\S]*?<div[^>]*class=["'][^"']*_2yz50[^"']*["'][^>]*>\s*<span[^>]*>([\s\S]*?)<\/span>[\s\S]*?<\/label>/gi;
+
+  function addSpec(match: RegExpMatchArray) {
+    const rawName = match[1];
+    const rawValue = match[2];
+    const label = stripTags(rawName);
+    const value = stripTags(rawValue);
+
+    if (!label || !value || label === value) return;
+
+    const group = extractSpecGroup(html.slice(Math.max(0, (match.index ?? 0) - 8_000), match.index));
+    const dedupeKey = `${label}:${value}`;
+    const existingIndex = seen.get(dedupeKey);
+
+    if (existingIndex !== undefined) {
+      if (group && !specs[existingIndex]?.group) {
+        specs[existingIndex] = { ...specs[existingIndex], group };
+      }
+
+      return;
+    }
+
+    seen.set(dedupeKey, specs.length);
+    specs.push({
+      ...(group ? { group } : {}),
+      label,
+      value,
+    });
+  }
+
+  for (const match of html.matchAll(currentSpecPattern)) {
+    addSpec(match);
+  }
+
+  for (const match of html.matchAll(legacySpecPattern)) {
+    addSpec(match);
+  }
+
+  for (const match of html.matchAll(compactSpecPattern)) {
+    addSpec(match);
+  }
+
+  return specs;
+}
+
+function specsRecord(technicalSpecs: ParsedTechnicalSpec[]) {
+  const record: Record<string, string | number | boolean> = {};
+
+  for (const spec of technicalSpecs) {
+    record[spec.label] = spec.value;
+  }
+
+  return record;
+}
+
+function extractTitle(html: string, product?: Record<string, unknown>) {
+  const jsonTitle = getObjectString(product?.name);
+  const ogTitle = readMetaContent(html, 'og:title');
+  const titleTag = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1];
+
+  return jsonTitle ?? ogTitle ?? (titleTag ? stripTags(titleTag) : undefined);
+}
+
+function createFallbackProductId(sourceUrl: string, title: string) {
+  return createHash('sha1').update(`${sourceUrl}:${title}`).digest('hex').slice(0, 16);
+}
+
+export function discoverYandexMarketProductsFromCategory(
+  html: string,
+  categoryUrl: string,
+  discoveredAt = new Date().toISOString(),
+): DiscoveredProduct[] {
+  const products = [...html.matchAll(productUrlPattern)]
+    .map((match) => normalizeProductUrlCandidate(match[0]))
+    .filter((value): value is NonNullable<ReturnType<typeof normalizeProductUrlCandidate>> => !!value)
+    .map((value) => ({
+      source: 'yandex_market' as const,
+      categoryUrl,
+      productUrl: value.productUrl,
+      sourceProductId: value.sourceProductId,
+      titlePreview: value.titlePreview,
+      discoveredAt,
+    }));
+
+  const byUrl = new Map<string, DiscoveredProduct>();
+
+  for (const product of products) {
+    byUrl.set(product.productUrl, product);
+  }
+
+  return [...byUrl.values()];
+}
+
+export function parseYandexMarketProduct(html: string, sourceUrl: string): ParsedMarketProduct {
+  const product = extractJsonLdProduct(html);
+  const title = extractTitle(html, product);
+
+  if (!title) {
+    throw new Error('Yandex Market product title was not found');
+  }
+
+  const canonicalUrl = normalizeProductUrlCandidate(getObjectString(product?.url) ?? sourceUrl)?.productUrl ?? sourceUrl;
+  const sourceProductId =
+    sourceProductIdFromUrl(canonicalUrl) ??
+    (typeof product?.sku === 'string' ? product.sku : undefined) ??
+    createFallbackProductId(canonicalUrl, title);
+  const brand = getObjectString(product?.brand);
+  const images = uniqueStable([...extractJsonLdImages(product), ...extractMetaImages(html)]).filter((url) =>
+    url.includes('/get-mpic/'),
+  );
+  const rawSpecs = extractTechnicalSpecsFromDom(html);
+
+  return {
+    source: 'yandex_market',
+    sourceProductId,
+    title: normalizeWhitespace(title),
+    ...(brand ? { brand } : {}),
+    images,
+    specs: specsRecord(rawSpecs),
+    rawSpecs,
+    sourceUrl: canonicalUrl,
+    aggregateRating: extractRating(product?.aggregateRating),
+  };
+}
+
+export function canonicalizeYandexMarketProductUrl(rawUrl: string) {
+  return canonicalizeMarketUrl(rawUrl);
+}

ingestion/src/stableHash.ts

@@ -0,0 +1,35 @@
+import { createHash } from 'node:crypto';
+import type { NormalizedMarketProduct } from './types.ts';
+
+function stableJson(value: unknown): string {
+  if (Array.isArray(value)) {
+    return `[${value.map(stableJson).join(',')}]`;
+  }
+
+  if (value && typeof value === 'object') {
+    const object = value as Record<string, unknown>;
+
+    return `{${Object.keys(object)
+      .sort()
+      .map((key) => `${JSON.stringify(key)}:${stableJson(object[key])}`)
+      .join(',')}}`;
+  }
+
+  return JSON.stringify(value);
+}
+
+export function createStableContentHash(product: NormalizedMarketProduct) {
+  const semanticPayload = {
+    sourceProductId: product.sourceProductId,
+    canonicalProductUrl: product.sourceUrl,
+    title: product.title,
+    brand: product.brand,
+    model: product.model,
+    canonicalName: product.canonicalName,
+    specs: product.specs,
+    rawSpecs: product.rawSpecs,
+    images: product.images,
+  };
+
+  return createHash('sha256').update(stableJson(semanticPayload)).digest('hex');
+}

ingestion/src/types.ts

@@ -0,0 +1,126 @@
+import type { ExportedSpecValue } from '../../web-front/src/entities/dashcam-catalog/model/schema.ts';
+
+export type MarketSource = 'yandex_market' | 'mvideo';
+
+export type CrawlJobType = 'category' | 'product' | 'sitemap';
+
+export type CrawlJobState =
+  | 'pending'
+  | 'fetching'
+  | 'fetched'
+  | 'parsed'
+  | 'normalized'
+  | 'deduplicated'
+  | 'needs_review'
+  | 'approved'
+  | 'exported'
+  | CrawlFailureReason;
+
+export type CrawlFailureReason =
+  | 'rate_limited'
+  | 'blocked'
+  | 'captcha'
+  | 'not_found'
+  | 'parse_failed'
+  | 'needs_manual_review';
+
+export type DiscoveredProduct = {
+  source: MarketSource;
+  categoryUrl?: string;
+  productUrl: string;
+  sourceProductId?: string;
+  titlePreview?: string;
+  discoveredAt: string;
+};
+
+export type ParsedTechnicalSpec = {
+  group?: string;
+  label: string;
+  value: string | number | boolean;
+};
+
+export type NormalizedTechnicalSpec = ParsedTechnicalSpec & {
+  normalizedKey?: string;
+  normalizedValue?: ExportedSpecValue;
+};
+
+export type ParsedMarketProduct = {
+  source: MarketSource;
+  sourceProductId?: string;
+  title: string;
+  brand?: string;
+  model?: string;
+  aggregateRating?: {
+    value: number;
+    scale?: number;
+    ratingCount?: number;
+  };
+  images: string[];
+  specs: Record<string, string | number | boolean>;
+  rawSpecs: ParsedTechnicalSpec[];
+  sourceUrl: string;
+};
+
+export type NormalizedMarketProduct = {
+  source: MarketSource;
+  sourceProductId?: string;
+  sourceUrl: string;
+  title: string;
+  brand: string;
+  model: string;
+  canonicalModel: string;
+  canonicalName: string;
+  specs: Record<string, ExportedSpecValue>;
+  rawSpecs: NormalizedTechnicalSpec[];
+  images: string[];
+  aggregateRating?: ParsedMarketProduct['aggregateRating'];
+};
+
+export type ProductSourceMetadata = {
+  source: MarketSource;
+  sourceProductId?: string;
+  url: string;
+  fetchedAt?: string;
+  stableContentHash?: string;
+  title: string;
+  aggregateRating?: ParsedMarketProduct['aggregateRating'];
+};
+
+export type SpecConflict = {
+  key: string;
+  values: Array<{
+    source: MarketSource;
+    url: string;
+    value: ExportedSpecValue;
+  }>;
+};
+
+export type ProductSourceSnapshot = {
+  source: MarketSource;
+  url: string;
+  fetchedAt: string;
+  httpStatus: number;
+  rawSnapshotHash: string;
+  stableContentHash?: string;
+  rawHtmlPath?: string;
+  parsedJson: Record<string, unknown>;
+};
+
+export type FetchClassification =
+  | { ok: true }
+  | {
+      ok: false;
+      reason: CrawlFailureReason;
+      retryable: boolean;
+    };
+
+export type CrawlJobRecord = {
+  id: string;
+  source: MarketSource;
+  url: string;
+  jobType: CrawlJobType;
+  state: CrawlJobState;
+  attempts: number;
+  nextRunAt: string;
+  lastError?: string;
+};

ingestion/src/worker.ts

@@ -0,0 +1,281 @@
+import { ingestionConfig } from './config.ts';
+import { createPgPool } from './db.ts';
+import { fetchRawSnapshot, getFetchFailureReason, randomDelayMs, type RawFetchSnapshot } from './fetcher.ts';
+import { normalizeMarketProduct } from './normalizer.ts';
+import { CrawlQueue, dailyBudgetForJob } from './queue.ts';
+import { IngestionRepository } from './repository.ts';
+import {
+  buildCookieHeader,
+  buildMvideoBffUrl,
+  buildMvideoProductUrl,
+  discoverMvideoProductIds,
+  extractMvideoBffBasePath,
+  extractMvideoSearchQuery,
+  parseMvideoProduct,
+} from './sources/mvideo.ts';
+import { discoverYandexMarketProductsFromCategory, parseYandexMarketProduct } from './sources/yandexMarket.ts';
+import { createStableContentHash } from './stableHash.ts';
+import type { CrawlFailureReason, CrawlJobRecord, DiscoveredProduct } from './types.ts';
+
+function sleep(ms: number) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function isStopReason(reason: string | undefined) {
+  return reason === 'captcha' || reason === 'blocked' || reason === 'rate_limited';
+}
+
+function sourceError(reason: CrawlFailureReason, message: string) {
+  const error = new Error(message) as Error & { reason?: CrawlFailureReason };
+  error.reason = reason;
+  return error;
+}
+
+function parseJsonSnapshot(snapshot: RawFetchSnapshot, label: string) {
+  try {
+    return JSON.parse(snapshot.html) as unknown;
+  } catch (error) {
+    throw sourceError('parse_failed', `Expected JSON from ${label}: ${(error as Error).message}`);
+  }
+}
+
+async function recordSnapshot(
+  repository: IngestionRepository,
+  snapshot: RawFetchSnapshot,
+  parsedJson: Record<string, unknown>,
+  stableContentHash?: string,
+) {
+  await repository.recordSnapshot({
+    source: snapshot.source,
+    url: snapshot.url,
+    fetchedAt: snapshot.fetchedAt,
+    httpStatus: snapshot.httpStatus,
+    rawSnapshotHash: snapshot.rawSnapshotHash,
+    rawHtmlPath: snapshot.rawHtmlPath,
+    parsedJson,
+    ...(stableContentHash ? { stableContentHash } : {}),
+  });
+}
+
+async function processYandexCategory(
+  job: CrawlJobRecord,
+  repository: IngestionRepository,
+  queue: CrawlQueue,
+) {
+  const snapshot = await fetchRawSnapshot({
+    source: job.source,
+    url: job.url,
+    snapshotDir: ingestionConfig.snapshotDir,
+  });
+  const products = discoverYandexMarketProductsFromCategory(snapshot.html, job.url, snapshot.fetchedAt);
+
+  await recordSnapshot(repository, snapshot, { discoveredProducts: products });
+  await repository.recordDiscoveredProducts(products);
+
+  for (const product of products) {
+    await queue.enqueue({
+      source: product.source,
+      url: product.productUrl,
+      jobType: 'product',
+    });
+  }
+
+  await queue.markState(job.id, 'parsed');
+  console.log(`yandex_market category parsed: ${products.length} product candidates`);
+}
+
+async function processMvideoCategory(
+  job: CrawlJobRecord,
+  repository: IngestionRepository,
+  queue: CrawlQueue,
+) {
+  const categorySnapshot = await fetchRawSnapshot({
+    source: job.source,
+    url: job.url,
+    snapshotDir: ingestionConfig.snapshotDir,
+  });
+  const bffBasePath = extractMvideoBffBasePath(categorySnapshot.html);
+  const searchQuery = extractMvideoSearchQuery(categorySnapshot.url);
+  const searchUrl = buildMvideoBffUrl(bffBasePath, 'products/v2/search');
+  const cookieHeader = buildCookieHeader(categorySnapshot.setCookieHeaders);
+
+  searchUrl.searchParams.set('query', searchQuery);
+  searchUrl.searchParams.set('offset', '0');
+  searchUrl.searchParams.set('limit', String(ingestionConfig.productDailyBudget));
+
+  const searchSnapshot = await fetchRawSnapshot({
+    source: job.source,
+    url: searchUrl.toString(),
+    snapshotDir: ingestionConfig.snapshotDir,
+    accept: 'application/json,text/plain,*/*',
+    ...(cookieHeader ? { headers: { cookie: cookieHeader } } : {}),
+  });
+  const productIds = discoverMvideoProductIds(parseJsonSnapshot(searchSnapshot, 'MVideo search'));
+  const products: DiscoveredProduct[] = productIds.map((sourceProductId) => ({
+    source: 'mvideo',
+    sourceProductId,
+    productUrl: buildMvideoProductUrl(sourceProductId),
+    categoryUrl: categorySnapshot.url,
+    discoveredAt: categorySnapshot.fetchedAt,
+  }));
+
+  await recordSnapshot(repository, categorySnapshot, {
+    bffBasePath,
+    searchQuery,
+    discoveredProducts: products,
+  });
+  await recordSnapshot(repository, searchSnapshot, { productIds });
+  await repository.recordDiscoveredProducts(products);
+
+  for (const product of products) {
+    await queue.enqueue({
+      source: product.source,
+      url: product.productUrl,
+      jobType: 'product',
+    });
+  }
+
+  await queue.markState(job.id, 'parsed');
+  console.log(`mvideo category parsed: ${products.length} product candidates`);
+}
+
+async function processYandexProduct(job: CrawlJobRecord, repository: IngestionRepository, queue: CrawlQueue) {
+  const snapshot = await fetchRawSnapshot({
+    source: job.source,
+    url: job.url,
+    snapshotDir: ingestionConfig.snapshotDir,
+  });
+  const parsed = parseYandexMarketProduct(snapshot.html, job.url);
+  const normalized = normalizeMarketProduct(parsed);
+  const stableContentHash = createStableContentHash(normalized);
+
+  await recordSnapshot(repository, snapshot, { parsed, normalized }, stableContentHash);
+  await repository.upsertNormalizedProduct(normalized, stableContentHash, snapshot.fetchedAt);
+  await queue.markState(job.id, 'needs_review');
+  console.log(`yandex_market product parsed: ${normalized.brand} ${normalized.canonicalModel}`);
+}
+
+async function processMvideoProduct(job: CrawlJobRecord, repository: IngestionRepository, queue: CrawlQueue) {
+  const productPageSnapshot = await fetchRawSnapshot({
+    source: job.source,
+    url: job.url,
+    snapshotDir: ingestionConfig.snapshotDir,
+  });
+  const bffBasePath = extractMvideoBffBasePath(productPageSnapshot.html);
+  const cookieHeader = buildCookieHeader(productPageSnapshot.setCookieHeaders);
+  const sourceProductId = productPageSnapshot.url.match(/(\d+)(?:\/?(?:\?.*)?)$/)?.[1];
+
+  if (!sourceProductId) {
+    throw sourceError('parse_failed', `MVideo product id was not found in ${productPageSnapshot.url}`);
+  }
+
+  const detailsUrl = buildMvideoBffUrl(bffBasePath, 'product-details');
+  detailsUrl.searchParams.set('productId', sourceProductId);
+
+  const detailsSnapshot = await fetchRawSnapshot({
+    source: job.source,
+    url: detailsUrl.toString(),
+    snapshotDir: ingestionConfig.snapshotDir,
+    accept: 'application/json,text/plain,*/*',
+    ...(cookieHeader ? { headers: { cookie: cookieHeader } } : {}),
+  });
+  const parsed = parseMvideoProduct(parseJsonSnapshot(detailsSnapshot, 'MVideo product details'));
+  const normalized = normalizeMarketProduct(parsed);
+  const stableContentHash = createStableContentHash(normalized);
+
+  await recordSnapshot(
+    repository,
+    productPageSnapshot,
+    { productApiUrl: detailsUrl.toString(), parsed, normalized },
+    stableContentHash,
+  );
+  await recordSnapshot(repository, detailsSnapshot, { parsed, normalized }, stableContentHash);
+  await repository.upsertNormalizedProduct(normalized, stableContentHash, productPageSnapshot.fetchedAt);
+  await queue.markState(job.id, 'needs_review');
+  console.log(`mvideo product parsed: ${normalized.brand} ${normalized.canonicalModel}`);
+}
+
+async function processJob(job: CrawlJobRecord, queue: CrawlQueue, repository: IngestionRepository) {
+  const reserved = await queue.reserveDailyBudget(job.source, job.jobType, dailyBudgetForJob(job.jobType));
+
+  if (!reserved) {
+    await queue.markFailure(job, 'rate_limited', `Daily ${job.jobType} budget is exhausted`);
+    return;
+  }
+
+  try {
+    if (job.source === 'yandex_market' && job.jobType === 'category') {
+      await processYandexCategory(job, repository, queue);
+      return;
+    }
+
+    if (job.source === 'mvideo' && job.jobType === 'category') {
+      await processMvideoCategory(job, repository, queue);
+      return;
+    }
+
+    if (job.source === 'yandex_market' && job.jobType === 'product') {
+      await processYandexProduct(job, repository, queue);
+      return;
+    }
+
+    if (job.source === 'mvideo' && job.jobType === 'product') {
+      await processMvideoProduct(job, repository, queue);
+      return;
+    }
+
+    await queue.markFailure(job, 'needs_manual_review', `Unsupported job: ${job.source}/${job.jobType}`);
+  } catch (error) {
+    const reason = getFetchFailureReason(error) ?? 'parse_failed';
+
+    await queue.markFailure(job, reason, (error as Error).message);
+
+    if (isStopReason(reason)) {
+      await queue.pauseSource(job.source, reason, new Date(Date.now() + 24 * 60 * 60 * 1000));
+    }
+  }
+}
+
+export async function runWorker() {
+  const pool = createPgPool();
+  const queue = new CrawlQueue(pool);
+  const repository = new IngestionRepository(pool);
+
+  for (const seed of ingestionConfig.sourceSeeds) {
+    if (!seed.implemented) {
+      console.warn(`source seed configured but crawler is not implemented yet: ${seed.source} ${seed.categoryUrl}`);
+      continue;
+    }
+
+    await queue.enqueue({
+      source: seed.source,
+      url: seed.categoryUrl,
+      jobType: 'category',
+    });
+  }
+
+  try {
+    do {
+      let processed = 0;
+
+      for (let index = 0; index < ingestionConfig.workerBatchSize; index += 1) {
+        const job = await queue.leaseNext();
+
+        if (!job) break;
+
+        await processJob(job, queue, repository);
+        processed += 1;
+        await sleep(randomDelayMs());
+      }
+
+      if (process.env.INGESTION_ONCE === '1') break;
+      if (processed === 0) await sleep(ingestionConfig.workerIdleMs);
+    } while (true);
+  } finally {
+    await pool.end();
+  }
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  await runWorker();
+}

package.json

@@ -4,17 +4,26 @@
   "version": "0.1.0",
   "type": "module",
   "scripts": {
-    "dev": "next dev",
+    "dev": "next dev web-front",
-    "build": "next build",
+    "build": "next build web-front",
-    "start": "next start",
+    "start": "next start web-front",
     "typecheck": "tsc --noEmit",
     "test": "vitest run",
+    "catalog:import": "node --experimental-strip-types web-front/scripts/importCatalogExport.ts",
+    "ingestion:migrate": "node --experimental-strip-types ingestion/src/migrate.ts",
+    "ingestion:worker": "node --experimental-strip-types ingestion/src/worker.ts",
+    "ingestion:admin": "node --experimental-strip-types ingestion/src/adminServer.ts",
+    "ingestion:export-catalog": "node --experimental-strip-types ingestion/src/exportCatalog.ts",
+    "ingestion:smoke-yandex": "node --experimental-strip-types ingestion/src/smokeFetchYandexMarket.ts",
+    "ingestion:smoke-mvideo": "node --experimental-strip-types ingestion/src/smokeFetchMvideo.ts",
     "ci:quality": "npm run typecheck && npm test && npm run build"
   },
   "dependencies": {
     "next": "16.2.4",
+    "pg": "8.20.0",
     "react": "19.2.5",
-    "react-dom": "19.2.5"
+    "react-dom": "19.2.5",
+    "zod": "4.4.2"
   },
   "optionalDependencies": {
     "@next/swc-darwin-arm64": "16.2.4",
@@ -31,6 +40,7 @@
     "@emnapi/runtime": "1.10.0",
     "@tailwindcss/postcss": "4.2.4",
     "@types/node": "24.10.1",
+    "@types/pg": "8.20.0",
     "@types/react": "19.2.14",
     "@types/react-dom": "19.2.3",
     "tailwindcss": "4.2.4",

src/app/globals.css

@@ -1,308 +0,0 @@
-:root {
-  color-scheme: light;
-  --bg: #f6f6f3;
-  --panel: #ffffff;
-  --panel-muted: #efede7;
-  --text: #1f2528;
-  --muted: #667075;
-  --line: #d8d5cc;
-  --accent: #156f5b;
-  --accent-strong: #0c4f41;
-  --warning: #9a5c12;
-  --danger: #a33a3a;
-  --info: #345f9f;
-  --shadow: 0 10px 24px rgb(31 37 40 / 8%);
-}
-
-* {
-  box-sizing: border-box;
-}
-
-html,
-body {
-  min-height: 100%;
-}
-
-body {
-  margin: 0;
-  background: var(--bg);
-  color: var(--text);
-  font-family:
-    Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
-}
-
-button,
-input,
-select {
-  font: inherit;
-}
-
-.page-shell {
-  width: min(1180px, calc(100% - 32px));
-  margin: 0 auto;
-  padding: 24px 0 40px;
-}
-
-.topbar {
-  display: flex;
-  align-items: flex-start;
-  justify-content: space-between;
-  gap: 24px;
-  border-bottom: 1px solid var(--line);
-  padding-bottom: 18px;
-}
-
-.eyebrow {
-  margin: 0 0 6px;
-  color: var(--accent);
-  font-size: 0.78rem;
-  font-weight: 700;
-  text-transform: uppercase;
-}
-
-h1,
-h2,
-h3,
-p {
-  overflow-wrap: anywhere;
-}
-
-h1 {
-  margin: 0;
-  font-size: 2rem;
-  line-height: 1.1;
-}
-
-h2 {
-  margin: 0;
-  font-size: 1.1rem;
-  line-height: 1.25;
-}
-
-h3 {
-  margin: 0;
-  font-size: 0.95rem;
-  line-height: 1.25;
-}
-
-.workbench {
-  display: grid;
-  grid-template-columns: minmax(280px, 360px) minmax(0, 1fr);
-  gap: 20px;
-  align-items: start;
-  padding-top: 22px;
-}
-
-.panel {
-  background: var(--panel);
-  border: 1px solid var(--line);
-  border-radius: 8px;
-  box-shadow: var(--shadow);
-}
-
-.panel-header {
-  display: flex;
-  align-items: flex-start;
-  justify-content: space-between;
-  gap: 16px;
-  border-bottom: 1px solid var(--line);
-  padding: 16px;
-}
-
-.panel-body {
-  padding: 16px;
-}
-
-.form-grid {
-  display: grid;
-  gap: 14px;
-}
-
-.field {
-  display: grid;
-  gap: 6px;
-}
-
-.field span {
-  color: var(--muted);
-  font-size: 0.82rem;
-  font-weight: 650;
-}
-
-.field input,
-.field select {
-  min-height: 42px;
-  width: 100%;
-  border: 1px solid var(--line);
-  border-radius: 6px;
-  background: #fff;
-  color: var(--text);
-  padding: 9px 10px;
-}
-
-.field input:focus,
-.field select:focus {
-  border-color: var(--accent);
-  outline: 3px solid rgb(21 111 91 / 16%);
-}
-
-.submit-button {
-  min-height: 42px;
-  border: 0;
-  border-radius: 6px;
-  background: var(--accent);
-  color: #fff;
-  cursor: pointer;
-  font-weight: 750;
-}
-
-.submit-button:hover {
-  background: var(--accent-strong);
-}
-
-.stats-grid {
-  display: grid;
-  grid-template-columns: repeat(4, minmax(110px, 1fr));
-  gap: 10px;
-}
-
-.metric {
-  min-height: 76px;
-  border: 1px solid var(--line);
-  border-radius: 8px;
-  background: var(--panel-muted);
-  padding: 12px;
-}
-
-.metric span {
-  display: block;
-  color: var(--muted);
-  font-size: 0.76rem;
-  font-weight: 700;
-  text-transform: uppercase;
-}
-
-.metric strong {
-  display: block;
-  margin-top: 7px;
-  font-size: 1.45rem;
-}
-
-.video-list {
-  display: grid;
-  gap: 10px;
-  margin-top: 16px;
-}
-
-.video-card {
-  display: grid;
-  grid-template-columns: minmax(0, 1fr) auto;
-  gap: 14px;
-  border: 1px solid var(--line);
-  border-radius: 8px;
-  background: #fff;
-  padding: 14px;
-}
-
-.video-meta {
-  display: flex;
-  flex-wrap: wrap;
-  gap: 8px;
-  margin-top: 8px;
-}
-
-.pill {
-  min-height: 24px;
-  border-radius: 999px;
-  background: var(--panel-muted);
-  color: var(--muted);
-  padding: 4px 9px;
-  font-size: 0.76rem;
-  font-weight: 700;
-}
-
-.pill.status-draft {
-  color: var(--warning);
-}
-
-.pill.status-published {
-  color: var(--accent);
-}
-
-.pill.status-archived {
-  color: var(--danger);
-}
-
-.cache-panel {
-  display: grid;
-  grid-template-columns: repeat(3, auto);
-  gap: 8px;
-  justify-content: end;
-}
-
-.cache-item {
-  min-height: 42px;
-  border: 1px solid var(--line);
-  border-radius: 8px;
-  background: var(--panel);
-  padding: 7px 10px;
-}
-
-.cache-item span {
-  display: block;
-  color: var(--muted);
-  font-size: 0.7rem;
-  font-weight: 700;
-  text-transform: uppercase;
-}
-
-.cache-item strong {
-  display: block;
-  margin-top: 2px;
-  font-size: 0.84rem;
-}
-
-.empty-state {
-  border: 1px dashed var(--line);
-  border-radius: 8px;
-  color: var(--muted);
-  padding: 22px;
-  text-align: center;
-}
-
-@media (max-width: 900px) {
-  .topbar,
-  .workbench {
-    grid-template-columns: 1fr;
-  }
-
-  .topbar {
-    display: grid;
-  }
-
-  .cache-panel {
-    justify-content: stretch;
-    grid-template-columns: repeat(3, minmax(0, 1fr));
-  }
-
-  .stats-grid {
-    grid-template-columns: repeat(2, minmax(0, 1fr));
-  }
-}
-
-@media (max-width: 560px) {
-  .page-shell {
-    width: min(100% - 20px, 1180px);
-    padding-top: 16px;
-  }
-
-  h1 {
-    font-size: 1.55rem;
-  }
-
-  .cache-panel,
-  .stats-grid,
-  .video-card {
-    grid-template-columns: 1fr;
-  }
-}

src/app/layout.tsx

@@ -1,19 +0,0 @@
-import type { Metadata } from 'next';
-import './globals.css';
-
-export const metadata: Metadata = {
-  title: 'Video Registry',
-  description: 'Operational video registry with primitive storage and aggressive caching.',
-};
-
-type RootLayoutProps = {
-  children: React.ReactNode;
-};
-
-export default function RootLayout({ children }: RootLayoutProps) {
-  return (
-    <html lang="en">
-      <body>{children}</body>
-    </html>
-  );
-}

src/app/page.tsx

@@ -1,27 +0,0 @@
-import { getCachedVideoDashboard } from '@/entities/video/api/videoCache';
-import { RegisterVideoForm } from '@/features/register-video/ui/RegisterVideoForm';
-import { CachePanel } from '@/widgets/cache-panel/ui/CachePanel';
-import { VideoDashboard } from '@/widgets/video-dashboard/ui/VideoDashboard';
-
-export const revalidate = 3600;
-
-export default async function Home() {
-  const dashboard = await getCachedVideoDashboard();
-
-  return (
-    <main className="page-shell">
-      <header className="topbar">
-        <div>
-          <p className="eyebrow">videoreg.ru</p>
-          <h1>Video Registry</h1>
-        </div>
-        <CachePanel metadata={dashboard.cache} />
-      </header>
-
-      <div className="workbench">
-        <RegisterVideoForm />
-        <VideoDashboard dashboard={dashboard} />
-      </div>
-    </main>
-  );
-}

tasks/002-crawler.md

@@ -0,0 +1,656 @@
+# Спека: Yandex Market Dashcam Catalog Ingestion
+
+## 1. Цель
+
+Построить поддерживаемый ingestion engine для каталога автомобильных видеорегистраторов.
+
+Целевая категория:
+
+`Автомобильные видеорегистраторы`
+
+Публичное приложение должно получать только approved canonical catalog:
+
+* название
+* бренд
+* модель
+* нормализованные характеристики
+* изображения
+* source metadata
+* дату первого обнаружения
+* дату последнего обновления
+
+В MVP не парсим и не экспортируем:
+
+* цены
+* продавцов
+* наличие
+* тексты отзывов
+* авторов отзывов
+* review content
+
+Общий aggregate rating можно сохранять внутри ingestion как source signal, но он не входит в публичный export и не участвует в `stableContentHash`.
+
+Главная мысль: не "краулер, который пролезет", а каталоговый ingestion engine, который можно поддерживать годами.
+
+---
+
+## 2. Границы и запреты
+
+Не реализовывать:
+
+* обход rate limiter
+* маскировку под реальных пользователей
+* ротацию User-Agent / browser fingerprint
+* прокси-фермы
+* headless stealth
+* обход captcha
+* агрессивную параллельную загрузку
+* регулярный browser-based crawl
+
+В MVP fetcher работает только через обычный HTTP fetch.
+
+Headless browser допустим только как ручной diagnostic tool для анализа структуры страницы. Он не является fallback в регулярном crawl path.
+
+---
+
+## 3. Источники
+
+### Primary
+
+* публичные category pages Яндекс Маркета
+* публичные product pages
+* JSON-LD, если доступен в HTML
+* embedded state, если доступен в HTML
+* sitemap, но только в ограниченных границах
+
+### Preferred official fallback
+
+* Yandex Market Partner API, если доступен по условиям проекта
+* фиды продавцов, если доступны легально
+* фиды брендов
+* официальные сайты производителей
+
+---
+
+## 4. Deployment Model
+
+Ingestion деплоится в то же облако, что и web-приложение (один Swarm/Portainer
+environment), а не на отдельный сервер. Раньше предполагался отдельный cloud, но
+управлять двумя независимыми облаками оказалось слишком сложно. См. решение
+`videoreg-nnv.3`.
+
+Код хранится в этом же репозитории:
+
+```text
+ingestion/
+  src/
+  migrations/
+  docker-compose.yml
+  Dockerfile
+```
+
+Минимальные сервисы:
+
+```text
+postgres
+ingestion-worker
+ingestion-admin
+```
+
+`export-server` больше не нужен: ingestion-worker пишет валидированный
+`catalog.json` + `manifest.json` в общий (shared) export-том, а web читает их
+напрямую с этого тома. Никакого HTTP-экспорта и Bearer-токена.
+
+Основное Next.js-приложение по-прежнему не подключается напрямую к ingestion DB —
+единственный контакт с каталогом идёт через export-артефакт на shared-томе.
+
+---
+
+## 5. Source of Truth и Bridge
+
+Source of truth для каталога: PostgreSQL ingestion stack.
+
+Текущее примитивное хранилище приложения (`data/videoreg.json`) является только read-model cache.
+
+Bridge работает со стороны приложения через общий export-том:
+
+1. ingestion пишет versioned export (`manifest.json` + `catalog.json`) на shared-том;
+2. приложение/сборка читает `manifest.json` с того же тома;
+3. проверяет `schemaVersion`, `exportId`, `createdAt`, `sha256`, `itemCount`;
+4. читает `catalog.json` с тома;
+5. сверяет `sha256` артефакта;
+6. валидирует payload через shared Zod schema;
+7. атомарно заменяет `data/videoreg.json` (или используется как build-time источник).
+
+В MVP import запускается вручную оператором, не по расписанию. HTTP/токен в этом
+обмене не участвуют — оба сервиса в одном облаке и делят том.
+
+---
+
+## 6. Export Contract
+
+Export публикуется как файлы на общем (shared) export-томе: `manifest.json` и
+`artifacts/<exportId>/catalog.json`. HTTP-эндпоинт и Bearer-токен не используются —
+ingestion и web живут в одном облаке и делят том. Целостность гарантируется
+`sha256` в манифесте и shared Zod-схемой при чтении.
+
+### Manifest
+
+```ts
+type CatalogExportManifest = {
+  schemaVersion: 1;
+  exportId: string;
+  createdAt: string;
+  artifactPath: string;
+  sha256: string;
+  itemCount: number;
+};
+```
+
+### Catalog Artifact
+
+```ts
+type DashcamCatalogExport = {
+  schemaVersion: 1;
+  exportId: string;
+  createdAt: string;
+  products: ExportedDashcamProduct[];
+};
+
+type ExportedDashcamProduct = {
+  id: string;
+  brand: string;
+  model: string;
+  canonicalName?: string;
+  title: string;
+  specs: Record<string, string | number | boolean>;
+  images: ExportedProductImage[];
+  sources: ExportedProductSource[];
+  discoveredAt: string;
+  updatedAt: string;
+};
+
+type ExportedProductImage = {
+  url: string;
+  originalUrl: string;
+  source: MarketSource;
+  sha256?: string;
+  width?: number;
+  height?: number;
+};
+
+type ExportedProductSource = {
+  source: MarketSource;
+  sourceProductId?: string;
+  url: string;
+  stableContentHash: string;
+  fetchedAt: string;
+};
+
+type MarketSource = "yandex_market" | "mvideo";
+```
+
+Runtime validation: shared Zod schema в репозитории. TypeScript-типы без runtime-валидации недостаточны.
+
+---
+
+## 7. Image Policy
+
+Ingestion кеширует approved images и публикует versioned image artifact URLs.
+
+Export содержит:
+
+* URL versioned image artifact
+* original source URL
+* source metadata
+* optional checksum/dimensions
+
+Приложение не должно hotlink-ить source images напрямую.
+
+Юридическое решение о хранении и показе изображений должно быть принято отдельно до production-публикации.
+
+---
+
+## 8. Crawl Policy
+
+Нагрузка:
+
+* 1 запрос за 5-15 секунд
+* максимум 100-300 страниц в сутки на источник
+* jitter между запросами
+* exponential backoff при 429/403/5xx
+* остановка источника при captcha
+* отдельный дневной budget на category pages, product pages, sitemap
+* не скачивать повторно страницы, если stable content не изменился
+
+HTTP-only fetcher должен явно детектить:
+
+```text
+rate_limited
+blocked
+captcha
+not_found
+parse_failed
+needs_manual_review
+```
+
+При `captcha`, `403`, `429` источник ставится на паузу. Не пытаться обходить ограничение.
+
+---
+
+## 9. Discovery Strategy
+
+Discovery работает sitemap-first, но не обходит широкий sitemap всего Маркета.
+
+Границы discovery задают:
+
+* ручные seed category URLs
+* ручные seed product URL patterns
+* product URLs, найденные из category HTTP discovery
+* уже известные product URLs
+
+Sitemap используется для:
+
+* diff известных URL
+* refresh кандидатных URL
+* обнаружение URL рядом с уже известными границами
+
+Category pages используются вторично:
+
+* первая страница категории
+* пагинация, если доступна без JS
+* top listings
+* новые product URLs, если они есть в HTML
+
+Если category HTML бедный без JS, discovery не должен переходить в regular headless crawl. Такие случаи уходят в `needs_manual_review` или official fallback.
+
+```ts
+type DiscoveredProduct = {
+  source: MarketSource;
+  categoryUrl?: string;
+  productUrl: string;
+  sourceProductId?: string;
+  titlePreview?: string;
+  discoveredAt: string;
+};
+```
+
+---
+
+## 10. Fetch Queue
+
+Очередь реализуется в PostgreSQL, без Redis/BullMQ в MVP.
+
+State machine:
+
+```text
+pending
+ -> fetching
+ -> fetched
+ -> parsed
+ -> normalized
+ -> deduplicated
+ -> needs_review
+ -> approved
+ -> exported
+```
+
+Ошибочные состояния:
+
+```text
+rate_limited
+blocked
+captcha
+not_found
+parse_failed
+needs_manual_review
+```
+
+Для каждой задачи хранить:
+
+* source
+* URL
+* job type
+* state
+* attempts
+* nextRunAt
+* lastError
+* createdAt
+* updatedAt
+
+Backoff и daily budgets считаются в PostgreSQL.
+
+---
+
+## 11. Parser
+
+Parser извлекает только разрешенные поля:
+
+* title
+* brand
+* model
+* specs
+* images
+* sourceProductId
+* aggregate rating как internal-only signal
+
+Приоритет источников внутри страницы:
+
+```text
+JSON-LD > embedded state > semantic DOM > fallback DOM
+```
+
+Не извлекать:
+
+* prices
+* sellers
+* availability
+* review texts
+* review authors
+
+```ts
+type ParsedMarketProduct = {
+  source: MarketSource;
+  sourceProductId?: string;
+  title: string;
+  brand?: string;
+  model?: string;
+  aggregateRating?: {
+    value: number;
+    scale?: number;
+    ratingCount?: number;
+  };
+  images: string[];
+  specs: Record<string, string | number | boolean>;
+  sourceUrl: string;
+};
+```
+
+---
+
+## 12. Snapshot и Checksum Model
+
+Сырой snapshot хранить всегда. Парсер должен уметь переобрабатывать старые страницы.
+
+Нельзя использовать hash всего HTML как сигнал "страница изменилась": динамическая верстка, цены, рекомендации, счетчики и timestamps будут постоянно шуметь.
+
+Для каждого snapshot хранить два хэша:
+
+```ts
+type ProductSourceSnapshot = {
+  source: MarketSource;
+  url: string;
+  fetchedAt: string;
+  httpStatus: number;
+  rawSnapshotHash: string;
+  stableContentHash?: string;
+  rawHtmlPath?: string;
+  parsedJson: Record<string, unknown>;
+};
+```
+
+### `rawSnapshotHash`
+
+Хэш точного raw snapshot для аудита и debug.
+
+### `stableContentHash`
+
+Source-specific semantic hash. Для Яндекс Маркета в MVP включать:
+
+* sourceProductId
+* canonical product URL
+* title
+* brand
+* model
+* normalized specs
+* image identifiers / original image URLs
+
+Исключать:
+
+* price
+* sellers
+* availability
+* aggregate rating
+* review count
+* timestamps
+* layout
+* recommendations
+* counters
+* tracking data
+
+---
+
+## 13. Normalization
+
+Нужно привести хаотичные характеристики к единому виду.
+
+Пример:
+
+```json
+{
+  "resolution_front": "3840x2160",
+  "channels": 2,
+  "sensor_front": "Sony STARVIS 2",
+  "gps": true,
+  "wifi": true,
+  "screen": true,
+  "parking_mode": true,
+  "memory_card_max_gb": 256,
+  "view_angle_degrees": 140
+}
+```
+
+Компоненты normalizer:
+
+* brand dictionary
+* model cleaner
+* spec mapper
+* unit converter
+* boolean detector
+* alias dictionary
+
+---
+
+## 14. Deduplication
+
+Дедупликация работает до модерации. Единица модерации: canonical product, уже сгруппированный из вариантов и source URLs.
+
+Сравнивать:
+
+* brand
+* normalized model name
+* aliases
+* sourceProductId
+* image perceptual hash
+* specs similarity
+* product URL history
+
+Статусы:
+
+```text
+new_product
+same_product
+possible_duplicate
+variant
+oem_clone
+```
+
+`possible_duplicate`, `variant`, `oem_clone` не должны попадать в public export без ручного решения.
+
+---
+
+## 15. Moderation Admin
+
+Админка живет внутри отдельного ingestion stack.
+
+Она работает напрямую с ingestion PostgreSQL, snapshots, очередями и canonical products.
+
+MVP-экраны:
+
+* новые canonical products
+* bulk select
+* "разрешить все"
+* товары с `parse_failed`
+* возможные дубли
+* конфликтующие характеристики
+* история изменений
+* ручное объединение моделей
+* ручное редактирование canonical specs
+
+Публикация в export разрешена только после ручной модерации. Bulk approve допустим, но утверждает canonical products, а не отдельные source URLs.
+
+---
+
+## 16. Database
+
+Минимальная PostgreSQL-модель:
+
+```sql
+products
+product_variants
+product_sources
+source_snapshots
+product_specs
+product_images
+source_ratings
+crawl_jobs
+crawl_errors
+moderation_queue
+export_runs
+```
+
+Принципы:
+
+* raw HTML/JSON snapshots и image artifacts хранить на mounted volume
+* в PostgreSQL хранить пути, checksum, metadata и связи
+* уникальность по `source + url`
+* ingestion DB является source of truth
+* `data/videoreg.json` приложения является replaceable read-model cache
+
+---
+
+## 17. Sync Schedule
+
+### Частый легкий sync
+
+Каждые 6-12 часов:
+
+* первая страница категории
+* sitemap diff в пределах известных границ
+* новые product URLs
+* изменение top listings, если доступно без JS
+
+### Глубокий sync
+
+Раз в 7-14 дней:
+
+* все известные карточки
+* характеристики
+* изображения
+* source metadata
+
+### New product detection
+
+Каждый день:
+
+* category diff
+* bounded sitemap diff
+* known model aliases search
+* official fallback feeds, если доступны
+
+---
+
+## 18. MVP Phases
+
+### Phase 1 - Legal discovery
+
+* проверить robots.txt
+* проверить sitemap
+* проверить доступность API/фидов
+* описать crawl budget
+* описать allowed URL patterns
+* зафиксировать image storage policy
+
+### Phase 2 - Shared contract
+
+* добавить shared Zod schema для manifest/catalog export
+* добавить TypeScript-типы из схем
+* описать `schemaVersion`
+* добавить app import command
+* заменить `data/videoreg.json` validated catalog payload
+
+### Phase 3 - Ingestion stack skeleton
+
+* создать `ingestion/`
+* добавить `docker-compose.yml`
+* поднять PostgreSQL
+* добавить migrations
+* добавить mounted volume для snapshots/images
+* добавить healthcheck
+
+### Phase 4 - Queue and fetcher
+
+* реализовать PostgreSQL-backed queue
+* реализовать HTTP-only fetcher
+* добавить rate limits, retry, backoff
+* stop on captcha / 403 / 429
+* сохранять raw snapshots
+* считать `rawSnapshotHash`
+
+### Phase 5 - Parser
+
+* category parser
+* bounded sitemap discovery
+* product parser
+* specs parser
+* image parser
+* aggregate rating parser as internal-only
+* source-specific `stableContentHash`
+
+### Phase 6 - Normalizer
+
+* brand dictionary
+* model cleaner
+* spec mapper
+* unit converter
+* boolean detector
+
+### Phase 7 - Deduplication and moderation
+
+* exact match
+* fuzzy match
+* alias match
+* canonical product grouping
+* moderation queue
+* bulk approve
+
+### Phase 8 - Export and import
+
+* publish versioned manifest
+* publish versioned catalog artifact
+* publish versioned image artifacts
+* write export artifacts to the shared export volume (no HTTP, no token)
+* verify checksum in app import
+* atomic replace of `data/videoreg.json`
+
+---
+
+## 19. Definition of Done
+
+Система считается готовой, если:
+
+* ingestion и web деплоятся в одно облако; export идёт через shared-том, без HTTP/токена
+* PostgreSQL является source of truth
+* приложение импортирует только approved canonical catalog
+* `data/videoreg.json` заменяется только после manifest/checksum/schema validation
+* crawler не создает заметной нагрузки на источник
+* crawler останавливается при ограничениях
+* crawler не использует regular headless/browser stealth
+* raw snapshots хранятся всегда
+* есть `rawSnapshotHash` и source-specific `stableContentHash`
+* старые snapshots можно переобработать новым parser
+* дубли попадают в moderation queue
+* bulk approve работает по canonical products
+* public export не содержит цены, продавцов, наличие и тексты отзывов
+* public export не содержит aggregate rating в MVP
+* approved images отдаются как versioned image artifacts

tsconfig.json

@@ -13,6 +13,7 @@
     "esModuleInterop": true,
     "module": "esnext",
     "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
     "resolveJsonModule": true,
     "isolatedModules": true,
     "jsx": "react-jsx",
@@ -24,18 +25,17 @@
     ],
     "paths": {
       "@/*": [
-        "./src/*"
+        "./web-front/src/*"
       ]
     }
   },
   "include": [
-    "next-env.d.ts",
+    "web-front/**/*.ts",
-    "**/*.ts",
+    "web-front/**/*.tsx",
-    "**/*.tsx",
+    "ingestion/**/*.ts"
-    ".next/types/**/*.ts",
-    ".next/dev/types/**/*.ts"
   ],
   "exclude": [
-    "node_modules"
+    "node_modules",
+    "web-front/.next"
   ]
 }

vitest.config.ts

@@ -4,11 +4,11 @@ import { defineConfig } from 'vitest/config';
 export default defineConfig({
   test: {
     environment: 'node',
-    include: ['src/**/*.test.ts'],
+    include: ['web-front/src/**/*.test.ts', 'ingestion/src/**/*.test.ts'],
   },
   resolve: {
     alias: {
-      '@': path.resolve(__dirname, 'src'),
+      '@': path.resolve(__dirname, 'web-front/src'),
     },
   },
 });

web-front/postcss.config.mjs

@@ -0,0 +1,7 @@
+const config = {
+  plugins: {
+    '@tailwindcss/postcss': {},
+  },
+};
+
+export default config;

web-front/public/dashcam-product.svg

@@ -0,0 +1,18 @@
+<svg width="1200" height="720" viewBox="0 0 1200 720" fill="none" xmlns="http://www.w3.org/2000/svg">
+  <rect width="1200" height="720" fill="#151A1C"/>
+  <path d="M0 515C206 448 412 418 618 424C824 430 1018 473 1200 552V720H0V515Z" fill="#202629"/>
+  <path d="M170 574C356 516 552 492 758 504C918 513 1065 548 1200 608V720H0V646C55 618 112 594 170 574Z" fill="#2B3235"/>
+  <path d="M592 515L548 720H653L615 515H592Z" fill="#D7D1C4" fill-opacity=".72"/>
+  <path d="M423 292C423 246.713 459.713 210 505 210H740C785.287 210 822 246.713 822 292V396C822 441.287 785.287 478 740 478H505C459.713 478 423 441.287 423 396V292Z" fill="#0F1314"/>
+  <path d="M423 292C423 246.713 459.713 210 505 210H740C785.287 210 822 246.713 822 292V396C822 441.287 785.287 478 740 478H505C459.713 478 423 441.287 423 396V292Z" stroke="#6F7779" stroke-width="3"/>
+  <path d="M486 210L518 164H716L748 210H486Z" fill="#1E2527" stroke="#6F7779" stroke-width="3"/>
+  <circle cx="623" cy="344" r="93" fill="#1B2224" stroke="#A7B8BD" stroke-width="5"/>
+  <circle cx="623" cy="344" r="62" fill="#0B0D0E" stroke="#56798A" stroke-width="4"/>
+  <circle cx="623" cy="344" r="31" fill="#223037"/>
+  <circle cx="642" cy="324" r="13" fill="#D9E7EA" fill-opacity=".55"/>
+  <rect x="483" y="278" width="72" height="19" rx="4" fill="#56798A" fill-opacity=".55"/>
+  <rect x="692" y="278" width="72" height="19" rx="4" fill="#8F7660" fill-opacity=".46"/>
+  <rect x="488" y="410" width="270" height="14" rx="7" fill="#D7D1C4" fill-opacity=".25"/>
+  <path d="M289 185C410 109 567 75 760 92C895 104 1005 146 1090 218" stroke="#D9E7EA" stroke-opacity=".16" stroke-width="2"/>
+  <path d="M183 446C304 405 440 385 591 385C797 385 968 423 1104 499" stroke="#D9E7EA" stroke-opacity=".14" stroke-width="2"/>
+</svg>

web-front/scripts/importCatalogExport.ts

@@ -0,0 +1,23 @@
+import { importCatalogExport } from '../src/entities/dashcam-catalog/api/importCatalogExport.ts';
+
+const exportDir = process.env.DASHCAM_CATALOG_EXPORT_DIR ?? process.argv[2];
+
+if (!exportDir) {
+  throw new Error('Set DASHCAM_CATALOG_EXPORT_DIR or pass the shared export directory as the first argument');
+}
+
+const result = await importCatalogExport({ exportDir });
+
+console.log(
+  JSON.stringify(
+    {
+      status: 'imported',
+      exportId: result.exportId,
+      itemCount: result.itemCount,
+      sha256: result.sha256,
+      dbFilePath: result.dbFilePath,
+    },
+    null,
+    2,
+  ),
+);

web-front/src/app/(dev)/[devRoot]/showcase/layout.tsx

@@ -0,0 +1,19 @@
+import type { Metadata } from 'next';
+
+export const metadata: Metadata = {
+  title: 'Design System Showcase | videoreg.ru',
+  description:
+    'Hidden design-system showcase for the Best Dashcams 2026 comparison experience.',
+  robots: {
+    follow: false,
+    index: false,
+  },
+};
+
+type ShowcaseLayoutProps = {
+  children: React.ReactNode;
+};
+
+export default function ShowcaseLayout({ children }: ShowcaseLayoutProps) {
+  return children;
+}

web-front/src/app/(dev)/[devRoot]/showcase/page.tsx

@@ -0,0 +1,26 @@
+import { notFound } from 'next/navigation';
+import { dashcamProducts } from '@/entities/dashcam/model/data';
+import { BestDashcamsPage } from '@/shared/ui/templates/BestDashcamsPage';
+
+export const dynamicParams = false;
+export const revalidate = 3600;
+
+export function generateStaticParams() {
+  return [{ devRoot: '_dev' }];
+}
+
+type ShowcasePageProps = {
+  params: Promise<{
+    devRoot: string;
+  }>;
+};
+
+export default async function ShowcasePage({ params }: ShowcasePageProps) {
+  const { devRoot } = await params;
+
+  if (devRoot !== '_dev') {
+    notFound();
+  }
+
+  return <BestDashcamsPage products={dashcamProducts} />;
+}

web-front/src/app/globals.css

@@ -0,0 +1,656 @@
+@import "tailwindcss";
+
+:root {
+  color-scheme: light;
+  --font-ui: Geist, Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI",
+    sans-serif;
+  --font-tech: "IBM Plex Mono", Iosevka, ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas,
+    "Liberation Mono", monospace;
+
+  --surface-0: #f2eee6;
+  --surface-1: #fbfaf6;
+  --surface-2: #e8e1d5;
+  --surface-3: #d8cfc0;
+  --surface-inverse: #151a1c;
+
+  --text-primary: #202528;
+  --text-muted: #667076;
+  --text-subtle: #858076;
+  --text-inverse: #f7f3ea;
+  --text-on-accent: #f8fbfb;
+
+  --border-subtle: #d7cec0;
+  --border-strong: #bdb3a4;
+
+  --accent: #56798a;
+  --accent-strong: #385f70;
+  --accent-soft: #dbe7e9;
+  --accent-soft-border: #b7cbd1;
+
+  --warning: #8f7044;
+  --warning-soft: #efe5d6;
+  --warning-soft-border: #d7bea0;
+
+  --danger: #884b46;
+  --danger-soft: #eadbd8;
+  --danger-soft-border: #c9aaa5;
+
+  --success: #58705c;
+  --success-soft: #dde8dd;
+  --success-soft-border: #b8ccb9;
+
+  --focus: #56798a;
+  --radius-sm: 4px;
+  --radius-md: 6px;
+  --radius-lg: 8px;
+  --shadow-soft: 0 14px 34px rgb(32 37 40 / 8%);
+  --shadow-raised: 0 22px 48px rgb(32 37 40 / 12%);
+}
+
+@theme inline {
+  --font-sans: var(--font-ui);
+  --font-mono: var(--font-tech);
+}
+
+@media (prefers-color-scheme: dark) {
+  :root {
+    color-scheme: dark;
+    --surface-0: #101415;
+    --surface-1: #171d1f;
+    --surface-2: #202729;
+    --surface-3: #2d3538;
+    --surface-inverse: #f4efe6;
+
+    --text-primary: #f3efe8;
+    --text-muted: #b7b2aa;
+    --text-subtle: #948f87;
+    --text-inverse: #161a1c;
+    --text-on-accent: #f8fbfb;
+
+    --border-subtle: #30383b;
+    --border-strong: #4b5559;
+
+    --accent: #7f9ead;
+    --accent-strong: #b5ced7;
+    --accent-soft: #1d3139;
+    --accent-soft-border: #31505b;
+
+    --warning: #d0b184;
+    --warning-soft: #352a1e;
+    --warning-soft-border: #5a462b;
+
+    --danger: #d7aaa5;
+    --danger-soft: #351f1f;
+    --danger-soft-border: #583433;
+
+    --success: #a8c2aa;
+    --success-soft: #1f3022;
+    --success-soft-border: #38523b;
+
+    --shadow-soft: 0 14px 34px rgb(0 0 0 / 24%);
+    --shadow-raised: 0 22px 48px rgb(0 0 0 / 32%);
+  }
+}
+
+* {
+  box-sizing: border-box;
+}
+
+html {
+  min-height: 100%;
+  scroll-behavior: smooth;
+}
+
+body {
+  min-height: 100%;
+  margin: 0;
+  background: var(--surface-0);
+  color: var(--text-primary);
+  font-family: var(--font-ui);
+  text-rendering: optimizeLegibility;
+}
+
+button,
+input,
+select {
+  font: inherit;
+}
+
+a {
+  color: inherit;
+  text-decoration: none;
+}
+
+img {
+  max-width: 100%;
+}
+
+h1,
+h2,
+h3,
+p,
+td,
+th {
+  overflow-wrap: anywhere;
+}
+
+::selection {
+  background: var(--accent-soft);
+  color: var(--text-primary);
+}
+
+.page-shell {
+  width: min(1180px, calc(100% - 32px));
+  margin: 0 auto;
+  padding: 24px 0 40px;
+}
+
+.topbar {
+  display: flex;
+  align-items: flex-start;
+  justify-content: space-between;
+  gap: 24px;
+  border-bottom: 1px solid var(--border-subtle);
+  padding-bottom: 18px;
+}
+
+.eyebrow {
+  margin: 0 0 6px;
+  color: var(--accent-strong);
+  font-size: 0.78rem;
+  font-weight: 700;
+  text-transform: uppercase;
+}
+
+.page-shell h1 {
+  margin: 0;
+  font-size: 2rem;
+  line-height: 1.1;
+}
+
+.page-shell h2 {
+  margin: 0;
+  font-size: 1.1rem;
+  line-height: 1.25;
+}
+
+.page-shell h3 {
+  margin: 0;
+  font-size: 0.95rem;
+  line-height: 1.25;
+}
+
+.workbench {
+  display: grid;
+  grid-template-columns: minmax(280px, 360px) minmax(0, 1fr);
+  gap: 20px;
+  align-items: start;
+  padding-top: 22px;
+}
+
+.panel {
+  background: var(--surface-1);
+  border: 1px solid var(--border-subtle);
+  border-radius: var(--radius-lg);
+  box-shadow: var(--shadow-soft);
+}
+
+.panel-header {
+  display: flex;
+  align-items: flex-start;
+  justify-content: space-between;
+  gap: 16px;
+  border-bottom: 1px solid var(--border-subtle);
+  padding: 16px;
+}
+
+.panel-body {
+  padding: 16px;
+}
+
+.form-grid {
+  display: grid;
+  gap: 14px;
+}
+
+.field {
+  display: grid;
+  gap: 6px;
+}
+
+.field span {
+  color: var(--text-muted);
+  font-size: 0.82rem;
+  font-weight: 650;
+}
+
+.field input,
+.field select {
+  min-height: 42px;
+  width: 100%;
+  border: 1px solid var(--border-subtle);
+  border-radius: var(--radius-md);
+  background: var(--surface-1);
+  color: var(--text-primary);
+  padding: 9px 10px;
+}
+
+.field input:focus,
+.field select:focus {
+  border-color: var(--accent);
+  outline: 3px solid color-mix(in srgb, var(--accent) 22%, transparent);
+}
+
+.submit-button {
+  min-height: 42px;
+  border: 0;
+  border-radius: var(--radius-md);
+  background: var(--accent);
+  color: var(--text-on-accent);
+  cursor: pointer;
+  font-weight: 750;
+}
+
+.submit-button:hover {
+  background: var(--accent-strong);
+}
+
+.stats-grid {
+  display: grid;
+  grid-template-columns: repeat(4, minmax(110px, 1fr));
+  gap: 10px;
+}
+
+.metric {
+  min-height: 76px;
+  border: 1px solid var(--border-subtle);
+  border-radius: var(--radius-lg);
+  background: var(--surface-2);
+  padding: 12px;
+}
+
+.metric span {
+  display: block;
+  color: var(--text-muted);
+  font-size: 0.76rem;
+  font-weight: 700;
+  text-transform: uppercase;
+}
+
+.metric strong {
+  display: block;
+  margin-top: 7px;
+  font-size: 1.45rem;
+}
+
+.video-list {
+  display: grid;
+  gap: 10px;
+  margin-top: 16px;
+}
+
+.video-card {
+  display: grid;
+  grid-template-columns: minmax(0, 1fr) auto;
+  gap: 14px;
+  border: 1px solid var(--border-subtle);
+  border-radius: var(--radius-lg);
+  background: var(--surface-1);
+  padding: 14px;
+}
+
+.video-meta {
+  display: flex;
+  flex-wrap: wrap;
+  gap: 8px;
+  margin-top: 8px;
+}
+
+.pill {
+  min-height: 24px;
+  border-radius: 999px;
+  background: var(--surface-2);
+  color: var(--text-muted);
+  padding: 4px 9px;
+  font-size: 0.76rem;
+  font-weight: 700;
+}
+
+.pill.status-draft {
+  color: var(--warning);
+}
+
+.pill.status-published {
+  color: var(--success);
+}
+
+.pill.status-archived {
+  color: var(--danger);
+}
+
+.cache-panel {
+  display: grid;
+  grid-template-columns: repeat(3, auto);
+  gap: 8px;
+  justify-content: end;
+}
+
+.cache-item {
+  min-height: 42px;
+  border: 1px solid var(--border-subtle);
+  border-radius: var(--radius-lg);
+  background: var(--surface-1);
+  padding: 7px 10px;
+}
+
+.cache-item span {
+  display: block;
+  color: var(--text-muted);
+  font-size: 0.7rem;
+  font-weight: 700;
+  text-transform: uppercase;
+}
+
+.cache-item strong {
+  display: block;
+  margin-top: 2px;
+  font-size: 0.84rem;
+}
+
+.empty-state {
+  border: 1px dashed var(--border-subtle);
+  border-radius: var(--radius-lg);
+  color: var(--text-muted);
+  padding: 22px;
+  text-align: center;
+}
+
+.catalog-page {
+  width: min(1180px, calc(100% - 32px));
+  margin: 0 auto;
+  padding: 28px 0 56px;
+}
+
+.catalog-header {
+  display: flex;
+  align-items: flex-end;
+  justify-content: space-between;
+  gap: 24px;
+  border-bottom: 1px solid var(--border-subtle);
+  padding-bottom: 20px;
+}
+
+.catalog-header h1 {
+  margin: 0;
+  font-size: clamp(2rem, 4vw, 3.25rem);
+  line-height: 1.05;
+}
+
+.catalog-header__eyebrow,
+.catalog-product__brand {
+  margin: 0 0 8px;
+  color: var(--accent-strong);
+  font-size: 0.75rem;
+  font-weight: 750;
+  letter-spacing: 0;
+  text-transform: uppercase;
+}
+
+.catalog-header__subtitle {
+  max-width: 46rem;
+  margin: 12px 0 0;
+  color: var(--text-muted);
+  font-size: 1rem;
+  line-height: 1.5;
+}
+
+.catalog-header__meta {
+  display: grid;
+  min-width: 126px;
+  border-left: 1px solid var(--border-subtle);
+  padding-left: 20px;
+}
+
+.catalog-header__meta strong {
+  font-family: var(--font-tech);
+  font-size: 2rem;
+  line-height: 1;
+}
+
+.catalog-header__meta span,
+.catalog-status {
+  color: var(--text-muted);
+  font-size: 0.78rem;
+}
+
+.catalog-status {
+  display: flex;
+  flex-wrap: wrap;
+  gap: 8px 16px;
+  padding: 14px 0;
+}
+
+.catalog-status span + span::before {
+  content: '•';
+  margin-right: 16px;
+  color: var(--border-strong);
+}
+
+.catalog-grid {
+  display: grid;
+  grid-template-columns: repeat(2, minmax(0, 1fr));
+  gap: 16px;
+}
+
+.catalog-product {
+  display: grid;
+  grid-template-columns: 150px minmax(0, 1fr);
+  overflow: hidden;
+  border: 1px solid var(--border-subtle);
+  border-radius: var(--radius-lg);
+  background: var(--surface-1);
+  box-shadow: var(--shadow-soft);
+}
+
+.catalog-product__media {
+  display: grid;
+  place-items: center;
+  min-height: 182px;
+  background: var(--surface-inverse);
+  padding: 18px;
+}
+
+.catalog-product__media img {
+  width: 100%;
+  max-height: 132px;
+  object-fit: contain;
+  opacity: 0.84;
+}
+
+.catalog-product__body {
+  display: grid;
+  gap: 14px;
+  padding: 18px;
+}
+
+.catalog-product__heading {
+  display: flex;
+  align-items: flex-start;
+  justify-content: space-between;
+  gap: 12px;
+}
+
+.catalog-product h2 {
+  margin: 0;
+  font-size: 1.25rem;
+  line-height: 1.2;
+}
+
+.catalog-product__source-count {
+  flex: none;
+  border: 1px solid var(--border-subtle);
+  border-radius: var(--radius-sm);
+  color: var(--text-muted);
+  padding: 4px 6px;
+  font-family: var(--font-tech);
+  font-size: 0.68rem;
+}
+
+.catalog-product__title {
+  margin: 0;
+  color: var(--text-muted);
+  font-size: 0.85rem;
+  line-height: 1.45;
+}
+
+.catalog-product__specs,
+.catalog-product__details dl {
+  display: grid;
+  grid-template-columns: repeat(2, minmax(0, 1fr));
+  gap: 8px;
+  margin: 0;
+}
+
+.catalog-product__specs div,
+.catalog-product__details dl div {
+  min-width: 0;
+  border-top: 1px solid var(--border-subtle);
+  padding-top: 7px;
+}
+
+.catalog-product__specs dt,
+.catalog-product__details dt {
+  color: var(--text-muted);
+  font-size: 0.7rem;
+}
+
+.catalog-product__specs dd,
+.catalog-product__details dd {
+  margin: 3px 0 0;
+  font-size: 0.82rem;
+  font-weight: 650;
+}
+
+.catalog-product__details summary {
+  cursor: pointer;
+  color: var(--accent-strong);
+  font-size: 0.8rem;
+  font-weight: 700;
+}
+
+.catalog-product__details dl {
+  margin-top: 10px;
+}
+
+.catalog-product__sources {
+  display: flex;
+  flex-wrap: wrap;
+  gap: 8px;
+  border-top: 1px solid var(--border-subtle);
+  padding-top: 12px;
+}
+
+.catalog-product__sources a {
+  border-bottom: 1px solid var(--accent-soft-border);
+  color: var(--accent-strong);
+  font-size: 0.78rem;
+  font-weight: 700;
+}
+
+.catalog-empty {
+  border: 1px dashed var(--border-subtle);
+  border-radius: var(--radius-lg);
+  background: var(--surface-1);
+  padding: 44px 24px;
+  text-align: center;
+}
+
+.catalog-empty h2,
+.catalog-empty p {
+  margin: 0;
+}
+
+.catalog-empty p {
+  margin-top: 8px;
+  color: var(--text-muted);
+}
+
+@media (max-width: 900px) {
+  .topbar,
+  .workbench {
+    grid-template-columns: 1fr;
+  }
+
+  .topbar {
+    display: grid;
+  }
+
+  .cache-panel {
+    justify-content: stretch;
+    grid-template-columns: repeat(3, minmax(0, 1fr));
+  }
+
+  .stats-grid {
+    grid-template-columns: repeat(2, minmax(0, 1fr));
+  }
+
+  .catalog-grid {
+    grid-template-columns: 1fr;
+  }
+}
+
+@media (max-width: 560px) {
+  .page-shell {
+    width: min(100% - 20px, 1180px);
+    padding-top: 16px;
+  }
+
+  .page-shell h1 {
+    font-size: 1.55rem;
+  }
+
+  .cache-panel,
+  .stats-grid,
+  .video-card {
+    grid-template-columns: 1fr;
+  }
+
+  .catalog-page {
+    width: min(100% - 20px, 1180px);
+    padding-top: 18px;
+  }
+
+  .catalog-header {
+    align-items: flex-start;
+    flex-direction: column;
+  }
+
+  .catalog-header__meta {
+    border-left: 0;
+    border-top: 1px solid var(--border-subtle);
+    padding: 14px 0 0;
+  }
+
+  .catalog-product {
+    grid-template-columns: 112px minmax(0, 1fr);
+  }
+
+  .catalog-product__media {
+    min-height: 0;
+    padding: 12px;
+  }
+
+  .catalog-product__specs,
+  .catalog-product__details dl {
+    grid-template-columns: 1fr;
+  }
+}
+
+@media (prefers-reduced-motion: reduce) {
+  *,
+  *::before,
+  *::after {
+    scroll-behavior: auto !important;
+    transition-duration: 0.01ms !important;
+  }
+}

web-front/src/app/layout.tsx

@@ -0,0 +1,45 @@
+import type { Metadata } from 'next';
+import Script from 'next/script';
+import './globals.css';
+
+const YANDEX_METRIKA_ID = 109268682;
+
+export const metadata: Metadata = {
+  title: 'videoreg.ru - автомобильные видеорегистраторы',
+  description: 'Каталог автомобильных видеорегистраторов из одобренных source cards.',
+};
+
+type RootLayoutProps = {
+  children: React.ReactNode;
+};
+
+export default function RootLayout({ children }: RootLayoutProps) {
+  return (
+    <html id="top" lang="ru">
+      <body>
+        {children}
+        <Script id="yandex-metrika" strategy="afterInteractive">
+          {`
+            (function(m,e,t,r,i,k,a){
+                m[i]=m[i]||function(){(m[i].a=m[i].a||[]).push(arguments)};
+                m[i].l=1*new Date();
+                for (var j = 0; j < document.scripts.length; j++) {if (document.scripts[j].src === r) { return; }}
+                k=e.createElement(t),a=e.getElementsByTagName(t)[0],k.async=1,k.src=r,a.parentNode.insertBefore(k,a)
+            })(window, document,'script','https://mc.yandex.ru/metrika/tag.js?id=${YANDEX_METRIKA_ID}', 'ym');
+
+            ym(${YANDEX_METRIKA_ID}, 'init', {ssr:true, webvisor:true, clickmap:true, ecommerce:"dataLayer", referrer: document.referrer, url: location.href, accurateTrackBounce:true, trackLinks:true});
+          `}
+        </Script>
+        <noscript>
+          <div>
+            <img
+              src={`https://mc.yandex.ru/watch/${YANDEX_METRIKA_ID}`}
+              style={{ position: 'absolute', left: '-9999px' }}
+              alt=""
+            />
+          </div>
+        </noscript>
+      </body>
+    </html>
+  );
+}

web-front/src/app/page.tsx

@@ -0,0 +1,17 @@
+import { readDashcamCatalog } from '@/entities/dashcam-catalog/api/catalogRepository';
+import { DashcamCatalog } from '@/widgets/dashcam-catalog/ui/DashcamCatalog';
+
+export const dynamic = 'force-dynamic';
+
+export default async function Home() {
+  const catalog = await readDashcamCatalog();
+
+  return (
+    <DashcamCatalog
+      catalog={catalog.catalog}
+      exportInfo={catalog.export}
+      hasSharedExportIssue={catalog.error === 'shared_export_unavailable'}
+      source={catalog.source}
+    />
+  );
+}

web-front/src/entities/dashcam-catalog/api/catalogRepository.ts

@@ -0,0 +1,62 @@
+import 'server-only';
+
+import { readPrimitiveDb } from '@/shared/db/primitiveJsonDb';
+import {
+  readCatalogExport,
+  type ReadCatalogExportResult,
+} from './importCatalogExport';
+import type { DashcamCatalogExport } from '../model/schema';
+
+export type DashcamCatalogReadModel = {
+  catalog?: DashcamCatalogExport;
+  error?: 'shared_export_unavailable';
+  source: 'shared_export' | 'read_model' | 'empty';
+  export?: Pick<ReadCatalogExportResult, 'createdAt' | 'exportId' | 'itemCount' | 'sha256'>;
+};
+
+export async function readDashcamCatalog(): Promise<DashcamCatalogReadModel> {
+  const exportDir = process.env.DASHCAM_CATALOG_EXPORT_DIR;
+
+  if (exportDir) {
+    try {
+      const result = await readCatalogExport({ exportDir });
+
+      return {
+        catalog: result.catalog,
+        source: 'shared_export',
+        export: {
+          createdAt: result.createdAt,
+          exportId: result.exportId,
+          itemCount: result.itemCount,
+          sha256: result.sha256,
+        },
+      };
+    } catch {
+      const readModel = await readPrimitiveDb();
+
+      if (readModel.dashcamCatalog) {
+        return {
+          catalog: readModel.dashcamCatalog,
+          source: 'read_model',
+          error: 'shared_export_unavailable',
+        };
+      }
+
+      return {
+        source: 'empty',
+        error: 'shared_export_unavailable',
+      };
+    }
+  }
+
+  const readModel = await readPrimitiveDb();
+
+  if (readModel.dashcamCatalog) {
+    return {
+      catalog: readModel.dashcamCatalog,
+      source: 'read_model',
+    };
+  }
+
+  return { source: 'empty' };
+}

web-front/src/entities/dashcam-catalog/api/importCatalogExport.test.ts

@@ -0,0 +1,82 @@
+import { createHash } from 'node:crypto';
+import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { describe, expect, it } from 'vitest';
+import { importCatalogExport } from './importCatalogExport';
+
+describe('importCatalogExport', () => {
+  it('validates a shared-volume manifest, checksum and atomically updates the primitive read model', async () => {
+    const tempDir = await mkdtemp(path.join(os.tmpdir(), 'videoreg-catalog-import-'));
+    const dbFilePath = path.join(tempDir, 'videoreg.json');
+    const exportDir = path.join(tempDir, 'exports');
+
+    try {
+      const catalog = {
+        schemaVersion: 1,
+        exportId: 'export_1',
+        createdAt: '2026-05-16T21:00:00.000Z',
+        products: [
+          {
+            id: 'ibox-f5',
+            brand: 'iBOX',
+            model: 'F5 WiFi',
+            canonicalName: 'iBOX F5 Wi-Fi',
+            title: 'Видеорегистратор для автомобиля iBOX F5 + WiFi с радар-детектором',
+            specs: {
+              channels: 1,
+              wifi: true,
+            },
+            images: [],
+            sources: [
+              {
+                source: 'yandex_market',
+                sourceProductId: '5197387201',
+                url: 'https://market.yandex.ru/card/ibox-f5-wifi-videoregistrator-s-signaturnym-radar-detektorom/5197387201',
+                stableContentHash: 'stable-hash',
+                fetchedAt: '2026-05-16T20:00:00.000Z',
+              },
+            ],
+            discoveredAt: '2026-05-16T20:00:00.000Z',
+            updatedAt: '2026-05-16T20:00:00.000Z',
+          },
+        ],
+      };
+      const catalogBytes = Buffer.from(JSON.stringify(catalog));
+      const digest = createHash('sha256').update(catalogBytes).digest('hex');
+      const artifactPath = path.join(exportDir, 'artifacts', 'export_1', 'catalog.json');
+
+      await mkdir(path.dirname(artifactPath), { recursive: true });
+      await writeFile(artifactPath, catalogBytes);
+      await writeFile(
+        path.join(exportDir, 'manifest.json'),
+        JSON.stringify({
+          schemaVersion: 1,
+          exportId: 'export_1',
+          createdAt: '2026-05-16T21:00:00.000Z',
+          artifactPath: 'artifacts/export_1/catalog.json',
+          sha256: digest,
+          itemCount: 1,
+        }),
+      );
+
+      const result = await importCatalogExport({
+        exportDir,
+        dbFilePath,
+        now: new Date('2026-05-16T22:00:00.000Z'),
+      });
+      const stored = JSON.parse(await readFile(dbFilePath, 'utf8')) as Record<string, unknown>;
+
+      expect(result).toMatchObject({
+        exportId: 'export_1',
+        itemCount: 1,
+        sha256: digest,
+      });
+      expect(stored.dashcamCatalog).toEqual(catalog);
+      expect(stored.dashcamCatalogImportedAt).toBe('2026-05-16T22:00:00.000Z');
+      expect(stored.videos).toEqual([]);
+    } finally {
+      await rm(tempDir, { recursive: true, force: true });
+    }
+  });
+});

web-front/src/entities/dashcam-catalog/api/importCatalogExport.ts

@@ -0,0 +1,178 @@
+import { createHash } from 'node:crypto';
+import { mkdir, readFile, rename, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import {
+  parseCatalogExportManifest,
+  parseDashcamCatalogExport,
+  type DashcamCatalogExport,
+} from '../model/schema.ts';
+
+type PrimitiveDbCache = {
+  videos?: unknown[];
+  dashcamCatalog?: DashcamCatalogExport;
+  dashcamCatalogImportedAt?: string;
+};
+
+export type ReadCatalogExportOptions = {
+  exportDir?: string;
+};
+
+export type ImportCatalogExportOptions = ReadCatalogExportOptions & {
+  dbFilePath?: string;
+  now?: Date;
+};
+
+export type ImportCatalogExportResult = {
+  exportId: string;
+  itemCount: number;
+  sha256: string;
+  dbFilePath: string;
+};
+
+export type ReadCatalogExportResult = {
+  catalog: DashcamCatalogExport;
+  createdAt: string;
+  exportDir: string;
+  exportId: string;
+  itemCount: number;
+  sha256: string;
+};
+
+function getDefaultDbFilePath() {
+  if (process.env.VIDEOREG_DB_FILE) {
+    return process.env.VIDEOREG_DB_FILE;
+  }
+
+  return path.join(process.cwd(), 'data', 'videoreg.json');
+}
+
+function getDefaultExportDir() {
+  if (process.env.DASHCAM_CATALOG_EXPORT_DIR) {
+    return process.env.DASHCAM_CATALOG_EXPORT_DIR;
+  }
+
+  return path.join(process.cwd(), 'data', 'catalog-exports');
+}
+
+function sha256(buffer: Buffer) {
+  return createHash('sha256').update(buffer).digest('hex');
+}
+
+async function readExistingDb(filePath: string): Promise<PrimitiveDbCache> {
+  try {
+    const raw = await readFile(filePath, 'utf8');
+    const parsed = JSON.parse(raw) as PrimitiveDbCache;
+
+    return {
+      ...parsed,
+      videos: Array.isArray(parsed.videos) ? parsed.videos : [],
+    };
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
+      return { videos: [] };
+    }
+
+    throw error;
+  }
+}
+
+async function writeJsonAtomically(filePath: string, payload: PrimitiveDbCache) {
+  const temporaryPath = `${filePath}.${process.pid}.tmp`;
+
+  await mkdir(path.dirname(filePath), { recursive: true });
+  await writeFile(temporaryPath, `${JSON.stringify(payload, null, 2)}\n`, 'utf8');
+  await rename(temporaryPath, filePath);
+}
+
+function parseJson(buffer: Buffer, label: string) {
+  try {
+    return JSON.parse(buffer.toString('utf8')) as unknown;
+  } catch (error) {
+    throw new Error(`Invalid ${label} JSON: ${(error as Error).message}`);
+  }
+}
+
+function resolveArtifactPath(exportDir: string, artifactPath: string) {
+  if (path.isAbsolute(artifactPath)) {
+    throw new Error('Catalog manifest artifactPath must be relative to the shared export directory');
+  }
+
+  const resolvedExportDir = path.resolve(exportDir);
+  const resolvedArtifactPath = path.resolve(resolvedExportDir, artifactPath);
+  const relativePath = path.relative(resolvedExportDir, resolvedArtifactPath);
+
+  if (relativePath === '' || relativePath === '..' || relativePath.startsWith(`..${path.sep}`)) {
+    throw new Error('Catalog manifest artifactPath escapes the shared export directory');
+  }
+
+  return resolvedArtifactPath;
+}
+
+function validateExport(manifestBytes: Buffer, artifactBytes: Buffer) {
+  const manifest = parseCatalogExportManifest(parseJson(manifestBytes, 'manifest'));
+  const artifactHash = sha256(artifactBytes);
+
+  if (artifactHash.toLowerCase() !== manifest.sha256.toLowerCase()) {
+    throw new Error(`Catalog artifact checksum mismatch: expected ${manifest.sha256}, got ${artifactHash}`);
+  }
+
+  const catalog = parseDashcamCatalogExport(parseJson(artifactBytes, 'catalog artifact'));
+
+  if (catalog.exportId !== manifest.exportId) {
+    throw new Error(`Catalog exportId mismatch: manifest=${manifest.exportId}, artifact=${catalog.exportId}`);
+  }
+
+  if (catalog.createdAt !== manifest.createdAt) {
+    throw new Error(`Catalog createdAt mismatch: manifest=${manifest.createdAt}, artifact=${catalog.createdAt}`);
+  }
+
+  if (catalog.products.length !== manifest.itemCount) {
+    throw new Error(`Catalog itemCount mismatch: manifest=${manifest.itemCount}, artifact=${catalog.products.length}`);
+  }
+
+  return {
+    catalog,
+    manifest,
+    sha256: artifactHash,
+  };
+}
+
+export async function readCatalogExport(
+  options: ReadCatalogExportOptions = {},
+): Promise<ReadCatalogExportResult> {
+  const exportDir = path.resolve(options.exportDir ?? getDefaultExportDir());
+  const manifestBytes = await readFile(path.join(exportDir, 'manifest.json'));
+  const manifest = parseCatalogExportManifest(parseJson(manifestBytes, 'manifest'));
+  const artifactPath = manifest.artifactPath ?? path.posix.join('artifacts', manifest.exportId, 'catalog.json');
+  const artifactBytes = await readFile(resolveArtifactPath(exportDir, artifactPath));
+  const validated = validateExport(manifestBytes, artifactBytes);
+
+  return {
+    catalog: validated.catalog,
+    createdAt: validated.manifest.createdAt,
+    exportDir,
+    exportId: validated.manifest.exportId,
+    itemCount: validated.manifest.itemCount,
+    sha256: validated.sha256,
+  };
+}
+
+export async function importCatalogExport(options: ImportCatalogExportOptions = {}): Promise<ImportCatalogExportResult> {
+  const dbFilePath = options.dbFilePath ?? getDefaultDbFilePath();
+  const importedAt = (options.now ?? new Date()).toISOString();
+  const exportResult = await readCatalogExport(options);
+
+  const current = await readExistingDb(dbFilePath);
+  await writeJsonAtomically(dbFilePath, {
+    ...current,
+    dashcamCatalog: exportResult.catalog,
+    dashcamCatalogImportedAt: importedAt,
+  });
+
+  return {
+    exportId: exportResult.exportId,
+    itemCount: exportResult.itemCount,
+    sha256: exportResult.sha256,
+    dbFilePath,
+  };
+}

web-front/src/entities/dashcam-catalog/model/schema.test.ts

@@ -0,0 +1,111 @@
+import { describe, expect, it } from 'vitest';
+import { CatalogExportManifestSchema, DashcamCatalogExportSchema } from './schema';
+
+const baseProduct = {
+  id: 'ibox-f5',
+  brand: 'iBOX',
+  model: 'F5 WiFi',
+  canonicalName: 'iBOX F5 Wi-Fi',
+  title: 'Видеорегистратор для автомобиля iBOX F5 + WiFi с радар-детектором',
+  specs: {
+    channels: 1,
+    wifi: true,
+    resolution_front: '2304x1296',
+  },
+  images: [
+    {
+      url: 'https://cdn.example.test/images/ibox-f5/orig.jpg',
+      originalUrl: 'https://avatars.mds.yandex.net/get-mpic/18787582/2a/orig',
+      source: 'yandex_market',
+    },
+  ],
+  sources: [
+    {
+      source: 'yandex_market',
+      sourceProductId: '5197387201',
+      url: 'https://market.yandex.ru/card/ibox-f5-wifi-videoregistrator-s-signaturnym-radar-detektorom/5197387201',
+      stableContentHash: 'abc123',
+      fetchedAt: '2026-05-16T20:00:00.000Z',
+    },
+  ],
+  discoveredAt: '2026-05-16T20:00:00.000Z',
+  updatedAt: '2026-05-16T20:00:00.000Z',
+};
+
+describe('dashcam catalog export schema', () => {
+  it('accepts approved canonical catalog payload', () => {
+    expect(
+      DashcamCatalogExportSchema.parse({
+        schemaVersion: 1,
+        exportId: 'export_1',
+        createdAt: '2026-05-16T21:00:00.000Z',
+        products: [baseProduct],
+      }),
+    ).toEqual({
+      schemaVersion: 1,
+      exportId: 'export_1',
+      createdAt: '2026-05-16T21:00:00.000Z',
+      products: [baseProduct],
+    });
+  });
+
+  it('rejects internal-only aggregate rating in public export', () => {
+    expect(() =>
+      DashcamCatalogExportSchema.parse({
+        schemaVersion: 1,
+        exportId: 'export_1',
+        createdAt: '2026-05-16T21:00:00.000Z',
+        products: [
+          {
+            ...baseProduct,
+            aggregateRating: {
+              value: 4.8,
+              ratingCount: 17,
+            },
+          },
+        ],
+      }),
+    ).toThrow();
+  });
+
+  it('accepts a local shared-volume artifact path and MVideo source metadata', () => {
+    expect(
+      CatalogExportManifestSchema.parse({
+        schemaVersion: 1,
+        exportId: 'export_1',
+        createdAt: '2026-05-16T21:00:00.000Z',
+        artifactPath: 'artifacts/export_1/catalog.json',
+        sha256: 'a'.repeat(64),
+        itemCount: 1,
+      }),
+    ).toMatchObject({
+      artifactPath: 'artifacts/export_1/catalog.json',
+    });
+
+    expect(
+      DashcamCatalogExportSchema.parse({
+        schemaVersion: 1,
+        exportId: 'export_1',
+        createdAt: '2026-05-16T21:00:00.000Z',
+        products: [
+          {
+            ...baseProduct,
+            sources: [
+              {
+                ...baseProduct.sources[0],
+                source: 'mvideo',
+                url: 'https://www.mvideo.ru/products/test-4254724',
+              },
+            ],
+          },
+        ],
+      }),
+    ).toMatchObject({
+      products: [
+        {
+          sources: [{ source: 'mvideo' }],
+        },
+      ],
+    });
+  });
+});

web-front/src/entities/dashcam-catalog/model/schema.ts

@@ -0,0 +1,84 @@
+import { z } from 'zod';
+
+const isoDateTime = z.string().refine((value) => !Number.isNaN(Date.parse(value)), {
+  message: 'Expected an ISO-compatible date-time string',
+});
+
+const exportedSpecValueSchema = z.union([z.string(), z.number(), z.boolean()]);
+export const MarketSourceSchema = z.enum(['yandex_market', 'mvideo']);
+
+export const ExportedProductImageSchema = z
+  .object({
+    url: z.string().url(),
+    originalUrl: z.string().url(),
+    source: MarketSourceSchema,
+    sha256: z.string().min(1).optional(),
+    width: z.number().int().positive().optional(),
+    height: z.number().int().positive().optional(),
+  })
+  .strict();
+
+export const ExportedProductSourceSchema = z
+  .object({
+    source: MarketSourceSchema,
+    sourceProductId: z.string().min(1).optional(),
+    url: z.string().url(),
+    stableContentHash: z.string().min(1),
+    fetchedAt: isoDateTime,
+  })
+  .strict();
+
+export const ExportedDashcamProductSchema = z
+  .object({
+    id: z.string().min(1),
+    brand: z.string().min(1),
+    model: z.string().min(1),
+    canonicalName: z.string().min(1).optional(),
+    title: z.string().min(1),
+    specs: z.record(z.string().min(1), exportedSpecValueSchema),
+    images: z.array(ExportedProductImageSchema),
+    sources: z.array(ExportedProductSourceSchema).min(1),
+    discoveredAt: isoDateTime,
+    updatedAt: isoDateTime,
+  })
+  .strict();
+
+export const DashcamCatalogExportSchema = z
+  .object({
+    schemaVersion: z.literal(1),
+    exportId: z.string().min(1),
+    createdAt: isoDateTime,
+    products: z.array(ExportedDashcamProductSchema),
+  })
+  .strict();
+
+export const CatalogExportManifestSchema = z
+  .object({
+    schemaVersion: z.literal(1),
+    exportId: z.string().min(1),
+    createdAt: isoDateTime,
+    artifactPath: z.string().min(1).optional(),
+    artifactUrl: z.string().url().optional(),
+    sha256: z.string().regex(/^[a-f0-9]{64}$/i),
+    itemCount: z.number().int().nonnegative(),
+  })
+  .strict()
+  .refine((manifest) => Boolean(manifest.artifactPath || manifest.artifactUrl), {
+    message: 'Expected artifactPath or artifactUrl',
+  });
+
+export type CatalogExportManifest = z.infer<typeof CatalogExportManifestSchema>;
+export type DashcamCatalogExport = z.infer<typeof DashcamCatalogExportSchema>;
+export type ExportedDashcamProduct = z.infer<typeof ExportedDashcamProductSchema>;
+export type ExportedProductImage = z.infer<typeof ExportedProductImageSchema>;
+export type ExportedProductSource = z.infer<typeof ExportedProductSourceSchema>;
+export type ExportedSpecValue = z.infer<typeof exportedSpecValueSchema>;
+export type MarketSource = z.infer<typeof MarketSourceSchema>;
+
+export function parseCatalogExportManifest(input: unknown) {
+  return CatalogExportManifestSchema.parse(input);
+}
+
+export function parseDashcamCatalogExport(input: unknown) {
+  return DashcamCatalogExportSchema.parse(input);
+}

web-front/src/entities/dashcam/model/data.ts

@@ -0,0 +1,401 @@
+import type { DashcamProduct, ScoreDimension, SpecKey } from './types';
+
+export const scoreDimensions: Array<{ key: ScoreDimension; label: string }> = [
+  { key: 'video', label: 'Video' },
+  { key: 'night', label: 'Night' },
+  { key: 'parking', label: 'Parking' },
+  { key: 'reliability', label: 'Reliability' },
+  { key: 'usability', label: 'Usability' },
+  { key: 'value', label: 'Value' },
+];
+
+export const comparisonSpecRows: Array<{ key: SpecKey; label: string }> = [
+  { key: 'sensor', label: 'Sensor' },
+  { key: 'resolution', label: 'Resolution / FPS' },
+  { key: 'bitrate', label: 'Bitrate' },
+  { key: 'hdr', label: 'HDR' },
+  { key: 'lens', label: 'Lens / FOV' },
+  { key: 'storage', label: 'Storage' },
+  { key: 'power', label: 'Capacitor / battery' },
+  { key: 'channels', label: 'Channels' },
+];
+
+export const dashcamProducts: DashcamProduct[] = [
+  {
+    id: 'kaito-vx9-pro',
+    brand: 'Kaito',
+    model: 'VX9 Pro',
+    award: 'Overall pick',
+    tagline: 'The most balanced front and rear system for daily drivers.',
+    score: 9.2,
+    price: '$329',
+    priceNote: 'typical kit price',
+    seller: 'Verified retailers',
+    ctaLabel: 'Check sellers',
+    mediaTone: 'petrol',
+    evidence: '46 logged drives, 12 night routes, 3 heat cycles',
+    editorialNote: 'Wins because the night image holds plate detail without making the app or install feel fragile.',
+    strengths: ['stable HDR', 'clear night plates', 'clean app pairing'],
+    knownIssues: ['rear cable is stiff in cold weather'],
+    scores: {
+      video: 9.4,
+      night: 9.1,
+      parking: 8.8,
+      reliability: 9.3,
+      usability: 9.0,
+      value: 8.8,
+    },
+    specs: {
+      sensor: {
+        label: 'Sensor',
+        value: '1/1.8 in stacked CMOS',
+        trust: 'verified',
+        method: 'teardown',
+        source: 'bench teardown',
+      },
+      resolution: {
+        label: 'Resolution / FPS',
+        value: '4K 60 front + 2K rear',
+        trust: 'verified',
+        method: 'measured',
+        source: 'sample footage',
+      },
+      bitrate: {
+        label: 'Bitrate',
+        value: '82 Mbps front',
+        trust: 'verified',
+        method: 'measured',
+        source: 'file analysis',
+      },
+      hdr: {
+        label: 'HDR',
+        value: 'dual-exposure HDR',
+        trust: 'verified',
+        method: 'firmware-docs',
+        source: 'firmware notes',
+      },
+      lens: {
+        label: 'Lens / FOV',
+        value: 'f/1.7, 146 deg',
+        trust: 'claimed',
+        method: 'manufacturer',
+      },
+      storage: {
+        label: 'Storage',
+        value: '512 GB microSD',
+        trust: 'verified',
+        method: 'measured',
+      },
+      power: {
+        label: 'Capacitor / battery',
+        value: 'supercapacitor',
+        trust: 'verified',
+        method: 'teardown',
+      },
+      channels: {
+        label: 'Channels',
+        value: '2-channel',
+        trust: 'verified',
+        method: 'multi-source',
+      },
+    },
+  },
+  {
+    id: 'renga-night-s2',
+    brand: 'Renga',
+    model: 'Night S2',
+    award: 'Best night detail',
+    tagline: 'A low-light specialist for unlit roads and wet city glass.',
+    score: 9.1,
+    price: '$289',
+    priceNote: 'front unit',
+    seller: 'Camera specialists',
+    ctaLabel: 'View pricing',
+    mediaTone: 'graphite',
+    evidence: '31 logged night routes, rain test, tunnel exposure test',
+    editorialNote: 'Plate recovery is excellent, but the parking mode setup is less polished than the overall pick.',
+    strengths: ['low noise image', 'fast exposure recovery', 'strong windshield glare control'],
+    knownIssues: ['parking menus need clearer wording'],
+    scores: {
+      video: 9.0,
+      night: 9.7,
+      parking: 7.9,
+      reliability: 8.7,
+      usability: 8.2,
+      value: 8.9,
+    },
+    specs: {
+      sensor: {
+        label: 'Sensor',
+        value: 'large-format night CMOS',
+        trust: 'claimed',
+        method: 'manufacturer',
+      },
+      resolution: {
+        label: 'Resolution / FPS',
+        value: '4K 30 front',
+        trust: 'verified',
+        method: 'measured',
+      },
+      bitrate: {
+        label: 'Bitrate',
+        value: '76 Mbps front',
+        trust: 'verified',
+        method: 'measured',
+      },
+      hdr: {
+        label: 'HDR',
+        value: 'night HDR',
+        trust: 'claimed',
+        method: 'manufacturer',
+      },
+      lens: {
+        label: 'Lens / FOV',
+        value: 'f/1.6, 142 deg',
+        trust: 'claimed',
+        method: 'manufacturer',
+      },
+      storage: {
+        label: 'Storage',
+        value: '256 GB microSD',
+        trust: 'verified',
+        method: 'measured',
+      },
+      power: {
+        label: 'Capacitor / battery',
+        value: 'supercapacitor',
+        trust: 'verified',
+        method: 'teardown',
+      },
+      channels: {
+        label: 'Channels',
+        value: '1-channel',
+        trust: 'verified',
+        method: 'multi-source',
+      },
+    },
+  },
+  {
+    id: 'aster-park-duo',
+    brand: 'Aster',
+    model: 'Park Duo',
+    award: 'Best parking mode',
+    tagline: 'The safest choice for parked cars and apartment garages.',
+    score: 8.9,
+    price: '$379',
+    priceNote: 'hardwire kit included',
+    seller: 'Install partners',
+    ctaLabel: 'Find kit',
+    mediaTone: 'brass',
+    evidence: '18 overnight sessions, 8 voltage cut-off cycles, heat soak test',
+    editorialNote: 'The image is not the sharpest, but the buffered parking behavior is dependable and easy to audit.',
+    strengths: ['buffered impact clips', 'stable voltage cut-off', 'excellent event labeling'],
+    knownIssues: ['front 4K mode increases cabin heat'],
+    scores: {
+      video: 8.4,
+      night: 8.5,
+      parking: 9.6,
+      reliability: 9.2,
+      usability: 8.7,
+      value: 8.1,
+    },
+    specs: {
+      sensor: {
+        label: 'Sensor',
+        value: 'dual 5 MP CMOS',
+        trust: 'unknown',
+      },
+      resolution: {
+        label: 'Resolution / FPS',
+        value: '2K 60 front + 2K rear',
+        trust: 'verified',
+        method: 'measured',
+      },
+      bitrate: {
+        label: 'Bitrate',
+        value: '64 Mbps combined',
+        trust: 'verified',
+        method: 'measured',
+      },
+      hdr: {
+        label: 'HDR',
+        value: 'parking HDR',
+        trust: 'claimed',
+        method: 'manufacturer',
+      },
+      lens: {
+        label: 'Lens / FOV',
+        value: 'f/1.8, 140 deg',
+        trust: 'claimed',
+        method: 'manufacturer',
+      },
+      storage: {
+        label: 'Storage',
+        value: '512 GB microSD',
+        trust: 'verified',
+        method: 'measured',
+      },
+      power: {
+        label: 'Capacitor / battery',
+        value: 'supercapacitor',
+        trust: 'verified',
+        method: 'teardown',
+      },
+      channels: {
+        label: 'Channels',
+        value: '2-channel',
+        trust: 'verified',
+        method: 'multi-source',
+      },
+    },
+  },
+  {
+    id: 'mori-value-4k',
+    brand: 'Mori',
+    model: 'Value 4K',
+    award: 'Best value',
+    tagline: 'Strong front-camera quality without paying for cloud extras.',
+    score: 8.6,
+    price: '$179',
+    priceNote: 'front unit',
+    seller: 'Marketplace verified',
+    ctaLabel: 'Compare price',
+    mediaTone: 'titanium',
+    evidence: '22 daytime routes, 9 night routes, 2 firmware versions',
+    editorialNote: 'A credible budget pick because it avoids the usual cheap-camera failure: unreadable evening plates.',
+    strengths: ['sharp daytime video', 'simple menus', 'good included mount'],
+    knownIssues: ['voice alerts sound compressed'],
+    scores: {
+      video: 8.8,
+      night: 8.1,
+      parking: 7.5,
+      reliability: 8.4,
+      usability: 8.8,
+      value: 9.5,
+    },
+    specs: {
+      sensor: {
+        label: 'Sensor',
+        value: '8 MP CMOS',
+        trust: 'unknown',
+      },
+      resolution: {
+        label: 'Resolution / FPS',
+        value: '4K 30 front',
+        trust: 'verified',
+        method: 'measured',
+      },
+      bitrate: {
+        label: 'Bitrate',
+        value: '58 Mbps front',
+        trust: 'verified',
+        method: 'measured',
+      },
+      hdr: {
+        label: 'HDR',
+        value: 'basic HDR',
+        trust: 'claimed',
+        method: 'manufacturer',
+      },
+      lens: {
+        label: 'Lens / FOV',
+        value: 'f/1.8, 150 deg',
+        trust: 'claimed',
+        method: 'manufacturer',
+      },
+      storage: {
+        label: 'Storage',
+        value: '256 GB microSD',
+        trust: 'verified',
+        method: 'measured',
+      },
+      power: {
+        label: 'Capacitor / battery',
+        value: 'supercapacitor',
+        trust: 'verified',
+        method: 'teardown',
+      },
+      channels: {
+        label: 'Channels',
+        value: '1-channel',
+        trust: 'verified',
+        method: 'multi-source',
+      },
+    },
+  },
+  {
+    id: 'haven-fleet-lte',
+    brand: 'Haven',
+    model: 'Fleet LTE',
+    award: 'Best connected',
+    tagline: 'For rideshare drivers who need location, upload, and incident logs.',
+    score: 8.7,
+    price: '$449',
+    priceNote: 'LTE kit',
+    seller: 'Fleet resellers',
+    ctaLabel: 'See plans',
+    mediaTone: 'oxblood',
+    evidence: '14 LTE upload tests, 20 app sessions, 6 parking alerts',
+    editorialNote: 'The subscription is not cheap, but remote event review is genuinely useful for shared vehicles.',
+    strengths: ['reliable LTE upload', 'good incident timeline', 'strong cabin audio controls'],
+    knownIssues: ['cloud plan pricing needs care'],
+    scores: {
+      video: 8.5,
+      night: 8.2,
+      parking: 8.9,
+      reliability: 8.8,
+      usability: 9.0,
+      value: 7.4,
+    },
+    specs: {
+      sensor: {
+        label: 'Sensor',
+        value: 'dual 5 MP CMOS',
+        trust: 'claimed',
+        method: 'manufacturer',
+      },
+      resolution: {
+        label: 'Resolution / FPS',
+        value: '2K 30 front + 1080p cabin',
+        trust: 'verified',
+        method: 'measured',
+      },
+      bitrate: {
+        label: 'Bitrate',
+        value: '48 Mbps combined',
+        trust: 'verified',
+        method: 'measured',
+      },
+      hdr: {
+        label: 'HDR',
+        value: 'front HDR',
+        trust: 'verified',
+        method: 'firmware-docs',
+      },
+      lens: {
+        label: 'Lens / FOV',
+        value: '145 deg front, 165 deg cabin',
+        trust: 'claimed',
+        method: 'manufacturer',
+      },
+      storage: {
+        label: 'Storage',
+        value: '256 GB microSD + cloud',
+        trust: 'verified',
+        method: 'multi-source',
+      },
+      power: {
+        label: 'Capacitor / battery',
+        value: 'supercapacitor',
+        trust: 'verified',
+        method: 'teardown',
+      },
+      channels: {
+        label: 'Channels',
+        value: '2-channel cabin',
+        trust: 'verified',
+        method: 'multi-source',
+      },
+    },
+  },
+];

web-front/src/entities/dashcam/model/types.ts

@@ -0,0 +1,48 @@
+export type TrustState = 'verified' | 'claimed' | 'unknown';
+
+export type VerificationMethod =
+  | 'measured'
+  | 'firmware-docs'
+  | 'teardown'
+  | 'multi-source'
+  | 'manufacturer';
+
+export type ScoreDimension = 'video' | 'night' | 'parking' | 'reliability' | 'usability' | 'value';
+
+export type SpecKey =
+  | 'sensor'
+  | 'resolution'
+  | 'bitrate'
+  | 'hdr'
+  | 'lens'
+  | 'storage'
+  | 'power'
+  | 'channels';
+
+export type SpecFact = {
+  label: string;
+  value: string;
+  trust: TrustState;
+  method?: VerificationMethod;
+  source?: string;
+};
+
+export type DashcamProduct = {
+  id: string;
+  brand: string;
+  model: string;
+  award: string;
+  tagline: string;
+  score: number;
+  price: string;
+  priceNote: string;
+  seller: string;
+  ctaLabel: string;
+  mediaTone: 'petrol' | 'graphite' | 'brass' | 'oxblood' | 'titanium';
+  evidence: string;
+  editorialNote: string;
+  strengths: string[];
+  knownIssues: string[];
+  scores: Record<ScoreDimension, number>;
+  specs: Record<SpecKey, SpecFact>;
+};