feat: complete dashcam catalog ingestion demo

ingestion/migrations/001_init.sql

@@ -0,0 +1,225 @@
+create extension if not exists pgcrypto;
+
+create table if not exists manufacturers (
+  id uuid primary key default gen_random_uuid(),
+  slug text not null unique,
+  name text not null,
+  trust_status text not null default 'unknown' check (trust_status in ('trusted', 'unknown', 'untrusted')),
+  notes text,
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now()
+);
+
+create table if not exists products (
+  id text primary key,
+  manufacturer_id uuid references manufacturers(id) on delete set null,
+  brand text not null,
+  model text not null,
+  canonical_name text not null,
+  title text not null,
+  moderation_status text not null default 'needs_review'
+    check (moderation_status in ('needs_review', 'approved', 'rejected', 'duplicate', 'variant', 'oem_clone')),
+  discovered_at timestamptz not null default now(),
+  updated_at timestamptz not null default now()
+);
+
+create table if not exists product_variants (
+  id uuid primary key default gen_random_uuid(),
+  product_id text references products(id) on delete cascade,
+  variant_kind text not null,
+  title text not null,
+  created_at timestamptz not null default now()
+);
+
+create table if not exists product_sources (
+  id uuid primary key default gen_random_uuid(),
+  product_id text references products(id) on delete set null,
+  source text not null check (source in ('yandex_market')),
+  url text not null,
+  source_product_id text,
+  category_url text,
+  title_preview text,
+  stable_content_hash text,
+  first_seen_at timestamptz not null default now(),
+  last_fetched_at timestamptz,
+  parsed_json jsonb not null default '{}'::jsonb,
+  state text not null default 'discovered',
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now(),
+  unique (source, url)
+);
+
+create index if not exists product_sources_product_id_idx on product_sources(product_id);
+create index if not exists product_sources_source_product_id_idx on product_sources(source, source_product_id);
+
+create table if not exists source_snapshots (
+  id uuid primary key default gen_random_uuid(),
+  source text not null check (source in ('yandex_market')),
+  url text not null,
+  fetched_at timestamptz not null,
+  http_status int not null,
+  raw_snapshot_hash text not null,
+  stable_content_hash text,
+  raw_html_path text,
+  parsed_json jsonb not null default '{}'::jsonb,
+  created_at timestamptz not null default now()
+);
+
+create index if not exists source_snapshots_url_idx on source_snapshots(source, url, fetched_at desc);
+create index if not exists source_snapshots_stable_hash_idx on source_snapshots(stable_content_hash);
+
+create table if not exists product_specs (
+  id uuid primary key default gen_random_uuid(),
+  product_id text not null references products(id) on delete cascade,
+  key text not null,
+  value jsonb not null,
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now(),
+  unique (product_id, key)
+);
+
+create table if not exists product_source_specs (
+  id uuid primary key default gen_random_uuid(),
+  product_id text not null references products(id) on delete cascade,
+  source text not null check (source in ('yandex_market')),
+  source_url text not null,
+  source_product_id text,
+  raw_group text,
+  raw_label text not null,
+  raw_value jsonb not null,
+  normalized_key text,
+  normalized_value jsonb,
+  value_hash text not null,
+  fetched_at timestamptz not null,
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now(),
+  unique (product_id, source, source_url, raw_label)
+);
+
+create index if not exists product_source_specs_product_key_idx on product_source_specs(product_id, normalized_key);
+
+create table if not exists product_images (
+  id uuid primary key default gen_random_uuid(),
+  product_id text not null references products(id) on delete cascade,
+  source text not null check (source in ('yandex_market')),
+  original_url text not null,
+  artifact_url text,
+  sha256 text,
+  width int,
+  height int,
+  sort_order int not null default 0,
+  status text not null default 'needs_review' check (status in ('needs_review', 'approved', 'rejected')),
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now(),
+  unique (product_id, original_url)
+);
+
+create table if not exists source_ratings (
+  id uuid primary key default gen_random_uuid(),
+  product_id text references products(id) on delete cascade,
+  source text not null check (source in ('yandex_market')),
+  source_product_id text,
+  rating_value numeric,
+  rating_scale numeric,
+  rating_count int,
+  fetched_at timestamptz not null,
+  created_at timestamptz not null default now()
+);
+
+create table if not exists crawl_jobs (
+  id uuid primary key default gen_random_uuid(),
+  source text not null check (source in ('yandex_market')),
+  url text not null,
+  job_type text not null check (job_type in ('category', 'product', 'sitemap')),
+  state text not null default 'pending' check (
+    state in (
+      'pending',
+      'fetching',
+      'fetched',
+      'parsed',
+      'normalized',
+      'deduplicated',
+      'needs_review',
+      'approved',
+      'exported',
+      'rate_limited',
+      'blocked',
+      'captcha',
+      'not_found',
+      'parse_failed',
+      'needs_manual_review'
+    )
+  ),
+  attempts int not null default 0,
+  next_run_at timestamptz not null default now(),
+  last_error text,
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now(),
+  unique (source, url, job_type)
+);
+
+create index if not exists crawl_jobs_pending_idx on crawl_jobs(state, next_run_at, created_at);
+
+create table if not exists crawl_errors (
+  id uuid primary key default gen_random_uuid(),
+  job_id uuid references crawl_jobs(id) on delete set null,
+  source text not null,
+  url text not null,
+  reason text not null,
+  message text not null,
+  created_at timestamptz not null default now()
+);
+
+create table if not exists source_pauses (
+  source text primary key check (source in ('yandex_market')),
+  reason text not null,
+  paused_until timestamptz not null,
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now()
+);
+
+create table if not exists source_daily_budgets (
+  source text not null check (source in ('yandex_market')),
+  budget_bucket text not null check (budget_bucket in ('category', 'product', 'sitemap')),
+  budget_day date not null,
+  daily_limit int not null,
+  used_count int not null default 0,
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now(),
+  primary key (source, budget_bucket, budget_day)
+);
+
+create table if not exists moderation_queue (
+  id uuid primary key default gen_random_uuid(),
+  product_id text not null references products(id) on delete cascade,
+  reason text not null check (
+    reason in ('new_product', 'same_product', 'possible_duplicate', 'variant', 'oem_clone', 'conflicting_specs')
+  ),
+  state text not null default 'pending' check (state in ('pending', 'resolved', 'dismissed')),
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now(),
+  unique (product_id, reason)
+);
+
+create table if not exists moderation_conflicts (
+  id uuid primary key default gen_random_uuid(),
+  product_id text not null references products(id) on delete cascade,
+  conflict_type text not null check (conflict_type in ('spec_value')),
+  key text not null,
+  values_json jsonb not null,
+  state text not null default 'pending' check (state in ('pending', 'resolved', 'dismissed')),
+  created_at timestamptz not null default now(),
+  updated_at timestamptz not null default now(),
+  unique (product_id, conflict_type, key)
+);
+
+create table if not exists export_runs (
+  id text primary key,
+  schema_version int not null,
+  created_at timestamptz not null,
+  artifact_path text not null,
+  manifest_path text not null,
+  sha256 text not null,
+  item_count int not null,
+  published_at timestamptz not null default now()
+);