videoreg.ru/ingestion/migrations/001_init.sql

create extension if not exists pgcrypto;

create table if not exists manufacturers (
  id uuid primary key default gen_random_uuid(),
  slug text not null unique,
  name text not null,
  trust_status text not null default 'unknown' check (trust_status in ('trusted', 'unknown', 'untrusted')),
  notes text,
  created_at timestamptz not null default now(),
  updated_at timestamptz not null default now()
);

create table if not exists products (
  id text primary key,
  manufacturer_id uuid references manufacturers(id) on delete set null,
  brand text not null,
  model text not null,
  canonical_name text not null,
  title text not null,
  moderation_status text not null default 'needs_review'
    check (moderation_status in ('needs_review', 'approved', 'rejected', 'duplicate', 'variant', 'oem_clone')),
  discovered_at timestamptz not null default now(),
  updated_at timestamptz not null default now()
);

create table if not exists product_variants (
  id uuid primary key default gen_random_uuid(),
  product_id text references products(id) on delete cascade,
  variant_kind text not null,
  title text not null,
  created_at timestamptz not null default now()
);

create table if not exists product_sources (
  id uuid primary key default gen_random_uuid(),
  product_id text references products(id) on delete set null,
  source text not null check (source in ('yandex_market')),
  url text not null,
  source_product_id text,
  category_url text,
  title_preview text,
  stable_content_hash text,
  first_seen_at timestamptz not null default now(),
  last_fetched_at timestamptz,
  parsed_json jsonb not null default '{}'::jsonb,
  state text not null default 'discovered',
  created_at timestamptz not null default now(),
  updated_at timestamptz not null default now(),
  unique (source, url)
);

create index if not exists product_sources_product_id_idx on product_sources(product_id);
create index if not exists product_sources_source_product_id_idx on product_sources(source, source_product_id);

create table if not exists source_snapshots (
  id uuid primary key default gen_random_uuid(),
  source text not null check (source in ('yandex_market')),
  url text not null,
  fetched_at timestamptz not null,
  http_status int not null,
  raw_snapshot_hash text not null,
  stable_content_hash text,
  raw_html_path text,
  parsed_json jsonb not null default '{}'::jsonb,
  created_at timestamptz not null default now()
);

create index if not exists source_snapshots_url_idx on source_snapshots(source, url, fetched_at desc);
create index if not exists source_snapshots_stable_hash_idx on source_snapshots(stable_content_hash);

create table if not exists product_specs (
  id uuid primary key default gen_random_uuid(),
  product_id text not null references products(id) on delete cascade,
  key text not null,
  value jsonb not null,
  created_at timestamptz not null default now(),
  updated_at timestamptz not null default now(),
  unique (product_id, key)
);

create table if not exists product_source_specs (
  id uuid primary key default gen_random_uuid(),
  product_id text not null references products(id) on delete cascade,
  source text not null check (source in ('yandex_market')),
  source_url text not null,
  source_product_id text,
  raw_group text,
  raw_label text not null,
  raw_value jsonb not null,
  normalized_key text,
  normalized_value jsonb,
  value_hash text not null,
  fetched_at timestamptz not null,
  created_at timestamptz not null default now(),
  updated_at timestamptz not null default now(),
  unique (product_id, source, source_url, raw_label)
);

create index if not exists product_source_specs_product_key_idx on product_source_specs(product_id, normalized_key);

create table if not exists product_images (
  id uuid primary key default gen_random_uuid(),
  product_id text not null references products(id) on delete cascade,
  source text not null check (source in ('yandex_market')),
  original_url text not null,
  artifact_url text,
  sha256 text,
  width int,
  height int,
  sort_order int not null default 0,
  status text not null default 'needs_review' check (status in ('needs_review', 'approved', 'rejected')),
  created_at timestamptz not null default now(),
  updated_at timestamptz not null default now(),
  unique (product_id, original_url)
);

create table if not exists source_ratings (
  id uuid primary key default gen_random_uuid(),
  product_id text references products(id) on delete cascade,
  source text not null check (source in ('yandex_market')),
  source_product_id text,
  rating_value numeric,
  rating_scale numeric,
  rating_count int,
  fetched_at timestamptz not null,
  created_at timestamptz not null default now()
);

create table if not exists crawl_jobs (
  id uuid primary key default gen_random_uuid(),
  source text not null check (source in ('yandex_market')),
  url text not null,
  job_type text not null check (job_type in ('category', 'product', 'sitemap')),
  state text not null default 'pending' check (
    state in (
      'pending',
      'fetching',
      'fetched',
      'parsed',
      'normalized',
      'deduplicated',
      'needs_review',
      'approved',
      'exported',
      'rate_limited',
      'blocked',
      'captcha',
      'not_found',
      'parse_failed',
      'needs_manual_review'
    )
  ),
  attempts int not null default 0,
  next_run_at timestamptz not null default now(),
  last_error text,
  created_at timestamptz not null default now(),
  updated_at timestamptz not null default now(),
  unique (source, url, job_type)
);

create index if not exists crawl_jobs_pending_idx on crawl_jobs(state, next_run_at, created_at);

create table if not exists crawl_errors (
  id uuid primary key default gen_random_uuid(),
  job_id uuid references crawl_jobs(id) on delete set null,
  source text not null,
  url text not null,
  reason text not null,
  message text not null,
  created_at timestamptz not null default now()
);

create table if not exists source_pauses (
  source text primary key check (source in ('yandex_market')),
  reason text not null,
  paused_until timestamptz not null,
  created_at timestamptz not null default now(),
  updated_at timestamptz not null default now()
);

create table if not exists source_daily_budgets (
  source text not null check (source in ('yandex_market')),
  budget_bucket text not null check (budget_bucket in ('category', 'product', 'sitemap')),
  budget_day date not null,
  daily_limit int not null,
  used_count int not null default 0,
  created_at timestamptz not null default now(),
  updated_at timestamptz not null default now(),
  primary key (source, budget_bucket, budget_day)
);

create table if not exists moderation_queue (
  id uuid primary key default gen_random_uuid(),
  product_id text not null references products(id) on delete cascade,
  reason text not null check (
    reason in ('new_product', 'same_product', 'possible_duplicate', 'variant', 'oem_clone', 'conflicting_specs')
  ),
  state text not null default 'pending' check (state in ('pending', 'resolved', 'dismissed')),
  created_at timestamptz not null default now(),
  updated_at timestamptz not null default now(),
  unique (product_id, reason)
);

create table if not exists moderation_conflicts (
  id uuid primary key default gen_random_uuid(),
  product_id text not null references products(id) on delete cascade,
  conflict_type text not null check (conflict_type in ('spec_value')),
  key text not null,
  values_json jsonb not null,
  state text not null default 'pending' check (state in ('pending', 'resolved', 'dismissed')),
  created_at timestamptz not null default now(),
  updated_at timestamptz not null default now(),
  unique (product_id, conflict_type, key)
);

create table if not exists export_runs (
  id text primary key,
  schema_version int not null,
  created_at timestamptz not null,
  artifact_path text not null,
  manifest_path text not null,
  sha256 text not null,
  item_count int not null,
  published_at timestamptz not null default now()
);