create extension if not exists pgcrypto; create table if not exists manufacturers ( id uuid primary key default gen_random_uuid(), slug text not null unique, name text not null, trust_status text not null default 'unknown' check (trust_status in ('trusted', 'unknown', 'untrusted')), notes text, created_at timestamptz not null default now(), updated_at timestamptz not null default now() ); create table if not exists products ( id text primary key, manufacturer_id uuid references manufacturers(id) on delete set null, brand text not null, model text not null, canonical_name text not null, title text not null, moderation_status text not null default 'needs_review' check (moderation_status in ('needs_review', 'approved', 'rejected', 'duplicate', 'variant', 'oem_clone')), discovered_at timestamptz not null default now(), updated_at timestamptz not null default now() ); create table if not exists product_variants ( id uuid primary key default gen_random_uuid(), product_id text references products(id) on delete cascade, variant_kind text not null, title text not null, created_at timestamptz not null default now() ); create table if not exists product_sources ( id uuid primary key default gen_random_uuid(), product_id text references products(id) on delete set null, source text not null check (source in ('yandex_market')), url text not null, source_product_id text, category_url text, title_preview text, stable_content_hash text, first_seen_at timestamptz not null default now(), last_fetched_at timestamptz, parsed_json jsonb not null default '{}'::jsonb, state text not null default 'discovered', created_at timestamptz not null default now(), updated_at timestamptz not null default now(), unique (source, url) ); create index if not exists product_sources_product_id_idx on product_sources(product_id); create index if not exists product_sources_source_product_id_idx on product_sources(source, source_product_id); create table if not exists source_snapshots ( id uuid primary key default gen_random_uuid(), source text not null check (source in ('yandex_market')), url text not null, fetched_at timestamptz not null, http_status int not null, raw_snapshot_hash text not null, stable_content_hash text, raw_html_path text, parsed_json jsonb not null default '{}'::jsonb, created_at timestamptz not null default now() ); create index if not exists source_snapshots_url_idx on source_snapshots(source, url, fetched_at desc); create index if not exists source_snapshots_stable_hash_idx on source_snapshots(stable_content_hash); create table if not exists product_specs ( id uuid primary key default gen_random_uuid(), product_id text not null references products(id) on delete cascade, key text not null, value jsonb not null, created_at timestamptz not null default now(), updated_at timestamptz not null default now(), unique (product_id, key) ); create table if not exists product_source_specs ( id uuid primary key default gen_random_uuid(), product_id text not null references products(id) on delete cascade, source text not null check (source in ('yandex_market')), source_url text not null, source_product_id text, raw_group text, raw_label text not null, raw_value jsonb not null, normalized_key text, normalized_value jsonb, value_hash text not null, fetched_at timestamptz not null, created_at timestamptz not null default now(), updated_at timestamptz not null default now(), unique (product_id, source, source_url, raw_label) ); create index if not exists product_source_specs_product_key_idx on product_source_specs(product_id, normalized_key); create table if not exists product_images ( id uuid primary key default gen_random_uuid(), product_id text not null references products(id) on delete cascade, source text not null check (source in ('yandex_market')), original_url text not null, artifact_url text, sha256 text, width int, height int, sort_order int not null default 0, status text not null default 'needs_review' check (status in ('needs_review', 'approved', 'rejected')), created_at timestamptz not null default now(), updated_at timestamptz not null default now(), unique (product_id, original_url) ); create table if not exists source_ratings ( id uuid primary key default gen_random_uuid(), product_id text references products(id) on delete cascade, source text not null check (source in ('yandex_market')), source_product_id text, rating_value numeric, rating_scale numeric, rating_count int, fetched_at timestamptz not null, created_at timestamptz not null default now() ); create table if not exists crawl_jobs ( id uuid primary key default gen_random_uuid(), source text not null check (source in ('yandex_market')), url text not null, job_type text not null check (job_type in ('category', 'product', 'sitemap')), state text not null default 'pending' check ( state in ( 'pending', 'fetching', 'fetched', 'parsed', 'normalized', 'deduplicated', 'needs_review', 'approved', 'exported', 'rate_limited', 'blocked', 'captcha', 'not_found', 'parse_failed', 'needs_manual_review' ) ), attempts int not null default 0, next_run_at timestamptz not null default now(), last_error text, created_at timestamptz not null default now(), updated_at timestamptz not null default now(), unique (source, url, job_type) ); create index if not exists crawl_jobs_pending_idx on crawl_jobs(state, next_run_at, created_at); create table if not exists crawl_errors ( id uuid primary key default gen_random_uuid(), job_id uuid references crawl_jobs(id) on delete set null, source text not null, url text not null, reason text not null, message text not null, created_at timestamptz not null default now() ); create table if not exists source_pauses ( source text primary key check (source in ('yandex_market')), reason text not null, paused_until timestamptz not null, created_at timestamptz not null default now(), updated_at timestamptz not null default now() ); create table if not exists source_daily_budgets ( source text not null check (source in ('yandex_market')), budget_bucket text not null check (budget_bucket in ('category', 'product', 'sitemap')), budget_day date not null, daily_limit int not null, used_count int not null default 0, created_at timestamptz not null default now(), updated_at timestamptz not null default now(), primary key (source, budget_bucket, budget_day) ); create table if not exists moderation_queue ( id uuid primary key default gen_random_uuid(), product_id text not null references products(id) on delete cascade, reason text not null check ( reason in ('new_product', 'same_product', 'possible_duplicate', 'variant', 'oem_clone', 'conflicting_specs') ), state text not null default 'pending' check (state in ('pending', 'resolved', 'dismissed')), created_at timestamptz not null default now(), updated_at timestamptz not null default now(), unique (product_id, reason) ); create table if not exists moderation_conflicts ( id uuid primary key default gen_random_uuid(), product_id text not null references products(id) on delete cascade, conflict_type text not null check (conflict_type in ('spec_value')), key text not null, values_json jsonb not null, state text not null default 'pending' check (state in ('pending', 'resolved', 'dismissed')), created_at timestamptz not null default now(), updated_at timestamptz not null default now(), unique (product_id, conflict_type, key) ); create table if not exists export_runs ( id text primary key, schema_version int not null, created_at timestamptz not null, artifact_path text not null, manifest_path text not null, sha256 text not null, item_count int not null, published_at timestamptz not null default now() );