diff --git a/Makefile b/Makefile index 1d97657..1745386 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ PYTHON ?= python3 -.PHONY: contracts dev down smoke +.PHONY: contracts dev down migrate seed smoke contracts: $(PYTHON) scripts/generate_openapi_contracts.py @@ -11,5 +11,11 @@ dev: down: docker compose down --remove-orphans +migrate: + alembic upgrade head + +seed: + PYTHONPATH=apps/backend $(PYTHON) -m src.infrastructure.seed_data + smoke: bash tests/smoke/public-health.sh diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..c348a40 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,41 @@ +[alembic] +script_location = apps/backend/alembic +prepend_sys_path = apps/backend +path_separator = os +sqlalchemy.url = postgresql+psycopg://pipeline:pipeline_local@localhost:5432/pipeline + +[post_write_hooks] + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/apps/backend/Dockerfile b/apps/backend/Dockerfile index bf6a379..db1e8f8 100644 --- a/apps/backend/Dockerfile +++ b/apps/backend/Dockerfile @@ -8,6 +8,8 @@ WORKDIR /app COPY apps/backend/requirements.txt ./requirements.txt RUN pip install --no-cache-dir -r requirements.txt +COPY alembic.ini ./alembic.ini +COPY apps/backend/alembic ./apps/backend/alembic COPY apps/backend/src ./src EXPOSE 8000 diff --git a/apps/backend/alembic/env.py b/apps/backend/alembic/env.py new file mode 100644 index 0000000..f0738f2 --- /dev/null +++ b/apps/backend/alembic/env.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import os +import sys +from logging.config import fileConfig +from pathlib import Path + +from alembic import context +from sqlalchemy import engine_from_config, pool + + +BACKEND_ROOT = Path(__file__).resolve().parents[1] +for import_root in (BACKEND_ROOT, BACKEND_ROOT.parents[1]): + if (import_root / "src").exists(): + sys.path.insert(0, str(import_root)) + +config = context.config + +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +target_metadata = None + + +def _database_url() -> str: + configured = os.environ.get("DATABASE_URL") or os.environ.get("POSTGRES_DSN") + if not configured: + return config.get_main_option("sqlalchemy.url") + + if configured.startswith("postgresql://"): + return "postgresql+psycopg://" + configured.removeprefix("postgresql://") + if configured.startswith("postgres://"): + return "postgresql+psycopg://" + configured.removeprefix("postgres://") + return configured + + +def run_migrations_offline() -> None: + context.configure( + url=_database_url(), + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + configuration = config.get_section(config.config_ini_section, {}) + configuration["sqlalchemy.url"] = _database_url() + connectable = engine_from_config( + configuration, + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/apps/backend/alembic/script.py.mako b/apps/backend/alembic/script.py.mako new file mode 100644 index 0000000..c39cb68 --- /dev/null +++ b/apps/backend/alembic/script.py.mako @@ -0,0 +1,26 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} +""" + +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + + +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/apps/backend/alembic/versions/202605210003_initial_postgres_schema.py b/apps/backend/alembic/versions/202605210003_initial_postgres_schema.py new file mode 100644 index 0000000..d552a5f --- /dev/null +++ b/apps/backend/alembic/versions/202605210003_initial_postgres_schema.py @@ -0,0 +1,45 @@ +"""initial postgres schema + +Revision ID: 202605210003 +Revises: +Create Date: 2026-05-21 00:03:00.000000 +""" + +from __future__ import annotations + +from alembic import op + +from src.infrastructure.schema import POSTGRES_SCHEMA_STATEMENTS + + +revision = "202605210003" +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade() -> None: + for statement in POSTGRES_SCHEMA_STATEMENTS: + op.execute(statement) + + +def downgrade() -> None: + for table_name in ( + "prompt_versions", + "publish_commits", + "research_run_manifests", + "agent_jobs", + "workflow_events", + "assets", + "article_drafts", + "claims", + "evidence_items", + "plan_sections", + "article_plans", + "boundary_questions", + "articles", + "script_config_versions", + "target_sites", + "users", + ): + op.execute(f"DROP TABLE IF EXISTS {table_name} CASCADE") diff --git a/apps/backend/requirements.txt b/apps/backend/requirements.txt index c5282c4..b14314d 100644 --- a/apps/backend/requirements.txt +++ b/apps/backend/requirements.txt @@ -1,5 +1,7 @@ fastapi==0.115.6 uvicorn[standard]==0.34.0 +alembic==1.14.0 +SQLAlchemy==2.0.36 psycopg[binary]==3.2.3 redis==5.2.1 boto3==1.35.90 diff --git a/apps/backend/src/application/seed_data.py b/apps/backend/src/application/seed_data.py new file mode 100644 index 0000000..645eb0a --- /dev/null +++ b/apps/backend/src/application/seed_data.py @@ -0,0 +1,129 @@ +from __future__ import annotations + +from datetime import UTC, datetime +from hashlib import sha256 +from uuid import NAMESPACE_URL, UUID, uuid5 + +from src.domain.contracts import ( + PublishingRules, + Role, + ScriptConfigVersionStatus, +) +from src.domain.schema import ( + SEEDED_ADMIN_EMAIL, + SEEDED_EDITOR_EMAIL, + SEEDED_TARGET_SITE_SLUG, +) + + +SEED_TIMESTAMP = datetime(2026, 1, 1, tzinfo=UTC) + + +def seed_reference_data(repository: object) -> None: + admin_id = _stable_uuid("user:admin") + editor_id = _stable_uuid("user:editor") + target_site_id = _stable_uuid(f"target-site:{SEEDED_TARGET_SITE_SLUG}") + script_config_version_id = _stable_uuid( + f"script-config-version:{SEEDED_TARGET_SITE_SLUG}:1" + ) + + repository.users.upsert( + user_id=admin_id, + email=SEEDED_ADMIN_EMAIL, + display_name="Admin User", + role=Role.ADMIN, + created_at=SEED_TIMESTAMP, + updated_at=SEED_TIMESTAMP, + ) + repository.users.upsert( + user_id=editor_id, + email=SEEDED_EDITOR_EMAIL, + display_name="Editor User", + role=Role.EDITOR, + created_at=SEED_TIMESTAMP, + updated_at=SEED_TIMESTAMP, + ) + + publishing_rules = PublishingRules( + repository_url="git@github.com:example/site.git", + production_branch="main", + content_format="mdx", + content_path_template="content/articles/{slug}.mdx", + asset_path_template="public/articles/{slug}/{filename}", + frontmatter_mapping={ + "title": "title", + "description": "meta_description", + "publishedAt": "published_at", + }, + transform_script_version_id=script_config_version_id, + dry_run_renderer="next-mdx", + ) + repository.target_sites.upsert( + site_id=target_site_id, + name="B2B SaaS Blog", + slug=SEEDED_TARGET_SITE_SLUG, + publishing_type="git_next", + default_language="en", + brand_voice="Clear, practical, evidence-led B2B editorial voice.", + audience="B2B SaaS founders and growth teams.", + seo_rules={ + "title_max_length": 60, + "meta_description_max_length": 155, + "primary_keyword_required": True, + }, + visual_rules={ + "hero_style": "product editorial", + "diagram_format": "mermaid", + }, + source_rules={ + "minimum_sources": 3, + "preferred_source_types": ["primary", "analyst", "vendor_docs"], + }, + publishing_rules=publishing_rules, + active_script_config_version_id=script_config_version_id, + created_at=SEED_TIMESTAMP, + updated_at=SEED_TIMESTAMP, + ) + + publishing_yaml = """target: b2b_saas_blog +repository: + url: git@github.com:example/site.git + branch: main +content: + format: mdx + path_template: content/articles/{slug}.mdx +assets: + path_template: public/articles/{slug}/{filename} +""" + transform_script = """export function transformArticle(article) { + return { + frontmatter: article.frontmatter, + body: article.body, + assets: article.assets, + }; +} +""" + repository.script_config_versions.upsert( + version_id=script_config_version_id, + target_site_id=target_site_id, + version=1, + status=ScriptConfigVersionStatus.ACTIVE, + created_by=admin_id, + created_at=SEED_TIMESTAMP, + updated_at=SEED_TIMESTAMP, + diff={"summary": "Initial Git-backed Next.js publishing configuration."}, + rollback_target_version_id=None, + activated_at=SEED_TIMESTAMP, + publishing_yaml=publishing_yaml, + publishing_yaml_hash=_sha256(publishing_yaml), + transform_script=transform_script, + transform_script_hash=_sha256(transform_script), + ) + + +def _stable_uuid(value: str) -> UUID: + return uuid5(NAMESPACE_URL, f"ai-content-pipeline:{value}") + + +def _sha256(value: str) -> str: + return sha256(value.encode("utf-8")).hexdigest() diff --git a/apps/backend/src/domain/schema.py b/apps/backend/src/domain/schema.py new file mode 100644 index 0000000..943d964 --- /dev/null +++ b/apps/backend/src/domain/schema.py @@ -0,0 +1,44 @@ +from __future__ import annotations + + +USERS_TABLE = "users" +TARGET_SITES_TABLE = "target_sites" +SCRIPT_CONFIG_VERSIONS_TABLE = "script_config_versions" +ARTICLES_TABLE = "articles" +BOUNDARY_QUESTIONS_TABLE = "boundary_questions" +ARTICLE_PLANS_TABLE = "article_plans" +PLAN_SECTIONS_TABLE = "plan_sections" +EVIDENCE_ITEMS_TABLE = "evidence_items" +CLAIMS_TABLE = "claims" +ARTICLE_DRAFTS_TABLE = "article_drafts" +ASSETS_TABLE = "assets" +WORKFLOW_EVENTS_TABLE = "workflow_events" +AGENT_JOBS_TABLE = "agent_jobs" +RESEARCH_RUN_MANIFESTS_TABLE = "research_run_manifests" +PUBLISH_COMMITS_TABLE = "publish_commits" +PROMPT_VERSIONS_TABLE = "prompt_versions" + + +CORE_TABLES: tuple[str, ...] = ( + USERS_TABLE, + TARGET_SITES_TABLE, + SCRIPT_CONFIG_VERSIONS_TABLE, + ARTICLES_TABLE, + BOUNDARY_QUESTIONS_TABLE, + ARTICLE_PLANS_TABLE, + PLAN_SECTIONS_TABLE, + EVIDENCE_ITEMS_TABLE, + CLAIMS_TABLE, + ARTICLE_DRAFTS_TABLE, + ASSETS_TABLE, + WORKFLOW_EVENTS_TABLE, + AGENT_JOBS_TABLE, + RESEARCH_RUN_MANIFESTS_TABLE, + PUBLISH_COMMITS_TABLE, + PROMPT_VERSIONS_TABLE, +) + + +SEEDED_ADMIN_EMAIL = "admin@example.com" +SEEDED_EDITOR_EMAIL = "editor@example.com" +SEEDED_TARGET_SITE_SLUG = "b2b_saas_blog" diff --git a/apps/backend/src/infrastructure/README.md b/apps/backend/src/infrastructure/README.md index 118813a..c92bcb8 100644 --- a/apps/backend/src/infrastructure/README.md +++ b/apps/backend/src/infrastructure/README.md @@ -2,3 +2,17 @@ Database, cache, object storage, queue, and other external adapters belong in this layer. + +## Database + +Run migrations from the repository root: + +```bash +DATABASE_URL=postgresql://pipeline:pipeline_local@localhost:5432/pipeline alembic upgrade head +``` + +`POSTGRES_DSN` is also supported. Reference data can be seeded with: + +```bash +DATABASE_URL=postgresql://pipeline:pipeline_local@localhost:5432/pipeline make seed +``` diff --git a/apps/backend/src/infrastructure/repositories.py b/apps/backend/src/infrastructure/repositories.py new file mode 100644 index 0000000..4146495 --- /dev/null +++ b/apps/backend/src/infrastructure/repositories.py @@ -0,0 +1,566 @@ +from __future__ import annotations + +import json +import sqlite3 +from collections.abc import Iterator +from contextlib import contextmanager +from datetime import datetime +from typing import Any +from uuid import UUID + +from src.domain.contracts import ( + PublishingRules, + Role, + ScriptConfigVersionStatus, + TargetSiteConfig, + UserSummary, +) +from src.infrastructure.schema import setup_database + + +JsonObject = dict[str, Any] + + +def open_backend_repository(dsn: str) -> "BackendRepository": + return BackendRepository(dsn) + + +class BackendRepository: + def __init__(self, dsn: str) -> None: + self.dsn = dsn + self.dialect = "sqlite" if dsn.startswith("sqlite:///") else "postgres" + self.users = UsersRepository(self) + self.target_sites = TargetSitesRepository(self) + self.script_config_versions = ScriptConfigVersionsRepository(self) + self.schema = SchemaRepository(self) + + def setup(self) -> None: + setup_database(self.dsn) + + @contextmanager + def connection(self) -> Iterator[Any]: + connection = self._connect() + try: + with connection: + yield connection + finally: + connection.close() + + def _connect(self) -> Any: + if self.dialect == "sqlite": + sqlite_path = self.dsn.removeprefix("sqlite:///") + connection = sqlite3.connect(sqlite_path) + connection.row_factory = sqlite3.Row + return connection + + import psycopg + from psycopg.rows import dict_row + + return psycopg.connect(self.dsn, row_factory=dict_row) + + def placeholder(self) -> str: + if self.dialect == "sqlite": + return "?" + return "%s" + + def json_cast(self) -> str: + if self.dialect == "sqlite": + return "" + return "::jsonb" + + +class UsersRepository: + def __init__(self, repository: BackendRepository) -> None: + self._repository = repository + + def upsert( + self, + *, + user_id: UUID, + email: str, + display_name: str, + role: Role, + created_at: datetime, + updated_at: datetime, + ) -> UserSummary: + placeholder = self._repository.placeholder() + sql = f""" + INSERT INTO users ( + id, email, display_name, role, created_at, updated_at + ) + VALUES ( + {placeholder}, {placeholder}, {placeholder}, {placeholder}, + {placeholder}, {placeholder} + ) + ON CONFLICT (email) DO UPDATE SET + display_name = excluded.display_name, + role = excluded.role, + updated_at = excluded.updated_at + """ + params = ( + str(user_id), + email, + display_name, + role.value, + _datetime_value(created_at), + _datetime_value(updated_at), + ) + with self._repository.connection() as connection: + connection.execute(sql, params) + + return self.get_by_email(email) + + def get_by_email(self, email: str) -> UserSummary: + placeholder = self._repository.placeholder() + with self._repository.connection() as connection: + row = connection.execute( + f""" + SELECT id, display_name, role + FROM users + WHERE email = {placeholder} + """, + (email,), + ).fetchone() + + if row is None: + raise LookupError(f"User not found: {email}") + return _user_from_row(row) + + def list(self) -> list[UserSummary]: + with self._repository.connection() as connection: + rows = connection.execute( + """ + SELECT id, display_name, role + FROM users + ORDER BY email + """ + ).fetchall() + + return [_user_from_row(row) for row in rows] + + def count_by_email(self, email: str) -> int: + placeholder = self._repository.placeholder() + with self._repository.connection() as connection: + row = connection.execute( + f"SELECT COUNT(*) AS count FROM users WHERE email = {placeholder}", + (email,), + ).fetchone() + + return int(_row_value(row, "count")) + + +class TargetSitesRepository: + def __init__(self, repository: BackendRepository) -> None: + self._repository = repository + + def upsert( + self, + *, + site_id: UUID, + name: str, + slug: str, + publishing_type: str, + default_language: str, + brand_voice: str, + audience: str, + seo_rules: JsonObject, + visual_rules: JsonObject, + source_rules: JsonObject, + publishing_rules: PublishingRules, + active_script_config_version_id: UUID | None, + created_at: datetime, + updated_at: datetime, + ) -> TargetSiteConfig: + placeholder = self._repository.placeholder() + json_cast = self._repository.json_cast() + sql = f""" + INSERT INTO target_sites ( + id, + name, + slug, + publishing_type, + default_language, + brand_voice, + audience, + seo_rules, + visual_rules, + source_rules, + publishing_rules, + active_script_config_version_id, + created_at, + updated_at + ) + VALUES ( + {placeholder}, + {placeholder}, + {placeholder}, + {placeholder}, + {placeholder}, + {placeholder}, + {placeholder}, + {placeholder}{json_cast}, + {placeholder}{json_cast}, + {placeholder}{json_cast}, + {placeholder}{json_cast}, + {placeholder}, + {placeholder}, + {placeholder} + ) + ON CONFLICT (slug) DO UPDATE SET + name = excluded.name, + publishing_type = excluded.publishing_type, + default_language = excluded.default_language, + brand_voice = excluded.brand_voice, + audience = excluded.audience, + seo_rules = excluded.seo_rules, + visual_rules = excluded.visual_rules, + source_rules = excluded.source_rules, + publishing_rules = excluded.publishing_rules, + active_script_config_version_id = excluded.active_script_config_version_id, + updated_at = excluded.updated_at + """ + params = ( + str(site_id), + name, + slug, + publishing_type, + default_language, + brand_voice, + audience, + _json_value(seo_rules), + _json_value(visual_rules), + _json_value(source_rules), + _json_value(publishing_rules.model_dump(mode="json")), + _uuid_value(active_script_config_version_id), + _datetime_value(created_at), + _datetime_value(updated_at), + ) + with self._repository.connection() as connection: + connection.execute(sql, params) + + return self.get_by_slug(slug) + + def get_by_slug(self, slug: str) -> TargetSiteConfig: + placeholder = self._repository.placeholder() + with self._repository.connection() as connection: + row = connection.execute( + f""" + SELECT + id, + name, + slug, + publishing_type, + default_language, + brand_voice, + audience, + seo_rules, + visual_rules, + source_rules, + publishing_rules, + active_script_config_version_id, + created_at, + updated_at + FROM target_sites + WHERE slug = {placeholder} + """, + (slug,), + ).fetchone() + + if row is None: + raise LookupError(f"Target site not found: {slug}") + return _target_site_from_row(row) + + def list(self) -> list[TargetSiteConfig]: + with self._repository.connection() as connection: + rows = connection.execute( + """ + SELECT + id, + name, + slug, + publishing_type, + default_language, + brand_voice, + audience, + seo_rules, + visual_rules, + source_rules, + publishing_rules, + active_script_config_version_id, + created_at, + updated_at + FROM target_sites + ORDER BY slug + """ + ).fetchall() + + return [_target_site_from_row(row) for row in rows] + + +class ScriptConfigVersionsRepository: + def __init__(self, repository: BackendRepository) -> None: + self._repository = repository + + def upsert( + self, + *, + version_id: UUID, + target_site_id: UUID, + version: int, + status: ScriptConfigVersionStatus, + created_by: UUID, + created_at: datetime, + updated_at: datetime, + diff: JsonObject, + rollback_target_version_id: UUID | None, + activated_at: datetime | None, + publishing_yaml: str, + publishing_yaml_hash: str, + transform_script: str, + transform_script_hash: str, + ) -> dict[str, Any]: + placeholder = self._repository.placeholder() + json_cast = self._repository.json_cast() + sql = f""" + INSERT INTO script_config_versions ( + id, + target_site_id, + version, + status, + created_by, + created_at, + updated_at, + diff, + rollback_target_version_id, + activated_at, + publishing_yaml, + publishing_yaml_hash, + transform_script, + transform_script_hash + ) + VALUES ( + {placeholder}, + {placeholder}, + {placeholder}, + {placeholder}, + {placeholder}, + {placeholder}, + {placeholder}, + {placeholder}{json_cast}, + {placeholder}, + {placeholder}, + {placeholder}, + {placeholder}, + {placeholder}, + {placeholder} + ) + ON CONFLICT (target_site_id, version) DO UPDATE SET + status = excluded.status, + created_by = excluded.created_by, + updated_at = excluded.updated_at, + diff = excluded.diff, + rollback_target_version_id = excluded.rollback_target_version_id, + activated_at = excluded.activated_at, + publishing_yaml = excluded.publishing_yaml, + publishing_yaml_hash = excluded.publishing_yaml_hash, + transform_script = excluded.transform_script, + transform_script_hash = excluded.transform_script_hash + """ + params = ( + str(version_id), + str(target_site_id), + version, + status.value, + str(created_by), + _datetime_value(created_at), + _datetime_value(updated_at), + _json_value(diff), + _uuid_value(rollback_target_version_id), + _datetime_value(activated_at), + publishing_yaml, + publishing_yaml_hash, + transform_script, + transform_script_hash, + ) + with self._repository.connection() as connection: + connection.execute(sql, params) + + return self.get_by_site_and_version(target_site_id=target_site_id, version=version) + + def get_by_site_and_version( + self, *, target_site_id: UUID, version: int + ) -> dict[str, Any]: + placeholder = self._repository.placeholder() + with self._repository.connection() as connection: + row = connection.execute( + f""" + SELECT * + FROM script_config_versions + WHERE target_site_id = {placeholder} AND version = {placeholder} + """, + (str(target_site_id), version), + ).fetchone() + + if row is None: + raise LookupError( + f"Script config version not found: {target_site_id} v{version}" + ) + return _plain_row(row) + + def list_for_site(self, target_site_id: UUID) -> list[dict[str, Any]]: + placeholder = self._repository.placeholder() + with self._repository.connection() as connection: + rows = connection.execute( + f""" + SELECT * + FROM script_config_versions + WHERE target_site_id = {placeholder} + ORDER BY version + """, + (str(target_site_id),), + ).fetchall() + + return [_plain_row(row) for row in rows] + + +class SchemaRepository: + def __init__(self, repository: BackendRepository) -> None: + self._repository = repository + + def list_tables(self) -> set[str]: + with self._repository.connection() as connection: + if self._repository.dialect == "sqlite": + rows = connection.execute( + """ + SELECT name + FROM sqlite_master + WHERE type = 'table' AND name NOT LIKE 'sqlite_%' + """ + ).fetchall() + else: + rows = connection.execute( + """ + SELECT table_name AS name + FROM information_schema.tables + WHERE table_schema = 'public' AND table_type = 'BASE TABLE' + """ + ).fetchall() + + return {_row_value(row, "name") for row in rows} + + def list_columns(self, table_name: str) -> set[str]: + with self._repository.connection() as connection: + if self._repository.dialect == "sqlite": + rows = connection.execute(f"PRAGMA table_info({table_name})").fetchall() + return {_row_value(row, "name") for row in rows} + + rows = connection.execute( + """ + SELECT column_name + FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = %s + """, + (table_name,), + ).fetchall() + + return {_row_value(row, "column_name") for row in rows} + + def list_indexes(self, table_name: str) -> set[str]: + with self._repository.connection() as connection: + if self._repository.dialect == "sqlite": + rows = connection.execute(f"PRAGMA index_list({table_name})").fetchall() + return {_row_value(row, "name") for row in rows} + + rows = connection.execute( + """ + SELECT indexname + FROM pg_indexes + WHERE schemaname = 'public' AND tablename = %s + """, + (table_name,), + ).fetchall() + + return {_row_value(row, "indexname") for row in rows} + + def count_rows(self, table_name: str) -> int: + with self._repository.connection() as connection: + row = connection.execute(f"SELECT COUNT(*) AS count FROM {table_name}").fetchone() + + return int(_row_value(row, "count")) + + +def _user_from_row(row: Any) -> UserSummary: + return UserSummary( + id=_row_value(row, "id"), + display_name=_row_value(row, "display_name"), + role=_row_value(row, "role"), + ) + + +def _target_site_from_row(row: Any) -> TargetSiteConfig: + return TargetSiteConfig( + id=_row_value(row, "id"), + name=_row_value(row, "name"), + slug=_row_value(row, "slug"), + publishing_type=_row_value(row, "publishing_type"), + default_language=_row_value(row, "default_language"), + brand_voice=_row_value(row, "brand_voice"), + audience=_row_value(row, "audience"), + seo_rules=_json_from_row(row, "seo_rules"), + visual_rules=_json_from_row(row, "visual_rules"), + source_rules=_json_from_row(row, "source_rules"), + publishing_rules=_json_from_row(row, "publishing_rules"), + active_script_config_version_id=_row_value( + row, "active_script_config_version_id" + ), + created_at=_row_value(row, "created_at"), + updated_at=_row_value(row, "updated_at"), + ) + + +def _plain_row(row: Any) -> dict[str, Any]: + if isinstance(row, sqlite3.Row): + result = dict(row) + else: + result = dict(row) + + for key in ( + "diff", + "seo_rules", + "visual_rules", + "source_rules", + "publishing_rules", + ): + if key in result: + result[key] = _json_decode(result[key]) + return result + + +def _row_value(row: Any, key: str) -> Any: + return row[key] + + +def _json_from_row(row: Any, key: str) -> JsonObject: + return _json_decode(_row_value(row, key)) + + +def _json_decode(value: Any) -> Any: + if isinstance(value, str): + return json.loads(value) + return value + + +def _json_value(value: Any) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":")) + + +def _uuid_value(value: UUID | None) -> str | None: + if value is None: + return None + return str(value) + + +def _datetime_value(value: datetime | None) -> str | None: + if value is None: + return None + return value.isoformat() diff --git a/apps/backend/src/infrastructure/schema.py b/apps/backend/src/infrastructure/schema.py new file mode 100644 index 0000000..093d9a4 --- /dev/null +++ b/apps/backend/src/infrastructure/schema.py @@ -0,0 +1,606 @@ +from __future__ import annotations + +import sqlite3 +from collections.abc import Iterable +from contextlib import closing + +from src.domain.contracts import ( + AgentJobErrorCategory, + AgentJobStatus, + AgentJobType, + ArticleWorkflowStatus, + AssetStatus, + AssetType, + ClaimRiskLevel, + ClaimSupportStatus, + PlanReviewStatus, + PublishingStatus, + Role, + ScriptConfigVersionStatus, +) + + +def _quoted_values(values: Iterable[str]) -> str: + return ", ".join(f"'{value}'" for value in values) + + +ROLE_VALUES = _quoted_values(role.value for role in Role) +ARTICLE_STATUS_VALUES = _quoted_values(status.value for status in ArticleWorkflowStatus) +PUBLISHING_STATUS_VALUES = _quoted_values(status.value for status in PublishingStatus) +SCRIPT_CONFIG_STATUS_VALUES = _quoted_values( + status.value for status in ScriptConfigVersionStatus +) +PLAN_REVIEW_STATUS_VALUES = _quoted_values(status.value for status in PlanReviewStatus) +CLAIM_SUPPORT_STATUS_VALUES = _quoted_values( + status.value for status in ClaimSupportStatus +) +CLAIM_RISK_LEVEL_VALUES = _quoted_values(level.value for level in ClaimRiskLevel) +ASSET_TYPE_VALUES = _quoted_values(asset_type.value for asset_type in AssetType) +ASSET_STATUS_VALUES = _quoted_values(status.value for status in AssetStatus) +AGENT_JOB_TYPE_VALUES = _quoted_values(job_type.value for job_type in AgentJobType) +AGENT_JOB_STATUS_VALUES = _quoted_values(status.value for status in AgentJobStatus) +AGENT_JOB_ERROR_CATEGORY_VALUES = _quoted_values( + category.value for category in AgentJobErrorCategory +) + + +POSTGRES_SCHEMA_STATEMENTS: tuple[str, ...] = ( + 'CREATE EXTENSION IF NOT EXISTS "pgcrypto"', + f""" + CREATE TABLE IF NOT EXISTS users ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + email TEXT NOT NULL UNIQUE, + display_name TEXT NOT NULL, + role TEXT NOT NULL CHECK (role IN ({ROLE_VALUES})), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """, + """ + CREATE TABLE IF NOT EXISTS target_sites ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name TEXT NOT NULL, + slug TEXT NOT NULL UNIQUE, + publishing_type TEXT NOT NULL DEFAULT 'git_next', + default_language TEXT NOT NULL DEFAULT 'en', + brand_voice TEXT NOT NULL, + audience TEXT NOT NULL, + seo_rules JSONB NOT NULL DEFAULT '{}'::jsonb, + visual_rules JSONB NOT NULL DEFAULT '{}'::jsonb, + source_rules JSONB NOT NULL DEFAULT '{}'::jsonb, + publishing_rules JSONB NOT NULL DEFAULT '{}'::jsonb, + active_script_config_version_id UUID, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """, + f""" + CREATE TABLE IF NOT EXISTS script_config_versions ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + target_site_id UUID NOT NULL REFERENCES target_sites(id) ON DELETE CASCADE, + version INTEGER NOT NULL, + status TEXT NOT NULL CHECK (status IN ({SCRIPT_CONFIG_STATUS_VALUES})), + created_by UUID NOT NULL REFERENCES users(id), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + diff JSONB NOT NULL DEFAULT '{{}}'::jsonb, + rollback_target_version_id UUID REFERENCES script_config_versions(id), + activated_at TIMESTAMPTZ, + publishing_yaml TEXT NOT NULL, + publishing_yaml_hash TEXT NOT NULL, + transform_script TEXT NOT NULL, + transform_script_hash TEXT NOT NULL, + UNIQUE (target_site_id, version) + ) + """, + f""" + CREATE TABLE IF NOT EXISTS articles ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + target_site_id UUID NOT NULL REFERENCES target_sites(id), + status TEXT NOT NULL CHECK (status IN ({ARTICLE_STATUS_VALUES})), + publishing_status TEXT NOT NULL CHECK ( + publishing_status IN ({PUBLISHING_STATUS_VALUES}) + ), + brief_description TEXT NOT NULL, + working_title TEXT, + language TEXT NOT NULL DEFAULT 'en', + content_type TEXT NOT NULL DEFAULT 'longform_guide', + primary_keyword TEXT, + assigned_editor_id UUID REFERENCES users(id), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """, + """ + CREATE TABLE IF NOT EXISTS boundary_questions ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + article_id UUID NOT NULL REFERENCES articles(id) ON DELETE CASCADE, + sort_order INTEGER NOT NULL, + question TEXT NOT NULL, + answer TEXT, + is_required BOOLEAN NOT NULL DEFAULT true, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (article_id, sort_order) + ) + """, + f""" + CREATE TABLE IF NOT EXISTS article_plans ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + article_id UUID NOT NULL REFERENCES articles(id) ON DELETE CASCADE, + version INTEGER NOT NULL, + status TEXT NOT NULL CHECK (status IN ({PLAN_REVIEW_STATUS_VALUES})), + recommended_title TEXT, + search_intent TEXT, + thesis TEXT, + claims_to_prove JSONB NOT NULL DEFAULT '[]'::jsonb, + risks JSONB NOT NULL DEFAULT '[]'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (article_id, version) + ) + """, + """ + CREATE TABLE IF NOT EXISTS plan_sections ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + article_plan_id UUID NOT NULL REFERENCES article_plans(id) ON DELETE CASCADE, + sort_order INTEGER NOT NULL, + heading TEXT NOT NULL, + purpose TEXT, + claims_to_support JSONB NOT NULL DEFAULT '[]'::jsonb, + target_word_count INTEGER, + UNIQUE (article_plan_id, sort_order) + ) + """, + """ + CREATE TABLE IF NOT EXISTS evidence_items ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + article_id UUID NOT NULL REFERENCES articles(id) ON DELETE CASCADE, + source_title TEXT NOT NULL, + source_url TEXT NOT NULL, + source_type TEXT NOT NULL, + source_quality_score NUMERIC(4, 3) NOT NULL, + summary TEXT NOT NULL, + supports_claims JSONB NOT NULL DEFAULT '[]'::jsonb, + artifact_manifest_id UUID, + retrieved_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """, + f""" + CREATE TABLE IF NOT EXISTS claims ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + article_id UUID NOT NULL REFERENCES articles(id) ON DELETE CASCADE, + section_id UUID REFERENCES plan_sections(id) ON DELETE SET NULL, + claim_text TEXT NOT NULL, + support_status TEXT NOT NULL CHECK ( + support_status IN ({CLAIM_SUPPORT_STATUS_VALUES}) + ), + risk_level TEXT NOT NULL CHECK (risk_level IN ({CLAIM_RISK_LEVEL_VALUES})), + evidence_item_ids JSONB NOT NULL DEFAULT '[]'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """, + f""" + CREATE TABLE IF NOT EXISTS article_drafts ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + article_id UUID NOT NULL REFERENCES articles(id) ON DELETE CASCADE, + version INTEGER NOT NULL, + title TEXT NOT NULL, + slug TEXT NOT NULL, + meta_title TEXT, + meta_description TEXT, + body_object_key TEXT, + status TEXT NOT NULL CHECK (status IN ({ARTICLE_STATUS_VALUES})), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (article_id, version) + ) + """, + f""" + CREATE TABLE IF NOT EXISTS assets ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + article_id UUID NOT NULL REFERENCES articles(id) ON DELETE CASCADE, + asset_type TEXT NOT NULL CHECK (asset_type IN ({ASSET_TYPE_VALUES})), + title TEXT NOT NULL, + prompt TEXT, + file_url TEXT, + alt_text TEXT, + caption TEXT, + status TEXT NOT NULL CHECK (status IN ({ASSET_STATUS_VALUES})), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """, + f""" + CREATE TABLE IF NOT EXISTS workflow_events ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + article_id UUID NOT NULL REFERENCES articles(id) ON DELETE CASCADE, + event_type TEXT NOT NULL, + from_status TEXT CHECK (from_status IN ({ARTICLE_STATUS_VALUES})), + to_status TEXT CHECK (to_status IN ({ARTICLE_STATUS_VALUES})), + actor_user_id UUID REFERENCES users(id), + payload JSONB NOT NULL DEFAULT '{{}}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """, + f""" + CREATE TABLE IF NOT EXISTS agent_jobs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + article_id UUID REFERENCES articles(id) ON DELETE SET NULL, + job_type TEXT NOT NULL CHECK (job_type IN ({AGENT_JOB_TYPE_VALUES})), + agent_profile TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ({AGENT_JOB_STATUS_VALUES})), + workspace_path TEXT, + input_files JSONB NOT NULL DEFAULT '[]'::jsonb, + output_files JSONB NOT NULL DEFAULT '[]'::jsonb, + error_category TEXT CHECK ( + error_category IS NULL + OR error_category IN ({AGENT_JOB_ERROR_CATEGORY_VALUES}) + ), + error_message TEXT, + queued_at TIMESTAMPTZ NOT NULL DEFAULT now(), + started_at TIMESTAMPTZ, + finished_at TIMESTAMPTZ + ) + """, + """ + CREATE TABLE IF NOT EXISTS research_run_manifests ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + article_id UUID NOT NULL REFERENCES articles(id) ON DELETE CASCADE, + agent_job_id UUID NOT NULL REFERENCES agent_jobs(id) ON DELETE CASCADE, + s3_prefix TEXT NOT NULL, + source_urls JSONB NOT NULL DEFAULT '[]'::jsonb, + object_keys JSONB NOT NULL DEFAULT '[]'::jsonb, + content_hashes JSONB NOT NULL DEFAULT '[]'::jsonb, + artifact_types JSONB NOT NULL DEFAULT '[]'::jsonb, + metadata_references JSONB NOT NULL DEFAULT '[]'::jsonb, + artifacts JSONB NOT NULL DEFAULT '[]'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """, + f""" + CREATE TABLE IF NOT EXISTS publish_commits ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + article_id UUID NOT NULL REFERENCES articles(id) ON DELETE CASCADE, + target_site_id UUID NOT NULL REFERENCES target_sites(id), + repository_url TEXT NOT NULL, + branch TEXT NOT NULL, + commit_sha TEXT, + content_bundle_manifest JSONB NOT NULL DEFAULT '{{}}'::jsonb, + status TEXT NOT NULL CHECK (status IN ({PUBLISHING_STATUS_VALUES})), + deployment_status TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """, + f""" + CREATE TABLE IF NOT EXISTS prompt_versions ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + prompt_key TEXT NOT NULL, + version INTEGER NOT NULL, + status TEXT NOT NULL CHECK (status IN ({SCRIPT_CONFIG_STATUS_VALUES})), + body TEXT NOT NULL, + created_by UUID NOT NULL REFERENCES users(id), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + activated_at TIMESTAMPTZ, + diff JSONB NOT NULL DEFAULT '{{}}'::jsonb, + UNIQUE (prompt_key, version) + ) + """, + "CREATE INDEX IF NOT EXISTS idx_target_sites_slug ON target_sites (slug)", + """ + CREATE INDEX IF NOT EXISTS idx_articles_dashboard + ON articles (target_site_id, status, publishing_status, updated_at DESC) + """, + """ + CREATE INDEX IF NOT EXISTS idx_articles_editor_dashboard + ON articles (assigned_editor_id, updated_at DESC) + """, + """ + CREATE INDEX IF NOT EXISTS idx_agent_jobs_queue + ON agent_jobs (status, queued_at) + """, + """ + CREATE INDEX IF NOT EXISTS idx_agent_jobs_article_timeline + ON agent_jobs (article_id, job_type, queued_at) + """, + """ + CREATE INDEX IF NOT EXISTS idx_workflow_events_timeline + ON workflow_events (article_id, created_at) + """, + """ + CREATE INDEX IF NOT EXISTS idx_research_run_manifests_article + ON research_run_manifests (article_id, created_at) + """, + """ + CREATE INDEX IF NOT EXISTS idx_publish_commits_article + ON publish_commits (article_id, created_at) + """, +) + + +SQLITE_SCHEMA_STATEMENTS: tuple[str, ...] = ( + """ + CREATE TABLE IF NOT EXISTS users ( + id TEXT PRIMARY KEY, + email TEXT NOT NULL UNIQUE, + display_name TEXT NOT NULL, + role TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP + ) + """, + """ + CREATE TABLE IF NOT EXISTS target_sites ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + slug TEXT NOT NULL UNIQUE, + publishing_type TEXT NOT NULL DEFAULT 'git_next', + default_language TEXT NOT NULL DEFAULT 'en', + brand_voice TEXT NOT NULL, + audience TEXT NOT NULL, + seo_rules TEXT NOT NULL DEFAULT '{}', + visual_rules TEXT NOT NULL DEFAULT '{}', + source_rules TEXT NOT NULL DEFAULT '{}', + publishing_rules TEXT NOT NULL DEFAULT '{}', + active_script_config_version_id TEXT, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP + ) + """, + """ + CREATE TABLE IF NOT EXISTS script_config_versions ( + id TEXT PRIMARY KEY, + target_site_id TEXT NOT NULL, + version INTEGER NOT NULL, + status TEXT NOT NULL, + created_by TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + diff TEXT NOT NULL DEFAULT '{}', + rollback_target_version_id TEXT, + activated_at TEXT, + publishing_yaml TEXT NOT NULL, + publishing_yaml_hash TEXT NOT NULL, + transform_script TEXT NOT NULL, + transform_script_hash TEXT NOT NULL, + UNIQUE (target_site_id, version) + ) + """, + """ + CREATE TABLE IF NOT EXISTS articles ( + id TEXT PRIMARY KEY, + target_site_id TEXT NOT NULL, + status TEXT NOT NULL, + publishing_status TEXT NOT NULL, + brief_description TEXT NOT NULL, + working_title TEXT, + language TEXT NOT NULL DEFAULT 'en', + content_type TEXT NOT NULL DEFAULT 'longform_guide', + primary_keyword TEXT, + assigned_editor_id TEXT, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP + ) + """, + """ + CREATE TABLE IF NOT EXISTS boundary_questions ( + id TEXT PRIMARY KEY, + article_id TEXT NOT NULL, + sort_order INTEGER NOT NULL, + question TEXT NOT NULL, + answer TEXT, + is_required INTEGER NOT NULL DEFAULT 1, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE (article_id, sort_order) + ) + """, + """ + CREATE TABLE IF NOT EXISTS article_plans ( + id TEXT PRIMARY KEY, + article_id TEXT NOT NULL, + version INTEGER NOT NULL, + status TEXT NOT NULL, + recommended_title TEXT, + search_intent TEXT, + thesis TEXT, + claims_to_prove TEXT NOT NULL DEFAULT '[]', + risks TEXT NOT NULL DEFAULT '[]', + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE (article_id, version) + ) + """, + """ + CREATE TABLE IF NOT EXISTS plan_sections ( + id TEXT PRIMARY KEY, + article_plan_id TEXT NOT NULL, + sort_order INTEGER NOT NULL, + heading TEXT NOT NULL, + purpose TEXT, + claims_to_support TEXT NOT NULL DEFAULT '[]', + target_word_count INTEGER, + UNIQUE (article_plan_id, sort_order) + ) + """, + """ + CREATE TABLE IF NOT EXISTS evidence_items ( + id TEXT PRIMARY KEY, + article_id TEXT NOT NULL, + source_title TEXT NOT NULL, + source_url TEXT NOT NULL, + source_type TEXT NOT NULL, + source_quality_score REAL NOT NULL, + summary TEXT NOT NULL, + supports_claims TEXT NOT NULL DEFAULT '[]', + artifact_manifest_id TEXT, + retrieved_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP + ) + """, + """ + CREATE TABLE IF NOT EXISTS claims ( + id TEXT PRIMARY KEY, + article_id TEXT NOT NULL, + section_id TEXT, + claim_text TEXT NOT NULL, + support_status TEXT NOT NULL, + risk_level TEXT NOT NULL, + evidence_item_ids TEXT NOT NULL DEFAULT '[]', + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP + ) + """, + """ + CREATE TABLE IF NOT EXISTS article_drafts ( + id TEXT PRIMARY KEY, + article_id TEXT NOT NULL, + version INTEGER NOT NULL, + title TEXT NOT NULL, + slug TEXT NOT NULL, + meta_title TEXT, + meta_description TEXT, + body_object_key TEXT, + status TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE (article_id, version) + ) + """, + """ + CREATE TABLE IF NOT EXISTS assets ( + id TEXT PRIMARY KEY, + article_id TEXT NOT NULL, + asset_type TEXT NOT NULL, + title TEXT NOT NULL, + prompt TEXT, + file_url TEXT, + alt_text TEXT, + caption TEXT, + status TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP + ) + """, + """ + CREATE TABLE IF NOT EXISTS workflow_events ( + id TEXT PRIMARY KEY, + article_id TEXT NOT NULL, + event_type TEXT NOT NULL, + from_status TEXT, + to_status TEXT, + actor_user_id TEXT, + payload TEXT NOT NULL DEFAULT '{}', + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP + ) + """, + """ + CREATE TABLE IF NOT EXISTS agent_jobs ( + id TEXT PRIMARY KEY, + article_id TEXT, + job_type TEXT NOT NULL, + agent_profile TEXT NOT NULL, + status TEXT NOT NULL, + workspace_path TEXT, + input_files TEXT NOT NULL DEFAULT '[]', + output_files TEXT NOT NULL DEFAULT '[]', + error_category TEXT, + error_message TEXT, + queued_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + started_at TEXT, + finished_at TEXT + ) + """, + """ + CREATE TABLE IF NOT EXISTS research_run_manifests ( + id TEXT PRIMARY KEY, + article_id TEXT NOT NULL, + agent_job_id TEXT NOT NULL, + s3_prefix TEXT NOT NULL, + source_urls TEXT NOT NULL DEFAULT '[]', + object_keys TEXT NOT NULL DEFAULT '[]', + content_hashes TEXT NOT NULL DEFAULT '[]', + artifact_types TEXT NOT NULL DEFAULT '[]', + metadata_references TEXT NOT NULL DEFAULT '[]', + artifacts TEXT NOT NULL DEFAULT '[]', + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP + ) + """, + """ + CREATE TABLE IF NOT EXISTS publish_commits ( + id TEXT PRIMARY KEY, + article_id TEXT NOT NULL, + target_site_id TEXT NOT NULL, + repository_url TEXT NOT NULL, + branch TEXT NOT NULL, + commit_sha TEXT, + content_bundle_manifest TEXT NOT NULL DEFAULT '{}', + status TEXT NOT NULL, + deployment_status TEXT, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP + ) + """, + """ + CREATE TABLE IF NOT EXISTS prompt_versions ( + id TEXT PRIMARY KEY, + prompt_key TEXT NOT NULL, + version INTEGER NOT NULL, + status TEXT NOT NULL, + body TEXT NOT NULL, + created_by TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + activated_at TEXT, + diff TEXT NOT NULL DEFAULT '{}', + UNIQUE (prompt_key, version) + ) + """, + "CREATE INDEX IF NOT EXISTS idx_target_sites_slug ON target_sites (slug)", + """ + CREATE INDEX IF NOT EXISTS idx_articles_dashboard + ON articles (target_site_id, status, publishing_status, updated_at) + """, + """ + CREATE INDEX IF NOT EXISTS idx_articles_editor_dashboard + ON articles (assigned_editor_id, updated_at) + """, + """ + CREATE INDEX IF NOT EXISTS idx_agent_jobs_queue + ON agent_jobs (status, queued_at) + """, + """ + CREATE INDEX IF NOT EXISTS idx_agent_jobs_article_timeline + ON agent_jobs (article_id, job_type, queued_at) + """, + """ + CREATE INDEX IF NOT EXISTS idx_workflow_events_timeline + ON workflow_events (article_id, created_at) + """, + """ + CREATE INDEX IF NOT EXISTS idx_research_run_manifests_article + ON research_run_manifests (article_id, created_at) + """, + """ + CREATE INDEX IF NOT EXISTS idx_publish_commits_article + ON publish_commits (article_id, created_at) + """, +) + + +def schema_statements_for_dsn(dsn: str) -> tuple[str, ...]: + if dsn.startswith("sqlite:///"): + return SQLITE_SCHEMA_STATEMENTS + return POSTGRES_SCHEMA_STATEMENTS + + +def setup_database(dsn: str) -> None: + statements = schema_statements_for_dsn(dsn) + if dsn.startswith("sqlite:///"): + sqlite_path = dsn.removeprefix("sqlite:///") + with closing(sqlite3.connect(sqlite_path)) as connection: + for statement in statements: + connection.execute(statement) + connection.commit() + return + + import psycopg + + with psycopg.connect(dsn) as connection: + with connection.cursor() as cursor: + for statement in statements: + cursor.execute(statement) diff --git a/apps/backend/src/infrastructure/seed_data.py b/apps/backend/src/infrastructure/seed_data.py new file mode 100644 index 0000000..282ec85 --- /dev/null +++ b/apps/backend/src/infrastructure/seed_data.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +import os + +from src.application.seed_data import seed_reference_data +from src.infrastructure.repositories import open_backend_repository + + +def main() -> None: + dsn = os.environ.get("DATABASE_URL") or os.environ.get("POSTGRES_DSN") + if not dsn: + raise RuntimeError("DATABASE_URL or POSTGRES_DSN must be set") + + repository = open_backend_repository(dsn) + repository.setup() + seed_reference_data(repository) + + +if __name__ == "__main__": + main() diff --git a/apps/backend/tests/integration/__init__.py b/apps/backend/tests/integration/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/apps/backend/tests/integration/__init__.py @@ -0,0 +1 @@ + diff --git a/apps/backend/tests/integration/test_postgres_alembic_migration.py b/apps/backend/tests/integration/test_postgres_alembic_migration.py new file mode 100644 index 0000000..2cd21ea --- /dev/null +++ b/apps/backend/tests/integration/test_postgres_alembic_migration.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import os +import subprocess +import sys +import unittest +from importlib.util import find_spec +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[4] +BACKEND_ROOT = REPO_ROOT / "apps" / "backend" +sys.path.insert(0, str(BACKEND_ROOT)) + + +class PostgresAlembicMigrationIntegrationTest(unittest.TestCase): + def test_alembic_upgrade_head_creates_schema_on_configured_postgres(self) -> None: + dsn = os.environ.get("PIPELINE_TEST_DATABASE_DSN") + if not dsn: + self.skipTest("PIPELINE_TEST_DATABASE_DSN is not set") + if not dsn.startswith(("postgresql://", "postgres://")): + self.skipTest("Postgres DSN is required for Alembic migration test") + if find_spec("alembic") is None: + self.skipTest("alembic is not installed") + + env = os.environ.copy() + env["DATABASE_URL"] = dsn + subprocess.run( + [sys.executable, "-m", "alembic", "upgrade", "head"], + cwd=REPO_ROOT, + env=env, + check=True, + ) + + from src.domain.schema import CORE_TABLES + from src.infrastructure.repositories import open_backend_repository + + repository = open_backend_repository(dsn) + self.assertTrue(set(CORE_TABLES).issubset(repository.schema.list_tables())) + + +if __name__ == "__main__": + unittest.main() diff --git a/apps/backend/tests/integration/test_schema_storage_contracts.py b/apps/backend/tests/integration/test_schema_storage_contracts.py new file mode 100644 index 0000000..d345b36 --- /dev/null +++ b/apps/backend/tests/integration/test_schema_storage_contracts.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +import os +import sys +import tempfile +import unittest +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path + + +BACKEND_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(BACKEND_ROOT)) + + +@contextmanager +def repository_dsn() -> Iterator[str]: + configured_dsn = os.environ.get("PIPELINE_TEST_DATABASE_DSN") + if configured_dsn: + yield configured_dsn + return + + with tempfile.TemporaryDirectory() as tmp_dir: + yield f"sqlite:///{Path(tmp_dir) / 'pipeline-schema.db'}" + + +class SchemaStorageContractsIntegrationTest(unittest.TestCase): + def test_schema_contains_task_tables_and_operational_indexes(self) -> None: + from src.domain.schema import CORE_TABLES + from src.infrastructure.repositories import open_backend_repository + + with repository_dsn() as dsn: + repository = open_backend_repository(dsn) + repository.setup() + + self.assertTrue(set(CORE_TABLES).issubset(repository.schema.list_tables())) + self.assertIn( + "idx_target_sites_slug", + repository.schema.list_indexes("target_sites"), + ) + self.assertIn( + "idx_articles_dashboard", + repository.schema.list_indexes("articles"), + ) + self.assertIn( + "idx_agent_jobs_queue", + repository.schema.list_indexes("agent_jobs"), + ) + self.assertIn( + "idx_workflow_events_timeline", + repository.schema.list_indexes("workflow_events"), + ) + + def test_required_domain_storage_fields_exist(self) -> None: + from src.infrastructure.repositories import open_backend_repository + + with repository_dsn() as dsn: + repository = open_backend_repository(dsn) + repository.setup() + + self.assertTrue( + {"status", "publishing_status"}.issubset( + repository.schema.list_columns("articles") + ) + ) + self.assertIn("status", repository.schema.list_columns("article_drafts")) + self.assertTrue( + { + "created_by", + "created_at", + "diff", + "rollback_target_version_id", + "activated_at", + "publishing_yaml", + "transform_script", + }.issubset(repository.schema.list_columns("script_config_versions")) + ) + self.assertTrue( + { + "s3_prefix", + "source_urls", + "object_keys", + "content_hashes", + "artifact_types", + "metadata_references", + "artifacts", + }.issubset(repository.schema.list_columns("research_run_manifests")) + ) + self.assertTrue( + { + "repository_url", + "branch", + "commit_sha", + "content_bundle_manifest", + "status", + }.issubset(repository.schema.list_columns("publish_commits")) + ) + + def test_seed_data_is_idempotent_for_users_site_and_script_version(self) -> None: + from src.application.seed_data import seed_reference_data + from src.domain.schema import ( + SEEDED_ADMIN_EMAIL, + SEEDED_EDITOR_EMAIL, + SEEDED_TARGET_SITE_SLUG, + ) + from src.infrastructure.repositories import open_backend_repository + + with repository_dsn() as dsn: + repository = open_backend_repository(dsn) + repository.setup() + + seed_reference_data(repository) + seed_reference_data(repository) + + site = repository.target_sites.get_by_slug(SEEDED_TARGET_SITE_SLUG) + seeded_sites = [ + candidate + for candidate in repository.target_sites.list() + if candidate.slug == SEEDED_TARGET_SITE_SLUG + ] + script_versions = repository.script_config_versions.list_for_site(site.id) + + self.assertEqual(1, repository.users.count_by_email(SEEDED_ADMIN_EMAIL)) + self.assertEqual(1, repository.users.count_by_email(SEEDED_EDITOR_EMAIL)) + self.assertEqual([site.id], [candidate.id for candidate in seeded_sites]) + self.assertEqual(1, len(script_versions)) + self.assertEqual("ACTIVE", script_versions[0]["status"]) + self.assertEqual( + str(site.active_script_config_version_id), str(script_versions[0]["id"]) + ) + self.assertIn("repository:", script_versions[0]["publishing_yaml"]) + self.assertIn("transformArticle", script_versions[0]["transform_script"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/apps/backend/tests/integration/test_seeded_target_site_lookup.py b/apps/backend/tests/integration/test_seeded_target_site_lookup.py new file mode 100644 index 0000000..a2d8b45 --- /dev/null +++ b/apps/backend/tests/integration/test_seeded_target_site_lookup.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import os +import sys +import tempfile +import unittest +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path + + +BACKEND_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(BACKEND_ROOT)) + + +SEEDED_SITE_SLUG = "b2b_saas_blog" + + +@contextmanager +def repository_dsn() -> Iterator[str]: + configured_dsn = os.environ.get("PIPELINE_TEST_DATABASE_DSN") + if configured_dsn: + yield configured_dsn + return + + with tempfile.TemporaryDirectory() as tmp_dir: + yield f"sqlite:///{Path(tmp_dir) / 'pipeline-seed-red.db'}" + + +class SeededTargetSiteLookupIntegrationTest(unittest.TestCase): + def test_seeded_git_next_target_site_lookup_by_slug_is_idempotent(self) -> None: + from src.application.seed_data import seed_reference_data + from src.infrastructure.repositories import open_backend_repository + + with repository_dsn() as dsn: + repository = open_backend_repository(dsn) + repository.setup() + + seed_reference_data(repository) + seed_reference_data(repository) + + site = repository.target_sites.get_by_slug(SEEDED_SITE_SLUG) + seeded_sites = [ + candidate + for candidate in repository.target_sites.list() + if candidate.slug == SEEDED_SITE_SLUG + ] + + from src.domain.contracts import TargetSiteConfig + + self.assertIsInstance(site, TargetSiteConfig) + self.assertEqual([site.id], [candidate.id for candidate in seeded_sites]) + self.assertEqual("B2B SaaS Blog", site.name) + self.assertEqual("git_next", site.publishing_type) + self.assertEqual("en", site.default_language) + self.assertEqual( + "git@github.com:example/site.git", + site.publishing_rules.repository_url, + ) + self.assertEqual("main", site.publishing_rules.production_branch) + self.assertEqual("mdx", site.publishing_rules.content_format) + self.assertEqual( + "content/articles/{slug}.mdx", + site.publishing_rules.content_path_template, + ) + self.assertEqual( + "public/articles/{slug}/{filename}", + site.publishing_rules.asset_path_template, + ) + self.assertIsNotNone(site.active_script_config_version_id) + + +if __name__ == "__main__": + unittest.main() diff --git a/tasks/003-postgres-schema-and-seed-data.md b/tasks/003-postgres-schema-and-seed-data.md index 0036c75..fcab10b 100644 --- a/tasks/003-postgres-schema-and-seed-data.md +++ b/tasks/003-postgres-schema-and-seed-data.md @@ -38,13 +38,13 @@ Development description: Create the initial relational schema, migrations, index ## Acceptance Criteria -- [ ] TDD pre-requirement: before implementing migrations, write one failing integration test through repository/public backend interfaces for seeded site lookup; add further tests one behavior at a time and record red-green evidence in `Result`. -- [ ] Migrations run cleanly on an empty Postgres database. -- [ ] Seed data is idempotent. -- [ ] Article/domain tables hold authoritative article statuses. -- [ ] Script/config versions store author, timestamp, diff, rollback target, activation timestamp, YAML, and transform script. -- [ ] Research manifests store S3 prefixes, source URLs, object keys, content hashes, artifact types, and metadata references. -- [ ] Publish commits store repository URL, branch, commit SHA, content bundle manifest, and status. +- [x] TDD pre-requirement: before implementing migrations, write one failing integration test through repository/public backend interfaces for seeded site lookup; add further tests one behavior at a time and record red-green evidence in `Result`. +- [x] Migrations run cleanly on an empty Postgres database. +- [x] Seed data is idempotent. +- [x] Article/domain tables hold authoritative article statuses. +- [x] Script/config versions store author, timestamp, diff, rollback target, activation timestamp, YAML, and transform script. +- [x] Research manifests store S3 prefixes, source URLs, object keys, content hashes, artifact types, and metadata references. +- [x] Publish commits store repository URL, branch, commit SHA, content bundle manifest, and status. ## Verification @@ -54,9 +54,133 @@ Development description: Create the initial relational schema, migrations, index ## Result -- Status: Pending execution. -- TDD plan: To be filled during execution. -- Red evidence: To be filled during execution. -- Green evidence: To be filled during execution. -- Refactor notes: To be filled during execution. -- Verification output: To be filled during execution. +- Status: Completed. +- TDD plan: + - Add one integration-style unittest slice before schema/seed implementation. + - Exercise the future public backend repository setup and application seed use case through `open_backend_repository(dsn)` and `seed_reference_data(repository)`. + - Use `PIPELINE_TEST_DATABASE_DSN` when provided so the same test can later run against Postgres; otherwise use a temporary `sqlite:///...` DSN only as a disposable Red-phase repository substitute. + - Verify observable seeded-site behavior: + - running repository setup plus seed makes the Git-backed Next target site lookupable by slug `b2b_saas_blog`; + - running seed twice leaves exactly one site with that slug; + - the returned site exposes the public `TargetSiteConfig` shape with Git/Next publishing fields and an active script config version. +- Red evidence: + - Command: `env PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s apps/backend/tests/integration -p test_seeded_target_site_lookup.py` + - Actual output: + ```text + E + ====================================================================== + ERROR: test_seeded_git_next_target_site_lookup_by_slug_is_idempotent (test_seeded_target_site_lookup.SeededTargetSiteLookupIntegrationTest.test_seeded_git_next_target_site_lookup_by_slug_is_idempotent) + ---------------------------------------------------------------------- + Traceback (most recent call last): + File "/Users/gavrilovdev/tmp/pupline/apps/backend/tests/integration/test_seeded_target_site_lookup.py", line 32, in test_seeded_git_next_target_site_lookup_by_slug_is_idempotent + from src.application.seed_data import seed_reference_data + ModuleNotFoundError: No module named 'src.application.seed_data' + + ---------------------------------------------------------------------- + Ran 1 test in 0.000s + + FAILED (errors=1) + ``` + - Result: failed as expected because seed data and repository interfaces are not implemented yet. +- Green evidence: + - Pre-requirement command: + `docker compose run --rm --no-deps -v /Users/gavrilovdev/tmp/pupline:/workspace -w /workspace backend python -m unittest discover -s apps/backend/tests/integration -p test_seeded_target_site_lookup.py` + - Actual output: + ```text + . + ---------------------------------------------------------------------- + Ran 1 test in 0.216s + + OK + ``` + - Added focused storage tests for the follow-up behaviors: + - task table/index shape; + - authoritative article status columns; + - script config audit/rollback/activation/YAML/script fields; + - research manifest S3/source/object/hash/type/metadata fields; + - publish commit repository/branch/SHA/manifest/status fields; + - seed idempotency for users, site, and active script config version. +- Refactor notes: + - Added domain schema/reference constants only under `src/domain/schema.py`. + - Added `seed_reference_data(repository)` in application and kept DB connections/repository/DDL in infrastructure. + - Added Alembic under `apps/backend/alembic` with root `alembic.ini`; `DATABASE_URL` and `POSTGRES_DSN` are supported. + - Fixed Alembic env import path so the same migration runs from repo root and inside the backend container. + - Added `make migrate` and `make seed`; public root command remains `alembic upgrade head`. +- Verification output: + - Backend tests: + ```text + Command: + docker compose run --rm --no-deps -v /Users/gavrilovdev/tmp/pupline:/workspace -w /workspace backend python -m unittest discover -s apps/backend/tests -p 'test*.py' + + Output: + ......s.... + ---------------------------------------------------------------------- + Ran 11 tests in 0.123s + + OK (skipped=1) + ``` + - Empty Postgres migration: + ```text + Command: + env POSTGRES_PORT=55433 docker compose -p pupline-task003 run --rm --no-deps -e DATABASE_URL=postgresql://pipeline:pipeline_local@postgres:5432/pipeline backend alembic upgrade head + + Output: + INFO [alembic.runtime.migration] Context impl PostgresqlImpl. + INFO [alembic.runtime.migration] Will assume transactional DDL. + INFO [alembic.runtime.migration] Running upgrade -> 202605210003, initial postgres schema + ``` + - Postgres Alembic integration test: + ```text + Command: + env POSTGRES_PORT=55433 docker compose -p pupline-task003 run --rm --no-deps -v /Users/gavrilovdev/tmp/pupline:/workspace -w /workspace -e PIPELINE_TEST_DATABASE_DSN=postgresql://pipeline:pipeline_local@postgres:5432/pipeline backend python -m unittest discover -s apps/backend/tests/integration -p test_postgres_alembic_migration.py + + Output: + . + ---------------------------------------------------------------------- + Ran 1 test in 0.693s + + OK + ``` + - Postgres repository/seed tests: + ```text + Command: + env POSTGRES_PORT=55433 docker compose -p pupline-task003 run --rm --no-deps -v /Users/gavrilovdev/tmp/pupline:/workspace -w /workspace -e PIPELINE_TEST_DATABASE_DSN=postgresql://pipeline:pipeline_local@postgres:5432/pipeline backend python -m unittest discover -s apps/backend/tests/integration -p test_seeded_target_site_lookup.py + + Output: + . + ---------------------------------------------------------------------- + Ran 1 test in 0.269s + + OK + + Command: + env POSTGRES_PORT=55433 docker compose -p pupline-task003 run --rm --no-deps -v /Users/gavrilovdev/tmp/pupline:/workspace -w /workspace -e PIPELINE_TEST_DATABASE_DSN=postgresql://pipeline:pipeline_local@postgres:5432/pipeline backend python -m unittest discover -s apps/backend/tests/integration -p test_schema_storage_contracts.py + + Output: + ... + ---------------------------------------------------------------------- + Ran 3 tests in 0.348s + + OK + ``` + - Seed command twice/no duplicates: + ```text + Commands: + env POSTGRES_PORT=55433 docker compose -p pupline-task003 run --rm --no-deps -e POSTGRES_DSN=postgresql://pipeline:pipeline_local@postgres:5432/pipeline backend python -m src.infrastructure.seed_data + env POSTGRES_PORT=55433 docker compose -p pupline-task003 run --rm --no-deps -e POSTGRES_DSN=postgresql://pipeline:pipeline_local@postgres:5432/pipeline backend python -m src.infrastructure.seed_data + + Verification output: + admin=1 editor=1 sites=1 scripts=1 active=ACTIVE + ``` + - Smoke: + ```text + Command: + bash tests/smoke/public-health.sh + + Output: + backend health ok + runner health ok + frontend health ok + backend dependencies ok + public health smoke ok + ```