huncode/content-factory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Task 003: add postgres schema and seed data

Browse Source
This commit is contained in:
huncode
2026-05-21 18:11:29 +03:00
parent a0ae06adfa
commit 7a11d50aa7
18 changed files with 1961 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+7 -1
View File
@@ -1,6 +1,6 @@
PYTHON ?= python3
.PHONY: contracts dev down smoke
.PHONY: contracts dev down migrate seed smoke
contracts:
$(PYTHON) scripts/generate_openapi_contracts.py
@@ -11,5 +11,11 @@ dev:
down:
docker compose down --remove-orphans
migrate:
alembic upgrade head
seed:
PYTHONPATH=apps/backend $(PYTHON) -m src.infrastructure.seed_data
smoke:
bash tests/smoke/public-health.sh
alembic.ini
+41
View File
@@ -0,0 +1,41 @@
[alembic]
script_location = apps/backend/alembic
prepend_sys_path = apps/backend
path_separator = os
sqlalchemy.url = postgresql+psycopg://pipeline:pipeline_local@localhost:5432/pipeline
[post_write_hooks]
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S
apps/backend/Dockerfile
+2
View File
@@ -8,6 +8,8 @@ WORKDIR /app
COPY apps/backend/requirements.txt ./requirements.txt
RUN pip install --no-cache-dir -r requirements.txt
COPY alembic.ini ./alembic.ini
COPY apps/backend/alembic ./apps/backend/alembic
COPY apps/backend/src ./src
EXPOSE 8000
apps/backend/alembic/env.py
+68
View File
@@ -0,0 +1,68 @@
from __future__ import annotations
import os
import sys
from logging.config import fileConfig
from pathlib import Path
from alembic import context
from sqlalchemy import engine_from_config, pool
BACKEND_ROOT = Path(__file__).resolve().parents[1]
for import_root in (BACKEND_ROOT, BACKEND_ROOT.parents[1]):
if (import_root / "src").exists():
sys.path.insert(0, str(import_root))
config = context.config
if config.config_file_name is not None:
fileConfig(config.config_file_name)
target_metadata = None
def _database_url() -> str:
configured = os.environ.get("DATABASE_URL") or os.environ.get("POSTGRES_DSN")
if not configured:
return config.get_main_option("sqlalchemy.url")
if configured.startswith("postgresql://"):
return "postgresql+psycopg://" + configured.removeprefix("postgresql://")
if configured.startswith("postgres://"):
return "postgresql+psycopg://" + configured.removeprefix("postgres://")
return configured
def run_migrations_offline() -> None:
context.configure(
url=_database_url(),
target_metadata=target_metadata,
literal_binds=True,
dialect_opts={"paramstyle": "named"},
)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online() -> None:
configuration = config.get_section(config.config_ini_section, {})
configuration["sqlalchemy.url"] = _database_url()
connectable = engine_from_config(
configuration,
prefix="sqlalchemy.",
poolclass=pool.NullPool,
)
with connectable.connect() as connection:
context.configure(connection=connection, target_metadata=target_metadata)
with context.begin_transaction():
context.run_migrations()
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()
apps/backend/alembic/script.py.mako
+26
View File
@@ -0,0 +1,26 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from __future__ import annotations
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
revision = ${repr(up_revision)}
down_revision = ${repr(down_revision)}
branch_labels = ${repr(branch_labels)}
depends_on = ${repr(depends_on)}
def upgrade() -> None:
${upgrades if upgrades else "pass"}
def downgrade() -> None:
${downgrades if downgrades else "pass"}
apps/backend/alembic/versions/202605210003_initial_postgres_schema.py
+45
View File
@@ -0,0 +1,45 @@
"""initial postgres schema
Revision ID: 202605210003
Revises:
Create Date: 2026-05-21 00:03:00.000000
"""
from __future__ import annotations
from alembic import op
from src.infrastructure.schema import POSTGRES_SCHEMA_STATEMENTS
revision = "202605210003"
down_revision = None
branch_labels = None
depends_on = None
def upgrade() -> None:
for statement in POSTGRES_SCHEMA_STATEMENTS:
op.execute(statement)
def downgrade() -> None:
for table_name in (
"prompt_versions",
"publish_commits",
"research_run_manifests",
"agent_jobs",
"workflow_events",
"assets",
"article_drafts",
"claims",
"evidence_items",
"plan_sections",
"article_plans",
"boundary_questions",
"articles",
"script_config_versions",
"target_sites",
"users",
):
op.execute(f"DROP TABLE IF EXISTS {table_name} CASCADE")
apps/backend/requirements.txt
+2
View File
@@ -1,5 +1,7 @@
fastapi==0.115.6
uvicorn[standard]==0.34.0
alembic==1.14.0
SQLAlchemy==2.0.36
psycopg[binary]==3.2.3
redis==5.2.1
boto3==1.35.90
apps/backend/src/application/seed_data.py
+129
View File
@@ -0,0 +1,129 @@
from __future__ import annotations
from datetime import UTC, datetime
from hashlib import sha256
from uuid import NAMESPACE_URL, UUID, uuid5
from src.domain.contracts import (
PublishingRules,
Role,
ScriptConfigVersionStatus,
)
from src.domain.schema import (
SEEDED_ADMIN_EMAIL,
SEEDED_EDITOR_EMAIL,
SEEDED_TARGET_SITE_SLUG,
)
SEED_TIMESTAMP = datetime(2026, 1, 1, tzinfo=UTC)
def seed_reference_data(repository: object) -> None:
admin_id = _stable_uuid("user:admin")
editor_id = _stable_uuid("user:editor")
target_site_id = _stable_uuid(f"target-site:{SEEDED_TARGET_SITE_SLUG}")
script_config_version_id = _stable_uuid(
f"script-config-version:{SEEDED_TARGET_SITE_SLUG}:1"
)
repository.users.upsert(
user_id=admin_id,
email=SEEDED_ADMIN_EMAIL,
display_name="Admin User",
role=Role.ADMIN,
created_at=SEED_TIMESTAMP,
updated_at=SEED_TIMESTAMP,
)
repository.users.upsert(
user_id=editor_id,
email=SEEDED_EDITOR_EMAIL,
display_name="Editor User",
role=Role.EDITOR,
created_at=SEED_TIMESTAMP,
updated_at=SEED_TIMESTAMP,
)
publishing_rules = PublishingRules(
repository_url="git@github.com:example/site.git",
production_branch="main",
content_format="mdx",
content_path_template="content/articles/{slug}.mdx",
asset_path_template="public/articles/{slug}/{filename}",
frontmatter_mapping={
"title": "title",
"description": "meta_description",
"publishedAt": "published_at",
},
transform_script_version_id=script_config_version_id,
dry_run_renderer="next-mdx",
)
repository.target_sites.upsert(
site_id=target_site_id,
name="B2B SaaS Blog",
slug=SEEDED_TARGET_SITE_SLUG,
publishing_type="git_next",
default_language="en",
brand_voice="Clear, practical, evidence-led B2B editorial voice.",
audience="B2B SaaS founders and growth teams.",
seo_rules={
"title_max_length": 60,
"meta_description_max_length": 155,
"primary_keyword_required": True,
},
visual_rules={
"hero_style": "product editorial",
"diagram_format": "mermaid",
},
source_rules={
"minimum_sources": 3,
"preferred_source_types": ["primary", "analyst", "vendor_docs"],
},
publishing_rules=publishing_rules,
active_script_config_version_id=script_config_version_id,
created_at=SEED_TIMESTAMP,
updated_at=SEED_TIMESTAMP,
)
publishing_yaml = """target: b2b_saas_blog
repository:
url: git@github.com:example/site.git
branch: main
content:
format: mdx
path_template: content/articles/{slug}.mdx
assets:
path_template: public/articles/{slug}/{filename}
"""
transform_script = """export function transformArticle(article) {
return {
frontmatter: article.frontmatter,
body: article.body,
assets: article.assets,
};
}
"""
repository.script_config_versions.upsert(
version_id=script_config_version_id,
target_site_id=target_site_id,
version=1,
status=ScriptConfigVersionStatus.ACTIVE,
created_by=admin_id,
created_at=SEED_TIMESTAMP,
updated_at=SEED_TIMESTAMP,
diff={"summary": "Initial Git-backed Next.js publishing configuration."},
rollback_target_version_id=None,
activated_at=SEED_TIMESTAMP,
publishing_yaml=publishing_yaml,
publishing_yaml_hash=_sha256(publishing_yaml),
transform_script=transform_script,
transform_script_hash=_sha256(transform_script),
)
def _stable_uuid(value: str) -> UUID:
return uuid5(NAMESPACE_URL, f"ai-content-pipeline:{value}")
def _sha256(value: str) -> str:
return sha256(value.encode("utf-8")).hexdigest()
apps/backend/src/domain/schema.py
+44
View File
@@ -0,0 +1,44 @@
from __future__ import annotations
USERS_TABLE = "users"
TARGET_SITES_TABLE = "target_sites"
SCRIPT_CONFIG_VERSIONS_TABLE = "script_config_versions"
ARTICLES_TABLE = "articles"
BOUNDARY_QUESTIONS_TABLE = "boundary_questions"
ARTICLE_PLANS_TABLE = "article_plans"
PLAN_SECTIONS_TABLE = "plan_sections"
EVIDENCE_ITEMS_TABLE = "evidence_items"
CLAIMS_TABLE = "claims"
ARTICLE_DRAFTS_TABLE = "article_drafts"
ASSETS_TABLE = "assets"
WORKFLOW_EVENTS_TABLE = "workflow_events"
AGENT_JOBS_TABLE = "agent_jobs"
RESEARCH_RUN_MANIFESTS_TABLE = "research_run_manifests"
PUBLISH_COMMITS_TABLE = "publish_commits"
PROMPT_VERSIONS_TABLE = "prompt_versions"
CORE_TABLES: tuple[str, ...] = (
USERS_TABLE,
TARGET_SITES_TABLE,
SCRIPT_CONFIG_VERSIONS_TABLE,
ARTICLES_TABLE,
BOUNDARY_QUESTIONS_TABLE,
ARTICLE_PLANS_TABLE,
PLAN_SECTIONS_TABLE,
EVIDENCE_ITEMS_TABLE,
CLAIMS_TABLE,
ARTICLE_DRAFTS_TABLE,
ASSETS_TABLE,
WORKFLOW_EVENTS_TABLE,
AGENT_JOBS_TABLE,
RESEARCH_RUN_MANIFESTS_TABLE,
PUBLISH_COMMITS_TABLE,
PROMPT_VERSIONS_TABLE,
)
SEEDED_ADMIN_EMAIL = "admin@example.com"
SEEDED_EDITOR_EMAIL = "editor@example.com"
SEEDED_TARGET_SITE_SLUG = "b2b_saas_blog"
apps/backend/src/infrastructure/README.md
+14
View File
@@ -2,3 +2,17 @@
Database, cache, object storage, queue, and other external adapters belong in
this layer.
## Database
Run migrations from the repository root:
```bash
DATABASE_URL=postgresql://pipeline:pipeline_local@localhost:5432/pipeline alembic upgrade head
```
`POSTGRES_DSN` is also supported. Reference data can be seeded with:
```bash
DATABASE_URL=postgresql://pipeline:pipeline_local@localhost:5432/pipeline make seed
```
apps/backend/src/infrastructure/repositories.py
+566
View File
@@ -0,0 +1,566 @@
from __future__ import annotations
import json
import sqlite3
from collections.abc import Iterator
from contextlib import contextmanager
from datetime import datetime
from typing import Any
from uuid import UUID
from src.domain.contracts import (
PublishingRules,
Role,
ScriptConfigVersionStatus,
TargetSiteConfig,
UserSummary,
)
from src.infrastructure.schema import setup_database
JsonObject = dict[str, Any]
def open_backend_repository(dsn: str) -> "BackendRepository":
return BackendRepository(dsn)
class BackendRepository:
def __init__(self, dsn: str) -> None:
self.dsn = dsn
self.dialect = "sqlite" if dsn.startswith("sqlite:///") else "postgres"
self.users = UsersRepository(self)
self.target_sites = TargetSitesRepository(self)
self.script_config_versions = ScriptConfigVersionsRepository(self)
self.schema = SchemaRepository(self)
def setup(self) -> None:
setup_database(self.dsn)
@contextmanager
def connection(self) -> Iterator[Any]:
connection = self._connect()
try:
with connection:
yield connection
finally:
connection.close()
def _connect(self) -> Any:
if self.dialect == "sqlite":
sqlite_path = self.dsn.removeprefix("sqlite:///")
connection = sqlite3.connect(sqlite_path)
connection.row_factory = sqlite3.Row
return connection
import psycopg
from psycopg.rows import dict_row
return psycopg.connect(self.dsn, row_factory=dict_row)
def placeholder(self) -> str:
if self.dialect == "sqlite":
return "?"
return "%s"
def json_cast(self) -> str:
if self.dialect == "sqlite":
return ""
return "::jsonb"
class UsersRepository:
def __init__(self, repository: BackendRepository) -> None:
self._repository = repository
def upsert(
self,
*,
user_id: UUID,
email: str,
display_name: str,
role: Role,
created_at: datetime,
updated_at: datetime,
) -> UserSummary:
placeholder = self._repository.placeholder()
sql = f"""
INSERT INTO users (
id, email, display_name, role, created_at, updated_at
)
VALUES (
{placeholder}, {placeholder}, {placeholder}, {placeholder},
{placeholder}, {placeholder}
)
ON CONFLICT (email) DO UPDATE SET
display_name = excluded.display_name,
role = excluded.role,
updated_at = excluded.updated_at
"""
params = (
str(user_id),
email,
display_name,
role.value,
_datetime_value(created_at),
_datetime_value(updated_at),
)
with self._repository.connection() as connection:
connection.execute(sql, params)
return self.get_by_email(email)
def get_by_email(self, email: str) -> UserSummary:
placeholder = self._repository.placeholder()
with self._repository.connection() as connection:
row = connection.execute(
f"""
SELECT id, display_name, role
FROM users
WHERE email = {placeholder}
""",
(email,),
).fetchone()
if row is None:
raise LookupError(f"User not found: {email}")
return _user_from_row(row)
def list(self) -> list[UserSummary]:
with self._repository.connection() as connection:
rows = connection.execute(
"""
SELECT id, display_name, role
FROM users
ORDER BY email
"""
).fetchall()
return [_user_from_row(row) for row in rows]
def count_by_email(self, email: str) -> int:
placeholder = self._repository.placeholder()
with self._repository.connection() as connection:
row = connection.execute(
f"SELECT COUNT(*) AS count FROM users WHERE email = {placeholder}",
(email,),
).fetchone()
return int(_row_value(row, "count"))
class TargetSitesRepository:
def __init__(self, repository: BackendRepository) -> None:
self._repository = repository
def upsert(
self,
*,
site_id: UUID,
name: str,
slug: str,
publishing_type: str,
default_language: str,
brand_voice: str,
audience: str,
seo_rules: JsonObject,
visual_rules: JsonObject,
source_rules: JsonObject,
publishing_rules: PublishingRules,
active_script_config_version_id: UUID | None,
created_at: datetime,
updated_at: datetime,
) -> TargetSiteConfig:
placeholder = self._repository.placeholder()
json_cast = self._repository.json_cast()
sql = f"""
INSERT INTO target_sites (
id,
name,
slug,
publishing_type,
default_language,
brand_voice,
audience,
seo_rules,
visual_rules,
source_rules,
publishing_rules,
active_script_config_version_id,
created_at,
updated_at
)
VALUES (
{placeholder},
{placeholder},
{placeholder},
{placeholder},
{placeholder},
{placeholder},
{placeholder},
{placeholder}{json_cast},
{placeholder}{json_cast},
{placeholder}{json_cast},
{placeholder}{json_cast},
{placeholder},
{placeholder},
{placeholder}
)
ON CONFLICT (slug) DO UPDATE SET
name = excluded.name,
publishing_type = excluded.publishing_type,
default_language = excluded.default_language,
brand_voice = excluded.brand_voice,
audience = excluded.audience,
seo_rules = excluded.seo_rules,
visual_rules = excluded.visual_rules,
source_rules = excluded.source_rules,
publishing_rules = excluded.publishing_rules,
active_script_config_version_id = excluded.active_script_config_version_id,
updated_at = excluded.updated_at
"""
params = (
str(site_id),
name,
slug,
publishing_type,
default_language,
brand_voice,
audience,
_json_value(seo_rules),
_json_value(visual_rules),
_json_value(source_rules),
_json_value(publishing_rules.model_dump(mode="json")),
_uuid_value(active_script_config_version_id),
_datetime_value(created_at),
_datetime_value(updated_at),
)
with self._repository.connection() as connection:
connection.execute(sql, params)
return self.get_by_slug(slug)
def get_by_slug(self, slug: str) -> TargetSiteConfig:
placeholder = self._repository.placeholder()
with self._repository.connection() as connection:
row = connection.execute(
f"""
SELECT
id,
name,
slug,
publishing_type,
default_language,
brand_voice,
audience,
seo_rules,
visual_rules,
source_rules,
publishing_rules,
active_script_config_version_id,
created_at,
updated_at
FROM target_sites
WHERE slug = {placeholder}
""",
(slug,),
).fetchone()
if row is None:
raise LookupError(f"Target site not found: {slug}")
return _target_site_from_row(row)
def list(self) -> list[TargetSiteConfig]:
with self._repository.connection() as connection:
rows = connection.execute(
"""
SELECT
id,
name,
slug,
publishing_type,
default_language,
brand_voice,
audience,
seo_rules,
visual_rules,
source_rules,
publishing_rules,
active_script_config_version_id,
created_at,
updated_at
FROM target_sites
ORDER BY slug
"""
).fetchall()
return [_target_site_from_row(row) for row in rows]
class ScriptConfigVersionsRepository:
def __init__(self, repository: BackendRepository) -> None:
self._repository = repository
def upsert(
self,
*,
version_id: UUID,
target_site_id: UUID,
version: int,
status: ScriptConfigVersionStatus,
created_by: UUID,
created_at: datetime,
updated_at: datetime,
diff: JsonObject,
rollback_target_version_id: UUID | None,
activated_at: datetime | None,
publishing_yaml: str,
publishing_yaml_hash: str,
transform_script: str,
transform_script_hash: str,
) -> dict[str, Any]:
placeholder = self._repository.placeholder()
json_cast = self._repository.json_cast()
sql = f"""
INSERT INTO script_config_versions (
id,
target_site_id,
version,
status,
created_by,
created_at,
updated_at,
diff,
rollback_target_version_id,
activated_at,
publishing_yaml,
publishing_yaml_hash,
transform_script,
transform_script_hash
)
VALUES (
{placeholder},
{placeholder},
{placeholder},
{placeholder},
{placeholder},
{placeholder},
{placeholder},
{placeholder}{json_cast},
{placeholder},
{placeholder},
{placeholder},
{placeholder},
{placeholder},
{placeholder}
)
ON CONFLICT (target_site_id, version) DO UPDATE SET
status = excluded.status,
created_by = excluded.created_by,
updated_at = excluded.updated_at,
diff = excluded.diff,
rollback_target_version_id = excluded.rollback_target_version_id,
activated_at = excluded.activated_at,
publishing_yaml = excluded.publishing_yaml,
publishing_yaml_hash = excluded.publishing_yaml_hash,
transform_script = excluded.transform_script,
transform_script_hash = excluded.transform_script_hash
"""
params = (
str(version_id),
str(target_site_id),
version,
status.value,
str(created_by),
_datetime_value(created_at),
_datetime_value(updated_at),
_json_value(diff),
_uuid_value(rollback_target_version_id),
_datetime_value(activated_at),
publishing_yaml,
publishing_yaml_hash,
transform_script,
transform_script_hash,
)
with self._repository.connection() as connection:
connection.execute(sql, params)
return self.get_by_site_and_version(target_site_id=target_site_id, version=version)
def get_by_site_and_version(
self, *, target_site_id: UUID, version: int
) -> dict[str, Any]:
placeholder = self._repository.placeholder()
with self._repository.connection() as connection:
row = connection.execute(
f"""
SELECT *
FROM script_config_versions
WHERE target_site_id = {placeholder} AND version = {placeholder}
""",
(str(target_site_id), version),
).fetchone()
if row is None:
raise LookupError(
f"Script config version not found: {target_site_id} v{version}"
)
return _plain_row(row)
def list_for_site(self, target_site_id: UUID) -> list[dict[str, Any]]:
placeholder = self._repository.placeholder()
with self._repository.connection() as connection:
rows = connection.execute(
f"""
SELECT *
FROM script_config_versions
WHERE target_site_id = {placeholder}
ORDER BY version
""",
(str(target_site_id),),
).fetchall()
return [_plain_row(row) for row in rows]
class SchemaRepository:
def __init__(self, repository: BackendRepository) -> None:
self._repository = repository
def list_tables(self) -> set[str]:
with self._repository.connection() as connection:
if self._repository.dialect == "sqlite":
rows = connection.execute(
"""
SELECT name
FROM sqlite_master
WHERE type = 'table' AND name NOT LIKE 'sqlite_%'
"""
).fetchall()
else:
rows = connection.execute(
"""
SELECT table_name AS name
FROM information_schema.tables
WHERE table_schema = 'public' AND table_type = 'BASE TABLE'
"""
).fetchall()
return {_row_value(row, "name") for row in rows}
def list_columns(self, table_name: str) -> set[str]:
with self._repository.connection() as connection:
if self._repository.dialect == "sqlite":
rows = connection.execute(f"PRAGMA table_info({table_name})").fetchall()
return {_row_value(row, "name") for row in rows}
rows = connection.execute(
"""
SELECT column_name
FROM information_schema.columns
WHERE table_schema = 'public' AND table_name = %s
""",
(table_name,),
).fetchall()
return {_row_value(row, "column_name") for row in rows}
def list_indexes(self, table_name: str) -> set[str]:
with self._repository.connection() as connection:
if self._repository.dialect == "sqlite":
rows = connection.execute(f"PRAGMA index_list({table_name})").fetchall()
return {_row_value(row, "name") for row in rows}
rows = connection.execute(
"""
SELECT indexname
FROM pg_indexes
WHERE schemaname = 'public' AND tablename = %s
""",
(table_name,),
).fetchall()
return {_row_value(row, "indexname") for row in rows}
def count_rows(self, table_name: str) -> int:
with self._repository.connection() as connection:
row = connection.execute(f"SELECT COUNT(*) AS count FROM {table_name}").fetchone()
return int(_row_value(row, "count"))
def _user_from_row(row: Any) -> UserSummary:
return UserSummary(
id=_row_value(row, "id"),
display_name=_row_value(row, "display_name"),
role=_row_value(row, "role"),
)
def _target_site_from_row(row: Any) -> TargetSiteConfig:
return TargetSiteConfig(
id=_row_value(row, "id"),
name=_row_value(row, "name"),
slug=_row_value(row, "slug"),
publishing_type=_row_value(row, "publishing_type"),
default_language=_row_value(row, "default_language"),
brand_voice=_row_value(row, "brand_voice"),
audience=_row_value(row, "audience"),
seo_rules=_json_from_row(row, "seo_rules"),
visual_rules=_json_from_row(row, "visual_rules"),
source_rules=_json_from_row(row, "source_rules"),
publishing_rules=_json_from_row(row, "publishing_rules"),
active_script_config_version_id=_row_value(
row, "active_script_config_version_id"
),
created_at=_row_value(row, "created_at"),
updated_at=_row_value(row, "updated_at"),
)
def _plain_row(row: Any) -> dict[str, Any]:
if isinstance(row, sqlite3.Row):
result = dict(row)
else:
result = dict(row)
for key in (
"diff",
"seo_rules",
"visual_rules",
"source_rules",
"publishing_rules",
):
if key in result:
result[key] = _json_decode(result[key])
return result
def _row_value(row: Any, key: str) -> Any:
return row[key]
def _json_from_row(row: Any, key: str) -> JsonObject:
return _json_decode(_row_value(row, key))
def _json_decode(value: Any) -> Any:
if isinstance(value, str):
return json.loads(value)
return value
def _json_value(value: Any) -> str:
return json.dumps(value, sort_keys=True, separators=(",", ":"))
def _uuid_value(value: UUID | None) -> str | None:
if value is None:
return None
return str(value)
def _datetime_value(value: datetime | None) -> str | None:
if value is None:
return None
return value.isoformat()
apps/backend/src/infrastructure/schema.py
+606
View File
@@ -0,0 +1,606 @@
from __future__ import annotations
import sqlite3
from collections.abc import Iterable
from contextlib import closing
from src.domain.contracts import (
AgentJobErrorCategory,
AgentJobStatus,
AgentJobType,
ArticleWorkflowStatus,
AssetStatus,
AssetType,
ClaimRiskLevel,
ClaimSupportStatus,
PlanReviewStatus,
PublishingStatus,
Role,
ScriptConfigVersionStatus,
)
def _quoted_values(values: Iterable[str]) -> str:
return ", ".join(f"'{value}'" for value in values)
ROLE_VALUES = _quoted_values(role.value for role in Role)
ARTICLE_STATUS_VALUES = _quoted_values(status.value for status in ArticleWorkflowStatus)
PUBLISHING_STATUS_VALUES = _quoted_values(status.value for status in PublishingStatus)
SCRIPT_CONFIG_STATUS_VALUES = _quoted_values(
status.value for status in ScriptConfigVersionStatus
)
PLAN_REVIEW_STATUS_VALUES = _quoted_values(status.value for status in PlanReviewStatus)
CLAIM_SUPPORT_STATUS_VALUES = _quoted_values(
status.value for status in ClaimSupportStatus
)
CLAIM_RISK_LEVEL_VALUES = _quoted_values(level.value for level in ClaimRiskLevel)
ASSET_TYPE_VALUES = _quoted_values(asset_type.value for asset_type in AssetType)
ASSET_STATUS_VALUES = _quoted_values(status.value for status in AssetStatus)
AGENT_JOB_TYPE_VALUES = _quoted_values(job_type.value for job_type in AgentJobType)
AGENT_JOB_STATUS_VALUES = _quoted_values(status.value for status in AgentJobStatus)
AGENT_JOB_ERROR_CATEGORY_VALUES = _quoted_values(
category.value for category in AgentJobErrorCategory
)
POSTGRES_SCHEMA_STATEMENTS: tuple[str, ...] = (
'CREATE EXTENSION IF NOT EXISTS "pgcrypto"',
f"""
CREATE TABLE IF NOT EXISTS users (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
email TEXT NOT NULL UNIQUE,
display_name TEXT NOT NULL,
role TEXT NOT NULL CHECK (role IN ({ROLE_VALUES})),
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
)
""",
"""
CREATE TABLE IF NOT EXISTS target_sites (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
name TEXT NOT NULL,
slug TEXT NOT NULL UNIQUE,
publishing_type TEXT NOT NULL DEFAULT 'git_next',
default_language TEXT NOT NULL DEFAULT 'en',
brand_voice TEXT NOT NULL,
audience TEXT NOT NULL,
seo_rules JSONB NOT NULL DEFAULT '{}'::jsonb,
visual_rules JSONB NOT NULL DEFAULT '{}'::jsonb,
source_rules JSONB NOT NULL DEFAULT '{}'::jsonb,
publishing_rules JSONB NOT NULL DEFAULT '{}'::jsonb,
active_script_config_version_id UUID,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
)
""",
f"""
CREATE TABLE IF NOT EXISTS script_config_versions (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
target_site_id UUID NOT NULL REFERENCES target_sites(id) ON DELETE CASCADE,
version INTEGER NOT NULL,
status TEXT NOT NULL CHECK (status IN ({SCRIPT_CONFIG_STATUS_VALUES})),
created_by UUID NOT NULL REFERENCES users(id),
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
diff JSONB NOT NULL DEFAULT '{{}}'::jsonb,
rollback_target_version_id UUID REFERENCES script_config_versions(id),
activated_at TIMESTAMPTZ,
publishing_yaml TEXT NOT NULL,
publishing_yaml_hash TEXT NOT NULL,
transform_script TEXT NOT NULL,
transform_script_hash TEXT NOT NULL,
UNIQUE (target_site_id, version)
)
""",
f"""
CREATE TABLE IF NOT EXISTS articles (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
target_site_id UUID NOT NULL REFERENCES target_sites(id),
status TEXT NOT NULL CHECK (status IN ({ARTICLE_STATUS_VALUES})),
publishing_status TEXT NOT NULL CHECK (
publishing_status IN ({PUBLISHING_STATUS_VALUES})
),
brief_description TEXT NOT NULL,
working_title TEXT,
language TEXT NOT NULL DEFAULT 'en',
content_type TEXT NOT NULL DEFAULT 'longform_guide',
primary_keyword TEXT,
assigned_editor_id UUID REFERENCES users(id),
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
)
""",
"""
CREATE TABLE IF NOT EXISTS boundary_questions (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
article_id UUID NOT NULL REFERENCES articles(id) ON DELETE CASCADE,
sort_order INTEGER NOT NULL,
question TEXT NOT NULL,
answer TEXT,
is_required BOOLEAN NOT NULL DEFAULT true,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
UNIQUE (article_id, sort_order)
)
""",
f"""
CREATE TABLE IF NOT EXISTS article_plans (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
article_id UUID NOT NULL REFERENCES articles(id) ON DELETE CASCADE,
version INTEGER NOT NULL,
status TEXT NOT NULL CHECK (status IN ({PLAN_REVIEW_STATUS_VALUES})),
recommended_title TEXT,
search_intent TEXT,
thesis TEXT,
claims_to_prove JSONB NOT NULL DEFAULT '[]'::jsonb,
risks JSONB NOT NULL DEFAULT '[]'::jsonb,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
UNIQUE (article_id, version)
)
""",
"""
CREATE TABLE IF NOT EXISTS plan_sections (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
article_plan_id UUID NOT NULL REFERENCES article_plans(id) ON DELETE CASCADE,
sort_order INTEGER NOT NULL,
heading TEXT NOT NULL,
purpose TEXT,
claims_to_support JSONB NOT NULL DEFAULT '[]'::jsonb,
target_word_count INTEGER,
UNIQUE (article_plan_id, sort_order)
)
""",
"""
CREATE TABLE IF NOT EXISTS evidence_items (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
article_id UUID NOT NULL REFERENCES articles(id) ON DELETE CASCADE,
source_title TEXT NOT NULL,
source_url TEXT NOT NULL,
source_type TEXT NOT NULL,
source_quality_score NUMERIC(4, 3) NOT NULL,
summary TEXT NOT NULL,
supports_claims JSONB NOT NULL DEFAULT '[]'::jsonb,
artifact_manifest_id UUID,
retrieved_at TIMESTAMPTZ NOT NULL DEFAULT now()
)
""",
f"""
CREATE TABLE IF NOT EXISTS claims (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
article_id UUID NOT NULL REFERENCES articles(id) ON DELETE CASCADE,
section_id UUID REFERENCES plan_sections(id) ON DELETE SET NULL,
claim_text TEXT NOT NULL,
support_status TEXT NOT NULL CHECK (
support_status IN ({CLAIM_SUPPORT_STATUS_VALUES})
),
risk_level TEXT NOT NULL CHECK (risk_level IN ({CLAIM_RISK_LEVEL_VALUES})),
evidence_item_ids JSONB NOT NULL DEFAULT '[]'::jsonb,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
)
""",
f"""
CREATE TABLE IF NOT EXISTS article_drafts (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
article_id UUID NOT NULL REFERENCES articles(id) ON DELETE CASCADE,
version INTEGER NOT NULL,
title TEXT NOT NULL,
slug TEXT NOT NULL,
meta_title TEXT,
meta_description TEXT,
body_object_key TEXT,
status TEXT NOT NULL CHECK (status IN ({ARTICLE_STATUS_VALUES})),
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
UNIQUE (article_id, version)
)
""",
f"""
CREATE TABLE IF NOT EXISTS assets (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
article_id UUID NOT NULL REFERENCES articles(id) ON DELETE CASCADE,
asset_type TEXT NOT NULL CHECK (asset_type IN ({ASSET_TYPE_VALUES})),
title TEXT NOT NULL,
prompt TEXT,
file_url TEXT,
alt_text TEXT,
caption TEXT,
status TEXT NOT NULL CHECK (status IN ({ASSET_STATUS_VALUES})),
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
)
""",
f"""
CREATE TABLE IF NOT EXISTS workflow_events (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
article_id UUID NOT NULL REFERENCES articles(id) ON DELETE CASCADE,
event_type TEXT NOT NULL,
from_status TEXT CHECK (from_status IN ({ARTICLE_STATUS_VALUES})),
to_status TEXT CHECK (to_status IN ({ARTICLE_STATUS_VALUES})),
actor_user_id UUID REFERENCES users(id),
payload JSONB NOT NULL DEFAULT '{{}}'::jsonb,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
)
""",
f"""
CREATE TABLE IF NOT EXISTS agent_jobs (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
article_id UUID REFERENCES articles(id) ON DELETE SET NULL,
job_type TEXT NOT NULL CHECK (job_type IN ({AGENT_JOB_TYPE_VALUES})),
agent_profile TEXT NOT NULL,
status TEXT NOT NULL CHECK (status IN ({AGENT_JOB_STATUS_VALUES})),
workspace_path TEXT,
input_files JSONB NOT NULL DEFAULT '[]'::jsonb,
output_files JSONB NOT NULL DEFAULT '[]'::jsonb,
error_category TEXT CHECK (
error_category IS NULL
OR error_category IN ({AGENT_JOB_ERROR_CATEGORY_VALUES})
),
error_message TEXT,
queued_at TIMESTAMPTZ NOT NULL DEFAULT now(),
started_at TIMESTAMPTZ,
finished_at TIMESTAMPTZ
)
""",
"""
CREATE TABLE IF NOT EXISTS research_run_manifests (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
article_id UUID NOT NULL REFERENCES articles(id) ON DELETE CASCADE,
agent_job_id UUID NOT NULL REFERENCES agent_jobs(id) ON DELETE CASCADE,
s3_prefix TEXT NOT NULL,
source_urls JSONB NOT NULL DEFAULT '[]'::jsonb,
object_keys JSONB NOT NULL DEFAULT '[]'::jsonb,
content_hashes JSONB NOT NULL DEFAULT '[]'::jsonb,
artifact_types JSONB NOT NULL DEFAULT '[]'::jsonb,
metadata_references JSONB NOT NULL DEFAULT '[]'::jsonb,
artifacts JSONB NOT NULL DEFAULT '[]'::jsonb,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
)
""",
f"""
CREATE TABLE IF NOT EXISTS publish_commits (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
article_id UUID NOT NULL REFERENCES articles(id) ON DELETE CASCADE,
target_site_id UUID NOT NULL REFERENCES target_sites(id),
repository_url TEXT NOT NULL,
branch TEXT NOT NULL,
commit_sha TEXT,
content_bundle_manifest JSONB NOT NULL DEFAULT '{{}}'::jsonb,
status TEXT NOT NULL CHECK (status IN ({PUBLISHING_STATUS_VALUES})),
deployment_status TEXT,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
)
""",
f"""
CREATE TABLE IF NOT EXISTS prompt_versions (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
prompt_key TEXT NOT NULL,
version INTEGER NOT NULL,
status TEXT NOT NULL CHECK (status IN ({SCRIPT_CONFIG_STATUS_VALUES})),
body TEXT NOT NULL,
created_by UUID NOT NULL REFERENCES users(id),
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
activated_at TIMESTAMPTZ,
diff JSONB NOT NULL DEFAULT '{{}}'::jsonb,
UNIQUE (prompt_key, version)
)
""",
"CREATE INDEX IF NOT EXISTS idx_target_sites_slug ON target_sites (slug)",
"""
CREATE INDEX IF NOT EXISTS idx_articles_dashboard
ON articles (target_site_id, status, publishing_status, updated_at DESC)
""",
"""
CREATE INDEX IF NOT EXISTS idx_articles_editor_dashboard
ON articles (assigned_editor_id, updated_at DESC)
""",
"""
CREATE INDEX IF NOT EXISTS idx_agent_jobs_queue
ON agent_jobs (status, queued_at)
""",
"""
CREATE INDEX IF NOT EXISTS idx_agent_jobs_article_timeline
ON agent_jobs (article_id, job_type, queued_at)
""",
"""
CREATE INDEX IF NOT EXISTS idx_workflow_events_timeline
ON workflow_events (article_id, created_at)
""",
"""
CREATE INDEX IF NOT EXISTS idx_research_run_manifests_article
ON research_run_manifests (article_id, created_at)
""",
"""
CREATE INDEX IF NOT EXISTS idx_publish_commits_article
ON publish_commits (article_id, created_at)
""",
)
SQLITE_SCHEMA_STATEMENTS: tuple[str, ...] = (
"""
CREATE TABLE IF NOT EXISTS users (
id TEXT PRIMARY KEY,
email TEXT NOT NULL UNIQUE,
display_name TEXT NOT NULL,
role TEXT NOT NULL,
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
)
""",
"""
CREATE TABLE IF NOT EXISTS target_sites (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
slug TEXT NOT NULL UNIQUE,
publishing_type TEXT NOT NULL DEFAULT 'git_next',
default_language TEXT NOT NULL DEFAULT 'en',
brand_voice TEXT NOT NULL,
audience TEXT NOT NULL,
seo_rules TEXT NOT NULL DEFAULT '{}',
visual_rules TEXT NOT NULL DEFAULT '{}',
source_rules TEXT NOT NULL DEFAULT '{}',
publishing_rules TEXT NOT NULL DEFAULT '{}',
active_script_config_version_id TEXT,
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
)
""",
"""
CREATE TABLE IF NOT EXISTS script_config_versions (
id TEXT PRIMARY KEY,
target_site_id TEXT NOT NULL,
version INTEGER NOT NULL,
status TEXT NOT NULL,
created_by TEXT NOT NULL,
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
diff TEXT NOT NULL DEFAULT '{}',
rollback_target_version_id TEXT,
activated_at TEXT,
publishing_yaml TEXT NOT NULL,
publishing_yaml_hash TEXT NOT NULL,
transform_script TEXT NOT NULL,
transform_script_hash TEXT NOT NULL,
UNIQUE (target_site_id, version)
)
""",
"""
CREATE TABLE IF NOT EXISTS articles (
id TEXT PRIMARY KEY,
target_site_id TEXT NOT NULL,
status TEXT NOT NULL,
publishing_status TEXT NOT NULL,
brief_description TEXT NOT NULL,
working_title TEXT,
language TEXT NOT NULL DEFAULT 'en',
content_type TEXT NOT NULL DEFAULT 'longform_guide',
primary_keyword TEXT,
assigned_editor_id TEXT,
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
)
""",
"""
CREATE TABLE IF NOT EXISTS boundary_questions (
id TEXT PRIMARY KEY,
article_id TEXT NOT NULL,
sort_order INTEGER NOT NULL,
question TEXT NOT NULL,
answer TEXT,
is_required INTEGER NOT NULL DEFAULT 1,
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
UNIQUE (article_id, sort_order)
)
""",
"""
CREATE TABLE IF NOT EXISTS article_plans (
id TEXT PRIMARY KEY,
article_id TEXT NOT NULL,
version INTEGER NOT NULL,
status TEXT NOT NULL,
recommended_title TEXT,
search_intent TEXT,
thesis TEXT,
claims_to_prove TEXT NOT NULL DEFAULT '[]',
risks TEXT NOT NULL DEFAULT '[]',
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
UNIQUE (article_id, version)
)
""",
"""
CREATE TABLE IF NOT EXISTS plan_sections (
id TEXT PRIMARY KEY,
article_plan_id TEXT NOT NULL,
sort_order INTEGER NOT NULL,
heading TEXT NOT NULL,
purpose TEXT,
claims_to_support TEXT NOT NULL DEFAULT '[]',
target_word_count INTEGER,
UNIQUE (article_plan_id, sort_order)
)
""",
"""
CREATE TABLE IF NOT EXISTS evidence_items (
id TEXT PRIMARY KEY,
article_id TEXT NOT NULL,
source_title TEXT NOT NULL,
source_url TEXT NOT NULL,
source_type TEXT NOT NULL,
source_quality_score REAL NOT NULL,
summary TEXT NOT NULL,
supports_claims TEXT NOT NULL DEFAULT '[]',
artifact_manifest_id TEXT,
retrieved_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
)
""",
"""
CREATE TABLE IF NOT EXISTS claims (
id TEXT PRIMARY KEY,
article_id TEXT NOT NULL,
section_id TEXT,
claim_text TEXT NOT NULL,
support_status TEXT NOT NULL,
risk_level TEXT NOT NULL,
evidence_item_ids TEXT NOT NULL DEFAULT '[]',
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
)
""",
"""
CREATE TABLE IF NOT EXISTS article_drafts (
id TEXT PRIMARY KEY,
article_id TEXT NOT NULL,
version INTEGER NOT NULL,
title TEXT NOT NULL,
slug TEXT NOT NULL,
meta_title TEXT,
meta_description TEXT,
body_object_key TEXT,
status TEXT NOT NULL,
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
UNIQUE (article_id, version)
)
""",
"""
CREATE TABLE IF NOT EXISTS assets (
id TEXT PRIMARY KEY,
article_id TEXT NOT NULL,
asset_type TEXT NOT NULL,
title TEXT NOT NULL,
prompt TEXT,
file_url TEXT,
alt_text TEXT,
caption TEXT,
status TEXT NOT NULL,
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
)
""",
"""
CREATE TABLE IF NOT EXISTS workflow_events (
id TEXT PRIMARY KEY,
article_id TEXT NOT NULL,
event_type TEXT NOT NULL,
from_status TEXT,
to_status TEXT,
actor_user_id TEXT,
payload TEXT NOT NULL DEFAULT '{}',
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
)
""",
"""
CREATE TABLE IF NOT EXISTS agent_jobs (
id TEXT PRIMARY KEY,
article_id TEXT,
job_type TEXT NOT NULL,
agent_profile TEXT NOT NULL,
status TEXT NOT NULL,
workspace_path TEXT,
input_files TEXT NOT NULL DEFAULT '[]',
output_files TEXT NOT NULL DEFAULT '[]',
error_category TEXT,
error_message TEXT,
queued_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
started_at TEXT,
finished_at TEXT
)
""",
"""
CREATE TABLE IF NOT EXISTS research_run_manifests (
id TEXT PRIMARY KEY,
article_id TEXT NOT NULL,
agent_job_id TEXT NOT NULL,
s3_prefix TEXT NOT NULL,
source_urls TEXT NOT NULL DEFAULT '[]',
object_keys TEXT NOT NULL DEFAULT '[]',
content_hashes TEXT NOT NULL DEFAULT '[]',
artifact_types TEXT NOT NULL DEFAULT '[]',
metadata_references TEXT NOT NULL DEFAULT '[]',
artifacts TEXT NOT NULL DEFAULT '[]',
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
)
""",
"""
CREATE TABLE IF NOT EXISTS publish_commits (
id TEXT PRIMARY KEY,
article_id TEXT NOT NULL,
target_site_id TEXT NOT NULL,
repository_url TEXT NOT NULL,
branch TEXT NOT NULL,
commit_sha TEXT,
content_bundle_manifest TEXT NOT NULL DEFAULT '{}',
status TEXT NOT NULL,
deployment_status TEXT,
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
)
""",
"""
CREATE TABLE IF NOT EXISTS prompt_versions (
id TEXT PRIMARY KEY,
prompt_key TEXT NOT NULL,
version INTEGER NOT NULL,
status TEXT NOT NULL,
body TEXT NOT NULL,
created_by TEXT NOT NULL,
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
activated_at TEXT,
diff TEXT NOT NULL DEFAULT '{}',
UNIQUE (prompt_key, version)
)
""",
"CREATE INDEX IF NOT EXISTS idx_target_sites_slug ON target_sites (slug)",
"""
CREATE INDEX IF NOT EXISTS idx_articles_dashboard
ON articles (target_site_id, status, publishing_status, updated_at)
""",
"""
CREATE INDEX IF NOT EXISTS idx_articles_editor_dashboard
ON articles (assigned_editor_id, updated_at)
""",
"""
CREATE INDEX IF NOT EXISTS idx_agent_jobs_queue
ON agent_jobs (status, queued_at)
""",
"""
CREATE INDEX IF NOT EXISTS idx_agent_jobs_article_timeline
ON agent_jobs (article_id, job_type, queued_at)
""",
"""
CREATE INDEX IF NOT EXISTS idx_workflow_events_timeline
ON workflow_events (article_id, created_at)
""",
"""
CREATE INDEX IF NOT EXISTS idx_research_run_manifests_article
ON research_run_manifests (article_id, created_at)
""",
"""
CREATE INDEX IF NOT EXISTS idx_publish_commits_article
ON publish_commits (article_id, created_at)
""",
)
def schema_statements_for_dsn(dsn: str) -> tuple[str, ...]:
if dsn.startswith("sqlite:///"):
return SQLITE_SCHEMA_STATEMENTS
return POSTGRES_SCHEMA_STATEMENTS
def setup_database(dsn: str) -> None:
statements = schema_statements_for_dsn(dsn)
if dsn.startswith("sqlite:///"):
sqlite_path = dsn.removeprefix("sqlite:///")
with closing(sqlite3.connect(sqlite_path)) as connection:
for statement in statements:
connection.execute(statement)
connection.commit()
return
import psycopg
with psycopg.connect(dsn) as connection:
with connection.cursor() as cursor:
for statement in statements:
cursor.execute(statement)
apps/backend/src/infrastructure/seed_data.py
+20
View File
@@ -0,0 +1,20 @@
from __future__ import annotations
import os
from src.application.seed_data import seed_reference_data
from src.infrastructure.repositories import open_backend_repository
def main() -> None:
dsn = os.environ.get("DATABASE_URL") or os.environ.get("POSTGRES_DSN")
if not dsn:
raise RuntimeError("DATABASE_URL or POSTGRES_DSN must be set")
repository = open_backend_repository(dsn)
repository.setup()
seed_reference_data(repository)
if __name__ == "__main__":
main()
apps/backend/tests/integration/__init__.py
+1
View File
@@ -0,0 +1 @@
apps/backend/tests/integration/test_postgres_alembic_migration.py
+43
View File
@@ -0,0 +1,43 @@
from __future__ import annotations
import os
import subprocess
import sys
import unittest
from importlib.util import find_spec
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[4]
BACKEND_ROOT = REPO_ROOT / "apps" / "backend"
sys.path.insert(0, str(BACKEND_ROOT))
class PostgresAlembicMigrationIntegrationTest(unittest.TestCase):
def test_alembic_upgrade_head_creates_schema_on_configured_postgres(self) -> None:
dsn = os.environ.get("PIPELINE_TEST_DATABASE_DSN")
if not dsn:
self.skipTest("PIPELINE_TEST_DATABASE_DSN is not set")
if not dsn.startswith(("postgresql://", "postgres://")):
self.skipTest("Postgres DSN is required for Alembic migration test")
if find_spec("alembic") is None:
self.skipTest("alembic is not installed")
env = os.environ.copy()
env["DATABASE_URL"] = dsn
subprocess.run(
[sys.executable, "-m", "alembic", "upgrade", "head"],
cwd=REPO_ROOT,
env=env,
check=True,
)
from src.domain.schema import CORE_TABLES
from src.infrastructure.repositories import open_backend_repository
repository = open_backend_repository(dsn)
self.assertTrue(set(CORE_TABLES).issubset(repository.schema.list_tables()))
if __name__ == "__main__":
unittest.main()
apps/backend/tests/integration/test_schema_storage_contracts.py
+136
View File
@@ -0,0 +1,136 @@
from __future__ import annotations
import os
import sys
import tempfile
import unittest
from collections.abc import Iterator
from contextlib import contextmanager
from pathlib import Path
BACKEND_ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(BACKEND_ROOT))
@contextmanager
def repository_dsn() -> Iterator[str]:
configured_dsn = os.environ.get("PIPELINE_TEST_DATABASE_DSN")
if configured_dsn:
yield configured_dsn
return
with tempfile.TemporaryDirectory() as tmp_dir:
yield f"sqlite:///{Path(tmp_dir) / 'pipeline-schema.db'}"
class SchemaStorageContractsIntegrationTest(unittest.TestCase):
def test_schema_contains_task_tables_and_operational_indexes(self) -> None:
from src.domain.schema import CORE_TABLES
from src.infrastructure.repositories import open_backend_repository
with repository_dsn() as dsn:
repository = open_backend_repository(dsn)
repository.setup()
self.assertTrue(set(CORE_TABLES).issubset(repository.schema.list_tables()))
self.assertIn(
"idx_target_sites_slug",
repository.schema.list_indexes("target_sites"),
)
self.assertIn(
"idx_articles_dashboard",
repository.schema.list_indexes("articles"),
)
self.assertIn(
"idx_agent_jobs_queue",
repository.schema.list_indexes("agent_jobs"),
)
self.assertIn(
"idx_workflow_events_timeline",
repository.schema.list_indexes("workflow_events"),
)
def test_required_domain_storage_fields_exist(self) -> None:
from src.infrastructure.repositories import open_backend_repository
with repository_dsn() as dsn:
repository = open_backend_repository(dsn)
repository.setup()
self.assertTrue(
{"status", "publishing_status"}.issubset(
repository.schema.list_columns("articles")
)
)
self.assertIn("status", repository.schema.list_columns("article_drafts"))
self.assertTrue(
{
"created_by",
"created_at",
"diff",
"rollback_target_version_id",
"activated_at",
"publishing_yaml",
"transform_script",
}.issubset(repository.schema.list_columns("script_config_versions"))
)
self.assertTrue(
{
"s3_prefix",
"source_urls",
"object_keys",
"content_hashes",
"artifact_types",
"metadata_references",
"artifacts",
}.issubset(repository.schema.list_columns("research_run_manifests"))
)
self.assertTrue(
{
"repository_url",
"branch",
"commit_sha",
"content_bundle_manifest",
"status",
}.issubset(repository.schema.list_columns("publish_commits"))
)
def test_seed_data_is_idempotent_for_users_site_and_script_version(self) -> None:
from src.application.seed_data import seed_reference_data
from src.domain.schema import (
SEEDED_ADMIN_EMAIL,
SEEDED_EDITOR_EMAIL,
SEEDED_TARGET_SITE_SLUG,
)
from src.infrastructure.repositories import open_backend_repository
with repository_dsn() as dsn:
repository = open_backend_repository(dsn)
repository.setup()
seed_reference_data(repository)
seed_reference_data(repository)
site = repository.target_sites.get_by_slug(SEEDED_TARGET_SITE_SLUG)
seeded_sites = [
candidate
for candidate in repository.target_sites.list()
if candidate.slug == SEEDED_TARGET_SITE_SLUG
]
script_versions = repository.script_config_versions.list_for_site(site.id)
self.assertEqual(1, repository.users.count_by_email(SEEDED_ADMIN_EMAIL))
self.assertEqual(1, repository.users.count_by_email(SEEDED_EDITOR_EMAIL))
self.assertEqual([site.id], [candidate.id for candidate in seeded_sites])
self.assertEqual(1, len(script_versions))
self.assertEqual("ACTIVE", script_versions[0]["status"])
self.assertEqual(
str(site.active_script_config_version_id), str(script_versions[0]["id"])
)
self.assertIn("repository:", script_versions[0]["publishing_yaml"])
self.assertIn("transformArticle", script_versions[0]["transform_script"])
if __name__ == "__main__":
unittest.main()
apps/backend/tests/integration/test_seeded_target_site_lookup.py
+74
View File
@@ -0,0 +1,74 @@
from __future__ import annotations
import os
import sys
import tempfile
import unittest
from collections.abc import Iterator
from contextlib import contextmanager
from pathlib import Path
BACKEND_ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(BACKEND_ROOT))
SEEDED_SITE_SLUG = "b2b_saas_blog"
@contextmanager
def repository_dsn() -> Iterator[str]:
configured_dsn = os.environ.get("PIPELINE_TEST_DATABASE_DSN")
if configured_dsn:
yield configured_dsn
return
with tempfile.TemporaryDirectory() as tmp_dir:
yield f"sqlite:///{Path(tmp_dir) / 'pipeline-seed-red.db'}"
class SeededTargetSiteLookupIntegrationTest(unittest.TestCase):
def test_seeded_git_next_target_site_lookup_by_slug_is_idempotent(self) -> None:
from src.application.seed_data import seed_reference_data
from src.infrastructure.repositories import open_backend_repository
with repository_dsn() as dsn:
repository = open_backend_repository(dsn)
repository.setup()
seed_reference_data(repository)
seed_reference_data(repository)
site = repository.target_sites.get_by_slug(SEEDED_SITE_SLUG)
seeded_sites = [
candidate
for candidate in repository.target_sites.list()
if candidate.slug == SEEDED_SITE_SLUG
]
from src.domain.contracts import TargetSiteConfig
self.assertIsInstance(site, TargetSiteConfig)
self.assertEqual([site.id], [candidate.id for candidate in seeded_sites])
self.assertEqual("B2B SaaS Blog", site.name)
self.assertEqual("git_next", site.publishing_type)
self.assertEqual("en", site.default_language)
self.assertEqual(
"git@github.com:example/site.git",
site.publishing_rules.repository_url,
)
self.assertEqual("main", site.publishing_rules.production_branch)
self.assertEqual("mdx", site.publishing_rules.content_format)
self.assertEqual(
"content/articles/{slug}.mdx",
site.publishing_rules.content_path_template,
)
self.assertEqual(
"public/articles/{slug}/{filename}",
site.publishing_rules.asset_path_template,
)
self.assertIsNotNone(site.active_script_config_version_id)
if __name__ == "__main__":
unittest.main()
tasks/003-postgres-schema-and-seed-data.md
+137 -13
View File
@@ -38,13 +38,13 @@ Development description: Create the initial relational schema, migrations, index
## Acceptance Criteria
- [ ] TDD pre-requirement: before implementing migrations, write one failing integration test through repository/public backend interfaces for seeded site lookup; add further tests one behavior at a time and record red-green evidence in `Result`.
- [ ] Migrations run cleanly on an empty Postgres database.
- [ ] Seed data is idempotent.
- [ ] Article/domain tables hold authoritative article statuses.
- [ ] Script/config versions store author, timestamp, diff, rollback target, activation timestamp, YAML, and transform script.
- [ ] Research manifests store S3 prefixes, source URLs, object keys, content hashes, artifact types, and metadata references.
- [ ] Publish commits store repository URL, branch, commit SHA, content bundle manifest, and status.
- [x] TDD pre-requirement: before implementing migrations, write one failing integration test through repository/public backend interfaces for seeded site lookup; add further tests one behavior at a time and record red-green evidence in `Result`.
- [x] Migrations run cleanly on an empty Postgres database.
- [x] Seed data is idempotent.
- [x] Article/domain tables hold authoritative article statuses.
- [x] Script/config versions store author, timestamp, diff, rollback target, activation timestamp, YAML, and transform script.
- [x] Research manifests store S3 prefixes, source URLs, object keys, content hashes, artifact types, and metadata references.
- [x] Publish commits store repository URL, branch, commit SHA, content bundle manifest, and status.
## Verification
@@ -54,9 +54,133 @@ Development description: Create the initial relational schema, migrations, index
## Result
- Status: Pending execution.
- TDD plan: To be filled during execution.
- Red evidence: To be filled during execution.
- Green evidence: To be filled during execution.
- Refactor notes: To be filled during execution.
- Verification output: To be filled during execution.
- Status: Completed.
- TDD plan:
- Add one integration-style unittest slice before schema/seed implementation.
- Exercise the future public backend repository setup and application seed use case through `open_backend_repository(dsn)` and `seed_reference_data(repository)`.
- Use `PIPELINE_TEST_DATABASE_DSN` when provided so the same test can later run against Postgres; otherwise use a temporary `sqlite:///...` DSN only as a disposable Red-phase repository substitute.
- Verify observable seeded-site behavior:
- running repository setup plus seed makes the Git-backed Next target site lookupable by slug `b2b_saas_blog`;
- running seed twice leaves exactly one site with that slug;
- the returned site exposes the public `TargetSiteConfig` shape with Git/Next publishing fields and an active script config version.
- Red evidence:
- Command: `env PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s apps/backend/tests/integration -p test_seeded_target_site_lookup.py`
- Actual output:
```text
E
======================================================================
ERROR: test_seeded_git_next_target_site_lookup_by_slug_is_idempotent (test_seeded_target_site_lookup.SeededTargetSiteLookupIntegrationTest.test_seeded_git_next_target_site_lookup_by_slug_is_idempotent)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/Users/gavrilovdev/tmp/pupline/apps/backend/tests/integration/test_seeded_target_site_lookup.py", line 32, in test_seeded_git_next_target_site_lookup_by_slug_is_idempotent
from src.application.seed_data import seed_reference_data
ModuleNotFoundError: No module named 'src.application.seed_data'
----------------------------------------------------------------------
Ran 1 test in 0.000s
FAILED (errors=1)
```
- Result: failed as expected because seed data and repository interfaces are not implemented yet.
- Green evidence:
- Pre-requirement command:
`docker compose run --rm --no-deps -v /Users/gavrilovdev/tmp/pupline:/workspace -w /workspace backend python -m unittest discover -s apps/backend/tests/integration -p test_seeded_target_site_lookup.py`
- Actual output:
```text
.
----------------------------------------------------------------------
Ran 1 test in 0.216s
OK
```
- Added focused storage tests for the follow-up behaviors:
- task table/index shape;
- authoritative article status columns;
- script config audit/rollback/activation/YAML/script fields;
- research manifest S3/source/object/hash/type/metadata fields;
- publish commit repository/branch/SHA/manifest/status fields;
- seed idempotency for users, site, and active script config version.
- Refactor notes:
- Added domain schema/reference constants only under `src/domain/schema.py`.
- Added `seed_reference_data(repository)` in application and kept DB connections/repository/DDL in infrastructure.
- Added Alembic under `apps/backend/alembic` with root `alembic.ini`; `DATABASE_URL` and `POSTGRES_DSN` are supported.
- Fixed Alembic env import path so the same migration runs from repo root and inside the backend container.
- Added `make migrate` and `make seed`; public root command remains `alembic upgrade head`.
- Verification output:
- Backend tests:
```text
Command:
docker compose run --rm --no-deps -v /Users/gavrilovdev/tmp/pupline:/workspace -w /workspace backend python -m unittest discover -s apps/backend/tests -p 'test*.py'
Output:
......s....
----------------------------------------------------------------------
Ran 11 tests in 0.123s
OK (skipped=1)
```
- Empty Postgres migration:
```text
Command:
env POSTGRES_PORT=55433 docker compose -p pupline-task003 run --rm --no-deps -e DATABASE_URL=postgresql://pipeline:pipeline_local@postgres:5432/pipeline backend alembic upgrade head
Output:
INFO [alembic.runtime.migration] Context impl PostgresqlImpl.
INFO [alembic.runtime.migration] Will assume transactional DDL.
INFO [alembic.runtime.migration] Running upgrade -> 202605210003, initial postgres schema
```
- Postgres Alembic integration test:
```text
Command:
env POSTGRES_PORT=55433 docker compose -p pupline-task003 run --rm --no-deps -v /Users/gavrilovdev/tmp/pupline:/workspace -w /workspace -e PIPELINE_TEST_DATABASE_DSN=postgresql://pipeline:pipeline_local@postgres:5432/pipeline backend python -m unittest discover -s apps/backend/tests/integration -p test_postgres_alembic_migration.py
Output:
.
----------------------------------------------------------------------
Ran 1 test in 0.693s
OK
```
- Postgres repository/seed tests:
```text
Command:
env POSTGRES_PORT=55433 docker compose -p pupline-task003 run --rm --no-deps -v /Users/gavrilovdev/tmp/pupline:/workspace -w /workspace -e PIPELINE_TEST_DATABASE_DSN=postgresql://pipeline:pipeline_local@postgres:5432/pipeline backend python -m unittest discover -s apps/backend/tests/integration -p test_seeded_target_site_lookup.py
Output:
.
----------------------------------------------------------------------
Ran 1 test in 0.269s
OK
Command:
env POSTGRES_PORT=55433 docker compose -p pupline-task003 run --rm --no-deps -v /Users/gavrilovdev/tmp/pupline:/workspace -w /workspace -e PIPELINE_TEST_DATABASE_DSN=postgresql://pipeline:pipeline_local@postgres:5432/pipeline backend python -m unittest discover -s apps/backend/tests/integration -p test_schema_storage_contracts.py
Output:
...
----------------------------------------------------------------------
Ran 3 tests in 0.348s
OK
```
- Seed command twice/no duplicates:
```text
Commands:
env POSTGRES_PORT=55433 docker compose -p pupline-task003 run --rm --no-deps -e POSTGRES_DSN=postgresql://pipeline:pipeline_local@postgres:5432/pipeline backend python -m src.infrastructure.seed_data
env POSTGRES_PORT=55433 docker compose -p pupline-task003 run --rm --no-deps -e POSTGRES_DSN=postgresql://pipeline:pipeline_local@postgres:5432/pipeline backend python -m src.infrastructure.seed_data
Verification output:
admin=1 editor=1 sites=1 scripts=1 active=ACTIVE
```
- Smoke:
```text
Command:
bash tests/smoke/public-health.sh
Output:
backend health ok
runner health ok
frontend health ok
backend dependencies ok
public health smoke ok
```