huncode/content-factory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(task-018): add observability retry cancel and audit trail

Browse Source
This commit is contained in:
huncode
2026-05-22 01:23:07 +03:00
parent 318ae790ef
commit 95fec3ba26
26 changed files with 1984 additions and 275 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/backend/tests/integration/test_observability_retry_cancel_audit_public_api.py
+493
View File
@@ -0,0 +1,493 @@
from __future__ import annotations
import os
import sys
import tempfile
import unittest
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from uuid import UUID
from fastapi.testclient import TestClient
BACKEND_ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(BACKEND_ROOT))
from src.application.seed_data import seed_reference_data # noqa: E402
from src.domain.contracts import AgentJobStatus, AgentJobType, PublishingStatus # noqa: E402
from src.infrastructure.repositories import open_backend_repository # noqa: E402
from src.presentation.dependencies import get_repository # noqa: E402
from src.presentation.main import app # noqa: E402
DEMO_EDITOR_EMAIL = "editor@example.com"
DEMO_ADMIN_EMAIL = "admin@example.com"
DEMO_USER_EMAIL_HEADER = "X-Demo-User-Email"
class ObservabilityRetryCancelAuditPublicApiTest(unittest.TestCase):
def setUp(self) -> None:
self.tmp_dir = tempfile.TemporaryDirectory()
os.environ["OBJECT_STORAGE_LOCAL_ROOT"] = str(Path(self.tmp_dir.name) / "objects")
dsn = f"sqlite:///{Path(self.tmp_dir.name) / 'observability.db'}"
self.repository = open_backend_repository(dsn)
self.repository.setup()
seed_reference_data(self.repository)
app.dependency_overrides[get_repository] = lambda: self.repository
self.client = TestClient(app)
def tearDown(self) -> None:
app.dependency_overrides.clear()
os.environ.pop("OBJECT_STORAGE_LOCAL_ROOT", None)
self.tmp_dir.cleanup()
def test_failed_job_appears_in_article_history_with_retry_eligibility(self) -> None:
article_id = self._prepare_article_for_parallel_production()
created_jobs = self._start_parallel_production(article_id)
section_job = [job for job in created_jobs if job["job_type"] == "SECTION_SCAFFOLD"][0]
failed_job = self._complete_job(
section_job["id"],
output={
"status": "SUCCEEDED",
"output_files": [{"path": "outputs/section-failed.md"}],
"payload": {
"unsupported_claims": [
{"claim_text": "Unverified claim", "risk_level": "high"}
]
},
},
)
self.assertEqual("FAILED", failed_job["status"])
detail_response = self.client.get(
f"/api/articles/{article_id}",
headers={DEMO_USER_EMAIL_HEADER: DEMO_EDITOR_EMAIL},
)
self.assertEqual(200, detail_response.status_code, detail_response.text)
article_jobs = detail_response.json()["agent_jobs"]
history_failed_job = [job for job in article_jobs if job["id"] == failed_job["id"]][0]
self.assertIn("retry_eligible", history_failed_job)
self.assertTrue(history_failed_job["retry_eligible"])
self.assertIsNone(history_failed_job["retry_block_reason"])
def test_article_timeline_mixes_user_system_and_agent_events_in_order(self) -> None:
article_id = self._prepare_article_for_parallel_production()
section_job = self._start_parallel_production(article_id)[0]
failed = self._complete_job(
section_job["id"],
output={
"status": "SUCCEEDED",
"output_files": [{"path": "outputs/section-failed.md"}],
"payload": {
"unsupported_claims": [{"claim_text": "unsupported claim", "risk_level": "high"}]
},
},
)
self.assertEqual("FAILED", failed["status"])
retry_response = self.client.post(
f"/api/agent-jobs/{failed['id']}/retry",
headers={DEMO_USER_EMAIL_HEADER: DEMO_ADMIN_EMAIL},
)
self.assertEqual(201, retry_response.status_code, retry_response.text)
detail_response = self.client.get(
f"/api/articles/{article_id}",
headers={DEMO_USER_EMAIL_HEADER: DEMO_EDITOR_EMAIL},
)
self.assertEqual(200, detail_response.status_code, detail_response.text)
timeline = detail_response.json()["timeline"]
self.assertGreaterEqual(len(timeline), 3)
created_at_values = [entry["created_at"] for entry in timeline]
self.assertEqual(created_at_values, sorted(created_at_values))
sources = {entry["source"] for entry in timeline}
self.assertIn("USER", sources)
self.assertIn("SYSTEM", sources)
self.assertIn("AGENT", sources)
failed_job_entry = [
entry
for entry in timeline
if entry["entry_type"] == "AGENT_JOB"
and entry.get("job_id") == failed["id"]
][0]
self.assertTrue(failed_job_entry["retry_eligible"])
self.assertEqual("FAILED", failed_job_entry["job_status"])
def test_sensitive_logs_are_redacted_and_editor_sees_only_safe_failure_summary(self) -> None:
article_id = self._prepare_article_for_parallel_production()
section_job = self._start_parallel_production(article_id)[0]
failed_job = self._complete_job(
section_job["id"],
exit_code=1,
stdout=(
"token=topsecret123 password=hunter2 "
"Authorization: Bearer very-secret-token\n"
),
stderr="api_key=prod-key-123\n",
output={
"status": "FAILED",
"error_message": "runner failed with sk-1234567890abcdef",
"payload": {"last_successful_step": "outline written"},
},
)
self.assertEqual("FAILED", failed_job["status"])
admin_detail = self.client.get(
f"/api/articles/{article_id}",
headers={DEMO_USER_EMAIL_HEADER: DEMO_ADMIN_EMAIL},
).json()
editor_detail = self.client.get(
f"/api/articles/{article_id}",
headers={DEMO_USER_EMAIL_HEADER: DEMO_EDITOR_EMAIL},
).json()
admin_job = [job for job in admin_detail["agent_jobs"] if job["id"] == failed_job["id"]][0]
editor_job = [job for job in editor_detail["agent_jobs"] if job["id"] == failed_job["id"]][0]
self.assertIn("[REDACTED]", admin_job["stdout"])
self.assertIn("[REDACTED]", admin_job["stderr"])
self.assertNotIn("topsecret123", admin_job["stdout"])
self.assertNotIn("hunter2", admin_job["stdout"])
self.assertNotIn("very-secret-token", admin_job["stdout"])
self.assertNotIn("prod-key-123", admin_job["stderr"])
self.assertNotIn("sk-1234567890abcdef", admin_job["error_message"] or "")
self.assertEqual("", editor_job["stdout"])
self.assertEqual("", editor_job["stderr"])
self.assertIn("failed", editor_job["safe_failure_summary"].lower())
self.assertNotIn("topsecret123", editor_job["safe_failure_summary"])
def test_retry_eligibility_and_cancel_eligibility_follow_policy(self) -> None:
article_id = self._prepare_article_for_parallel_production()
jobs = self._start_parallel_production(article_id)
section_job = [job for job in jobs if job["job_type"] == "SECTION_SCAFFOLD"][0]
section_failed = self._complete_job(
section_job["id"],
output={
"status": "SUCCEEDED",
"output_files": [{"path": "outputs/section-failed.md"}],
"payload": {
"unsupported_claims": [{"claim_text": "unsupported claim", "risk_level": "high"}]
},
},
)
self.assertEqual("FAILED", section_failed["status"])
now = datetime.now(UTC)
evidence_job = self.repository.agent_jobs.create(
article_id=UUID(article_id),
parent_job_id=None,
attempt=1,
job_type=AgentJobType.EVIDENCE_MATRIX,
agent_profile="manual-test",
status=AgentJobStatus.QUEUED,
input_files=[{"path": "inputs/evidence.json"}],
queued_at=now,
)
evidence_failed = self.repository.agent_jobs.complete(
job_id=evidence_job.id,
status=AgentJobStatus.FAILED,
workspace_path="/tmp/evidence",
output_files=[],
payload={"phase": "collect"},
error_category=None,
error_message="temporary failure",
stdout="",
stderr="",
exit_code=1,
duration_ms=2,
finished_at=now,
)
self.assertEqual(AgentJobStatus.FAILED, evidence_failed.status)
queued_section = [job for job in jobs if job["id"] != section_job["id"]][0]
detail = self.client.get(
f"/api/articles/{article_id}",
headers={DEMO_USER_EMAIL_HEADER: DEMO_ADMIN_EMAIL},
).json()
failed_section_view = [job for job in detail["agent_jobs"] if job["id"] == section_failed["id"]][0]
evidence_view = [job for job in detail["agent_jobs"] if job["id"] == str(evidence_failed.id)][0]
queued_view = [job for job in detail["agent_jobs"] if job["id"] == queued_section["id"]][0]
self.assertTrue(failed_section_view["retry_eligible"])
self.assertIsNone(failed_section_view["retry_block_reason"])
self.assertFalse(failed_section_view["cancel_eligible"])
self.assertFalse(evidence_view["retry_eligible"])
self.assertIn("not supported", evidence_view["retry_block_reason"].lower())
self.assertFalse(evidence_view["cancel_eligible"])
self.assertFalse(queued_view["retry_eligible"])
self.assertTrue(queued_view["cancel_eligible"])
def test_cancelling_running_or_queued_job_prevents_article_state_mutation(self) -> None:
article_id = self._prepare_article_for_parallel_production()
jobs = self._start_parallel_production(article_id)
first_job = jobs[0]
cancel_response = self.client.post(
f"/api/agent-jobs/{first_job['id']}/cancel",
headers={DEMO_USER_EMAIL_HEADER: DEMO_ADMIN_EMAIL},
)
self.assertEqual(200, cancel_response.status_code, cancel_response.text)
self.assertEqual("CANCELLED", cancel_response.json()["job"]["status"])
complete_response = self.client.post(
f"/internal/agent-jobs/{first_job['id']}/complete",
json={
"workspace_path": f"/tmp/{first_job['id']}",
"stdout": "completion after cancellation\n",
"stderr": "",
"exit_code": 0,
"duration_ms": 2,
"output": {
"status": "SUCCEEDED",
"output_files": [{"path": "outputs/cancelled.md"}],
"payload": {"draft_markdown": "should never apply"},
},
},
)
self.assertEqual(200, complete_response.status_code, complete_response.text)
self.assertEqual("CANCELLED", complete_response.json()["job"]["status"])
detail = self.client.get(
f"/api/articles/{article_id}",
headers={DEMO_USER_EMAIL_HEADER: DEMO_EDITOR_EMAIL},
).json()
self.assertEqual("PARALLEL_PRODUCTION_RUNNING", detail["article"]["status"])
cancelled_view = [job for job in detail["agent_jobs"] if job["id"] == first_job["id"]][0]
self.assertEqual("CANCELLED", cancelled_view["status"])
def test_publish_commit_retry_blocked_after_publish_commit_exists(self) -> None:
article = self.client.post(
"/api/articles",
headers={DEMO_USER_EMAIL_HEADER: DEMO_EDITOR_EMAIL},
json={
"target_site_id": self._first_site_id(),
"brief_description": "Publish retry policy test",
"working_title": "Publish Retry Policy",
"content_type": "longform_guide",
"primary_keyword": "publish retry policy",
},
).json()["article"]
article_id = article["id"]
now = datetime.now(UTC)
publish_job = self.repository.agent_jobs.create(
article_id=UUID(article_id),
parent_job_id=None,
attempt=1,
job_type=AgentJobType.PUBLISH_COMMIT,
agent_profile="manual-publish-commit",
status=AgentJobStatus.QUEUED,
input_files=[{"path": "inputs/publish.json"}],
queued_at=now,
)
publish_job = self.repository.agent_jobs.complete(
job_id=publish_job.id,
status=AgentJobStatus.FAILED,
workspace_path="/tmp/publish",
output_files=[],
payload={"stage": "git push"},
error_category=None,
error_message="git push failed",
stdout="",
stderr="",
exit_code=1,
duration_ms=5,
finished_at=now,
)
self.repository.publish_commits.create(
article_id=UUID(article_id),
target_site_id=UUID(self._first_site_id()),
repository_url="git@example.com/site.git",
branch="main",
commit_sha="abc123",
content_bundle_manifest={"validation": {"label": "ok"}},
status=PublishingStatus.PUBLISH_COMMIT_CREATED,
deployment_status=None,
created_at=now,
)
retry_response = self.client.post(
f"/api/agent-jobs/{publish_job.id}/retry",
headers={DEMO_USER_EMAIL_HEADER: DEMO_ADMIN_EMAIL},
)
self.assertEqual(409, retry_response.status_code, retry_response.text)
self.assertIn("already exists", retry_response.text.lower())
detail = self.client.get(
f"/api/articles/{article_id}",
headers={DEMO_USER_EMAIL_HEADER: DEMO_ADMIN_EMAIL},
).json()
publish_job_view = [job for job in detail["agent_jobs"] if job["id"] == str(publish_job.id)][0]
self.assertFalse(publish_job_view["retry_eligible"])
self.assertIn("already exists", publish_job_view["retry_block_reason"].lower())
def test_script_config_changes_are_audit_visible_with_diff_and_rollback_target(self) -> None:
site_id = self._first_site_id()
first = self.client.post(
f"/api/sites/{site_id}/publishing-config/versions",
headers={DEMO_USER_EMAIL_HEADER: DEMO_ADMIN_EMAIL},
json={
"publishing_yaml": "target: demo\n",
"transform_script": "export function transformArticle(a){return a;}\n",
"diff": {"summary": "v1 create"},
"activate": True,
},
)
self.assertEqual(201, first.status_code, first.text)
first_version = first.json()["version"]
second = self.client.post(
f"/api/sites/{site_id}/publishing-config/versions",
headers={DEMO_USER_EMAIL_HEADER: DEMO_ADMIN_EMAIL},
json={
"publishing_yaml": "target: demo\nbranch: main\n",
"transform_script": "export function transformArticle(a){return {...a, changed:true};}\n",
"diff": {"summary": "v2 create"},
"rollback_target_version_id": first_version["id"],
"activate": True,
},
)
self.assertEqual(201, second.status_code, second.text)
second_version = second.json()["version"]
rollback = self.client.post(
f"/api/sites/{site_id}/publishing-config/versions/{first_version['id']}/rollback",
headers={DEMO_USER_EMAIL_HEADER: DEMO_ADMIN_EMAIL},
)
self.assertEqual(200, rollback.status_code, rollback.text)
audit_response = self.client.get(
f"/api/sites/{site_id}/publishing-config/audit",
headers={DEMO_USER_EMAIL_HEADER: DEMO_ADMIN_EMAIL},
)
self.assertEqual(200, audit_response.status_code, audit_response.text)
events = audit_response.json()["events"]
self.assertGreaterEqual(len(events), 3)
for event in events:
self.assertIn("diff", event)
self.assertIn("rollback_target_version_id", event)
rollback_events = [event for event in events if event["event_type"] == "SCRIPT_CONFIG_VERSION_ROLLBACK"]
self.assertEqual(1, len(rollback_events))
rollback_event = rollback_events[0]
self.assertEqual(second_version["id"], rollback_event["rollback_target_version_id"])
def _start_parallel_production(self, article_id: str) -> list[dict[str, Any]]:
response = self.client.post(
f"/api/articles/{article_id}/draft/start",
headers={DEMO_USER_EMAIL_HEADER: DEMO_EDITOR_EMAIL},
)
self.assertEqual(202, response.status_code, response.text)
return response.json()["jobs"]
def _complete_job(
self,
job_id: str,
*,
output: dict[str, Any],
stdout: str = "fake section scaffolding runner\n",
stderr: str = "",
exit_code: int = 0,
) -> dict[str, Any]:
response = self.client.post(
f"/internal/agent-jobs/{job_id}/complete",
json={
"workspace_path": f"/tmp/{job_id}",
"stdout": stdout,
"stderr": stderr,
"exit_code": exit_code,
"duration_ms": 2,
"output": output,
},
)
self.assertEqual(200, response.status_code, response.text)
return response.json()["job"]
def _first_site_id(self) -> str:
response = self.client.get(
"/api/sites",
headers={DEMO_USER_EMAIL_HEADER: DEMO_EDITOR_EMAIL},
)
self.assertEqual(200, response.status_code, response.text)
return response.json()[0]["site"]["id"]
def _prepare_article_for_parallel_production(self) -> str:
article_id = self._create_article_with_approved_plan()
research_response = self.client.post(
f"/api/articles/{article_id}/research/start",
headers={DEMO_USER_EMAIL_HEADER: DEMO_EDITOR_EMAIL},
)
self.assertEqual(201, research_response.status_code, research_response.text)
evidence_response = self.client.get(
f"/api/articles/{article_id}/evidence",
headers={DEMO_USER_EMAIL_HEADER: DEMO_EDITOR_EMAIL},
)
self.assertEqual(200, evidence_response.status_code, evidence_response.text)
self.assertEqual("EVIDENCE_MATRIX_READY", evidence_response.json()["article"]["status"])
return article_id
def _create_article_with_approved_plan(self) -> str:
article_response = self.client.post(
"/api/articles",
headers={DEMO_USER_EMAIL_HEADER: DEMO_EDITOR_EMAIL},
json={
"target_site_id": self._first_site_id(),
"brief_description": "Observe failed job history and retry eligibility.",
"working_title": "Observability Retry Audit",
"content_type": "longform_guide",
"primary_keyword": "observability retry audit",
},
)
self.assertEqual(201, article_response.status_code, article_response.text)
article_id = article_response.json()["article"]["id"]
questions = self.client.post(
f"/api/articles/{article_id}/boundary-questions/generate",
headers={DEMO_USER_EMAIL_HEADER: DEMO_EDITOR_EMAIL},
).json()["questions"]
for question in questions:
if question["is_required"]:
patch_response = self.client.patch(
f"/api/articles/{article_id}/boundary-questions/{question['id']}",
headers={DEMO_USER_EMAIL_HEADER: DEMO_EDITOR_EMAIL},
json={"answer": f"Answer for {question['category']}"},
)
self.assertEqual(200, patch_response.status_code, patch_response.text)
submit_response = self.client.post(
f"/api/articles/{article_id}/boundary-questions/submit",
headers={DEMO_USER_EMAIL_HEADER: DEMO_EDITOR_EMAIL},
)
self.assertEqual(200, submit_response.status_code, submit_response.text)
plan_response = self.client.post(
f"/api/articles/{article_id}/plan/generate",
headers={DEMO_USER_EMAIL_HEADER: DEMO_EDITOR_EMAIL},
)
self.assertEqual(201, plan_response.status_code, plan_response.text)
plan = plan_response.json()["plan"]
approve_response = self.client.post(
f"/api/articles/{article_id}/plans/{plan['id']}/approve",
headers={DEMO_USER_EMAIL_HEADER: DEMO_EDITOR_EMAIL},
)
self.assertEqual(200, approve_response.status_code, approve_response.text)
return article_id
if __name__ == "__main__":
unittest.main()
apps/backend/tests/integration/test_script_config_audit.py
+2
View File
@@ -119,6 +119,8 @@ class ScriptConfigAuditTest(unittest.TestCase):
activate: bool = False,
rollback_target_version_id: UUID | None = None,
) -> ScriptConfigVersionCreateRequest:
from src.domain.contracts import ScriptConfigVersionCreateRequest
payload = ScriptConfigVersionCreateRequest(
publishing_yaml="target:\n repo: audit\n",
transform_script="export function transformArticle(article) { return article; }",