Task 011 implement evidence matrix and claim gate

2026-05-21 22:52:41 +03:00
parent 7d4a79acb5
commit 7e1b05067c
23 changed files with 2090 additions and 17 deletions
@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+COMPOSE_FILE="${ROOT_DIR}/docker-compose.yml"
+BACKEND_URL="${BACKEND_URL:-http://localhost:8000}"
+WAIT_SECONDS="${WAIT_SECONDS:-60}"
+PYTHON_BIN="${PYTHON_BIN:-python3}"
+EDITOR_EMAIL="${EDITOR_EMAIL:-editor@example.com}"
+
+cleanup() {
+  docker compose -f "${COMPOSE_FILE}" down --remove-orphans >/dev/null 2>&1 || true
+}
+
+request() {
+  local method="$1"
+  local path="$2"
+  local body="${3:-}"
+
+  if [[ -n "${body}" ]]; then
+    curl -fsS -X "${method}" -H "Content-Type: application/json" \
+      -H "X-Demo-User-Email: ${EDITOR_EMAIL}" --data "${body}" "${BACKEND_URL}${path}"
+    return
+  fi
+
+  curl -fsS -X "${method}" -H "X-Demo-User-Email: ${EDITOR_EMAIL}" "${BACKEND_URL}${path}"
+}
+
+wait_backend() {
+  local deadline=$((SECONDS + WAIT_SECONDS))
+  while (( SECONDS < deadline )); do
+    if curl -fsS "${BACKEND_URL}/health" >/dev/null 2>&1; then
+      return 0
+    fi
+    sleep 1
+  done
+  echo "Timed out waiting for backend at ${BACKEND_URL}" >&2
+  return 1
+}
+
+json_eval() {
+  local expression="$1"
+  "${PYTHON_BIN}" -c "import json,sys; print(${expression})"
+}
+
+create_researched_article() {
+  local suffix="$1"
+  local site_id article_body article_id questions_json commands question_id answer plan_id
+
+  site_id="$(request GET /api/sites | json_eval "json.load(sys.stdin)[0]['site']['id']")"
+  article_body="$(
+    SITE_ID="${site_id}" SUFFIX="${suffix}" "${PYTHON_BIN}" - <<'PY'
+import json
+import os
+import uuid
+
+suffix = os.environ["SUFFIX"]
+print(json.dumps({
+    "target_site_id": os.environ["SITE_ID"],
+    "brief_description": f"Evidence smoke {suffix} {uuid.uuid4()}",
+    "working_title": "Evidence Smoke Flow",
+    "content_type": "longform_guide",
+    "primary_keyword": suffix,
+}))
+PY
+  )"
+  article_id="$(request POST /api/articles "${article_body}" | json_eval "json.load(sys.stdin)['article']['id']")"
+  questions_json="$(request POST "/api/articles/${article_id}/boundary-questions/generate")"
+  commands="$(
+    QUESTIONS_JSON="${questions_json}" "${PYTHON_BIN}" - <<'PY'
+import json
+import os
+
+for question in json.loads(os.environ["QUESTIONS_JSON"])["questions"]:
+    if question["is_required"]:
+        print(f"{question['id']}\tAccepted answer for {question['category']}.")
+PY
+  )"
+  while IFS=$'\t' read -r question_id answer; do
+    [[ -z "${question_id}" ]] && continue
+    patch_body="$(ANSWER="${answer}" "${PYTHON_BIN}" - <<'PY'
+import json
+import os
+
+print(json.dumps({"answer": os.environ["ANSWER"]}))
+PY
+    )"
+    request PATCH "/api/articles/${article_id}/boundary-questions/${question_id}" "${patch_body}" >/dev/null
+  done <<< "${commands}"
+  request POST "/api/articles/${article_id}/boundary-questions/submit" >/dev/null
+  plan_id="$(request POST "/api/articles/${article_id}/plan/generate" | json_eval "json.load(sys.stdin)['plan']['id']")"
+  request POST "/api/articles/${article_id}/plans/${plan_id}/approve" >/dev/null
+  request POST "/api/articles/${article_id}/research/start" >/dev/null
+  printf '%s' "${article_id}"
+}
+
+cd "${ROOT_DIR}"
+trap cleanup EXIT
+
+docker compose -f "${COMPOSE_FILE}" up --build -d
+wait_backend
+
+sufficient_id="$(create_researched_article "sufficient evidence coverage")"
+sufficient_json="$(request GET "/api/articles/${sufficient_id}/evidence")"
+sufficient_reasons="$(printf '%s' "${sufficient_json}" | json_eval "len(json.load(sys.stdin)['insufficient_evidence_reasons'])")"
+if [[ "${sufficient_reasons}" != "0" ]]; then
+  echo "Expected sufficient flow without evidence reasons" >&2
+  exit 1
+fi
+
+insufficient_id="$(create_researched_article "insufficient evidence coverage")"
+insufficient_json="$(request GET "/api/articles/${insufficient_id}/evidence")"
+insufficient_status="$(printf '%s' "${insufficient_json}" | json_eval "json.load(sys.stdin)['article']['status']")"
+unsupported_count="$(printf '%s' "${insufficient_json}" | json_eval "len([claim for claim in json.load(sys.stdin)['claims'] if claim['support_status'] == 'UNSUPPORTED'])")"
+if [[ "${insufficient_status}" != "PLAN_REVISION_REQUIRED" || "${unsupported_count}" == "0" ]]; then
+  echo "Expected insufficient flow to return to PLAN_REVISION_REQUIRED with unsupported claims" >&2
+  exit 1
+fi
+
+echo "evidence flow smoke ok"