#!/usr/bin/env python3
"""Build the public University News census index without redistributing bodies."""
from __future__ import annotations

import csv
import hashlib
import json
import re
from datetime import date
from pathlib import Path

ROOT = Path(__file__).resolve().parents[2]
PRIVATE = ROOT / "work" / "private-university-corpus-final.jsonl"
SAMPLE = ROOT / "work" / "university-manual-sample.jsonl"
AUTO = ROOT / "work" / "university-automated-article-coding.csv"
OUT = ROOT / "research-output" / "articles.csv"
MANIFEST = ROOT / "research-output" / "corpus-manifest.json"

FIELDS = [
    "article_id", "title", "subtitle", "description", "author", "date", "section", "subdesk",
    "url", "retrieval_date", "topic_classifications", "leadership_period",
    "article_type", "discovery_method", "content_access", "content_hash",
    "word_count", "visible_update_or_correction", "coding_status", "exclusion_reason",
]

TOPICS = {
    "campus_protest": r"\b(protest|encampment|demonstrat|occupation|rally|walkout|cuad|sjp|jvp)\w*",
    "palestine_israel": r"\b(palestin|gaza|israel|hamas|antisemiti|zionis)\w*",
    "student_discipline": r"\b(disciplin|suspend|expel|interim measure|rules of university conduct)\w*",
    "policing_public_safety": r"\b(nypd|police|policing|public safety|security officer|arrest)\w*",
    "labor_unions": r"\b(union|bargain|strike|labor|worker|collective agreement|swc|uaw)\w*",
    "governance_leadership": r"\b(trustee|university senate|president|provost|dean|governance|resign|interim president)\w*",
    "federal_pressure": r"\b(trump|federal|congress|education department|justice department|funding freeze|title vi|title ix)\w*",
    "campus_access": r"\b(campus access|campus gate|lockdown|id check|restricted campus|access restriction)\w*",
    "speech_academic_freedom": r"\b(free speech|academic freedom|speech policy|expression|censure|aaup)\w*",
    "institutional_policy": r"\b(policy|rule|guideline|code of conduct|institutional neutrality)\w*",
    "academics_research": r"\b(curriculum|course|faculty|professor|research|academic|institute|school of)\w*",
    "student_life_services": r"\b(housing|dining|club|student life|health service|counseling|orientation|commencement)\w*",
}
COMPILED = {key: re.compile(pattern, re.I) for key, pattern in TOPICS.items()}


def load_jsonl(path: Path) -> list[dict]:
    if not path.exists():
        return []
    return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]


def article_id(url: str) -> str:
    return "spec-" + hashlib.sha256(url.encode()).hexdigest()[:16]


def leadership_period(day: str) -> str:
    value = date.fromisoformat(day)
    if value <= date(2023, 12, 12):
        return "147th"
    if value <= date(2024, 12, 10):
        return "148th"
    if value <= date(2025, 12, 9):
        return "149th"
    return "150th"


def topic_labels(row: dict) -> str:
    text = " ".join((row.get("title", ""), row.get("description", ""), row.get("body", "")))
    found = [key for key, pattern in COMPILED.items() if pattern.search(text)]
    return ";".join(found or ["other_university_news"])


def main() -> None:
    rows = load_jsonl(PRIVATE)
    assert len(rows) == 1527, f"expected 1,527 rows, found {len(rows)}"
    assert len({row["url"] for row in rows}) == len(rows)
    assert all(row.get("extraction_status") == "success" for row in rows)
    sampled = {row["article_id"] for row in load_jsonl(SAMPLE)}
    auto = {}
    if AUTO.exists():
        with AUTO.open(newline="", encoding="utf-8") as stream:
            auto = {row["article_id"]: row for row in csv.DictReader(stream)}
    public = []
    for row in sorted(rows, key=lambda item: (item["archive_date"], item["url"])):
        url = row.get("canonical_url") or row["url"]
        aid = article_id(url)
        subdesk = (row.get("archive_categories") or ["unknown"])[0]
        notes = row.get("update_correction_notes") or []
        retrieved = row.get("retrieved_at", "")[:10] or "2026-08-10"
        coded = auto.get(aid, {})
        public.append({
            "article_id": aid,
            "title": row.get("title", ""),
            "subtitle": row.get("subtitle", ""),
            "description": row.get("description", ""),
            "author": row.get("author", ""),
            "date": row["archive_date"],
            "section": "university_news",
            "subdesk": subdesk,
            "url": url,
            "retrieval_date": retrieved,
            "topic_classifications": coded.get("topic_labels", "").replace("|", ";") or topic_labels(row),
            "leadership_period": leadership_period(row["archive_date"]),
            "article_type": coded.get("article_type_proxy", "pending_validated_or_automated_classification"),
            "discovery_method": f"authorized official University News archive crawl; {subdesk}",
            "content_access": "full_text_private_working_corpus",
            "content_hash": row.get("body_sha256", ""),
            "word_count": row.get("word_count", ""),
            "visible_update_or_correction": coded.get("visible_update_correction_status", "") or int(bool(notes)),
            "coding_status": "manual_validation_sample" if aid in sampled else "automated_census_screen",
            "exclusion_reason": "",
        })
    with OUT.open("w", newline="", encoding="utf-8") as stream:
        writer = csv.DictWriter(stream, fieldnames=FIELDS)
        writer.writeheader()
        writer.writerows(public)
    manifest = {
        "study_start": "2023-10-01",
        "data_cutoff": "2026-08-10",
        "generated_at_utc": "2026-08-10T19:04:00Z",
        "university_news_unique_articles": len(public),
        "subdesk_counts": {key: sum(row["subdesk"] == key for row in public) for key in ("administration", "academics", "student_life")},
        "successful_full_text_records": len(public),
        "failed_records": 0,
        "observed_article_date_range": [min(row["date"] for row in public), max(row["date"] for row in public)],
        "normalized_body_words": sum(int(row["word_count"] or 0) for row in public),
        "private_corpus_sha256": hashlib.sha256(PRIVATE.read_bytes()).hexdigest(),
        "public_articles_sha256": hashlib.sha256(OUT.read_bytes()).hexdigest(),
        "private_corpus_redistributed": False,
        "manual_validation_sample_n": len(sampled),
        "manual_validation_sample_share": len(sampled) / len(public),
        "parser_repair_log": "corpus-parser-repair-log.md",
    }
    MANIFEST.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8")
    print(json.dumps(manifest, indent=2))


if __name__ == "__main__":
    main()
