#!/usr/bin/env python3
"""Select the frozen 230-article University News manual-validation sample."""
from __future__ import annotations

import hashlib
import json
import math
import re
from collections import defaultdict
from pathlib import Path

ROOT = Path(__file__).resolve().parent
CORPUS = ROOT / "private-university-corpus-final.jsonl"
SAMPLE = ROOT / "university-manual-sample.jsonl"
CODER_A = ROOT / "university-coder-a-assignment.jsonl"
CODER_B = ROOT / "university-coder-b-assignment.jsonl"
MANIFEST = ROOT / "university-manual-sample-manifest.json"
SEED = "university-news-manual-audit-2026-08-10-v1"
TARGET = 230
OVERLAP = 50

ADMIN = re.compile(r"\b(columbia|barnard|university|administration|administrator|trustee|president|provost|dean|public safety|office of institutional equity|shafik|armstrong|shipman|mnookin|rosenbury)\b", re.I)
CHALLENGER = re.compile(r"\b(student|students|sjp|jvp|cuad|protester|protesters|demonstrator|union|worker|workers|faculty|aaup|organizer|organizers|senator|senators)\b", re.I)
RELEVANT = re.compile(r"\b(protest|palestin|gaza|israel|disciplin|suspend|expel|polic|public safety|union|bargain|governance|trustee|president|federal|campus access|speech|policy|settlement|funding cut|immigration|ice|divest)\w*", re.I)
RELEASE = re.compile(r"\b(announc|news release|press release|spokesperson|statement to spectator|email to (?:the )?(?:university )?community|university webpage)\w*", re.I)
MARKED = re.compile(r"\b(stunning|sweeping|quietly|under fire|fail(?:s|ed|ure)?|refus(?:e|ed|es)|sidelines?|crisis|chaos|retaliat\w*|mismanage\w*|violat\w*|crackdown|controvers\w*|caves?|capitulat\w*)\b", re.I)


def read_jsonl(path):
    with path.open(encoding="utf-8") as stream:
        return [json.loads(line) for line in stream if line.strip()]


def stable(value):
    return hashlib.sha256(f"{SEED}|{value}".encode()).hexdigest()


def quarter(date):
    year, month = int(date[:4]), int(date[5:7])
    return f"{year}Q{(month - 1) // 3 + 1}"


def length_band(words):
    return "short" if words < 400 else "medium" if words < 900 else "long"


def enrich(row):
    text = " ".join([row.get("title", ""), row.get("description", ""), row.get("body", "")])
    first = " ".join([row.get("title", ""), row.get("description", ""), " ".join(row.get("body", "").split()[:250])])
    row = dict(row)
    row["article_id"] = "spec-" + hashlib.sha256(row["url"].encode()).hexdigest()[:16]
    row["subdesk"] = (row.get("archive_categories") or ["unknown"])[0]
    row["quarter"] = quarter(row["archive_date"])
    row["length_band"] = length_band(int(row.get("word_count", 0)))
    row["screen_relevant"] = int(bool(RELEVANT.search(text)))
    admin, challenger = bool(ADMIN.search(text)), bool(CHALLENGER.search(text))
    row["screen_actor_class"] = "both" if admin and challenger else "admin" if admin else "challenger" if challenger else "neither"
    row["screen_release_origin"] = int(bool(RELEASE.search(first)))
    row["screen_marked_language"] = int(bool(MARKED.search(first)))
    row["sample_stratum"] = f"{row['subdesk']}|{row['quarter']}"
    row["screen_signature"] = f"{row['screen_relevant']}|{row['screen_actor_class']}|{row['screen_release_origin']}|{row['screen_marked_language']}|{row['length_band']}"
    row["sample_rank"] = stable(row["article_id"])
    return row


def allocate(groups):
    total = sum(len(rows) for rows in groups.values())
    exact = {key: len(rows) * TARGET / total for key, rows in groups.items()}
    allocation = {key: min(len(groups[key]), max(1, math.floor(value))) for key, value in exact.items()}
    while sum(allocation.values()) < TARGET:
        candidates = [key for key in groups if allocation[key] < len(groups[key])]
        key = max(candidates, key=lambda item: (exact[item] - allocation[item], len(groups[item]), item))
        allocation[key] += 1
    while sum(allocation.values()) > TARGET:
        candidates = [key for key in groups if allocation[key] > 1]
        key = min(candidates, key=lambda item: (exact[item] - allocation[item], len(groups[item]), item))
        allocation[key] -= 1
    return allocation


def choose(rows, target):
    by_signature = defaultdict(list)
    for row in rows:
        by_signature[row["screen_signature"]].append(row)
    for values in by_signature.values():
        values.sort(key=lambda row: row["sample_rank"])
    signatures = sorted(by_signature, key=lambda key: (len(by_signature[key]), key))
    selected = []
    while len(selected) < target:
        progressed = False
        for signature in signatures:
            values = by_signature[signature]
            if values and len(selected) < target:
                selected.append(values.pop(0))
                progressed = True
        if not progressed:
            break
    return selected


def write_jsonl(path, rows):
    with path.open("w", encoding="utf-8") as stream:
        for row in rows:
            stream.write(json.dumps(row, ensure_ascii=False) + "\n")


def main():
    rows = [enrich(row) for row in read_jsonl(CORPUS)]
    assert len(rows) == 1527 and all(row.get("extraction_status") == "success" for row in rows)
    groups = defaultdict(list)
    for row in rows:
        groups[row["sample_stratum"]].append(row)
    allocation = allocate(groups)
    sample = []
    for key in sorted(groups):
        sample.extend(choose(groups[key], allocation[key]))
    sample.sort(key=lambda row: row["sample_rank"])
    assert len(sample) == TARGET and len({row["article_id"] for row in sample}) == TARGET

    overlap = sample[:OVERLAP]
    remaining = sample[OVERLAP:]
    coder_a = overlap + remaining[:90]
    coder_b = overlap + remaining[90:]
    assert len(coder_a) == 140 and len(coder_b) == 140
    assert len({row["article_id"] for row in coder_a} & {row["article_id"] for row in coder_b}) == OVERLAP

    write_jsonl(SAMPLE, sample)
    write_jsonl(CODER_A, coder_a)
    write_jsonl(CODER_B, coder_b)
    manifest = {
        "seed": SEED,
        "population_n": len(rows),
        "sample_n": len(sample),
        "sample_share": len(sample) / len(rows),
        "double_code_overlap_n": OVERLAP,
        "coder_a_n": len(coder_a),
        "coder_b_n": len(coder_b),
        "allocation": allocation,
        "sample_sha256": hashlib.sha256(SAMPLE.read_bytes()).hexdigest(),
        "coder_a_sha256": hashlib.sha256(CODER_A.read_bytes()).hexdigest(),
        "coder_b_sha256": hashlib.sha256(CODER_B.read_bytes()).hexdigest(),
    }
    MANIFEST.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8")
    print(json.dumps(manifest, indent=2))


if __name__ == "__main__":
    main()
