#!/usr/bin/env python3
"""Integrity, schema, reference, excerpt, and copyright checks for the public package."""
from __future__ import annotations

import csv
import hashlib
import json
import re
from pathlib import Path

try:
    from jsonschema import Draft202012Validator, FormatChecker
except ModuleNotFoundError:  # Reproduction environment may not have dependencies installed yet.
    Draft202012Validator = FormatChecker = None

OUT = Path(__file__).resolve().parents[1]
REQUIRED = [
    "executive-summary.md", "full-report.md", "preregistration.md",
    "preregistration-university-news-addendum.md", "methodology.md", "codebook.md",
    "limitations.md", "claim-ledger.csv", "articles.csv", "article-coding.csv",
    "actor-level-analysis.csv", "leadership-timeline.csv", "leadership.json", "results.json",
    "chart-data.json", "sources.csv", "corpus-readme.md", "corpus-manifest.json",
    "data-dictionary.md", "results.schema.json", "university-manual-audit.csv",
    "university-manual-intercoder-reliability.csv", "university-automation-validation.csv",
    "university-validation-results.json", "university-manual-results.json",
    "leadership-exploratory-results.json", "automated-screening-results.json",
    "automated-method-change-log.md", "corpus-parser-repair-log.md", "requirements.txt",
    "run-analysis.sh",
]


def read_csv(name):
    with (OUT / name).open(newline="", encoding="utf-8") as stream:
        reader = csv.DictReader(stream)
        return list(reader), reader.fieldnames


def main():
    missing = [name for name in REQUIRED if not (OUT / name).exists()]
    assert not missing, f"missing required files: {missing}"
    for name in ["results.json", "chart-data.json", "leadership.json", "corpus-manifest.json", "university-validation-results.json", "university-manual-results.json", "leadership-exploratory-results.json", "automated-screening-results.json"]:
        json.loads((OUT / name).read_text(encoding="utf-8"))

    results = json.loads((OUT / "results.json").read_text(encoding="utf-8"))
    schema = json.loads((OUT / "results.schema.json").read_text(encoding="utf-8"))
    if Draft202012Validator is not None:
        errors = sorted(Draft202012Validator(schema, format_checker=FormatChecker()).iter_errors(results), key=lambda error: list(error.path))
        assert not errors, "results schema errors: " + "; ".join(error.message for error in errors[:10])
        schema_status = "jsonschema_valid"
    else:
        assert set(schema["required"]) == set(results)
        assert schema["properties"]["corpus"]["properties"]["articles_n"]["const"] == 1527
        schema_status = "structural_fallback_valid; install requirements for full JSON Schema validation"

    articles, article_fields = read_csv("articles.csv")
    coding, coding_fields = read_csv("article-coding.csv")
    actors, actor_fields = read_csv("actor-level-analysis.csv")
    manual, manual_fields = read_csv("university-manual-audit.csv")
    claims, _ = read_csv("claim-ledger.csv")
    leadership, _ = read_csv("leadership-timeline.csv")
    sources, _ = read_csv("sources.csv")
    validation, _ = read_csv("university-automation-validation.csv")

    assert len(articles) == 1527 and len({r["article_id"] for r in articles}) == 1527
    assert len(coding) == 1527 and len({r["article_id"] for r in coding}) == 1527
    assert len(actors) == 6614
    assert len(manual) == 230 and len({r["article_id"] for r in manual}) == 230
    assert len(claims) == 60 and len(leadership) == 48 and len(sources) >= 52
    article_ids = {r["article_id"] for r in articles}
    assert {r["article_id"] for r in coding} == article_ids
    assert {r["article_id"] for r in actors} <= article_ids
    assert {r["article_id"] for r in manual} <= article_ids
    assert all(r["section"] == "university_news" for r in articles)
    assert CounterLike(r["subdesk"] for r in articles) == {"administration": 671, "academics": 266, "student_life": 590}
    assert min(r["date"] for r in articles) == "2023-10-02" and max(r["date"] for r in articles) == "2026-08-05"

    forbidden_columns = {"body", "article_body", "full_text", "html", "raw_html", "content_elements"}
    assert not forbidden_columns.intersection(article_fields or [])
    assert not forbidden_columns.intersection(coding_fields or [])
    assert not forbidden_columns.intersection(actor_fields or [])
    assert not forbidden_columns.intersection(manual_fields or [])
    assert all(not r.get("marked_evidence_excerpt_max_25_words") for r in coding)
    assert all(not r.get("marked_evidence_excerpt") for r in actors)
    assert all(len(r["evidence_excerpt"].split()) <= 25 for r in manual)
    assert all(r["coding_status"] == "automated_screening_only_failed_manual_semantic_gate" for r in coding)
    assert all(r["validation_status"] == "automated_screening_only_failed_manual_semantic_gate" for r in actors)

    gate = json.loads((OUT / "university-validation-results.json").read_text(encoding="utf-8"))["population_semantic_gate"]
    assert len(gate) == 19 and not any(gate.values())
    assert len(validation) == 19 and all(r["passes_0_60_gate"] == "0" for r in validation)
    assert results["corpus"]["articles_n"] == 1527
    assert results["validation"]["automated_semantic_measures_passing"] == 0
    assert results["study"]["main_hypothesis_assessment"].startswith("not_established")
    assert results["study"]["leadership_trend_assessment"].startswith("not_supported")

    manifest = json.loads((OUT / "corpus-manifest.json").read_text(encoding="utf-8"))
    assert manifest["university_news_unique_articles"] == 1527 and manifest["failed_records"] == 0
    assert manifest["private_corpus_sha256"] == "b697367b2f86dbd04d11c5a25a0d4113a4f94606febf18b2979e6bc8d628f37b"
    assert manifest["public_articles_sha256"] == hashlib.sha256((OUT / "articles.csv").read_bytes()).hexdigest()

    # Do not ship raw copyrighted/source formats or obvious private working corpora.
    forbidden_suffixes = {".docx", ".html", ".htm"}
    forbidden_names = {"private-university-corpus-final.jsonl", "private-university-corpus-final-v2.jsonl", "private-university-corpus-final-v3.jsonl"}
    leaked = [str(p.relative_to(OUT)) for p in OUT.rglob("*") if p.is_file() and (p.suffix.lower() in forbidden_suffixes or p.name in forbidden_names)]
    assert not leaked, f"forbidden public files: {leaked}"

    # Public short excerpts should not contain obvious direct-contact strings.
    excerpt_text = "\n".join(r["evidence_excerpt"] for r in manual)
    assert not re.search(r"[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}", excerpt_text)
    assert not re.search(r"(?:\+?1[-. ]?)?\(?\d{3}\)?[-. ]\d{3}[-. ]\d{4}", excerpt_text)

    print(json.dumps({
        "status": "pass", "articles": len(articles), "actor_rows": len(actors),
        "manual_rows": len(manual), "semantic_measures_suppressed": len(gate),
        "results_schema": schema_status, "copyright_body_leaks": 0,
    }, indent=2))


def CounterLike(values):
    out = {}
    for value in values:
        out[value] = out.get(value, 0) + 1
    return out


if __name__ == "__main__":
    main()
