#!/usr/bin/env python3
"""Repair interactive-body paragraph boundaries and HTTP-recovered descriptions.

The authorized crawl is rate limited globally to one request start per second.
Only records whose raw interactive HTML was flattened are re-requested. Standard
HTTP-recovered rows take their visible meta description from the first text body
element, matching the public page's description metadata.
"""
from __future__ import annotations

import concurrent.futures
import hashlib
import json
import re
import threading
import time
from pathlib import Path
from urllib.request import Request, urlopen

from lxml import html

ROOT = Path(__file__).resolve().parent
SOURCE = ROOT / "private-university-corpus-final.jsonl"
OUTPUT = ROOT / "private-university-corpus-final-v2.jsonl"
SUMMARY = ROOT / "http-record-repair-summary.json"
USER_AGENT = "Spectator-University-News-Audit/1.0 (authorized research crawl)"
EXTRA_TARGET_IDS = {"spec-61e6477d08299748"}


class Limiter:
    def __init__(self, seconds=1.0):
        self.seconds, self.lock, self.last = seconds, threading.Lock(), 0.0
    def acquire(self):
        with self.lock:
            wait = self.seconds - (time.monotonic() - self.last)
            if wait > 0:
                time.sleep(wait)
            self.last = time.monotonic()


LIMITER = Limiter()


def aid(url: str) -> str:
    return "spec-" + hashlib.sha256(url.encode()).hexdigest()[:16]


def clean(text: str) -> str:
    return re.sub(r"\s+", " ", text or "").strip()


def basic(value):
    return value.get("basic", "") if isinstance(value, dict) else value if isinstance(value, str) else ""


def global_content(page: str) -> dict:
    marker = "Fusion.globalContent="
    start = page.find(marker)
    if start < 0:
        raise ValueError("Fusion.globalContent not found")
    value, _ = json.JSONDecoder().raw_decode(page[start + len(marker):])
    return value


def has_ancestor_class(node, names: tuple[str, ...]) -> bool:
    for parent in node.iterancestors():
        classes = set((parent.get("class") or "").split())
        if classes.intersection(names):
            return True
    return False


def raw_html_story(raw: str) -> tuple[str, str, list[str]]:
    tree = html.fromstring(raw)
    title = clean(tree.xpath("string(//*[@id='headline'][1])")) or clean(tree.xpath("string(//title[1])"))
    description = clean(tree.xpath("string(//*[@id='dek'][1])"))
    candidates = tree.xpath("//*[contains(concat(' ', normalize-space(@class), ' '), ' story ')]//p")
    if not candidates:
        candidates = tree.xpath("//p")
    paragraphs = []
    for node in candidates:
        classes = set((node.get("class") or "").split())
        if classes.intersection({"byline", "pub-date", "cover-caption", "inline-credits", "article-credits"}):
            continue
        if has_ancestor_class(node, ("graphic", "ai2html", "footer", "article-credits", "cover", "story-top")):
            continue
        text = clean(node.text_content())
        if not text or text in {title, description}:
            continue
        if re.match(r"^(?:Want to keep up with breaking news|Produced with |University News Editor|Deputy News Editor|Staff Writer|Senior Staff Writer|Managing Editor|Editor in Chief).{0,260}(?:contacted at|newsletter|Spectator Graphics)", text, re.I):
            continue
        if " can be contacted at " in text and len(text.split()) < 80:
            continue
        if text.startswith("[Read More:"):
            continue
        paragraphs.append(text)
    # Remove consecutive exact duplicates without altering legitimate repeated quotations.
    deduped = []
    for text in paragraphs:
        if not deduped or text != deduped[-1]:
            deduped.append(text)
    if not description and deduped:
        description = deduped[0]
    return title, description, deduped


def repair_fetch(row: dict) -> dict:
    last = ""
    for attempt in range(1, 4):
        LIMITER.acquire()
        try:
            request = Request(row["url"], headers={"User-Agent": USER_AGENT})
            with urlopen(request, timeout=45) as response:
                page = response.read().decode("utf-8", "replace")
            content = global_content(page)
            raw = next((e.get("content", "") for e in content.get("content_elements", []) if e.get("type") == "raw_html"), "")
            if not raw:
                # The special concatenation check can be a standard article.
                paras = []
                for element in content.get("content_elements", []):
                    if element.get("type") == "text" and isinstance(element.get("content"), str):
                        paras.append(clean(html.fragment_fromstring(element["content"], create_parent="div").text_content()))
                title, description = basic(content.get("headlines")), basic(content.get("description"))
                if description.lower() in {"administration", "academics", "student life", "student_life"} and paras:
                    description = paras[0]
            else:
                title, description, paras = raw_html_story(raw)
            paras = [p for p in paras if p]
            body = "\n\n".join(paras)
            if not body:
                raise ValueError("empty repaired body")
            out = dict(row)
            out.update({
                "title": title or row.get("title", ""),
                "description": description,
                "body": body,
                "word_count": len(body.split()),
                "body_sha256": hashlib.sha256(body.encode()).hexdigest(),
                "retrieved_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
                "retrieval_method": "authorized_rate_limited_public_html_repaired",
                "repair_http_attempts": attempt,
                "repair_status": "success",
            })
            return out
        except Exception as exc:
            last = f"{type(exc).__name__}: {exc}"
            if attempt < 3:
                time.sleep(2 * attempt)
    out = dict(row)
    out["repair_status"] = "error"
    out["repair_error"] = last[:500]
    return out


def main():
    rows = [json.loads(line) for line in SOURCE.read_text(encoding="utf-8").splitlines() if line.strip()]
    target_indexes = [
        idx for idx, row in enumerate(rows)
        if (row.get("retrieval_method") == "authorized_rate_limited_public_html_payload" and "\n\n" not in row.get("body", ""))
        or aid(row["url"]) in EXTRA_TARGET_IDS
    ]
    targets = [(idx, rows[idx]) for idx in target_indexes]
    repaired = {}
    with concurrent.futures.ThreadPoolExecutor(max_workers=5) as pool:
        futures = {pool.submit(repair_fetch, row): idx for idx, row in targets}
        for done, future in enumerate(concurrent.futures.as_completed(futures), 1):
            idx = futures[future]
            repaired[idx] = future.result()
            if done % 10 == 0 or done == len(futures):
                print(f"repaired {done}/{len(futures)}", flush=True)
    assert all(row.get("repair_status") == "success" for row in repaired.values())
    output_rows = []
    standard_description_repairs = 0
    for idx, row in enumerate(rows):
        if idx in repaired:
            output_rows.append(repaired[idx])
        elif row.get("retrieval_method") == "authorized_rate_limited_public_html_payload":
            out = dict(row)
            paragraphs = row.get("body", "").split("\n\n")
            if paragraphs:
                out["description"] = clean(paragraphs[0])
                standard_description_repairs += 1
            out["repair_status"] = "description_repaired_from_first_structured_text_element"
            output_rows.append(out)
        else:
            output_rows.append(row)
    with OUTPUT.open("w", encoding="utf-8") as stream:
        for row in output_rows:
            stream.write(json.dumps(row, ensure_ascii=False) + "\n")
    summary = {
        "source_sha256": hashlib.sha256(SOURCE.read_bytes()).hexdigest(),
        "output_sha256": hashlib.sha256(OUTPUT.read_bytes()).hexdigest(),
        "rows": len(output_rows),
        "interactive_or_special_refetch_targets": len(targets),
        "interactive_or_special_successes": sum(r.get("repair_status") == "success" for r in repaired.values()),
        "standard_description_repairs": standard_description_repairs,
        "remaining_one_paragraph_http_records": sum(r.get("retrieval_method", "").startswith("authorized_rate_limited_public_html") and "\n\n" not in r.get("body", "") for r in output_rows),
        "empty_bodies": sum(not r.get("body") for r in output_rows),
    }
    SUMMARY.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
    print(json.dumps(summary, indent=2))


if __name__ == "__main__":
    main()
