#!/usr/bin/env python3
"""Remove update/correction labels from repaired description metadata."""
import hashlib
import json
import re
from pathlib import Path

ROOT = Path(__file__).resolve().parent
SOURCE = ROOT / "private-university-corpus-final.jsonl"
OUTPUT = ROOT / "private-university-corpus-final-v3.jsonl"
NOTE = re.compile(r"^(?:updated\b|correction:|editor(?:’|')s note:?)", re.I)
LABELS = {"administration", "academics", "student life", "student_life"}

rows = [json.loads(line) for line in SOURCE.read_text(encoding="utf-8").splitlines() if line.strip()]
changed = 0
for row in rows:
    desc = (row.get("description") or "").strip()
    if desc.lower() in LABELS or NOTE.match(desc):
        paragraphs = [p.strip() for p in row.get("body", "").split("\n\n") if p.strip()]
        replacement = next((p for p in paragraphs if not NOTE.match(p)), desc)
        if replacement != desc:
            row["description"] = replacement
            changed += 1
with OUTPUT.open("w", encoding="utf-8") as stream:
    for row in rows:
        stream.write(json.dumps(row, ensure_ascii=False) + "\n")
print(json.dumps({"rows": len(rows), "descriptions_changed": changed, "sha256": hashlib.sha256(OUTPUT.read_bytes()).hexdigest()}, indent=2))
