#!/usr/bin/env python3
"""Estudio "AI hallucinations in courts, Q3 2026". Reproducible: python3 analisis.py
Lee datos/charlotin-2026-10-21.csv (snapshot CC BY 4.0 de la base de Damien Charlotin)
y escribe salida/*.csv + salida/hallazgos.json. Solo stdlib."""
import csv, json, hashlib, collections, os, re, sys
HERE = os.path.dirname(os.path.abspath(__file__))
SNAP = os.path.join(HERE, "datos", "charlotin-2026-10-21.csv")
OLD = os.path.join(HERE, "datos", "charlotin-2026-04-24.csv")
OUT = os.path.join(HERE, "salida")
os.makedirs(OUT, exist_ok=True)
Q3 = ("2026-07", "2026-08", "2026-09")
Q2 = ("2026-04", "2026-05", "2026-06")

def sha(p):
    return hashlib.sha256(open(p, "rb").read()).hexdigest()

def load(p):
    return list(csv.DictReader(open(p, encoding="utf-8")))

def quarter(d):
    if not re.match(r"\d{4}-\d{2}", d or ""): return ""
    y, m = int(d[:4]), int(d[5:7])
    return f"{y}-Q{(m-1)//3+1}"

def party_class(p):
    s = {t.strip().lower() for t in (p or "").split(";") if t.strip()}
    if not s: return "unspecified"
    pro = "pro se litigant" in s
    law = bool(s & {"lawyer", "governement lawyer", "government lawyer", "prosecutor"})
    if pro and law: return "mixed"
    if pro: return "pro_se"
    if law and len(s) == 1 or (law and s <= {"lawyer", "governement lawyer", "government lawyer", "prosecutor"}): return "lawyer"
    return "other"   # judge, expert, arbitrator, lawyer+judge

ARB = {"domestic arbitration", "international arbitration", "wipo"}

def primary_country(v):
    toks = [t.strip() for t in (v or "").split(";") if t.strip()]
    real = [t for t in toks if t.lower() not in ARB]
    return real[0] if real else ("arbitration/other" if toks else "unspecified")

def sanction_class(r):
    """Most severe recorded consequence, first match in this order:
    monetary (a money amount >1 in the Monetary Penalty field) > professional > costs > filing_consequence
    > remedial > warning > pending_or_procedural > none_recorded > not_recorded > other."""
    o = (r["Outcome"] or "").lower().strip()
    amt = amount(r["Monetary Penalty"])
    if amt and amt > 1: return "monetary"
    if r["Professional Sanction"] == "Yes" or re.search(r"bar referral|reprimand|censure|disqualif|contempt|suspen|self-report|barred|pro hac vice|revoked|law society|notice of complaint", o): return "professional"
    if re.search(r"^no sanction|no sanctions|sanctions? motion denied|motion for sanctions denied|but no sanction|costs denied|no award of costs|no exclusion", o): return "none_recorded"
    if re.search(r"costs|fees|monetary|fine", o): return "costs_or_monetary_no_amount"
    if re.search(r"struck|dismiss|ignored|disregard|rejected|vacated|set aside|reversed|forfeit|remand|affirm|lost on merits|adverse inference|evidence", o): return "filing_consequence"
    if re.search(r"cle\b|certif|disclos|apolog|amend|correct|notify|training|withdrew|errata", o): return "remedial"
    if re.search(r"warning|admonish|caution|formal", o): return "warning"
    if re.search(r"show cause|explain|hearing|upcoming|ongoing", o): return "pending_or_procedural"
    if not o or o == "n/a": return "not_recorded"
    return "other"

def amount(mp):
    m = re.match(r"\s*([\d.,]+)\s*([A-Z]{3})?", mp or "")
    if not m: return None
    try: return float(m.group(1).replace(",", ""))
    except ValueError: return None

def usd(mp):
    m = re.match(r"\s*([\d.,]+)\s*USD\b", mp or "")
    return float(m.group(1).replace(",", "")) if m else None

def pct(a, b): return round(100.0 * a / b, 1) if b else None

def write(name, header, rows):
    with open(os.path.join(OUT, name), "w", newline="", encoding="utf-8") as f:
        w = csv.writer(f); w.writerow(header); w.writerows(rows)

rows = load(SNAP)
N = len(rows)
cases = []
for r in rows:
    d = (r["Date"] or "").strip()
    cases.append(dict(
        case=r["Case Name"], date=d, month=d[:7] if d else "", quarter=quarter(d),
        country=primary_country(r["State(s)"]),
        party=party_class(r["Party(ies)"]), sanction=sanction_class(r),
        usd=usd(r["Monetary Penalty"]), professional=r["Professional Sanction"] == "Yes",
        alleged=r["Alleged"] == "Yes", outcome_raw=r["Outcome"].strip()[:120],
        source=("https://www.damiencharlotin.com" + r["Source"]) if r["Source"].startswith("/") else r["Source"]))

# derived per-case CSV
write("casos_derivados.csv",
      ["case_name", "decision_date", "month", "quarter", "country", "party_class", "sanction_class", "usd_penalty", "professional_sanction", "alleged", "outcome_raw", "source_url"],
      [[c["case"], c["date"], c["month"], c["quarter"], c["country"], c["party"], c["sanction"],
        "" if c["usd"] is None else int(c["usd"]) if c["usd"] == int(c["usd"]) else c["usd"], c["professional"], c["alleged"], c["outcome_raw"], c["source"]] for c in cases])

dated = [c for c in cases if c["month"]]
# 1 volume by month (2025-01 onward shown in CSV; full history in the per-case file)
bym = collections.Counter(c["month"] for c in dated)
months = sorted(bym)
write("volumen_mensual.csv", ["month", "cases", "pro_se", "lawyer", "other_or_mixed"],
      [[m, bym[m], sum(1 for c in dated if c["month"] == m and c["party"] == "pro_se"),
        sum(1 for c in dated if c["month"] == m and c["party"] == "lawyer"),
        sum(1 for c in dated if c["month"] == m and c["party"] not in ("pro_se", "lawyer"))] for m in months])

# 2 party by quarter
qs = sorted({c["quarter"] for c in dated})
bq = {q: [c for c in dated if c["quarter"] == q] for q in qs}
write("trimestres.csv", ["quarter", "cases", "pro_se", "lawyer", "other_or_mixed", "pro_se_pct", "lawyer_pct", "monetary", "monetary_pct", "professional", "professional_pct"],
      [[q, len(v), (ps := sum(1 for c in v if c["party"] == "pro_se")), (lw := sum(1 for c in v if c["party"] == "lawyer")),
        len(v) - ps - lw, pct(ps, len(v)), pct(lw, len(v)),
        (mo := sum(1 for c in v if c["sanction"] == "monetary")), pct(mo, len(v)),
        (pr := sum(1 for c in v if c["professional"])), pct(pr, len(v))] for q, v in bq.items()])

# 3 sanction types
sc = collections.Counter(c["sanction"] for c in dated)
write("sanciones.csv", ["sanction_class", "cases", "pct_of_dated"], [[k, v, pct(v, len(dated))] for k, v in sc.most_common()])
write("sanciones_por_parte.csv", ["party_class", "cases", "monetary", "monetary_pct", "professional", "professional_pct", "warning", "warning_pct", "none_or_not_recorded_pct"],
      [[p, len(v), (mo := sum(1 for c in v if c["sanction"] == "monetary")), pct(mo, len(v)),
        (pr := sum(1 for c in v if c["professional"])), pct(pr, len(v)),
        (wa := sum(1 for c in v if c["sanction"] == "warning")), pct(wa, len(v)),
        pct(sum(1 for c in v if c["sanction"] in ("none_recorded", "not_recorded")), len(v))]
       for p in ("pro_se", "lawyer", "other", "mixed", "unspecified") if (v := [c for c in dated if c["party"] == p])])
usd_vals = sorted(c["usd"] for c in dated if c["usd"] and c["usd"] > 1)

# 4 jurisdictions
cc = collections.Counter(c["country"] for c in cases)
cq = {q: collections.Counter(c["country"] for c in bq.get(q, [])) for q in ("2026-Q2", "2026-Q3")}
write("jurisdicciones.csv", ["country", "cases", "pct_of_all", "cases_2026_Q2", "cases_2026_Q3"],
      [[k, v, pct(v, N), cq["2026-Q2"][k], cq["2026-Q3"][k]] for k, v in cc.most_common()])

# 5 Q3 vs Q2 2026, and the backfill measurement
def block(q):
    v = bq.get(q, []); n = len(v)
    return dict(cases=n, pro_se=sum(1 for c in v if c["party"] == "pro_se"), lawyer=sum(1 for c in v if c["party"] == "lawyer"),
                monetary=sum(1 for c in v if c["sanction"] == "monetary"), professional=sum(1 for c in v if c["professional"]),
                countries=len({c["country"] for c in v if c["country"] not in ("unspecified", "arbitration/other")}), us=sum(1 for c in v if c["country"] == "USA"))
q3, q2, q1 = block("2026-Q3"), block("2026-Q2"), block("2026-Q1")
old = [r for r in load(OLD) if (r["Date"] or "")[:7] in ("2026-01", "2026-02", "2026-03")]
old_q1 = len(old)
new_q1 = q1["cases"]
write("tendencia_3t_vs_2t.csv", ["metric", "2026-Q2", "2026-Q3", "change_abs", "change_pct"],
      [[k, q2[k], q3[k], q3[k] - q2[k], pct(q3[k] - q2[k], q2[k])] for k in q3])

H = dict(
    snapshot=dict(file="datos/charlotin-2026-10-21.csv", sha256=sha(SNAP), rows=N, dated_rows=len(dated), undated_rows=N - len(dated),
                  source="Damien Charlotin, AI Hallucination Cases Database, https://www.damiencharlotin.com/hallucinations/ (CC BY 4.0)",
                  latest_date=max(c["date"] for c in dated)),
    h1_volume=dict(total=N, countries=len([k for k in cc if k not in ("unspecified", "arbitration/other")]), q2=q2["cases"], q3=q3["cases"],
                   peak_month=max(months, key=lambda m: bym[m]), peak_month_cases=max(bym.values()),
                   q3_months={m: bym[m] for m in Q3}, jan_sep_2026=sum(bym[m] for m in months if "2026-01" <= m <= "2026-09"),
                   cases_2025=sum(bym[m] for m in months if m.startswith("2025")), cases_2026_to_date=sum(bym[m] for m in months if m.startswith("2026"))),
    h2_party=dict(all_pro_se=sum(1 for c in cases if c["party"] == "pro_se"), all_lawyer=sum(1 for c in cases if c["party"] == "lawyer"),
                  all_pro_se_pct=pct(sum(1 for c in cases if c["party"] == "pro_se"), N), all_lawyer_pct=pct(sum(1 for c in cases if c["party"] == "lawyer"), N),
                  q3_pro_se_pct=pct(q3["pro_se"], q3["cases"]), q3_lawyer_pct=pct(q3["lawyer"], q3["cases"]),
                  q2_pro_se_pct=pct(q2["pro_se"], q2["cases"]), q2_lawyer_pct=pct(q2["lawyer"], q2["cases"])),
    h3_sanctions=dict(classes={k: v for k, v in sc.items()}, usd_count=len(usd_vals), usd_total=int(sum(usd_vals)),
                      usd_median=usd_vals[len(usd_vals)//2] if usd_vals else None, usd_max=int(max(usd_vals)) if usd_vals else None,
                      professional_yes=sum(1 for c in cases if c["professional"]),
                      lawyer_monetary_pct=pct(sum(1 for c in dated if c["party"] == "lawyer" and c["sanction"] == "monetary"), sum(1 for c in dated if c["party"] == "lawyer")),
                      pro_se_monetary_pct=pct(sum(1 for c in dated if c["party"] == "pro_se" and c["sanction"] == "monetary"), sum(1 for c in dated if c["party"] == "pro_se")),
                      lawyer_professional_pct=pct(sum(1 for c in dated if c["party"] == "lawyer" and c["professional"]), sum(1 for c in dated if c["party"] == "lawyer")),
                      pro_se_professional_pct=pct(sum(1 for c in dated if c["party"] == "pro_se" and c["professional"]), sum(1 for c in dated if c["party"] == "pro_se"))),
    h4_jurisdictions=dict(top=cc.most_common(10), us=cc["USA"], us_pct=pct(cc["USA"], N), non_us=N - cc["USA"], non_us_pct=pct(N - cc["USA"], N),
                          q3_countries=q3["countries"], q2_countries=q2["countries"]),
    h5_q3_vs_q2=dict(q2=q2, q3=q3, cases_change_pct=pct(q3["cases"] - q2["cases"], q2["cases"]),
                     q3_us_pct=pct(q3["us"], q3["cases"]), q2_us_pct=pct(q2["us"], q2["cases"]),
                     q3_monetary_pct=pct(q3["monetary"], q3["cases"]), q2_monetary_pct=pct(q2["monetary"], q2["cases"]),
                     q3_professional_pct=pct(q3["professional"], q3["cases"]), q2_professional_pct=pct(q2["professional"], q2["cases"])),
    quarters={q: dict(cases=len(v), pro_se_pct=pct(sum(1 for c in v if c["party"] == "pro_se"), len(v)), lawyer_pct=pct(sum(1 for c in v if c["party"] == "lawyer"), len(v))) for q, v in bq.items() if q <= "2026-Q3"},
    limitations=dict(q1_2026_rows_in_snapshot_24apr=old_q1, q1_2026_rows_in_snapshot_21oct=new_q1,
                     q1_backfill_pct=pct(new_q1 - old_q1, old_q1), snapshot_gap_days=180, usd_only_note="USD-labelled amounts only; other currencies are not converted"),
)
json.dump(H, open(os.path.join(OUT, "hallazgos.json"), "w"), indent=1, ensure_ascii=False)
print(json.dumps(H, indent=1, ensure_ascii=False))