#!/usr/bin/env python3
"""객관 지표 분석: 형식 준수(앱 렌더링 호환), 안정성, 지연, 토큰."""
import json, re, statistics as st

SP = "/private/tmp/claude-501/-Users-ymjung-Project-wordbit2/4ac326f2-ee60-44f4-ab5c-4b7aff5d25e6/scratchpad"
SECTIONS = ["Giải thích từ vựng", "Câu ví dụ", "Từ vựng liên quan", "Mẹo phát âm", "Sự thật thú vị"]

def metrics(r):
    t = r.get("text", "") or ""
    m = {}
    m["sections"] = sum(1 for s in SECTIONS if s in t)
    m["has_all_sections"] = m["sections"] == 5
    m["b_tag"] = "<b>" in t
    m["md_bold"] = "**" in t                       # 앱이 파싱 못함 → 리터럴 노출
    m["md_header"] = bool(re.search(r"(?m)^#{1,4} ", t))  # 마크다운 헤더도 리터럴
    m["fg_highlight"] = "<fg:#4ED3F6>" in t or "<fg:#4ed3f6>" in t
    m["fg_closed"] = t.count("<fg:") == t.count("</fg>")
    m["ex_count"] = sum(1 for e in ["1️⃣", "2️⃣", "3️⃣"] if e in t)
    m["has_3_examples"] = m["ex_count"] == 3
    # 발음 대괄호: 한글 포함 [ ] 블록
    prons = re.findall(r"\[([^\[\]]{1,60})\]", t)
    kr_prons = [p for p in prons if re.search(r"[가-힣]", p)]
    m["kr_pron_blocks"] = len(kr_prons)
    m["digit_in_pron"] = sum(1 for p in kr_prons if re.search(r"\d", p))  # 발음표기에 숫자 금지 규칙
    m["target_highlighted"] = bool(re.search(r"<fg:#4[eE][dD]3[fF]6>\s*" + re.escape(r_word(r)), t, re.I)) if r_word(r) else False
    m["chars"] = len(t)
    u = r.get("usage") or {}
    m["in_tokens"] = u.get("input_tokens")
    m["out_tokens"] = u.get("output_tokens")
    m["reasoning_tokens"] = (u.get("output_tokens_details") or {}).get("reasoning_tokens")
    return m

def r_word(r):
    return (r.get("content") or "").strip()

def pct(xs, p):
    xs = sorted(xs)
    return xs[min(len(xs)-1, int(len(xs)*p))] if xs else None

def main():
    recs = [json.loads(l) for l in open(f"{SP}/results_final.jsonl")]
    by = {}
    for r in recs:
        by.setdefault(r["model"], []).append(r)
    report = {}
    for model, rs in sorted(by.items()):
        ok = [r for r in rs if r.get("ok")]
        fail = [r for r in rs if not r.get("ok")]
        ms = [metrics(r) for r in ok]
        n = len(ms) or 1
        agg = {
            "requests": len(rs), "ok": len(ok), "fail": len(fail),
            "fail_reasons": [f"{r.get('http')}:{(r.get('error') or r.get('final') or '')[:60]}" for r in fail][:5],
            "retried": sum(1 for r in rs if r.get("attempt") == 2),
            "incomplete": sum(1 for r in rs if r.get("final") == "response.incomplete"),
            "형식/섹션5개완비": f"{sum(m['has_all_sections'] for m in ms)}/{n}",
            "형식/b태그사용": f"{sum(m['b_tag'] for m in ms)}/{n}",
            "형식/마크다운볼드(결함)": f"{sum(m['md_bold'] for m in ms)}/{n}",
            "형식/마크다운헤더(결함)": f"{sum(m['md_header'] for m in ms)}/{n}",
            "형식/fg하이라이트": f"{sum(m['fg_highlight'] for m in ms)}/{n}",
            "형식/fg태그짝맞음": f"{sum(m['fg_closed'] for m in ms)}/{n}",
            "형식/예문3개": f"{sum(m['has_3_examples'] for m in ms)}/{n}",
            "형식/대상단어하이라이트": f"{sum(m['target_highlighted'] for m in ms)}/{n}",
            "형식/발음표기에숫자(결함건수)": sum(m['digit_in_pron'] for m in ms),
            "발음블록수/평균": round(st.mean([m['kr_pron_blocks'] for m in ms]), 1),
            "출력길이chars/중앙값": int(st.median([m['chars'] for m in ms])),
            "TTFB초/중앙값": round(st.median([r['ttfb'] for r in ok if r.get('ttfb')]), 2),
            "TTFB초/p95": round(pct([r['ttfb'] for r in ok if r.get('ttfb')], 0.95), 2),
            "총소요초/중앙값": round(st.median([r['elapsed'] for r in ok]), 1),
            "총소요초/p95": round(pct([r['elapsed'] for r in ok], 0.95), 1),
            "입력토큰/평균": int(st.mean([m['in_tokens'] for m in ms if m['in_tokens']])),
            "출력토큰/평균": int(st.mean([m['out_tokens'] for m in ms if m['out_tokens']])),
            "출력토큰/p95": int(pct([m['out_tokens'] for m in ms if m['out_tokens']], 0.95)),
            "reasoning토큰합": sum(m['reasoning_tokens'] or 0 for m in ms),
        }
        report[model] = agg
    print(json.dumps(report, ensure_ascii=False, indent=1))
    json.dump(report, open(f"{SP}/objective_report_final.json", "w"), ensure_ascii=False, indent=1)

if __name__ == "__main__":
    main()