"""Add transparent integer counts and archive provenance to scorer output.""" from pathlib import Path import importlib.util import json LAB = Path(__file__).resolve().parent spec = importlib.util.spec_from_file_location("scorer", LAB / "sources/lab_score.py") scorer = importlib.util.module_from_spec(spec) spec.loader.exec_module(scorer) def sentence_count(paragraphs, text): items = [s for p in paragraphs for s in scorer.sentences(p)] return {"hit": sum(scorer.norm(s) in scorer.norm(text) for s in items), "total": len(items)} def counts(truth, text): result = {} kind = truth["kind"] paragraphs = truth.get("paragraphs", []) if paragraphs: result["prose_sentences"] = sentence_count(paragraphs, text) anchors = [scorer.norm(" ".join(p.split()[:6])) for p in paragraphs] positions = [scorer.norm(text).find(a) for a in anchors] result["ordered_anchor_pairs"] = { "hit": sum(a >= 0 and b > a for a, b in zip(positions, positions[1:])), "total": max(0, len(positions) - 1)} tables = [truth] if kind == "table" else [] if kind == "scan": result["scan_sentences"] = sentence_count([truth["text"]], text) if kind == "mixed": tables = [p for p in truth["pages"] if p["kind"] == "table"] result["prose_sentences"] = sentence_count( [p for pg in truth["pages"] if pg["kind"] == "prose" for p in pg["paragraphs"]], text) result["scan_sentences"] = sentence_count([pg["text"] for pg in truth["pages"] if pg["kind"] == "scan"], text) anchors = truth["page_anchors"] result["page_anchors"] = {"hit": sum(scorer.norm(a) in scorer.norm(text) for a in anchors), "total": len(anchors)} if tables: rows = [r for t in tables for r in t["rows"]] lines = [scorer.norm(ln) for ln in text.splitlines() if ln.strip()] result["table_rows"] = {"hit": sum(any(all(scorer.norm(str(c)) in ln for c in row if str(c).strip()) for ln in lines) for row in rows), "total": len(rows)} cells = [str(c) for row in rows for c in row if str(c).strip()] result["table_cells_anywhere"] = {"hit": sum(scorer.norm(c) in scorer.norm(text) for c in cells), "total": len(cells)} if kind == "equation": result["normalized_expressions"] = {"hit": sum(scorer.math_norm(e) in scorer.math_norm(text) for e in truth["expressions"]), "total": len(truth["expressions"])} return result def main(): path = LAB / "scores.json" scores = json.loads(path.read_text()) runs = json.loads((LAB / "runs.json").read_text()) mapping = {(r["tool"], r["sample"]): r for r in runs["results"]} for cell in scores["cells"]: run = mapping[cell["tool"], cell["sample"]] text = (LAB / run["output"]).read_text() if run.get("output") else "" truth = json.loads((LAB / "corpus" / f"{cell['sample']}.truth.json").read_text()) cell["counts"] = counts(truth, text) scores["count_definitions"] = { "prose_sentences, scan_sentences": "Integer numerator and denominator of scorer prose_recall / scan_text_recall, with identical sentence split and normalization", "ordered_anchor_pairs": "Integer numerator and denominator of reading_order; not a complete ordering test", "table_rows": "Integer numerator and denominator of table_rows_intact; substring co-occurrence on any single physical output line", "table_cells_anywhere": "Loose count of individual truth data-cell strings anywhere in normalized output; not a cell-coordinate metric", "normalized_expressions": "Integer numerator and denominator of equation_recall using scorer.math_norm", "page_anchors": "Integer numerator and denominator of pages_covered; presence does not prove every word of the page survived"} scores["routing"] = json.loads((LAB / "routing.json").read_text()) snapshots = {} for c in scores["cells"]: if c["tool"] != "markovo": continue raw = json.loads((LAB / "markovo" / f"{c['sample']}.json").read_text()) job = raw["job"] estimate = raw["convert_response"].get("jobs", [{}])[0] snapshots[c["sample"]] = { "created_at": job.get("created_at"), "updated_at": job.get("updated_at"), "status": job.get("status"), "pages_total": job.get("pages_total"), "pages_ocr": job.get("pages_ocr"), "quality_score": job.get("quality_score"), "billing_pages_ocr": job.get("billing_metrics", {}).get("pagesOcr"), "estimated_ocr_page_indexes": estimate.get("billing_metrics", {}).get("ocrPageIndexes"), "provenance": "upstream archive, from_cache=true; no API submission for this guide"} scores["api_snapshots"] = snapshots scores["scope_notes"] = [ "PDF inputs only; non-PDF archive outputs are out of scope", "Markovo was not run on any of the three added diagnostic fixtures", "Job pages_ocr and billing pagesOcr disagree; no claim that every page used OCR", "Archived elapsed_s zero placeholders replaced by null; no remote speed comparison", "The first narrow-column intervention changes routing but not the poor sentence recall; the wider-gutter pair reproduces the readability regression", "HTML tables on one physical line satisfy loose row/header substring tests without proving grouping or grid correctness" ] path.write_text(json.dumps(scores, ensure_ascii=False, indent=2) + "\n") columns = ["prose_recall", "reading_order", "table_rows_intact", "table_header_intact", "table_pipe_detected", "pages_covered", "scan_text_recall", "equation_recall", "elapsed_s"] lines = ["| Tool | Sample | " + " | ".join(columns) + " |", "| --- | --- |" + " --- |" * len(columns)] for c in scores["cells"]: values = ["n/a" if c[k] is None else str(c[k]).lower() for k in columns] lines.append("| " + c["tool"] + " | " + c["sample"] + " | " + " | ".join(values) + " |") (LAB / "pdf-summary.md").write_text("\n".join(lines) + "\n") if __name__ == "__main__": main()