"""Reproduce source-text versus semantic Word conversion, without Word UI. Run from repository root with tools/lab-venv/bin/python . DOCX fixtures are generated locally; public evidence uses JSON/MD/TXT/PY. """ import hashlib import importlib.metadata as metadata import json import os import platform import subprocess import sys import time import zipfile from collections import Counter from datetime import datetime, timezone from html.parser import HTMLParser from pathlib import Path from xml.etree import ElementTree as ET from docx import Document from docx.shared import Inches, Pt from markitdown import MarkItDown ROOT = Path(__file__).resolve().parents[4] OUT = Path(__file__).resolve().parent LAB = OUT.parent ENV = dict(os.environ, PYTHONDONTWRITEBYTECODE="1") SOURCE = "# Review\n\nShip **today**.\n\n> Internal only.\n\n| Task | Owner |\n| --- | --- |\n| Ship | Mei |\n" NS = {"w": "http://schemas.openxmlformats.org/wordprocessingml/2006/main"} def call(command, source=None): start = time.perf_counter() result = subprocess.run(command, input=source, capture_output=True, text=True, check=True, env=ENV) return result.stdout, result.stderr, time.perf_counter() - start class Tags(HTMLParser): def __init__(self): super().__init__() self.counts = Counter() def handle_starttag(self, tag, attrs): self.counts[tag] += 1 def tag_counts(text): p = Tags() p.feed(text) return dict(p.counts) def relative(path): return str(path.relative_to(LAB.parent)) def save(path, data): path.parent.mkdir(parents=True, exist_ok=True) path.write_text(data, encoding="utf-8") def main(): fixture_dir = LAB / "word-fixtures" fixture_dir.mkdir(exist_ok=True) save(OUT / "input.md", SOURCE) for name in ("literal", "styled", "cosmetic"): doc = Document() if name == "literal": for line in SOURCE.splitlines(): doc.add_paragraph(line) else: if name == "styled": doc.add_heading("Review", level=1) else: run = doc.add_paragraph().add_run("Review") run.bold = True run.font.size = Pt(24) p = doc.add_paragraph("Ship ") p.add_run("today").bold = True p.add_run(".") if name == "styled": doc.add_paragraph("Internal only.", style="Quote") else: p = doc.add_paragraph("Internal only.") p.paragraph_format.left_indent = Inches(0.5) table = doc.add_table(rows=2, cols=2) table.style = "Table Grid" for row, values in zip(table.rows, (("Task", "Owner"), ("Ship", "Mei"))): for cell, text in zip(row.cells, values): cell.text = text path = fixture_dir / (name + ".docx") doc.save(path) with zipfile.ZipFile(path) as archive: xml = archive.read("word/document.xml") save(OUT / (name + ".word-xml.txt"), xml.decode()) report = { "kind": "markdown-rendering", "date": datetime.now(timezone.utc).isoformat(), "python": platform.python_version(), "versions": {n: metadata.version(n) for n in ("python-docx", "markitdown", "mammoth", "markdownify")}, "command": "PYTHONDONTWRITEBYTECODE=1 tools/lab-venv/bin/python " + str(Path(__file__).resolve().relative_to(ROOT)), "renderer_options": { "markdown-it-commonmark": {"preset": "commonmark", "html": True, "breaks": False}, "pandoc-gfm": ["-f", "gfm", "-t", "html", "--wrap=none"], }, "scope": "Constructed DOCX and local parsers only. No Word desktop/web UI, clipboard, add-in, Markovo API, GitHub.com or VS Code UI was executed. Counts describe syntax structure, not layout or fidelity.", "fixtures": [], "conversions": [], "renders": [], "commands": [], } version, _, _ = call(["pandoc", "--version"]) report["versions"]["pandoc"] = version.splitlines()[0] version, _, _ = call([str(ROOT / ".venv/bin/python"), "-c", "import importlib.metadata as m; print(m.version('markdown-it-py'))"]) report["versions"]["markdown-it-py"] = version.strip() converter = MarkItDown(enable_plugins=False) for name in ("literal", "styled", "cosmetic"): input_path = fixture_dir / (name + ".docx") xml = (OUT / (name + ".word-xml.txt")).read_text() tree = ET.fromstring(xml) pstyles = [p.attrib.get("{%s}val" % NS["w"]) for p in tree.findall(".//w:pStyle", NS)] report["fixtures"].append({"name": name, "input": relative(input_path), "document_xml": relative(OUT / (name + ".word-xml.txt")), "paragraph_styles": pstyles, "native_tables": len(tree.findall(".//w:tbl", NS)), "header_row_flags": len(tree.findall(".//w:tblHeader", NS)), "sha256": hashlib.sha256(input_path.read_bytes()).hexdigest()}) for tool in ("pandoc", "markitdown"): output_path = LAB / (tool + "-word") / (name + ".md") if tool == "pandoc": command = ["pandoc", "-f", "docx", "-t", "gfm", "--wrap=none", str(input_path)] text, stderr, elapsed = call(command) display = ["pandoc", "-f", "docx", "-t", "gfm", "--wrap=none", relative(input_path)] report["commands"].append(display) else: start = time.perf_counter() text = converter.convert(str(input_path)).text_content elapsed = time.perf_counter() - start stderr = "" display = "MarkItDown(enable_plugins=False).convert(input_path).text_content" save(output_path, text) ast, err, _ = call(["pandoc", "-f", "gfm", "-t", "json"], text) ast_path = OUT / (name + "--" + tool + ".ast.json") save(ast_path, ast) save(OUT / (name + "--" + tool + ".stderr.txt"), stderr + err) nodes = Counter() def walk(value): if isinstance(value, dict): if "t" in value: nodes[value["t"]] += 1 for child in value.values(): walk(child) elif isinstance(value, list): for child in value: walk(child) walk(json.loads(ast)["blocks"]) report["conversions"].append({"sample": name, "tool": tool, "output": relative(output_path), "ast": relative(ast_path), "elapsed_seconds": round(elapsed, 6), "ast_nodes": dict(nodes), "command": display}) for renderer in ("markdown-it-commonmark", "pandoc-gfm"): if renderer == "markdown-it-commonmark": cmd = [str(ROOT / ".venv/bin/python"), "-c", "from markdown_it import MarkdownIt; import sys; print(MarkdownIt('commonmark', {'html': True, 'breaks': False}).render(sys.stdin.read()), end='')"] else: cmd = ["pandoc", "-f", "gfm", "-t", "html", "--wrap=none"] html, _, _ = call(cmd, text) html_path = OUT / (name + "--" + tool + "--" + renderer + ".txt") save(html_path, html) report["renders"].append({"sample": name, "tool": tool, "renderer": renderer, "source": relative(output_path), "output": relative(html_path), "tags": tag_counts(html)}) for renderer in ("markdown-it-commonmark", "pandoc-gfm"): if renderer == "markdown-it-commonmark": cmd = [str(ROOT / ".venv/bin/python"), "-c", "from markdown_it import MarkdownIt; import sys; print(MarkdownIt('commonmark', {'html': True, 'breaks': False}).render(sys.stdin.read()), end='')"] else: cmd = ["pandoc", "-f", "gfm", "-t", "html", "--wrap=none"] html, _, _ = call(cmd, SOURCE) path = OUT / ("input--" + renderer + ".txt") save(path, html) report["renders"].append({"sample": "input", "tool": "direct", "renderer": renderer, "source": "lab/renderers/input.md", "output": relative(path), "tags": tag_counts(html)}) imported = fixture_dir / "imported.docx" command = ["pandoc", "-f", "gfm", str(OUT / "input.md"), "-o", str(imported)] _, stderr, _ = call(command) with zipfile.ZipFile(imported) as archive: xml = archive.read("word/document.xml").decode() save(OUT / "imported.word-xml.txt", xml) tree = ET.fromstring(xml) report["import"] = {"command": "pandoc -f gfm input.md -o output.docx", "source": "lab/renderers/input.md", "native_tables": len(tree.findall(".//w:tbl", NS)), "document_xml": "lab/renderers/imported.word-xml.txt", "paragraph_styles": [p.attrib.get("{%s}val" % NS["w"]) for p in tree.findall(".//w:pStyle", NS)], "stderr": stderr} report["example_equivalence"] = { "markitdown": "Path('output.md').write_text(MarkItDown(enable_plugins=False).convert('input.docx').text_content, encoding='utf-8') is the same executed conversion call with reader-facing paths", "pandoc_export": "pandoc -f docx -t gfm --wrap=none input.docx -o output.md is the executed stdout conversion with file output instead", } save(OUT / "results.json", json.dumps(report, ensure_ascii=False, indent=2) + "\n") print(json.dumps({"kind": report["kind"], "versions": report["versions"], "conversions": report["conversions"], "fixtures": report["fixtures"]}, indent=2)) if __name__ == "__main__": main()