"""Run only code-language cases; raw HTML is saved as allowlisted .txt.""" import argparse import hashlib import importlib.metadata import inspect import io import json import platform import subprocess from contextlib import redirect_stdout from datetime import datetime, timezone from html.parser import HTMLParser from pathlib import Path import markdown from markdown.extensions import fenced_code from markdown_it import MarkdownIt from markdown_it.renderer import RendererHTML FENCE = chr(96) * 3 BODY = 'name = "alpha"\nprint(name)\nprint("done")\n' AFTER = '\n## After code\n' SAMPLES = { 'python': FENCE + 'python\n' + BODY + FENCE + '\n' + AFTER, 'py-alias': FENCE + 'py\n' + BODY + FENCE + '\n' + AFTER, 'unknown': FENCE + 'not-a-language\n' + BODY + FENCE + '\n' + AFTER, 'plain-text': FENCE + 'text\n' + BODY + FENCE + '\n' + AFTER, 'unlabeled': FENCE + '\n' + BODY + FENCE + '\n' + AFTER, 'extra-info': FENCE + 'python title="input.py"\n' + BODY + FENCE + '\n' + AFTER, 'braced-language': FENCE + '{.python}\n' + BODY + FENCE + '\n' + AFTER, 'long-close': FENCE + 'python\n' + BODY + FENCE + chr(96) + '\n' + AFTER, 'tagged-close': FENCE + 'python\n' + BODY + FENCE + 'python\n' + AFTER, } OPTIONS = { 'markdown-it-commonmark': "MarkdownIt('commonmark'); highlight=None; default langPrefix", 'python-default': "markdown.markdown(source); no extensions", 'python-fenced': "markdown.markdown(source, extensions=['fenced_code']); no codehilite", 'pandoc-gfm': "pandoc -f gfm -t html; no filters, no custom syntax or theme", } class Structure(HTMLParser): def __init__(self): super().__init__(convert_charrefs=True) self.tags = {} self.code_classes = [] self.token_spans = 0 def handle_starttag(self, tag, attrs): self.tags[tag] = self.tags.get(tag, 0) + 1 attrs = dict(attrs) if tag == 'code': self.code_classes.append(attrs.get('class', '')) if tag == 'span' and attrs.get('class'): self.token_spans += 1 def file_sha(path): return hashlib.sha256(path.read_bytes()).hexdigest() def run(lab): out = lab.resolve() / 'renderers' out.mkdir(parents=True, exist_ok=True) rows = [] for name, source in SAMPLES.items(): source_path = out / f'language-{name}.md' source_path.write_text(source, encoding='utf-8') results = { 'markdown-it-commonmark': MarkdownIt('commonmark').render(source), 'python-default': markdown.markdown(source), 'python-fenced': markdown.markdown(source, extensions=['fenced_code']), } process = subprocess.run( ['pandoc', '-f', 'gfm', '-t', 'html'], input=source, text=True, capture_output=True, check=True, ) results['pandoc-gfm'] = process.stdout for renderer, html in results.items(): path = out / f'language-{name}--{renderer}.txt' path.write_text(html, encoding='utf-8') structure = Structure() structure.feed(html) rows.append({ 'sample': name, 'renderer': renderer, 'source': f'lab/renderers/{source_path.name}', 'output': f'lab/renderers/{path.name}', 'tags': structure.tags, 'token_spans': structure.token_spans, 'code_classes': structure.code_classes, 'source_sha256': file_sha(source_path), 'output_sha256': file_sha(path), }) source_facts = [] for obj, label, url in [ (RendererHTML.fence, 'markdown_it/renderer.py:RendererHTML.fence', 'https://github.com/executablebooks/markdown-it-py/blob/v4.0.0/markdown_it/renderer.py'), (fenced_code.FencedBlockPreprocessor, 'markdown/extensions/fenced_code.py:FencedBlockPreprocessor', 'https://github.com/Python-Markdown/markdown/blob/3.10.2/markdown/extensions/fenced_code.py'), ]: file = Path(inspect.getsourcefile(obj)) source_facts.append({ 'object': label, 'upstream': url, 'start_line': inspect.getsourcelines(obj)[1], 'installed_source_sha256': file_sha(file), }) languages = subprocess.check_output(['pandoc', '--list-highlight-languages'], text=True) (out / 'language-pandoc-registry.txt').write_text(languages, encoding='utf-8') execution = io.StringIO() with redirect_stdout(execution): exec(compile(BODY, '', 'exec'), {}) (out / 'language-python-execution.txt').write_text(execution.getvalue(), encoding='utf-8') report = { 'kind': 'markdown-rendering', 'date': datetime.now(timezone.utc).isoformat(), 'python': platform.python_version(), 'versions': { 'markdown-it-py': importlib.metadata.version('markdown-it-py'), 'Python-Markdown': importlib.metadata.version('Markdown'), 'Pandoc': subprocess.check_output(['pandoc', '--version'], text=True).splitlines()[0], }, 'options': OPTIONS, 'command': f'env PYTHONDONTWRITEBYTECODE=1 .venv/bin/python {lab.parent.as_posix()}/lab/code-language.py --lab {lab.as_posix()}', 'scope': 'Local HTML fragments only. Not GitHub.com, VS Code, browser CSS, arbitrary extensions, execution engines, or an exhaustive language registry test.', 'definitions': { 'pre': 'Number of HTML pre start tags', 'code': 'Number of HTML code start tags, including inline code', 'token_spans': 'Number of HTML span start tags with a nonempty class attribute; structural evidence of token markup, not a color or correctness score', 'h2': 'Number of h2 start tags; the After code sentinel is parsed as a heading when present', 'code_classes': 'Class attributes of all code elements; presence is not proof of a recognized grammar', }, 'source_audit': source_facts, 'results': rows, } (out / 'language-results.json').write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8') print(json.dumps({'cases': len(rows), 'versions': report['versions'], 'python': report['python']})) return report if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--lab', required=True, type=Path) run(parser.parse_args().lab)