"""Run only code-language cases; raw HTML is saved as allowlisted .txt."""
import argparse
import hashlib
import importlib.metadata
import inspect
import io
import json
import platform
import subprocess
from contextlib import redirect_stdout
from datetime import datetime, timezone
from html.parser import HTMLParser
from pathlib import Path
import markdown
from markdown.extensions import fenced_code
from markdown_it import MarkdownIt
from markdown_it.renderer import RendererHTML
FENCE = chr(96) * 3
BODY = 'name = "alpha"\nprint(name)\nprint("done")\n'
AFTER = '\n## After code\n'
SAMPLES = {
'python': FENCE + 'python\n' + BODY + FENCE + '\n' + AFTER,
'py-alias': FENCE + 'py\n' + BODY + FENCE + '\n' + AFTER,
'unknown': FENCE + 'not-a-language\n' + BODY + FENCE + '\n' + AFTER,
'plain-text': FENCE + 'text\n' + BODY + FENCE + '\n' + AFTER,
'unlabeled': FENCE + '\n' + BODY + FENCE + '\n' + AFTER,
'extra-info': FENCE + 'python title="input.py"\n' + BODY + FENCE + '\n' + AFTER,
'braced-language': FENCE + '{.python}\n' + BODY + FENCE + '\n' + AFTER,
'long-close': FENCE + 'python\n' + BODY + FENCE + chr(96) + '\n' + AFTER,
'tagged-close': FENCE + 'python\n' + BODY + FENCE + 'python\n' + AFTER,
}
OPTIONS = {
'markdown-it-commonmark': "MarkdownIt('commonmark'); highlight=None; default langPrefix",
'python-default': "markdown.markdown(source); no extensions",
'python-fenced': "markdown.markdown(source, extensions=['fenced_code']); no codehilite",
'pandoc-gfm': "pandoc -f gfm -t html; no filters, no custom syntax or theme",
}
class Structure(HTMLParser):
def __init__(self):
super().__init__(convert_charrefs=True)
self.tags = {}
self.code_classes = []
self.token_spans = 0
def handle_starttag(self, tag, attrs):
self.tags[tag] = self.tags.get(tag, 0) + 1
attrs = dict(attrs)
if tag == 'code':
self.code_classes.append(attrs.get('class', ''))
if tag == 'span' and attrs.get('class'):
self.token_spans += 1
def file_sha(path):
return hashlib.sha256(path.read_bytes()).hexdigest()
def run(lab):
out = lab.resolve() / 'renderers'
out.mkdir(parents=True, exist_ok=True)
rows = []
for name, source in SAMPLES.items():
source_path = out / f'language-{name}.md'
source_path.write_text(source, encoding='utf-8')
results = {
'markdown-it-commonmark': MarkdownIt('commonmark').render(source),
'python-default': markdown.markdown(source),
'python-fenced': markdown.markdown(source, extensions=['fenced_code']),
}
process = subprocess.run(
['pandoc', '-f', 'gfm', '-t', 'html'],
input=source, text=True, capture_output=True, check=True,
)
results['pandoc-gfm'] = process.stdout
for renderer, html in results.items():
path = out / f'language-{name}--{renderer}.txt'
path.write_text(html, encoding='utf-8')
structure = Structure()
structure.feed(html)
rows.append({
'sample': name, 'renderer': renderer,
'source': f'lab/renderers/{source_path.name}',
'output': f'lab/renderers/{path.name}',
'tags': structure.tags,
'token_spans': structure.token_spans,
'code_classes': structure.code_classes,
'source_sha256': file_sha(source_path),
'output_sha256': file_sha(path),
})
source_facts = []
for obj, label, url in [
(RendererHTML.fence, 'markdown_it/renderer.py:RendererHTML.fence',
'https://github.com/executablebooks/markdown-it-py/blob/v4.0.0/markdown_it/renderer.py'),
(fenced_code.FencedBlockPreprocessor, 'markdown/extensions/fenced_code.py:FencedBlockPreprocessor',
'https://github.com/Python-Markdown/markdown/blob/3.10.2/markdown/extensions/fenced_code.py'),
]:
file = Path(inspect.getsourcefile(obj))
source_facts.append({
'object': label, 'upstream': url,
'start_line': inspect.getsourcelines(obj)[1],
'installed_source_sha256': file_sha(file),
})
languages = subprocess.check_output(['pandoc', '--list-highlight-languages'], text=True)
(out / 'language-pandoc-registry.txt').write_text(languages, encoding='utf-8')
execution = io.StringIO()
with redirect_stdout(execution):
exec(compile(BODY, '', 'exec'), {})
(out / 'language-python-execution.txt').write_text(execution.getvalue(), encoding='utf-8')
report = {
'kind': 'markdown-rendering',
'date': datetime.now(timezone.utc).isoformat(),
'python': platform.python_version(),
'versions': {
'markdown-it-py': importlib.metadata.version('markdown-it-py'),
'Python-Markdown': importlib.metadata.version('Markdown'),
'Pandoc': subprocess.check_output(['pandoc', '--version'], text=True).splitlines()[0],
},
'options': OPTIONS,
'command': f'env PYTHONDONTWRITEBYTECODE=1 .venv/bin/python {lab.parent.as_posix()}/lab/code-language.py --lab {lab.as_posix()}',
'scope': 'Local HTML fragments only. Not GitHub.com, VS Code, browser CSS, arbitrary extensions, execution engines, or an exhaustive language registry test.',
'definitions': {
'pre': 'Number of HTML pre start tags',
'code': 'Number of HTML code start tags, including inline code',
'token_spans': 'Number of HTML span start tags with a nonempty class attribute; structural evidence of token markup, not a color or correctness score',
'h2': 'Number of h2 start tags; the After code sentinel is parsed as a heading when present',
'code_classes': 'Class attributes of all code elements; presence is not proof of a recognized grammar',
},
'source_audit': source_facts,
'results': rows,
}
(out / 'language-results.json').write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
print(json.dumps({'cases': len(rows), 'versions': report['versions'], 'python': report['python']}))
return report
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--lab', required=True, type=Path)
run(parser.parse_args().lab)