### markdown_it.rules_block.code; sha256=893031bf4535f8c0e1cfcf0cd66d698b6bf33a110c48444eb17328d90abefa45
"""Code block (4 spaces padded)."""
import logging
from .state_block import StateBlock
LOGGER = logging.getLogger(__name__)
def code(state: StateBlock, startLine: int, endLine: int, silent: bool) -> bool:
LOGGER.debug("entering code: %s, %s, %s, %s", state, startLine, endLine, silent)
if not state.is_code_block(startLine):
return False
last = nextLine = startLine + 1
while nextLine < endLine:
if state.isEmpty(nextLine):
nextLine += 1
continue
if state.is_code_block(nextLine):
nextLine += 1
last = nextLine
continue
break
state.line = last
token = state.push("code_block", "code", 0)
token.content = state.getLines(startLine, last, 4 + state.blkIndent, False) + "\n"
token.map = [startLine, state.line]
return True
### markdown_it.rules_block.paragraph; sha256=f69982c00ede32ee0b05d5787d62b30b811dc1a3a8686c3691f79849088bc9ef
"""Paragraph."""
import logging
from .state_block import StateBlock
LOGGER = logging.getLogger(__name__)
def paragraph(state: StateBlock, startLine: int, endLine: int, silent: bool) -> bool:
LOGGER.debug(
"entering paragraph: %s, %s, %s, %s", state, startLine, endLine, silent
)
nextLine = startLine + 1
ruler = state.md.block.ruler
terminatorRules = ruler.getRules("paragraph")
endLine = state.lineMax
oldParentType = state.parentType
state.parentType = "paragraph"
# jump line-by-line until empty one or EOF
while nextLine < endLine:
if state.isEmpty(nextLine):
break
# this would be a code block normally, but after paragraph
# it's considered a lazy continuation regardless of what's there
if state.sCount[nextLine] - state.blkIndent > 3:
nextLine += 1
continue
# quirk for blockquotes, this line should already be checked by that rule
if state.sCount[nextLine] < 0:
nextLine += 1
continue
# Some tags can terminate paragraph without empty line.
terminate = False
for terminatorRule in terminatorRules:
if terminatorRule(state, nextLine, endLine, True):
terminate = True
break
if terminate:
break
nextLine += 1
content = state.getLines(startLine, nextLine, state.blkIndent, False).strip()
state.line = nextLine
token = state.push("paragraph_open", "p", 1)
token.map = [startLine, state.line]
token = state.push("inline", "", 0)
token.content = content
token.map = [startLine, state.line]
token.children = []
token = state.push("paragraph_close", "p", -1)
state.parentType = oldParentType
return True
### markdown_it.rules_block.fence; sha256=049814f8fa99e2f0250aa19cadcf14b5d2e9249c8c79158df86f8ea7ecf1acc7
# fences (``` lang, ~~~ lang)
import logging
from .state_block import StateBlock
LOGGER = logging.getLogger(__name__)
def fence(state: StateBlock, startLine: int, endLine: int, silent: bool) -> bool:
LOGGER.debug("entering fence: %s, %s, %s, %s", state, startLine, endLine, silent)
haveEndMarker = False
pos = state.bMarks[startLine] + state.tShift[startLine]
maximum = state.eMarks[startLine]
if state.is_code_block(startLine):
return False
if pos + 3 > maximum:
return False
marker = state.src[pos]
if marker not in ("~", "`"):
return False
# scan marker length
mem = pos
pos = state.skipCharsStr(pos, marker)
length = pos - mem
if length < 3:
return False
markup = state.src[mem:pos]
params = state.src[pos:maximum]
if marker == "`" and marker in params:
return False
# Since start is found, we can report success here in validation mode
if silent:
return True
# search end of block
nextLine = startLine
while True:
nextLine += 1
if nextLine >= endLine:
# unclosed block should be autoclosed by end of document.
# also block seems to be autoclosed by end of parent
break
pos = mem = state.bMarks[nextLine] + state.tShift[nextLine]
maximum = state.eMarks[nextLine]
if pos < maximum and state.sCount[nextLine] < state.blkIndent:
# non-empty line with negative indent should stop the list:
# - ```
# test
break
try:
if state.src[pos] != marker:
continue
except IndexError:
break
if state.is_code_block(nextLine):
continue
pos = state.skipCharsStr(pos, marker)
# closing code fence must be at least as long as the opening one
if pos - mem < length:
continue
# make sure tail has spaces only
pos = state.skipSpaces(pos)
if pos < maximum:
continue
haveEndMarker = True
# found!
break
# If a fence has heading spaces, they should be removed from its inner block
length = state.sCount[startLine]
state.line = nextLine + (1 if haveEndMarker else 0)
token = state.push("fence", "code", 0)
token.info = params
token.content = state.getLines(startLine + 1, nextLine, length, True)
token.markup = markup
token.map = [startLine, state.line]
return True
### markdown_it.rules_inline.backticks; sha256=27b6dece38cdc625e52aabc77347c99076701fbfb69c1b1756370ac9d93c1383
# Parse backticks
import re
from .state_inline import StateInline
regex = re.compile("^ (.+) $")
def backtick(state: StateInline, silent: bool) -> bool:
pos = state.pos
if state.src[pos] != "`":
return False
start = pos
pos += 1
maximum = state.posMax
# scan marker length
while pos < maximum and (state.src[pos] == "`"):
pos += 1
marker = state.src[start:pos]
openerLength = len(marker)
if state.backticksScanned and state.backticks.get(openerLength, 0) <= start:
if not silent:
state.pending += marker
state.pos += openerLength
return True
matchStart = matchEnd = pos
# Nothing found in the cache, scan until the end of the line (or until marker is found)
while True:
try:
matchStart = state.src.index("`", matchEnd)
except ValueError:
break
matchEnd = matchStart + 1
# scan marker length
while matchEnd < maximum and (state.src[matchEnd] == "`"):
matchEnd += 1
closerLength = matchEnd - matchStart
if closerLength == openerLength:
# Found matching closer length.
if not silent:
token = state.push("code_inline", "code", 0)
token.markup = marker
token.content = state.src[pos:matchStart].replace("\n", " ")
if (
token.content.startswith(" ")
and token.content.endswith(" ")
and len(token.content.strip()) > 0
):
token.content = token.content[1:-1]
state.pos = matchEnd
return True
# Some different length found, put it in cache as upper limit of where closer can be found
state.backticks[closerLength] = matchStart
# Scanned through the end, didn't find anything
state.backticksScanned = True
if not silent:
state.pending += marker
state.pos += openerLength
return True
### markdown_it.renderer; sha256=2f3af4825a9de68c452f5d0339f8e35bc920e06a78d62750e2f88441a138ee80
"""
class Renderer
Generates HTML from parsed token stream. Each instance has independent
copy of rules. Those can be rewritten with ease. Also, you can add new
rules if you create plugin and adds new token types.
"""
from __future__ import annotations
from collections.abc import Sequence
import inspect
from typing import Any, ClassVar, Protocol
from .common.utils import escapeHtml, unescapeAll
from .token import Token
from .utils import EnvType, OptionsDict
class RendererProtocol(Protocol):
__output__: ClassVar[str]
def render(
self, tokens: Sequence[Token], options: OptionsDict, env: EnvType
) -> Any: ...
class RendererHTML(RendererProtocol):
"""Contains render rules for tokens. Can be updated and extended.
Example:
Each rule is called as independent static function with fixed signature:
::
class Renderer:
def token_type_name(self, tokens, idx, options, env) {
# ...
return renderedHTML
::
class CustomRenderer(RendererHTML):
def strong_open(self, tokens, idx, options, env):
return ''
def strong_close(self, tokens, idx, options, env):
return ''
md = MarkdownIt(renderer_cls=CustomRenderer)
result = md.render(...)
See https://github.com/markdown-it/markdown-it/blob/master/lib/renderer.js
for more details and examples.
"""
__output__ = "html"
def __init__(self, parser: Any = None):
self.rules = {
k: v
for k, v in inspect.getmembers(self, predicate=inspect.ismethod)
if not (k.startswith("render") or k.startswith("_"))
}
def render(
self, tokens: Sequence[Token], options: OptionsDict, env: EnvType
) -> str:
"""Takes token stream and generates HTML.
:param tokens: list on block tokens to render
:param options: params of parser instance
:param env: additional data from parsed input
"""
result = ""
for i, token in enumerate(tokens):
if token.type == "inline":
if token.children:
result += self.renderInline(token.children, options, env)
elif token.type in self.rules:
result += self.rules[token.type](tokens, i, options, env)
else:
result += self.renderToken(tokens, i, options, env)
return result
def renderInline(
self, tokens: Sequence[Token], options: OptionsDict, env: EnvType
) -> str:
"""The same as ``render``, but for single token of `inline` type.
:param tokens: list on block tokens to render
:param options: params of parser instance
:param env: additional data from parsed input (references, for example)
"""
result = ""
for i, token in enumerate(tokens):
if token.type in self.rules:
result += self.rules[token.type](tokens, i, options, env)
else:
result += self.renderToken(tokens, i, options, env)
return result
def renderToken(
self,
tokens: Sequence[Token],
idx: int,
options: OptionsDict,
env: EnvType,
) -> str:
"""Default token renderer.
Can be overridden by custom function
:param idx: token index to render
:param options: params of parser instance
"""
result = ""
needLf = False
token = tokens[idx]
# Tight list paragraphs
if token.hidden:
return ""
# Insert a newline between hidden paragraph and subsequent opening
# block-level tag.
#
# For example, here we should insert a newline before blockquote:
# - a
# >
#
if token.block and token.nesting != -1 and idx and tokens[idx - 1].hidden:
result += "\n"
# Add token name, e.g. ``.
#
needLf = False
result += ">\n" if needLf else ">"
return result
@staticmethod
def renderAttrs(token: Token) -> str:
"""Render token attributes to string."""
result = ""
for key, value in token.attrItems():
result += " " + escapeHtml(key) + '="' + escapeHtml(str(value)) + '"'
return result
def renderInlineAsText(
self,
tokens: Sequence[Token] | None,
options: OptionsDict,
env: EnvType,
) -> str:
"""Special kludge for image `alt` attributes to conform CommonMark spec.
Don't try to use it! Spec requires to show `alt` content with stripped markup,
instead of simple escaping.
:param tokens: list on block tokens to render
:param options: params of parser instance
:param env: additional data from parsed input
"""
result = ""
for token in tokens or []:
if token.type == "text":
result += token.content
elif token.type == "image":
if token.children:
result += self.renderInlineAsText(token.children, options, env)
elif token.type == "softbreak":
result += "\n"
return result
###################################################
def code_inline(
self, tokens: Sequence[Token], idx: int, options: OptionsDict, env: EnvType
) -> str:
token = tokens[idx]
return (
"
"
+ escapeHtml(tokens[idx].content)
+ ""
)
def code_block(
self,
tokens: Sequence[Token],
idx: int,
options: OptionsDict,
env: EnvType,
) -> str:
token = tokens[idx]
return (
"
"
+ escapeHtml(tokens[idx].content)
+ "\n"
)
def fence(
self,
tokens: Sequence[Token],
idx: int,
options: OptionsDict,
env: EnvType,
) -> str:
token = tokens[idx]
info = unescapeAll(token.info).strip() if token.info else ""
langName = ""
langAttrs = ""
if info:
arr = info.split(maxsplit=1)
langName = arr[0]
if len(arr) == 2:
langAttrs = arr[1]
if options.highlight:
highlighted = options.highlight(
token.content, langName, langAttrs
) or escapeHtml(token.content)
else:
highlighted = escapeHtml(token.content)
if highlighted.startswith(""
+ highlighted
+ "\n"
)
return (
""
+ highlighted
+ "\n"
)
def image(
self,
tokens: Sequence[Token],
idx: int,
options: OptionsDict,
env: EnvType,
) -> str:
token = tokens[idx]
# "alt" attr MUST be set, even if empty. Because it's mandatory and
# should be placed on proper position for tests.
if token.children:
token.attrSet("alt", self.renderInlineAsText(token.children, options, env))
else:
token.attrSet("alt", "")
return self.renderToken(tokens, idx, options, env)
def hardbreak(
self, tokens: Sequence[Token], idx: int, options: OptionsDict, env: EnvType
) -> str:
return ".*?)(?<=\n) # the code block
(?P=fence)[ ]*$ # closing fence
'''),
re.MULTILINE | re.DOTALL | re.VERBOSE
)
def __init__(self, md: Markdown, config: dict[str, Any]):
super().__init__(md)
self.config = config
self.checked_for_deps = False
self.codehilite_conf: dict[str, Any] = {}
self.use_attr_list = False
# List of options to convert to boolean values
self.bool_options = [
'linenums',
'guess_lang',
'noclasses',
'use_pygments'
]
def run(self, lines: list[str]) -> list[str]:
""" Match and store Fenced Code Blocks in the `HtmlStash`. """
# Check for dependent extensions
if not self.checked_for_deps:
for ext in self.md.registeredExtensions:
if isinstance(ext, CodeHiliteExtension):
self.codehilite_conf = ext.getConfigs()
if isinstance(ext, AttrListExtension):
self.use_attr_list = True
self.checked_for_deps = True
text = "\n".join(lines)
index = 0
while 1:
m = self.FENCED_BLOCK_RE.search(text, index)
if m:
lang, id, classes, config = None, '', [], {}
if m.group('attrs'):
attrs, remainder = get_attrs_and_remainder(m.group('attrs'))
if remainder: # Does not have correctly matching curly braces, so the syntax is invalid.
index = m.end('attrs') # Explicitly skip over this, to prevent an infinite loop.
continue
id, classes, config = self.handle_attrs(attrs)
if len(classes):
lang = classes.pop(0)
else:
if m.group('lang'):
lang = m.group('lang')
if m.group('hl_lines'):
# Support `hl_lines` outside of `attrs` for backward-compatibility
config['hl_lines'] = parse_hl_lines(m.group('hl_lines'))
# If `config` is not empty, then the `codehighlite` extension
# is enabled, so we call it to highlight the code
if self.codehilite_conf and self.codehilite_conf['use_pygments'] and config.get('use_pygments', True):
local_config = self.codehilite_conf.copy()
local_config.update(config)
# Combine classes with `cssclass`. Ensure `cssclass` is at end
# as Pygments appends a suffix under certain circumstances.
# Ignore ID as Pygments does not offer an option to set it.
if classes:
local_config['css_class'] = '{} {}'.format(
' '.join(classes),
local_config['css_class']
)
highliter = CodeHilite(
m.group('code'),
lang=lang,
style=local_config.pop('pygments_style', 'default'),
**local_config
)
code = highliter.hilite(shebang=False)
else:
id_attr = lang_attr = class_attr = kv_pairs = ''
if lang:
prefix = self.config.get('lang_prefix', 'language-')
lang_attr = f' class="{prefix}{_escape_attrib_html(lang)}"'
if classes:
class_attr = f' class="{_escape_attrib_html(" ".join(classes))}"'
if id:
id_attr = f' id="{_escape_attrib_html(id)}"'
if self.use_attr_list and config and not config.get('use_pygments', False):
# Only assign key/value pairs to code element if `attr_list` extension is enabled, key/value
# pairs were defined on the code block, and the `use_pygments` key was not set to `True`. The
# `use_pygments` key could be either set to `False` or not defined. It is omitted from output.
kv_pairs = ''.join(
f' {k}="{_escape_attrib_html(v)}"' for k, v in config.items() if k != 'use_pygments'
)
code = self._escape(m.group('code'))
code = f'{code}
'
placeholder = self.md.htmlStash.store(code)
text = f'{text[:m.start()]}\n{placeholder}\n{text[m.end():]}'
# Continue from after the replaced text in the next iteration.
index = m.start() + 1 + len(placeholder)
else:
break
return text.split("\n")
def handle_attrs(self, attrs: Iterable[tuple[str, str]]) -> tuple[str, list[str], dict[str, Any]]:
""" Return tuple: `(id, [list, of, classes], {configs})` """
id = ''
classes = []
configs = {}
for k, v in attrs:
if k == 'id':
id = v
elif k == '.':
classes.append(v)
elif k == 'hl_lines':
configs[k] = parse_hl_lines(v)
elif k in self.bool_options:
configs[k] = parseBoolValue(v, fail_on_errors=False, preserve_none=True)
else:
configs[k] = v
return id, classes, configs
def _escape(self, txt: str) -> str:
""" basic html escaping """
txt = txt.replace('&', '&')
txt = txt.replace('<', '<')
txt = txt.replace('>', '>')
txt = txt.replace('"', '"')
return txt
def makeExtension(**kwargs): # pragma: no cover
return FencedCodeExtension(**kwargs)