import re
import markdownify
from typing import Any, Optional
from urllib.parse import quote, urlparse, urlunparse
_PERCENT_ENCODED_OCTET = re.compile(r"%[0-9A-Fa-f]{2}")
def _quote_path_preserving_percent_encoded_octets(path: str) -> str:
"""Quote a URL path while preserving existing %HH byte encodings."""
parts: list[str] = []
last_end = 0
for match in _PERCENT_ENCODED_OCTET.finditer(path):
parts.append(quote(path[last_end : match.start()]))
parts.append(match.group(0))
last_end = match.end()
parts.append(quote(path[last_end:]))
return "".join(parts)
class _CustomMarkdownify(markdownify.MarkdownConverter):
"""
A custom version of markdownify's MarkdownConverter. Changes include:
- Altering the default heading style to use '#', '##', etc.
- Removing javascript hyperlinks.
- Truncating images with large data:uri sources.
- Ensuring URIs are properly escaped, and do not conflict with Markdown syntax
"""
def __init__(self, **options: Any):
options["heading_style"] = options.get("heading_style", markdownify.ATX)
options["keep_data_uris"] = options.get("keep_data_uris", False)
# Explicitly cast options to the expected type if necessary
super().__init__(**options)
def convert_hn(
self,
n: int,
el: Any,
text: str,
convert_as_inline: Optional[bool] = False,
**kwargs,
) -> str:
"""Same as usual, but be sure to start with a new line"""
if not convert_as_inline:
if not re.search(r"^\n", text):
return "\n" + super().convert_hn(n, el, text, convert_as_inline) # type: ignore
return super().convert_hn(n, el, text, convert_as_inline) # type: ignore
def convert_a(
self,
el: Any,
text: str,
convert_as_inline: Optional[bool] = False,
**kwargs,
):
"""Same as usual converter, but removes JavaScript links and escapes URIs."""
prefix, suffix, text = markdownify.chomp(text) # type: ignore
if not text:
return ""
if el.find_parent("pre") is not None:
return text
href = el.get("href")
title = el.get("title")
# Escape URIs and skip non-http or file schemes
if href:
try:
parsed_url = urlparse(href) # type: ignore
if parsed_url.scheme and parsed_url.scheme.lower() not in ["http", "https", "file"]: # type: ignore
return "%s%s%s" % (prefix, text, suffix)
href = urlunparse(
parsed_url._replace(
path=_quote_path_preserving_percent_encoded_octets(
parsed_url.path
)
)
) # type: ignore
except ValueError: # It's not clear if this ever gets thrown
return "%s%s%s" % (prefix, text, suffix)
# For the replacement see #29: text nodes underscores are escaped
if (
self.options["autolinks"]
and text.replace(r"\_", "_") == href
and not title
and not self.options["default_title"]
):
# Shortcut syntax
return "<%s>" % href
if self.options["default_title"] and not title:
title = href
title_part = ' "%s"' % title.replace('"', r"\"") if title else ""
return (
"%s[%s](%s%s)%s" % (prefix, text, href, title_part, suffix)
if href
else text
)
def convert_img(
self,
el: Any,
text: str,
convert_as_inline: Optional[bool] = False,
**kwargs,
) -> str:
"""Same as usual converter, but removes data URIs"""
alt = el.attrs.get("alt", None) or ""
src = el.attrs.get("src", None) or ""
data_src = el.attrs.get("data-src", None) or ""
# Lazy-loading libraries commonly leave a tiny placeholder data URI in
# src and put the real image in data-src. Prefer data-src when src
# isn't a usable URL, so the placeholder doesn't win over actual
# content. When keep_data_uris is set the caller explicitly wants the
# embedded bytes, so a data URI in src is left alone.
if data_src and (
not src
or (src[:5].lower() == "data:" and not self.options["keep_data_uris"])
):
src = data_src
title = el.attrs.get("title", None) or ""
title_part = ' "%s"' % title.replace('"', r"\"") if title else ""
# Remove all line breaks from alt
alt = alt.replace("\n", " ")
if (
convert_as_inline
and el.parent.name not in self.options["keep_inline_images_in"]
):
return alt
# Remove dataURIs
if src[:5].lower() == "data:" and not self.options["keep_data_uris"]:
src = src.split(",")[0] + "..."
return "" % (alt, src, title_part)
def convert_input(
self,
el: Any,
text: str,
convert_as_inline: Optional[bool] = False,
**kwargs,
) -> str:
"""Convert checkboxes to Markdown [x]/[ ] syntax."""
if el.get("type") == "checkbox":
return "[x] " if el.has_attr("checked") else "[ ] "
return ""
def convert_u(
self,
el: Any,
text: str,
convert_as_inline: Optional[bool] = False,
**kwargs,
) -> str:
if not text.strip():
return text
prefix, suffix, text = markdownify.chomp(text) # type: ignore
if not text:
return ""
return f"{prefix}{text}{suffix}"
def convert_strike(self, el: Any, text: str, *args, **kwargs) -> str:
"""Obsolete is still in the wild; treat it like /."""
return self.convert_s(el, text, *args, **kwargs) # type: ignore
def convert_soup(self, soup: Any) -> str:
return super().convert_soup(soup) # type: ignore