Merge pull request #44 from pydsigner/GH-31_rich_markdown_step

Add an extended Markdown rendering Step

Author: pydsigner

pyproject.toml

@@ -34,7 +34,15 @@ Source = "https://github.com/pydsigner/anchovy"
 [project.optional-dependencies]
 toml = ['tomli >= 2.0.1; python_version < "3.11"']
 jinja = ["Jinja2>=3.1.2"]
-markdown = ["anchovy[jinja]", "markdown_it_py>=3.0.0"]
+markdown = [
+    "anchovy[jinja]",
+    "anchovy[toml]",
+    "markdown_it_py>=3.0.0",
+    "mdit_py_plugins>=0.4.0",
+    # Pygments is a fairly large dependency and perhaps not critical; but we
+    # include as part of [base] via rich anyways.
+    "Pygments>=2.12.0",
+]
 css = ["tinycss2>=1.1.1"]
 pretty = ["rich>=12.5.1"]
 pillow = ["Pillow>=9.2.0"]

src/anchovy/__init__.py

@@ -9,7 +9,7 @@
 )
 from .images import CWebPStep, ImageMagickStep, IMThumbnailStep, PillowStep, OptipngStep
 from .include import RequestsFetchStep, UnpackArchiveStep, URLLibFetchStep
-from .jinja import JinjaMarkdownStep, JinjaRenderStep
+from .jinja import JinjaExtendedMarkdownStep, JinjaMarkdownStep, JinjaRenderStep
 from .minify import CSSMinifierStep, HTMLMinifierStep, ResourcePackerStep
 from .paths import DirPathCalc, OutputDirPathCalc, REMatcher, WorkingDirPathCalc
 from .simple import DirectCopyStep

src/anchovy/components/__init__.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+import sys
+import typing as t
+if sys.version_info < (3, 11):
+    import tomli as tomllib
+else:
+    import tomllib
+
+from markdown_it.common.utils import escapeHtml, unescapeAll
+from markdown_it.renderer import RendererHTML
+if t.TYPE_CHECKING:
+    from collections.abc import Sequence
+    from markdown_it.token import Token
+    from markdown_it.utils import EnvType, OptionsDict
+
+
+def get_container_renderer(container_name, html_tag):
+    def render(
+        self: RendererHTML,
+        tokens: Sequence[Token],
+        idx: int,
+        _options: OptionsDict,
+        env: EnvType,
+    ) -> str:
+        tokens[idx].tag = html_tag
+        # add a class to the opening tag
+        if tokens[idx].nesting == 1:
+            tokens[idx].attrJoin("class", container_name)
+            nt = tokens[idx+1]
+            if nt.type == 'paragraph_open':
+                nt.hidden = True
+                counter = idx + 2
+                while tokens[counter].type != 'paragraph_close' or tokens[counter].level != nt.level:
+                    counter += 1
+                tokens[counter].hidden = True
+
+        return self.renderToken(tokens, idx, _options, env)
+
+    render.__name__ = f'render_{container_name}_to_{html_tag}'
+    return render
+
+
+class AnchovyRendererHTML(RendererHTML):
+    # https://github.com/executablebooks/markdown-it-py/issues/256
+    def fence(self, tokens: Sequence[Token], idx: int, options: OptionsDict, env: EnvType):
+        token = tokens[idx]
+        info = unescapeAll(token.info).strip() if token.info else ''
+        langName = info.split(maxsplit=1)[0] if info else ''
+
+        return (
+            options.highlight
+            and options.highlight(token.content, langName, '')
+            or escapeHtml(token.content)
+        )
+
+    def front_matter(self, tokens: Sequence[Token], idx: int, options: OptionsDict, env: EnvType):
+        parsed = tomllib.loads(tokens[idx].content)
+        env['anchovy_meta'].update(parsed)
+        return ''

src/anchovy/components/md_rendering.py

@@ -1,26 +1,33 @@
 from __future__ import annotations
 
 import shutil
+import sys
 import typing as t
 from functools import reduce
 from pathlib import Path
 
-from .core import Context, Step
+from .core import Step
 from .dependencies import pip_dependency, Dependency
 
 if t.TYPE_CHECKING:
+    from collections.abc import Sequence
     from jinja2 import Environment
-
+    from markdown_it.renderer import RendererHTML
+    from markdown_it.token import Token
+    from markdown_it.utils import EnvType, OptionsDict
 
 MDProcessor = t.Callable[[str], str]
+MDContainerRenderer =  t.Callable[
+    ['RendererHTML', 'Sequence[Token]', int, 'OptionsDict', 'EnvType'],
+    str
+]
 
 
 class JinjaRenderStep(Step):
     """
     Abstract base class for Steps using Jinja rendering.
     """
     encoding = 'utf-8'
-    env: Environment
 
     @classmethod
     def get_dependencies(cls):
@@ -31,26 +38,24 @@ def get_dependencies(cls):
     def __init__(self,
                  env: Environment | None = None,
                  extra_globals: dict[str, t.Any] | None = None):
-        self._temporary_env = env
+        if env and extra_globals:
+            env.globals.update(extra_globals)
+        self._env = env
         self._extra_globals = extra_globals
 
-    def bind(self, context: Context):
-        """
-        Bind this Step to a specific context. Also initializes a Jinja
-        environment if none is set up already.
-        """
-        super().bind(context)
-
-        if self._temporary_env:
-            self.env = self._temporary_env
-        else:
-            from jinja2 import Environment, FileSystemLoader, select_autoescape
-            self.env = Environment(
-                loader=FileSystemLoader(context['input_dir']),
-                autoescape=select_autoescape()
-            )
+    @property
+    def env(self):
+        if self._env:
+            return self._env
+
+        from jinja2 import Environment, FileSystemLoader, select_autoescape
+        self._env = Environment(
+            loader=FileSystemLoader(self.context['input_dir']),
+            autoescape=select_autoescape()
+        )
         if self._extra_globals:
-            self.env.globals.update(self._extra_globals)
+            self._env.globals.update(self._extra_globals)
+        return self._env
 
     def render_template(self, template_name: str, meta: dict[str, t.Any], output_paths: list[Path]):
         """
@@ -184,3 +189,135 @@ def extract_metadata(self, text: str):
             i += 1
 
         return meta, '\n'.join(lines[i:])
+
+
+class JinjaExtendedMarkdownStep(JinjaRenderStep):
+    encoding = 'utf-8'
+
+    @classmethod
+    def get_dependencies(cls):
+        deps = super().get_dependencies() | {
+            pip_dependency('markdown-it-py', check_name='markdown_it'),
+            pip_dependency('mdit_py_plugins'),
+            pip_dependency('Pygments', check_name='pygments'),
+        }
+        if sys.version_info < (3, 11):
+            deps.add(pip_dependency('tomli'))
+        return deps
+
+    def __init__(self,
+                 default_template: str | None = None,
+                 jinja_env: Environment | None = None,
+                 jinja_globals: dict[str, t.Any] | None = None,
+                 *,
+                 container_types: list[tuple[str | None, list[str]]] | None = None,
+                 container_renderers: dict[str, MDContainerRenderer] | None = None,
+                 substitutions: dict[str, str] | None = None,
+                 auto_anchors: bool = False,
+                 auto_typography: bool = True,
+                 code_highlighting: bool = True,
+                 pygments_params: dict[str, t.Any] | None = None,
+                 wordcount: bool = False):
+        super().__init__(jinja_env, jinja_globals)
+        self.default_template = default_template
+        self.container_types = container_types or []
+        self.container_renderers = container_renderers or {}
+        self.substitutions = substitutions or {}
+        self.auto_anchors = auto_anchors
+        self.auto_typography = auto_typography
+        self.code_highlighting = code_highlighting
+        self.pygments_params = pygments_params or {}
+        self.wordcount = wordcount
+        self._md_processor: t.Callable[[str], tuple[str, dict[str, t.Any]]] | None = None
+
+    def __call__(self, path: Path, output_paths: list[Path]):
+        md, meta = self.md_processor(
+            self.apply_substitutions(
+                path.read_text(self.encoding).strip()
+            )
+        )
+
+        meta['rendered_markdown'] = md
+
+        template_path = self.render_template(
+            meta.get('template', self.default_template),
+            meta,
+            output_paths
+        )
+        if template_path:
+            return [path, Path(template_path)], output_paths
+
+    @property
+    def md_processor(self):
+        if not self._md_processor:
+            self._md_processor = self._build_processor()
+        return self._md_processor
+
+    def apply_substitutions(self, text: str):
+        for sub, value in self.substitutions.items():
+            text = text.replace('${{ ' + sub + ' }}', value)
+        return text
+
+    def highlight_code(self, code: str, lang: str, lang_attrs: str):
+        from pygments import highlight
+        from pygments.formatters import HtmlFormatter
+        from pygments.lexers import get_lexer_by_name, guess_lexer
+        from pygments.util import ClassNotFound
+        try:
+            lexer = get_lexer_by_name(lang)
+        except ClassNotFound:
+            try:
+                lexer = guess_lexer(code)
+            except ClassNotFound:
+                return ''
+
+        return highlight(code, lexer, HtmlFormatter(**self.pygments_params))
+
+    def _build_processor(self):
+        import markdown_it
+        # TODO Need for pyright suppression will be eliminated in the next
+        # release of mdit_py_plugins:
+        #  https://github.com/executablebooks/mdit-py-plugins/pull/91
+        from mdit_py_plugins.anchors import anchors_plugin  # type: ignore[reportPrivateImportUsage]
+        from mdit_py_plugins.attrs import attrs_block_plugin, attrs_plugin  # type: ignore[reportPrivateImportUsage]
+        from mdit_py_plugins.container import container_plugin  # type: ignore[reportPrivateImportUsage]
+        from mdit_py_plugins.front_matter import front_matter_plugin  # type: ignore[reportPrivateImportUsage]
+        from mdit_py_plugins.wordcount import wordcount_plugin  # type: ignore[reportPrivateImportUsage]
+        from .components import md_rendering
+
+        processor = markdown_it.MarkdownIt(
+            'commonmark',
+            {
+                'typographer': self.auto_typography,
+                'highlight': self.highlight_code if self.code_highlighting else None,
+            },
+            renderer_cls=md_rendering.AnchovyRendererHTML
+        )
+        processor.enable(['strikethrough', 'table'])
+        if self.auto_typography:
+            processor.enable(['smartquotes', 'replacements'])
+        if self.auto_anchors:
+            anchors_plugin(processor)
+        attrs_plugin(processor)
+        attrs_block_plugin(processor)
+        front_matter_plugin(processor)
+        if self.wordcount:
+            wordcount_plugin(processor)
+
+        for tag, names in self.container_types:
+            for name in names:
+                renderer = (
+                    self.container_renderers.get(name)
+                    or (md_rendering.get_container_renderer(name, tag) if tag else None)
+                )
+                container_plugin(processor, name, render=renderer)
+
+        def convert(md_string: str):
+            env = {'anchovy_meta': dict[str, t.Any]()}
+            md: str = processor.render(md_string, env=env)
+            meta = env['anchovy_meta']
+            if self.wordcount:
+                meta['wordcount'] = env['wordcount']
+            return md, meta
+
+        return convert