# Copyright Kevin Deldycke <kevin@deldycke.com> and contributors.
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
"""Generic Python execution directives for Sphinx documentation.
Sibling to {mod}`click_extra.sphinx.click`. The Click directives execute a
Click CLI and render its terminal output; these execute arbitrary Python and
render either captured `stdout` (`python:run`) or live document content
parsed from the captured output (`python:render`, `python:render-myst`,
`python:render-rst`).
The render trio lets a documentation page embed dynamically generated tables,
references, or whole sections without a separate `docs_update.py`
regenerator script: the content stays in sync with its source by construction.
Each render variant has a clear parser contract:
- `python:render` parses the captured output with whatever parser owns the
host document. Use it when the generated markup matches the host file
format (MyST inside `.md`, reST inside `.rst`).
- `python:render-myst` always parses the captured output as MyST,
regardless of host. Lets `.rst` documents embed MyST-generated content.
- `python:render-rst` always parses the captured output as reST, regardless
of host. Lets `.md` documents embed reST-generated content.
On top of live rendering, `python:render` accepts a `:mirror:` flag. A
mirror block keeps a copy of its generated Markdown in the source `.md`,
between {data}`MIRROR_MARKER_START` / {data}`MIRROR_MARKER_END` markers
directly below the fence, so the output is reviewable in place and renders on
GitHub. The committed region is refreshed by {func}`update_mirror_blocks`
(behind the `click-extra refresh-directives` command, alongside the
``{matrix}`` blocks of {mod}`click_extra.sphinx.matrix`), while Sphinx builds
always render the fresh output by regenerating the region in memory at
`source-read` time: the marker region of a `docs_update.py` script, with the
generator inlined into the page.
"""
from __future__ import annotations
import contextlib
import io
import re
import sys
from pathlib import Path
from docutils import nodes
from docutils.parsers.rst import Parser as RstParser
from docutils.utils import new_document
from ..markdown.blocks import (
fence_spans,
marker_res,
region_markers,
rewrite_fenced_regions,
rewrite_fences,
split_options,
update_blocks,
walk_fences,
)
from ._base import (
PLACEHOLDER_FILE,
StatelessDomain,
compile_directive,
make_cleanup,
parse_into_section,
)
from .click import ClickDirective
TYPE_CHECKING = False
if TYPE_CHECKING:
from collections.abc import Iterable
from types import CodeType
from typing import ClassVar
from docutils.frontend import Values
from docutils.parsers import Parser
from sphinx.application import Sphinx
from ..markdown.blocks import FenceSpan
MIRROR_MARKER_START = region_markers("mirror")[0]
"""Opening marker of a `python:render` `:mirror:` region.
Written on its own line, directly below the fence, and paired with
{data}`MIRROR_MARKER_END`. Everything between the two markers is regenerated
by {func}`update_mirror_blocks`: edit the Python block above the marker, never
the mirrored region itself. Same `<!-- name --> / <!-- name-end -->` grammar
as the `<!-- matrix … -->` regions of {mod}`click_extra.sphinx.matrix`.
"""
MIRROR_MARKER_END = region_markers("mirror")[1]
"""Closing marker of a `:mirror:` region. See {data}`MIRROR_MARKER_START`."""
_MIRROR_FENCE_OPEN_RE = re.compile(r"^[ \t]*`{3,}\{python:render\}[ \t]*\S*[ \t]*$")
"""Match a MyST `python:render` backtick-fence opening line.
Anchored on ``{python:render}`` so the sibling `python:render-myst` and
`python:render-rst` directives never match: `:mirror:` writes Markdown
back into a Markdown host, so it is scoped to the plain `python:render` form.
"""
MIRROR_SRC_MARKER_END = region_markers("mirror-src")[1]
"""Closing marker of a `<!-- mirror-src … -->` region.
The comment form of `:mirror:`: the generator Python lives inside an
`<!-- mirror-src … -->` HTML comment instead of a visible `{python:render}`
fence, so the generated body renders on GitHub or PyPI (a README diagram, a
compatibility table) while the generator stays invisible. Everything between the
opening comment's closing `-->` and this marker is regenerated by
{func}`update_mirror_blocks`; edit the comment, never the body.
"""
# Opening `<!-- mirror-src` comment line, left open to hold the generator body,
# and that body's closing `-->`. Neither is a marker of the shared grammar: the
# opening comment closes lines later, where `marker_res` wants it on one line.
_MIRROR_SRC_OPEN_RE = re.compile(r"^[ \t]*<!--[ \t]*mirror-src[ \t]*$")
_MIRROR_SRC_CODE_CLOSE_RE = re.compile(r"^[ \t]*-->[ \t]*$")
# The `<!-- mirror-src-end -->` region terminator, which is one.
_MIRROR_SRC_END_RE = marker_res("mirror-src")[1]
def _exec_capturing(code: CodeType, namespace: dict[str, object]) -> list[str]:
"""Run `code` in `namespace` and return the lines it printed to `stdout`.
The capture is `contextlib.redirect_stdout`, a sys-level mechanism with no
file-descriptor variant, so the `click_extra_run_capture` distinction the
Click runner makes does not apply here.
"""
buffer = io.StringIO()
with contextlib.redirect_stdout(buffer):
exec(code, namespace)
return buffer.getvalue().splitlines()
[docs]
class PythonRunner:
"""Stateful runner for `python:*` directives.
Holds a per-document namespace so consecutive directive blocks share
imports and variables. One instance is attached to `state.document` on
first use and dropped at `doctree-read` time by
{func}`cleanup_python_runner`.
"""
def __init__(self, capture: str | None = None) -> None:
# `capture` is accepted for interface parity with the shared
# {attr}`~click_extra.sphinx.click.ClickDirective.runner` property,
# which passes the `click_extra_run_capture` value to every runner
# factory. It is ignored here, see `_exec_capturing`.
self.namespace: dict[str, object] = {"__file__": PLACEHOLDER_FILE}
def _locate(self, directive: ClickDirective) -> None:
"""Tell the namespace where the documentation running it lives.
A block generating an asset has to write it somewhere, and the only
somewhere that means anything is the documentation source root. The
name sits beside `__file__` rather than replacing it: that one is
pinned to a placeholder every block already sees, and a block reading
it would not expect a real path to appear.
"""
self.namespace.setdefault("__srcdir__", str(directive.env.srcdir))
[docs]
def execute_source(self, directive: ClickDirective) -> None:
"""Execute the directive's content with no output capture.
Used by {class}`PythonSourceDirective` when the example is run only
for its side effects (typically to seed imports for a follow-up
`python:run` block).
"""
self._locate(directive)
exec(compile_directive(directive), self.namespace)
[docs]
def run_python(self, directive: ClickDirective) -> list[str]:
"""Execute the directive's content and capture `stdout`.
Returns the captured output as a list of lines, suitable for either
a code-block render (`python:run`) or a live nested-parse pass
(the `python:render` directives).
"""
self._locate(directive)
return _exec_capturing(compile_directive(directive), self.namespace)
[docs]
class PythonDirective(ClickDirective):
"""Base class for `python:*` directives.
Reuses the option spec and rendering scaffolding of
{class}`~click_extra.sphinx.click.ClickDirective`, but rebinds the
per-document runner so the Click and Python namespaces don't collide within
the same document.
"""
runner_attr = "python_runner"
runner_factory = PythonRunner
[docs]
class PythonSourceDirective(PythonDirective):
"""Render a Python source block and execute it silently.
Counterpart to {class}`~click_extra.sphinx.click.SourceDirective` for
non-Click code: show the source for teaching, run it as a side effect
to seed the runner namespace for subsequent `python:run` or
`python:render` blocks in the same document.
"""
default_language = "python"
show_source_by_default = True
show_results_by_default = False
runner_method = "execute_source"
screenshots_source = True
[docs]
class PythonRunDirective(PythonDirective):
"""Execute a Python block and render its captured `stdout`.
Default language for the result code block is `text` (plain, no
highlighting). Override with the `:language:` option when the output
is structured (`json`, `html`, `yaml`, etc.). The source, when shown, is
rendered as `python` whatever the result language, and a `:screenshot:`
pictures the captured output, as it does on `click:run`.
"""
default_language = "text"
show_source_by_default = False
show_results_by_default = True
runner_method = "run_python"
source_language = PythonSourceDirective.default_language
def _parse_with(
parser: Parser,
text: str,
settings: Values | None,
parent: nodes.Element,
source_path: str,
) -> None:
"""Parse `text` with `parser` and graft the resulting nodes into `parent`.
Builds a fresh sub-document that inherits the host document's settings,
runs the supplied parser on the captured text, then transfers the
sub-document's children to `parent`. This lets a directive force a
specific parser regardless of what owns the surrounding source file.
"""
sub_doc = new_document(source_path, settings)
parser.parse(text, sub_doc)
parent.extend(sub_doc.children)
[docs]
class PythonRenderBaseDirective(PythonDirective):
"""Common scaffolding for the `python:render` directive family.
Subclasses set `forced_parser` to either `None` (use the host
document's parser via `state.nested_parse`) or a
{class}`~docutils.parsers.Parser` factory that produces the parser to apply
to the captured output.
"""
default_language = "markdown"
show_source_by_default = False
show_results_by_default = True
runner_method = "run_python"
forced_parser: ClassVar[type[Parser] | None] = None
"""Parser class to instantiate for the captured output.
`None` means defer to the host document's parser via
`state.nested_parse` (preserving full Sphinx context). A concrete
class forces that parser regardless of host file format.
"""
[docs]
def run(self) -> list[nodes.Node]:
"""Render the captured stdout as live document content.
A `:mirror:` block is not run here. {func}`rewrite_python_mirror_regions`
writes its output as a raw Markdown region below the fence during the
`source-read` pass, and the host parser renders that region: the
directive shows at most its Python source, so the content is not
rendered twice.
"""
results = [] if "mirror" in self.options else self.runner.run_python(self)
children: list[nodes.Node] = []
if self.show_source:
children += parse_into_section(
self,
self.render_code_block(self.content, "python"),
)
if self.show_results and results:
if self.forced_parser is None:
# Host parser: reuse the surrounding state machine so cross
# references and Sphinx-aware roles resolve naturally.
children += parse_into_section(self, results)
else:
section = nodes.section()
source_file, _ = self.get_source_info()
_parse_with(
self.forced_parser(),
"\n".join(results),
self.state.document.settings,
section,
source_file,
)
children += section.children
return children
[docs]
class PythonRenderDirective(PythonRenderBaseDirective):
"""Execute a Python block and parse its `stdout` with the host parser.
The captured output is fed to whatever parser owns the surrounding
source file: MyST inside `.md` documents, reST inside `.rst`
documents. Use this when the generated markup matches the host file
format. Cross-references and Sphinx-aware roles resolve naturally
because the host state machine is reused.
For a host-independent contract, see {class}`PythonRenderMystDirective`
and {class}`PythonRenderRstDirective`.
Accepts an extra `:mirror:` flag on top of the shared option spec. When
set, the block keeps a copy of its generated Markdown in the source file,
between {data}`MIRROR_MARKER_START` / {data}`MIRROR_MARKER_END` markers,
so the output is reviewable in place and renders on GitHub. The committed
region is refreshed by {func}`update_mirror_blocks`; builds render fresh
output via {func}`rewrite_python_mirror_regions`.
"""
forced_parser = None
[docs]
class PythonRenderMystDirective(PythonRenderBaseDirective):
"""Execute a Python block and parse its `stdout` as MyST, regardless of host.
Lets a `.rst` document embed MyST-generated content: the captured
output goes through MyST parsing even when the surrounding file is
reST. Requires `myst-parser` to be importable; the import is
deferred to directive run time so consumers that never use this
directive don't pay the dependency cost.
Replaces the `docs_update.py` regenerator pattern (write generated
content between `<!-- start -->` / `<!-- end -->` markers) with
inline build-time execution. The content is always current and there
is no separate regeneration step.
"""
@property
def forced_parser(self) -> type[Parser]: # type: ignore[override]
"""The MyST docutils parser, imported on first use."""
# Deferred import: keeps myst-parser optional for a reST-only project.
from myst_parser.parsers.docutils_ import Parser as MystParser
return MystParser
[docs]
class PythonRenderRstDirective(PythonRenderBaseDirective):
"""Execute a Python block and parse its `stdout` as reST, regardless of host.
Lets a `.md` MyST document embed reST-generated content: the
captured output goes through the docutils reST parser even when the
surrounding file is MyST. No additional dependency: the reST parser
ships with docutils.
"""
forced_parser = RstParser
def _execute_mirror_block(
body: list[str],
namespace: dict[str, object],
location: str,
) -> list[str]:
"""Execute a mirror block's Python body and return its captured stdout lines.
Mirrors {meth}`PythonRunner.run_python` but works from raw source lines:
the offline refresher and the `source-read` pass both run before any
directive instance exists. Shares `namespace` across the blocks of one
document so a later block can reuse an earlier block's imports and
variables.
The source file's directory is importable while the block runs, matching
the Sphinx build where the `conf.py` directory rides `sys.path`: a block
importing a sibling helper module (a `docs_update.py` next to the page)
behaves the same offline as it does at build time.
"""
code = compile("\n".join(body), location, "exec")
directory = str(Path(location).resolve().parent)
sys.path.insert(0, directory)
try:
return _exec_capturing(code, namespace)
finally:
with contextlib.suppress(ValueError):
sys.path.remove(directory)
def _rewrite_mirror_regions(text: str, location: str) -> str:
"""Return `text` with every ``python:render {mirror}`` region refreshed.
Only a top-level `python:render` fence carrying a `:mirror:` option is
executed, its output written into the marker region directly below it. The
walk, the insertion and the idempotency are those of
{func}`click_extra.markdown.blocks.rewrite_fenced_regions`. The blocks of one
document share a namespace, so a later block can reuse an earlier block's
imports and variables.
```{note}
The mirrored region is raw Markdown, re-parsed by the host on every
build and reformatted by `mdformat` in the autofix pipeline. A
mirror block must therefore `print` `mdformat`-canonical Markdown
(like {func}`click_extra.table.render_table` in `GITHUB` mode) or
the generator and the formatter will fight over the region.
```
"""
namespace: dict[str, object] = {"__file__": PLACEHOLDER_FILE}
def generate(inner: list[str]) -> list[str] | None:
options, body = split_options(inner)
if "mirror" not in options:
return None
return _execute_mirror_block(body, namespace, location)
return rewrite_fenced_regions(text, "mirror", _MIRROR_FENCE_OPEN_RE, generate)
def _rewrite_mirror_src_regions(text: str, location: str) -> str:
"""Return `text` with every `<!-- mirror-src … -->` region refreshed.
The comment counterpart of {func}`_rewrite_mirror_regions`: instead of a
visible `{python:render}` `:mirror:` fence, the generator Python lives inside
an `<!-- mirror-src -->` HTML comment, so the generated body renders on
GitHub or PyPI while the generator stays invisible. The body runs through
{func}`_execute_mirror_block` (so a sibling helper module imports the same
way it does for a fence), and its output is written between the comment's
closing `-->` and {data}`MIRROR_SRC_MARKER_END`, which is inserted on first
sight. Idempotent: a region whose generator is unchanged round-trips.
Like {func}`_rewrite_mirror_regions`, the walk steps over backtick fences
through {func}`click_extra.markdown.blocks.rewrite_fences`, so a
`<!-- mirror-src -->` example nested inside a longer `code-block` fence (a
documented illustration) is copied verbatim, never executed or refreshed.
"""
lines = text.split("\n")
spans = fence_spans(lines)
total = len(lines)
namespace: dict[str, object] = {"__file__": PLACEHOLDER_FILE}
def refresh(index: int, span: FenceSpan | None) -> tuple[list[str], int] | None:
# A backtick fence is an opaque unit, never scanned inside for a
# mirror-src marker: a documented example must not run.
if span is not None or not _MIRROR_SRC_OPEN_RE.match(lines[index]):
return None
# Copy the opening comment and generator body verbatim, up to `-->`.
cursor = index + 1
while cursor < total and not _MIRROR_SRC_CODE_CLOSE_RE.match(lines[cursor]):
cursor += 1
if cursor >= total:
# Unterminated comment: leave the tail untouched.
return lines[index:], total
body = lines[index + 1 : cursor]
# Past the `-->` line.
cursor += 1
generated = _execute_mirror_block(body, namespace, location)
end = _mirror_src_end(lines, spans, cursor)
chunk = [*lines[index:cursor], "", *generated, ""]
if end is None:
# First sight: insert a fresh region; following content stays put.
chunk.append(MIRROR_SRC_MARKER_END)
return chunk, cursor
# Replace the existing body, consume through the terminator, and
# collapse the gap to the following content to a single blank line.
chunk.append(lines[end])
resume = end + 1
while resume < total and not lines[resume].strip():
resume += 1
if resume < total:
chunk.append("")
return chunk, resume
return "\n".join(rewrite_fences(lines, refresh, spans))
def _mirror_src_end(
lines: list[str], spans: dict[int, FenceSpan], start: int
) -> int | None:
"""Index of the terminator closing the region a generator writes from `start`.
`None` when the generator has no region yet. The walk stops at the next
`<!-- mirror-src` comment, since a terminator below it closes the region of
that other generator, and steps over fences, whose content is never a
marker.
"""
for index, span in walk_fences(lines, spans, start):
if span is not None:
continue
if _MIRROR_SRC_END_RE.match(lines[index]):
return index
if _MIRROR_SRC_OPEN_RE.match(lines[index]):
return None
return None
[docs]
def rewrite_mirrors(text: str, location: str) -> str:
"""Return `text` with both kinds of mirror region refreshed.
The fence form first (`_rewrite_mirror_regions`), then the comment form
(`_rewrite_mirror_src_regions`), each only when `text` can hold one: a walk
costs a pass over every line of the document.
"""
if "python:render" in text and ":mirror:" in text:
text = _rewrite_mirror_regions(text, location)
# Test for the bare name: `_MIRROR_SRC_OPEN_RE` accepts any spaces or tabs
# after `<!--`, or none.
if "mirror-src" in text:
text = _rewrite_mirror_src_regions(text, location)
return text
[docs]
def update_mirror_blocks(paths: Iterable[Path], *, check: bool = False) -> list[Path]:
"""Refresh every `python:render` `:mirror:` region in the given sources.
See {func}`click_extra.markdown.blocks.update_blocks` for the walk, write, and
`check`-mode contract.
```{danger}
Refreshing executes each mirror block's Python with the privileges of
the current process, exactly as a documentation build would. Only run
it on trusted sources.
```
:return: the files whose mirror regions were (or, under `check`, would
be) updated.
"""
return update_blocks(
paths,
lambda text, path: rewrite_mirrors(text, str(path)),
check=check,
)
[docs]
def rewrite_python_mirror_regions(
app: Sphinx,
docname: str,
source: list[str],
) -> None:
"""`source-read` handler refreshing `:mirror:` regions in memory.
Refreshes both kinds of region, below a visible `{python:render}`
`:mirror:` fence and below an invisible `<!-- mirror-src -->` comment, in the
in-memory `source` before the document is parsed, so every build renders the
fresh output even when the committed region is stale. Nothing is written to
disk: the committed region is refreshed offline by
{func}`update_mirror_blocks`, behind the `click-extra refresh-directives`
command.
```{danger}
Like the `python:*` directives, this executes arbitrary Python at
build time with the full privileges of the Sphinx process. It is
registered only when `click_extra_enable_exec_directives` is set.
```
"""
source[0] = rewrite_mirrors(source[0], str(app.env.doc2path(docname)))
[docs]
class PythonDomain(StatelessDomain):
"""Sphinx domain registering the `python:*` directives.
Provides:
- `python:source` to show and silently execute Python source.
- `python:run` to execute Python and render the captured stdout in a
code block.
- `python:render` to execute Python and parse the captured stdout
with the host document's parser.
- `python:render-myst` to force MyST parsing of the captured stdout.
- `python:render-rst` to force reST parsing of the captured stdout.
The domain name is `python`, distinct from Sphinx's built-in `py`
domain (which provides `py:function` / `py:class` / `py:mod` for
documenting Python API objects). Both coexist without collision.
"""
name = "python"
label = "Python (build-time execution)"
directives: ClassVar[dict] = {
"source": PythonSourceDirective,
"run": PythonRunDirective,
"render": PythonRenderDirective,
"render-myst": PythonRenderMystDirective,
"render-rst": PythonRenderRstDirective,
}
cleanup_python_runner = make_cleanup(PythonDirective.runner_attr)
"""Drop the {class}`PythonRunner` from the doctree once the document is read."""