Source code for click_extra.testing
# Copyright Kevin Deldycke <kevin@deldycke.com> and contributors.
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
"""CLI testing and simulation of their execution."""
from __future__ import annotations
import inspect
import logging
import re
import subprocess
from contextlib import nullcontext
from functools import cached_property, partial
from textwrap import indent
from unittest.mock import patch
import click
import click.testing
from boltons.iterutils import flatten
from boltons.strutils import strip_ansi
from boltons.tbutils import ExceptionInfo
from extra_platforms import is_windows
from . import Color, Style
from .colorize import default_theme
TYPE_CHECKING = False
if TYPE_CHECKING:
from collections.abc import Iterable
from contextlib import AbstractContextManager
from typing import IO, Any, Literal
from ._types import TArg, TEnvVars, TNestedArgs
PROMPT = (">" if is_windows() else "$") + " "
"""Prompt used to simulate the CLI execution.
.. hint::
Use ASCII characters to avoid issues with Windows terminals.
"""
INDENT = " " * len(PROMPT)
"""Constants for rendering of CLI execution."""
[docs]
def args_cleanup(*args: TArg | TNestedArgs) -> tuple[str, ...]:
"""Flatten recursive iterables, remove all ``None``, and cast each element to
strings.
Helps serialize :py:class:`pathlib.Path` and other objects.
It also allows for nested iterables and ``None`` values as CLI arguments for
convenience. We just need to flatten and filters them out.
"""
return tuple(str(arg) for arg in flatten(args) if arg is not None)
[docs]
def format_cli_prompt(
cmd_args: Iterable[str],
extra_env: TEnvVars | None = None,
) -> str:
"""Simulate the console prompt used to invoke the CLI."""
extra_env_string = ""
if extra_env:
extra_env_string = default_theme.envvar(
"".join(f"{k}={v} " for k, v in extra_env.items()),
)
cmd_str = default_theme.invoked_command(" ".join(cmd_args))
return PROMPT + extra_env_string + cmd_str
[docs]
def render_cli_run(
args: Iterable[str],
result: click.testing.Result | subprocess.CompletedProcess,
env: TEnvVars | None = None,
) -> str:
"""Generates the full simulation of CLI execution, including output.
Mostly used to print debug traces to user or in test results.
"""
prompt = format_cli_prompt(args, env)
stdout = ""
stderr = ""
output = ""
exit_code = None
if isinstance(result, click.testing.Result):
stdout = result.stdout
stderr = result.stderr
exit_code = result.exit_code
output = result.output
elif isinstance(result, subprocess.CompletedProcess):
stdout = result.stdout
stderr = result.stderr
exit_code = result.returncode
# Render the execution trace.
trace = []
trace.append(prompt)
if output:
trace.append(f"{Style(fg=Color.blue)('<output>')} stream:")
trace.append(indent(output, INDENT))
if stdout:
trace.append(f"{Style(fg=Color.green)('<stdout>')} stream:")
trace.append(indent(stdout, INDENT))
if stderr:
trace.append(f"{Style(fg=Color.red)('<stderr>')} stream:")
trace.append(indent(stderr, INDENT))
if exit_code is not None:
trace.append(f"{Style(fg=Color.yellow)('<exit_code>')}: {exit_code}")
return "\n".join(trace)
[docs]
def print_cli_run(
args: Iterable[str],
result: click.testing.Result | subprocess.CompletedProcess,
env: TEnvVars | None = None,
) -> None:
"""Prints the full simulation of CLI execution, including output."""
print(render_cli_run(args, result, env))
INVOKE_ARGS = set(inspect.getfullargspec(click.testing.CliRunner.invoke).args)
"""Parameter IDs of ``click.testing.CliRunner.invoke()``.
We need to collect them to help us identify which extra parameters passed to
``invoke()`` collides with its original signature.
.. warning::
This has been `reported upstream to Click project
<https://github.com/pallets/click/issues/2110>`_ but has been rejected and not
considered an issue worth fixing.
"""
[docs]
class ExtraResult(click.testing.Result):
"""A ``Result`` subclass with automatic traceback formatting.
Enhances ``__repr__`` so that pytest assertion failures show the full
traceback instead of just the exception type.
"""
[docs]
@cached_property
def formatted_exception(self) -> str | None:
"""Full formatted traceback, or ``None`` if no exception occurred."""
if self.exception is None:
return None
if self.exc_info:
return ExceptionInfo.from_exc_info(*self.exc_info).get_formatted()
return f"Exception occurred: {self.exception}"
def __repr__(self) -> str:
if self.formatted_exception:
return f"<{type(self).__name__}\n{self.formatted_exception}>"
exc_str = repr(self.exception) if self.exception else "okay"
return f"<{type(self).__name__} {exc_str}>"
[docs]
class ExtraCliRunner(click.testing.CliRunner):
"""Augment :class:`click.testing.CliRunner` with extra features and bug fixes."""
force_color: bool = False
"""Global class attribute to override the ``color`` parameter in ``invoke``."""
[docs]
def invoke( # type: ignore[override]
self,
cli: click.Command,
*args: TArg | TNestedArgs,
input: str | bytes | IO | None = None,
env: TEnvVars | None = None,
catch_exceptions: bool = True,
color: bool | Literal["forced"] | None = None,
**extra: Any,
) -> ExtraResult:
"""Same as ``click.testing.CliRunner.invoke()`` with extra features.
- The first positional parameter is the CLI to invoke. The remaining positional
parameters of the function are the CLI arguments. All other parameters are
required to be named.
- The CLI arguments can be nested iterables of arbitrary depth. This is
`useful for argument composition of test cases with @pytest.mark.parametrize
<https://docs.pytest.org/en/stable/example/parametrize.html>`_.
- Allow forcing of the ``color`` property at the class-level via
``force_color`` attribute.
- Adds a special case in the form of ``color="forced"`` parameter, which allows
colored output to be kept, while forcing the initialization of
``Context.color = True``. This is `not allowed in current implementation
<https://github.com/pallets/click/issues/2110>`_ of
``click.testing.CliRunner.invoke()`` because of colliding parameters.
- Strips all ANSI codes from results if ``color`` was explicirely set to
``False``.
- Always prints a simulation of the CLI execution as the user would see it in
its terminal. Including colors.
- Pretty-prints a formatted exception traceback if the command fails.
:param cli: CLI to invoke.
:param *args: can be nested iterables composed of ``str``,
:py:class:`pathlib.Path` objects and ``None`` values. The nested structure
will be flattened and ``None`` values will be filtered out. Then all
elements will be casted to ``str``. See :func:`args_cleanup` for details.
:param input: same as ``click.testing.CliRunner.invoke()``.
:param env: same as ``click.testing.CliRunner.invoke()``.
:param catch_exceptions: same as ``click.testing.CliRunner.invoke()``.
:param color: If a boolean, the parameter will be passed as-is to
``click.testing.CliRunner.isolation()``. If ``"forced"``, the parameter
will be passed as ``True`` to ``click.testing.CliRunner.isolation()`` and
an extra ``color=True`` parameter will be passed to the invoked CLI.
:param **extra: same as ``click.testing.CliRunner.invoke()``, but colliding
parameters are allowed and properly passed on to the invoked CLI.
"""
# Initialize ``extra`` if not provided.
if not extra:
extra = {}
# Pop out the ``args`` parameter from ``extra`` and append it to the positional
# arguments. This handles the case where ``args`` is passed as a keyword
# argument, as in vanilla Click's ``CliRunner.invoke()`` API.
cli_args = list(args)
if "args" in extra:
cli_args.extend(extra.pop("args"))
# Flatten and filters out CLI arguments.
clean_args = args_cleanup(*cli_args)
if color == "forced":
# Pass the color argument as an extra parameter to the invoked CLI.
# This works around Click issue #2110: ``CliRunner.invoke(color=True)``
# controls the test "terminal" but cannot simultaneously pass ``color``
# through to ``Context``.
extra["color"] = True
# The class attribute ``force_color`` overrides the ``color`` parameter.
if self.force_color:
isolation_color = True
# Cast to ``bool`` to avoid passing ``None`` or ``"forced"`` to ``invoke()``.
else:
isolation_color = bool(color)
# No-op context manager without any effects.
extra_params_bypass: AbstractContextManager = nullcontext()
# If ``extra`` contains parameters that collide with the original ``invoke()``
# parameters, we need to remove them from ``extra``, then use a monkeypatch to
# properly pass them to the CLI.
colliding_params = INVOKE_ARGS.intersection(extra)
if colliding_params:
# Transfer colliding parameters from ``extra`` to ``extra_bypass``.
extra_bypass = {pid: extra.pop(pid) for pid in colliding_params}
# Monkeypatch the original command's ``main()`` call to pass extra
# parameter for ``Context`` initialization. Because we cannot simply add
# colliding parameter IDs to ``**extra``.
extra_params_bypass = patch.object(
cli,
"main",
partial(cli.main, **extra_bypass),
)
with extra_params_bypass:
result = super().invoke(
cli=cli,
args=clean_args,
input=input,
env=env,
catch_exceptions=catch_exceptions,
color=isolation_color,
**extra,
)
# Upgrade the result to our subclass for automatic traceback formatting.
result.__class__ = ExtraResult
extra_result: ExtraResult = result # type: ignore[assignment]
# ``color`` has been explicitly set to ``False``, so strip all ANSI codes.
if color is False:
extra_result.stdout_bytes = strip_ansi(extra_result.stdout_bytes) # type: ignore[assignment,arg-type]
extra_result.stderr_bytes = strip_ansi(extra_result.stderr_bytes) # type: ignore[assignment,arg-type]
extra_result.output_bytes = strip_ansi(extra_result.output_bytes) # type: ignore[assignment,arg-type]
print_cli_run(
[self.get_default_prog_name(cli), *clean_args],
extra_result,
env=env,
)
if extra_result.formatted_exception:
print(extra_result.formatted_exception)
return extra_result
[docs]
def unescape_regex(text: str) -> str:
"""De-obfuscate a regex for better readability.
This is like the reverse of ``re.escape()``.
"""
char_map = {
escaped_char: chr(single_char)
for single_char, escaped_char in (
re._special_chars_map.items() # type: ignore[attr-defined]
)
}
char_map.update({r"\x1b": "\x1b"})
for escaped, char in char_map.items():
text = text.replace(escaped, char)
return text
[docs]
class RegexLineMismatch(AssertionError):
"""Raised when a regex line does not match the corresponding content line."""
def __init__(self, regex_line: str, content_line: str, line_number: int) -> None:
# De-obfuscate the regex to allow for comparison with the output.
self.regex_line = unescape_regex(regex_line)
self.content_line = content_line
self.line_number = line_number
message = (
f"Line #{self.line_number} does not match.\n"
f"Regex : {self.regex_line!r}\n"
f"Output: {self.content_line!r}"
)
super().__init__(message)
REGEX_NEWLINE = "\\n"
"""Newline representation in the regexes above."""
[docs]
def regex_fullmatch_line_by_line(regex: re.Pattern | str, content: str) -> None:
"""Check that the ``content`` matches the given ``regex``.
If the ``regex`` does not fully match the ``content``, raise an ``AssertionError``,
with a message showing the first mismatching line.
This is useful when comparing large walls of text, such as CLI output.
"""
# If the regex fully match the output right away, no need for a custom message.
if re.fullmatch(regex, content):
return
content_lines = content.splitlines(keepends=True)
if isinstance(regex, str):
regex_lines = [line + REGEX_NEWLINE for line in regex.split(REGEX_NEWLINE)]
else:
regex_lines = regex.pattern.splitlines(keepends=True)
line_indexes = range(max(len(regex_lines), len(content_lines)))
for i in line_indexes:
regex_line = regex_lines[i]
content_line = content_lines[i]
if re.fullmatch(regex_line, content_line):
logging.debug(
f"Line #{i + 1} match.\n"
f"Regex : {regex_line!r}\n"
f"Output: {content_line!r}"
)
else:
raise RegexLineMismatch(regex_line, content_line, i + 1)