# Copyright Kevin Deldycke <kevin@deldycke.com> and contributors.
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
"""Configuration file formats and their stateless content parsers.
Holds the {class}`ConfigFormat` enum, which probes on first use the modules that
decide whether a format is enabled, and {func}`parse_content`, the stateless
dispatch used by {class}`~click_extra.config.option.ConfigOption` for every format
that does not need the CLI parameter structure.
```{caution}
This module is imported early in the package's import graph
({mod}`~click_extra.table` reaches it before the
{mod}`~click_extra.parameters` / {mod}`~click_extra.context` chain has
settled), so it takes no top-level import from `click_extra` itself. A format
whose parsing needs the CLI structure or a binary file, like `INI`, `ARGFILE`
and `SQLITE`, lives as a `ConfigOption` method in
{mod}`~click_extra.config.option` instead of here.
```
"""
from __future__ import annotations
import importlib.util
import json
import logging
import re
from enum import Enum
from fnmatch import fnmatch
from functools import cache
from .._utils import missing_extra_message
TYPE_CHECKING = False
if TYPE_CHECKING:
from collections.abc import Iterable, Mapping
from pathlib import Path
from typing import Any
logger = logging.getLogger(__name__)
_NO_SQLITE_MESSAGE = (
"SQLite support disabled: this Python provides no sqlite3 module. "
"Install the sqlite3 package of your Python distribution to enable it."
)
"""Reason and remedy reported for a Python whose SQLite bindings are missing.
{func}`disabled_format_message` cannot derive it the way it does for the other
formats: `SQLITE` is backed by a standard library module, so no
`click-extra[extra]` install target enables it."""
_JSONC_COMMENT = r'(?s)("(?:\\.|[^"\\])*")|//[^\n]*|/\*.*?\*/'
"""Match a JSON string, kept, or a `//` line or `/* */` block comment, dropped.
A pattern string rather than a compiled pattern: {mod}`re` compiles and caches it
on the first {func}`parse_jsonc` call, which keeps the compilation off the
startup path of every CLI."""
_JSONC_TRAILING_COMMA = r'("(?:\\.|[^"\\])*")|,(\s*[}\]])'
"""Match a JSON string, kept, or the comma closing an object or an array, dropped.
Compiled on first use, like `_JSONC_COMMENT`."""
@cache
def _module_available(fmt: ConfigFormat) -> bool:
"""Probe the module backing `fmt`, without importing it.
Logs {func}`disabled_format_message` at debug level when the module is missing.
The parser itself is imported by {func}`parse_content`, on the first file it
parses.
"""
module = fmt.value[1]
if module is None or importlib.util.find_spec(module) is not None:
return True
logger.debug(disabled_format_message(fmt))
return False
SQLITE_CONFIG_TABLE = "config"
"""Name of the table {func}`click_extra.config.option.ConfigOption.load_sqlite_config`
reads a `SQLITE` configuration from.
The table holds `key`/`value` columns: dotted parameter paths and their
JSON-encoded values."""
def _toml_ready(data: Any) -> Any:
"""Replace every tuple in *data* with a list, at any depth.
`tomlrt` refuses a tuple, while Click returns one for a parameter with
`multiple=True` or `nargs`. TOML has a single array type, so the list writes
the same array.
"""
if isinstance(data, dict):
return {key: _toml_ready(value) for key, value in data.items()}
if isinstance(data, (list, tuple)):
return [_toml_ready(value) for value in data]
return data
[docs]
def parse_jsonc(content: str) -> Any:
"""Read a JSONC document: JSON with comments and trailing commas.
The comments are blanked and the trailing commas removed before {mod}`json`
reads the rest, so an error it raises still names the line it sits on. A
string keeps its content whole: a `//` or a comma inside it is not syntax.
"""
def blank_comment(match: re.Match[str]) -> str:
if match.group(1) is not None:
return match.group(1)
return "\n" * match.group().count("\n")
content = re.sub(_JSONC_COMMENT, blank_comment, content)
content = re.sub(
_JSONC_TRAILING_COMMA,
lambda match: match.group(1) if match.group(1) is not None else match.group(2),
content,
)
return json.loads(content)
[docs]
def parse_xml(content: str) -> dict[str, Any]:
"""Read an XML document into nested dictionaries, the root element outermost.
An element holding text alone reads as that text, stripped of the whitespace
around it, or as `None` when there is none. One holding attributes or child
elements reads as a dictionary: an attribute under its name prefixed with
`@`, a child under its tag, and the text under `#text`. Elements repeating
a tag under the same parent are collected into a list, in document order.
Comments and processing instructions are skipped, and a tag keeps its
namespace prefix as written. These are the conventions of `xmltodict`.
:raises xml.parsers.expat.ExpatError: on a malformed document.
"""
# Deferred import: keeps xml off the startup path of every CLI.
from xml.parsers import expat
root: dict[str, Any] = {}
# Each open element, as the dictionary it builds and its text so far.
stack: list[tuple[dict[str, Any], list[str]]] = [(root, [])]
def start_element(name: str, attributes: dict[str, str]) -> None:
stack.append(({f"@{key}": value for key, value in attributes.items()}, []))
def end_element(name: str) -> None:
node, chunks = stack.pop()
text = "".join(chunks).strip()
value: Any
if node:
if text:
node["#text"] = text
value = node
else:
value = text or None
parent = stack[-1][0]
if name in parent:
siblings = parent[name]
if isinstance(siblings, list):
siblings.append(value)
else:
parent[name] = [siblings, value]
else:
parent[name] = value
def character_data(data: str) -> None:
stack[-1][1].append(data)
parser = expat.ParserCreate()
parser.StartElementHandler = start_element
parser.EndElementHandler = end_element
parser.CharacterDataHandler = character_data
parser.Parse(content, True)
return root
def _xml_name(name: str) -> str:
"""Spell `name` as an XML element or attribute name.
A character invalid in a name becomes an underscore. A name that cannot start
an element, like one starting with a digit, gets a leading underscore: `0`
becomes `_0`. A colon stays, so a namespace prefix {func}`parse_xml` kept
writes back as it was read.
"""
safe = "".join(c if c.isalnum() or c in "_.-:" else "_" for c in name)
return safe if safe[:1].isalpha() or safe[:1] == "_" else f"_{safe}"
def _xml_text(value: Any) -> str:
"""Spell a value as XML text: `true` or `false` for a boolean, `str()` otherwise."""
if isinstance(value, bool):
return "true" if value else "false"
return str(value)
[docs]
def serialize_xml(data: Mapping[str, Any], *, indent: str = "\t") -> str:
"""Write nested dictionaries as an XML document, one element per level.
The inverse of {func}`parse_xml`, with the same conventions: a `@` key is an
attribute, `#text` the text of an element that also carries attributes or
children, a list a run of elements sharing a tag, and `None` or an empty
string an empty element. Each top-level key is a root element. Child
elements sit on their own lines, each level pushed by one `indent`. A key
XML does not accept as a name is respelled, like `0` into `_0` and
`my key` into `my_key`, so the document stays well-formed.
"""
# Deferred import: keeps xml off the startup path of every CLI.
from xml.sax.saxutils import escape, quoteattr
lines: list[str] = []
def emit(name: str, value: Any, depth: int) -> None:
if isinstance(value, list):
for item in value:
emit(name, item, depth)
return
attributes = ""
text: str | None = None
children: list[tuple[str, Any]] = []
if isinstance(value, dict):
for key, item in value.items():
if key.startswith("@"):
attributes += f" {_xml_name(key[1:])}={quoteattr(_xml_text(item))}"
elif key == "#text":
text = _xml_text(item)
else:
children.append((key, item))
elif value is not None:
text = _xml_text(value)
pad = indent * depth
tag = _xml_name(name)
opening = len(lines)
lines.append(f"{pad}<{tag}{attributes}>")
for key, item in children:
emit(key, item, depth + 1)
if len(lines) == opening + 1:
# Nothing was written under the element: close it on its own line.
lines[opening] += f"{escape(text) if text else ''}</{tag}>"
return
if text:
lines.append(f"{pad}{indent}{escape(text)}")
lines.append(f"{pad}</{tag}>")
for name, value in data.items():
emit(name, value, 0)
return "\n".join(lines)
[docs]
def parse_content(fmt: ConfigFormat, content: str) -> Any:
"""Parse content with a single stateless format.
`INI` is excluded: it needs the CLI parameter structure for type coercion and
is handled by {meth}`~click_extra.config.option.ConfigOption.load_ini_config`.
`ARGFILE` is excluded for the same reason: it maps command-line tokens to the
CLI's parameters and is handled by
{meth}`~click_extra.config.option.ConfigOption.load_argfile_config`. `SQLITE`
is excluded too: it is a binary format, read from its file path by
{meth}`~click_extra.config.option.ConfigOption.load_sqlite_config` instead of
a text payload.
`PLIST` parses here from its XML variant, the only one expressible as a text
payload. The binary variant is read from its file path by
{meth}`~click_extra.config.option.ConfigOption.load_plist_config`.
```{note}
Optional third-party parsers are imported at the point of use. Only enabled
formats reach this function (disabled ones are filtered out of
`ConfigOption.file_format_patterns`), so the import always resolves for the
formats parsed here.
```
"""
match fmt:
case ConfigFormat.TOML:
# Deferred import: keeps tomllib off the startup path of every CLI.
import tomllib
return tomllib.loads(content)
case ConfigFormat.YAML:
import yaml
return yaml.full_load(content)
case ConfigFormat.JSON:
return json.loads(content)
case ConfigFormat.JSON5:
import json5
return json5.loads(content)
case ConfigFormat.JSONC:
return parse_jsonc(content)
case ConfigFormat.HJSON:
import hjson
return hjson.loads(content)
case ConfigFormat.XML:
return parse_xml(content)
case ConfigFormat.PLIST:
# Deferred import: keeps plistlib off the startup path of every CLI.
import plistlib
return plistlib.loads(content.encode("UTF-8"))
case ConfigFormat.PYPROJECT_TOML:
# Deferred import: keeps tomllib off the startup path of every CLI.
import tomllib
return tomllib.loads(content).get("tool", {})
raise ValueError(f"{fmt!r} is not handled by parse_content().")
SERIALIZABLE_FORMATS: tuple[ConfigFormat, ...] = (
ConfigFormat.TOML,
ConfigFormat.YAML,
ConfigFormat.JSON,
ConfigFormat.JSON5,
ConfigFormat.JSONC,
ConfigFormat.HJSON,
ConfigFormat.XML,
ConfigFormat.PLIST,
)
"""Configuration formats {func}`serialize_content` can write, in priority order.
Every {class}`ConfigFormat` except {attr}`~ConfigFormat.INI`,
{attr}`~ConfigFormat.SQLITE`, {attr}`~ConfigFormat.ARGFILE` and
{attr}`~ConfigFormat.PYPROJECT_TOML`, which have no serializer. `JSON`,
`JSON5` and `JSONC` are emitted as plain JSON and `PLIST` through
{mod}`plistlib`, all from the standard library, so they need no optional
dependency; the others require their format's extra.
```{caution}
Keep this in sync with the `match` statement in {func}`serialize_content`.
```
"""
[docs]
def serialize_content(fmt: ConfigFormat, data: Any, **kwargs: Any) -> str:
"""Serialize a Python object to a string in the given format.
The dumping counterpart to {func}`parse_content`. Per-format defaults can be
overridden through `kwargs` (forwarded to the underlying serializer). JSON5
and JSONC are emitted as plain JSON, a valid subset of both.
```{caution}
Not every format round-trips: `TOML`, `XML` and `PLIST` have no null
type (`plistlib` even raises on `None` values), and `XML` expects a
single root mapping, so the caller is responsible for shaping `data`
accordingly. `INI`, `SQLITE`, `ARGFILE` and `PYPROJECT_TOML` have no
serializer.
```
```{note}
Optional third-party serializers are imported lazily, at the point of use.
Writing `TOML` uses [`tomlrt`](https://dimbleby.github.io/tomlrt/) (the
`[toml]` extra), unlike reading which relies on the built-in `tomllib`.
```
:raises ValueError: the format has no serializer.
"""
match fmt:
case ConfigFormat.JSON | ConfigFormat.JSON5 | ConfigFormat.JSONC:
return (
json.dumps(data, **{"ensure_ascii": False, "indent": 2, **kwargs})
+ "\n"
)
case ConfigFormat.YAML:
import yaml
return str(
yaml.dump(
data,
**{"allow_unicode": True, "default_flow_style": False, **kwargs},
)
)
case ConfigFormat.TOML:
import tomlrt
return tomlrt.dumps(_toml_ready(data))
case ConfigFormat.HJSON:
import hjson
return str(hjson.dumps(data, **{"ensure_ascii": False, **kwargs})) + "\n"
case ConfigFormat.XML:
return serialize_xml(data, **kwargs) + "\n"
case ConfigFormat.PLIST:
# Deferred import: keeps plistlib off the startup path of every CLI.
import plistlib
return plistlib.dumps(data, **kwargs).decode("UTF-8") + "\n"
raise ValueError(f"{fmt!r} is not handled by serialize_content().")
def _serialize_toml_with_unset(tree: dict[str, Any]) -> str:
"""Render *tree* as TOML, emitting `None` leaves as commented-out keys.
TOML has no null type, so parameters without a value cannot round-trip as
real entries. Instead of dropping them from the export, each one is
rendered as a `# key =` comment line: the generated file documents every
key that can be set, while leaving the unset ones inert.
Relies on `tomlrt` (the `[toml]` extra), like
{func}`~click_extra.config.formats.serialize_content` does for TOML.
`tomlrt` attaches a comment to the key or the section header below it. The
tree is walked in the order TOML renders it, the keys of a table before its
sub-tables. Each comment waits for the next key or header, or goes to the
end of the document.
`tomlrt` writes no header for a table that holds only sub-tables. When such a
table also has commented-out keys, they would read as keys of the table
above. A comment or a blank line set above a header pins it, so each section
gets a blank line there.
:raises ImportError: when `tomlrt` is not installed.
"""
import tomlrt
pending: list[str] = []
def fill(container: Any, mapping: dict[str, Any]) -> None:
for key, value in mapping.items():
if isinstance(value, dict):
continue
if value is None:
pending.append(f"{key} =")
continue
container[key] = _toml_ready(value)
if pending:
container.leading_comments[key] = tuple(pending)
pending.clear()
for key, value in mapping.items():
if not isinstance(value, dict):
continue
section = tomlrt.Table.section()
# A blank line above the header pins it, and separates the section
# from the one above.
section.header_leading_block = (None,)
container[key] = section
# Comments waiting for this header are set on the installed table:
# set on the detached one, tomlrt adds a second blank line above them.
table = container[key]
if pending:
table.header_leading_block = (*pending, None)
pending.clear()
fill(table, value)
doc = tomlrt.Document()
fill(doc, tree)
if pending:
doc.epilogue = tuple(pending)
return tomlrt.dumps(doc)
[docs]
def read_file(path: Path, formats: Iterable[ConfigFormat] | None = None) -> Any:
"""Read a file and parse it, picking the format from its name.
The format is resolved with {func}`format_from_path` over `formats` (every
{class}`~click_extra.config.formats.ConfigFormat` by default), then the
content is parsed with {func}`parse_content`.
:raises ValueError: the file name matches none of the candidate `formats`.
:raises ImportError: the matched format's parser is not available.
"""
fmt = format_from_path(path, formats)
if fmt is None:
raise ValueError(f"Unsupported file extension: {path.name!r}")
if not fmt.enabled:
raise ImportError(disabled_format_message(fmt))
return parse_content(fmt, path.read_text(encoding="UTF-8"))