# Copyright Kevin Deldycke <kevin@deldycke.com> and contributors.
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
"""End-to-end behavior of mail parsing, timestamps and hashing, through `mdedup`.
Unit-level tests of the mail object live in `test_mail.py`.
"""
from __future__ import annotations
import time
from mailbox import Maildir, mbox, mboxMessage
from textwrap import dedent
import pytest
from extra_platforms.pytest import skip_windows
from .conftest import MailFactory, check_box, metrics
invalid_windows_dates = skip_windows(
reason="Invalid dates produce negative timestamps on Windows."
)
# Some invalid dates are not supported on Windows as they produce negative
# timestamps. See:
# https://github.com/arrow-py/arrow/issues/675
# https://github.com/arrow-py/arrow/pull/745
# The `Thu, 13 Dec 101 15:30 WET` header comes from a real non-y2k-compliant mailer
# that once made the tool crash with "ValueError: year out of range". See:
# https://github.com/kdeldycke/mail-deduplicate/issues/54
invalid_date_mail_1 = MailFactory(date_rfc2822="Thu, 13 Dec 101 15:30 WET")
invalid_date_mail_2 = MailFactory(date_rfc2822="Thu, 13 Dec 102 15:30 WET")
DATELESS_HASH_HEADERS = (
"--hash-header=message-id",
"--hash-header=from",
"--hash-header=to",
"--hash-header=subject",
)
"""Four headers every test mail carries, and no `Date`: mails that differ by their
date alone land in the same duplicate set."""
[docs]
@invalid_windows_dates
@pytest.mark.parametrize(
("hash_headers", "remaining"),
(
# Each group shares its metadata, dates included, so no group has a newest
# mail to pick: nothing is deduplicated.
pytest.param(
(),
[invalid_date_mail_1] * 3 + [invalid_date_mail_2] * 2,
id="noop",
),
# Without the date in the hash, both dates share one set, and its newest
# mails are deleted.
pytest.param(DATELESS_HASH_HEADERS, [invalid_date_mail_1] * 3, id="dedup"),
),
)
def test_invalid_date_parsing(invoke, make_box, hash_headers, remaining):
"""Mails with odd non-standard dates are parsed and grouped into duplicate sets."""
box_path, box_type, _ = make_box(
Maildir,
[
invalid_date_mail_1,
invalid_date_mail_2,
invalid_date_mail_2,
invalid_date_mail_1,
invalid_date_mail_1,
],
)
result = invoke(
*hash_headers,
"--strategy=select-newest",
"--action=delete-selected",
box_path,
)
assert result.exit_code == 0
check_box(box_path, box_type, content=remaining)
undated_mail = MailFactory(date_rfc2822="invalid date")
"""A mail whose `Date` header cannot be parsed into a timestamp."""
dated_mail = MailFactory(date="2021-01-01")
"""A mail with a valid `Date` header."""
[docs]
@pytest.mark.parametrize(
("mails", "args"),
(
pytest.param(
[undated_mail, undated_mail],
("--strategy=select-oldest",),
id="all-undated",
),
# A single undated mail is enough to skip its whole set.
pytest.param(
[undated_mail, dated_mail],
(*DATELESS_HASH_HEADERS, "--strategy=select-newest"),
id="one-undated",
),
),
)
def test_unparsable_date_skips_time_strategy(invoke, make_box, mails, args):
"""Time-based strategies skip sets holding mails with no parseable `Date` header.
They name the offending mails instead of crashing.
See: https://github.com/kdeldycke/mail-deduplicate/issues/132
"""
box_path, box_type, _ = make_box(Maildir, mails)
result = invoke(*args, "--action=delete-selected", box_path)
assert result.exit_code == 0
assert "cannot compare mails without a timestamp" in result.stderr
assert "No timestamp for <MaildirDedupMail" in result.stderr
# No mail was removed.
check_box(box_path, box_type, content=mails)
[docs]
def test_unparsable_date_show_diff(invoke, make_box):
"""Rendering the diff of mails with no parseable `Date` header does not crash."""
undated_variant = MailFactory(date_rfc2822="invalid date", body="A different body.")
box_path, box_type, _ = make_box(Maildir, [undated_mail, undated_variant])
result = invoke(
"--content-threshold=0",
"--show-diff",
"--strategy=select-oldest",
"--action=delete-selected",
box_path,
)
assert result.exit_code == 0
assert "mails are too dissimilar in content" in result.stderr
# No mail was removed.
check_box(box_path, box_type, content=[undated_mail, undated_variant])
[docs]
@pytest.mark.parametrize(
("hash_body", "other_body", "deduplicated"),
(
pytest.param("skip", "Hello world\n", True, id="skip-spaced"),
pytest.param("skip", "Goodbye world\n", True, id="skip-different"),
pytest.param("raw", "Hello world\n", False, id="raw-spaced"),
pytest.param("raw", "Goodbye world\n", False, id="raw-different"),
pytest.param("decoded", "Hello world\n", False, id="decoded-spaced"),
pytest.param("decoded", "Goodbye world\n", False, id="decoded-different"),
pytest.param("normalized", "Hello world\n", True, id="normalized-spaced"),
pytest.param("normalized", "Goodbye world\n", False, id="normalized-different"),
),
)
def test_hash_body(invoke, make_box, hash_body, other_body, deduplicated):
"""--hash-body=skip groups mails by their headers alone, whatever their bodies.
=raw folds every byte of the body into the hash, =decoded folds in its decoded
text, and =normalized folds it in but ignores whitespace.
"""
a = MailFactory(
date="2021-01-01", message_id="<n@nohost.com>", body="Hello world\n"
)
b = MailFactory(date="2021-01-01", message_id="<n@nohost.com>", body=other_body)
box_path, _, _ = make_box(Maildir, [a, b])
result = invoke(
f"--hash-body={hash_body}",
"--strategy=select-one",
"--action=delete-discarded",
box_path,
)
assert result.exit_code == 0
assert len(Maildir(box_path, create=False)) == (1 if deduplicated else 2)
[docs]
def test_ctime_time_source(invoke, make_box):
"""--time-source=ctime dates each mail by the inode change time of its file.
A maildir with distinct file ctimes deduplicates by them.
"""
dup = MailFactory(body="Same body.\n", message_id="<ctime@nohost.com>")
# Deliver the two identical copies more than a second apart, so the second file is
# unambiguously the newest by ctime at any filesystem timestamp resolution. A
# sub-second gap ties on coarse-grained filesystems (e.g. overlayfs, 1 s), and on
# Windows getctime reads the immutable creation time, so touching a file's metadata
# in place (chmod) cannot reorder the copies there.
box_path, _, _ = make_box(Maildir, [dup])
time.sleep(1.1)
box = Maildir(box_path, create=False)
box.lock()
box.add(dup.render())
box.close()
result = invoke(
"--time-source=ctime",
"--strategy=select-newest",
"--action=delete-selected",
box_path,
)
assert result.exit_code == 0
# The newest copy (delivered last) was selected and deleted; one copy remains.
assert len(Maildir(box_path, create=False)) == 1
[docs]
@pytest.mark.parametrize(
"args",
(
pytest.param(("--strategy=select-one",), id="default"),
pytest.param(("--preset=conservative",), id="conservative"),
),
)
def test_crlf_and_lf_bodies_are_duplicates(invoke, tmp_path, args):
"""Two mails that differ only by LF or CRLF line endings are duplicates.
Line endings must not count toward size or content differences, nor toward the
bytes the conservative preset compares.
See: https://github.com/kdeldycke/mail-deduplicate/issues/844 (fixed by PR 845).
"""
headers = (
b"Date: Mon, 15 Jan 2024 10:30:45 +0000\n"
b"From: foo@bar.com\n"
b"To: baz@qux.com\n"
b"Subject: Line endings\n"
b"Message-ID: <crlf@nohost.com>\n"
b"Content-Type: text/plain\n"
b"\n"
)
body = b"Line one\nLine two\nLine three\n"
box_path = str(tmp_path / "box")
Maildir(box_path, create=True).close() # Lay down the new/cur/tmp structure.
# Write both copies straight into new/ in binary, so their LF and CRLF line
# endings survive verbatim: Maildir.add rewrites `\n` to os.linesep (CRLF on
# Windows), which would collapse the very distinction under test.
(tmp_path / "box" / "new" / "lf").write_bytes(headers + body)
(tmp_path / "box" / "new" / "crlf").write_bytes(
headers.replace(b"\n", b"\r\n") + body.replace(b"\n", b"\r\n")
)
result = invoke(*args, "--action=delete-discarded", box_path)
assert result.exit_code == 0
assert "too dissimilar" not in result.stderr
# The CRLF and LF copies collapsed into a single duplicate set; one copy remains.
assert len(Maildir(box_path, create=False)) == 1