"""The one robots.txt policy every collector applies.

Why one module: each collector once carried its own reading of robots.txt, and
they disagreed in ways that cost real runs. The stdlib parser predates wildcard
patterns, so it read "Disallow: /*/search/" as permission and a whole retail
family was collected for days on that misreading. The hand-written matcher that
replaced it keyed the bot group on the FIRST token of the User-Agent string,
"mozilla", so a group addressed to us by name (which is how a retailer says
"you specifically") was silently ignored (reproduced 2026-09-04). And each
collector answered "robots.txt unreadable" differently: one crawled on, one
stopped. Now there is one matcher, one identity and one policy, and every
collect() and read_one() opens with check_allowed().

The policy, decided 2026-09-04 (build plan §9, robots policy):

* A group applies to us when its agent value is a case-insensitive substring
  of BOT_NAME or BOT_NAME is a substring of it. Never the first UA token.
* Paths match RFC 9309 style: "*" spans anything including "/", "$" anchors.
  ANY matching Disallow is a no, in our own group or the "*" group, and a
  longer Allow does not out-lawyer it. Over-refusing is the only safe direction
  for a business built on relationships with these same retailers.
* Crawl-delay comes from our own group if it names one, else from "*". The
  run waits the slower of that and its own setting.
* An unreadable robots.txt has exactly three outcomes: 404/410 (and other
  plain 4xx) means the host publishes no rules, so we proceed and log the date;
  401/403 is the host refusing us, SourceBlocked; 5xx, 429 or a timeout means
  we cannot know, so the run stops and tries again next run (RobotsUnavailable).

Reads go through urllib directly rather than fetch.fetch(), because the fetch
port treats an empty body as a block, and an empty robots.txt is a perfectly
good "no rules". The fetch module is imported lazily: it imports BOT_NAME from
here, and a top-level import in both directions would be circular.
"""

import logging
import re
import time
import urllib.error
import urllib.request
from collections.abc import Callable, Iterable
from dataclasses import dataclass, field
from datetime import UTC, datetime
from urllib.parse import urlsplit

logger = logging.getLogger(__name__)

BOT_NAME = "DutyFreeProfessorBot"
READ_TIMEOUT = 20
# read_one() is called in a loop by verification, and one robots read per host
# per pass is polite enough; collect() always asks for a fresh read.
CACHE_TTL_SECONDS = 600

Fetcher = Callable[[str], tuple[int, bytes]]


class RobotsUnavailable(RuntimeError):
    """robots.txt could not be read for a reason that may clear by next run.

    Not a refusal, so not SourceBlocked; not something to work around either.
    The run records an error and the next scheduled run tries again.
    """


@dataclass(slots=True)
class _Group:
    agents: list[str] = field(default_factory=list)
    rules: list[tuple[str, str]] = field(default_factory=list)
    crawl_delay: float | None = None

    @property
    def is_star(self) -> bool:
        return "*" in self.agents

    @property
    def is_ours(self) -> bool:
        ours = BOT_NAME.lower()
        return any(a != "*" and (a in ours or ours in a) for a in self.agents)


@dataclass(slots=True)
class Robots:
    """A host's rules as they apply to this bot."""

    host: str
    disallows: list[str]
    crawl_delay: float | None
    status: int | None = None

    @classmethod
    def unrestricted(cls, host: str, status: int | None = None) -> "Robots":
        return cls(host=host, disallows=[], crawl_delay=None, status=status)

    def allows(self, target: str) -> bool:
        """Whether a path or full URL may be fetched. Any covering Disallow is a no."""
        path = _path_of(target)
        return not any(_pattern_matches(p, path) for p in self.disallows)

    def first_refusal(self, targets: Iterable[str]) -> str | None:
        return next((t for t in targets if not self.allows(t)), None)

    def delay_for(self, requested: float) -> float:
        """The slower of what the host asks and what we were going to do anyway."""
        return max(float(requested), self.crawl_delay or 0.0)


def _path_of(target: str) -> str:
    if "://" in target:
        parts = urlsplit(target)
        path = parts.path or "/"
        return f"{path}?{parts.query}" if parts.query else path
    return target or "/"


def pattern_regex(pattern: str) -> str:
    """One robots.txt path pattern as an anchored regex (RFC 9309: `*` spans any
    characters including `/`, a trailing `$` anchors the end).

    Public so the fetcher's sidecar route filter uses the same translation
    instead of a copy of it (two copies of a matcher drift; issue 2026-09-05).
    """
    anchored = pattern.endswith("$")
    if anchored:
        pattern = pattern[:-1]
    regex = "^" + "".join(".*" if ch == "*" else re.escape(ch) for ch in pattern)
    return regex + "$" if anchored else regex


def _pattern_matches(pattern: str, path: str) -> bool:
    """RFC 9309 path matching, through the one translation."""
    return re.match(pattern_regex(pattern), path) is not None


def parse(text: str, host: str = "", status: int | None = 200) -> Robots:
    """Evaluate robots.txt text against BOT_NAME. Pure; no network."""
    groups: list[_Group] = []
    current: _Group | None = None
    expecting_agents = False
    for line in text.splitlines():
        line = line.split("#", 1)[0].strip()
        if not line or ":" not in line:
            continue
        name, _, value = line.partition(":")
        name, value = name.strip().lower(), value.strip()
        if name == "user-agent":
            if current is None or not expecting_agents:
                current = _Group()
                groups.append(current)
            current.agents.append(value.lower())
            expecting_agents = True
        elif current is None:
            continue
        elif name in ("allow", "disallow"):
            current.rules.append((name, value))
            expecting_agents = False
        elif name == "crawl-delay":
            try:
                current.crawl_delay = float(value)
            except ValueError:
                pass
            expecting_agents = False
        # Anything else (Sitemap, Noindex, Host) is not a crawl rule.

    ours = [g for g in groups if g.is_ours]
    star = [g for g in groups if g.is_star]
    disallows = [
        pattern
        for group in (*ours, *star)
        for kind, pattern in group.rules
        if kind == "disallow" and pattern
    ]
    crawl_delay = next(
        (g.crawl_delay for g in (*ours, *star) if g.crawl_delay is not None), None
    )
    return Robots(host=host, disallows=disallows, crawl_delay=crawl_delay, status=status)


def _default_fetcher(url: str) -> tuple[int, bytes]:
    from app.services.collectors.fetch import USER_AGENT  # lazy: see module docstring

    request = urllib.request.Request(
        url, headers={"User-Agent": USER_AGENT, "Accept": "text/plain, */*;q=0.5"}
    )
    try:
        with urllib.request.urlopen(request, timeout=READ_TIMEOUT) as response:
            return response.status, response.read()
    except urllib.error.HTTPError as exc:
        return exc.code, exc.read() if exc.fp else b""


_cache: dict[str, tuple[float, Robots]] = {}


def read(base_url: str, *, fetcher: Fetcher | None = None, fresh: bool = True) -> Robots:
    """Read and evaluate a host's robots.txt under the three-outcome policy."""
    from app.services.collectors.fetch import SourceBlocked  # lazy: see module docstring

    host = base_url.rstrip("/")
    if not fresh:
        hit = _cache.get(host)
        if hit and time.monotonic() - hit[0] < CACHE_TTL_SECONDS:
            return hit[1]
    url = f"{host}/robots.txt"
    try:
        status, body = (fetcher or _default_fetcher)(url)
    except (urllib.error.URLError, TimeoutError, OSError) as exc:
        raise RobotsUnavailable(f"{url} unreadable ({exc}); run stops, retry next run") from exc

    if status in (401, 403):
        raise SourceBlocked(f"{url} answered HTTP {status}: the host refuses us")
    if status == 429 or status >= 500:
        raise RobotsUnavailable(f"{url} answered HTTP {status}; run stops, retry next run")
    if status >= 400:
        logger.info(
            "robots_absent host=%s status=%d date=%s (proceeding unrestricted)",
            host, status, datetime.now(UTC).date().isoformat(),
        )
        robots = Robots.unrestricted(host, status)
    else:
        robots = parse(body.decode("utf-8", "replace"), host=host, status=status)
    _cache[host] = (time.monotonic(), robots)
    return robots


def check_allowed(
    base_url: str,
    paths: Iterable[str],
    *,
    fetcher: Fetcher | None = None,
    fresh: bool = True,
) -> Robots:
    """Open every collect() and read_one() with this.

    Reads the host's rules (fresh for a run, cached briefly for per-listing
    reads), refuses with SourceBlocked if any of `paths` is disallowed, and
    returns the rules so the caller can honour crawl-delay and check further
    URLs offline with `allows()`.
    """
    from app.services.collectors.fetch import SourceBlocked  # lazy: see module docstring

    robots = read(base_url, fetcher=fetcher, fresh=fresh)
    refused = robots.first_refusal(paths)
    if refused is not None:
        raise SourceBlocked(f"{robots.host} robots.txt disallows {refused}")
    return robots