"""Our own robots.txt: what we tell other readers, by name.

The collectors' `robots.py` is how we READ a host's rules for our named bot;
this is the same policy in reverse. Groups are addressed to bots by name,
because that is what a well-behaved bot honours (it is what ours honours), and
the three Content Signals say the same thing in the vocabulary Cloudflare and
the AI crawlers have agreed on, so a reader that understands either gets one
consistent answer.

The stance is data, in one place: flip `CONTENT_SIGNALS["ai-train"]` and the
training-only groups below switch from Disallow to Allow with it. Search and
answer engines are always welcome: the site exists to be read by them.
"""

from dataclasses import dataclass

# Content Signals Policy (contentsignals.org, served by Cloudflare's managed
# robots.txt when a zone opts in): search = build an index and link here;
# ai-input = use the page as input to an answer (grounding, RAG); ai-train =
# train or fine-tune a model on it. The site is a price comparison meant to be
# cited live, so search and ai-input are yes; training gives nothing back and
# is the client's call to open later.
CONTENT_SIGNALS: dict[str, bool] = {"search": True, "ai-input": True, "ai-train": False}

# Paths no bot should crawl: JSON the pages read for themselves. The client
# and owner surfaces stay crawlable but noindexed (a disallowed URL can still
# be indexed by its address; a crawled noindex cannot). The same holds for every
# generated page a person has not approved for indexing (`noindex, follow`, plan
# W18): never Disallow them here, or the flag is never read and the links on
# them to the approved pages are never followed.
DISALLOW_ALL = ("/api/",)


@dataclass(frozen=True)
class BotRule:
    agent: str
    purpose: str
    #: The content signals this bot's crawl serves; it is allowed iff any of them is on.
    signals: tuple[str, ...]


# Bots whose crawl serves a named signal, so their access follows the switch.
# Search and user-initiated fetchers (Googlebot, Bingbot, ChatGPT-User,
# OAI-SearchBot, Claude-User, Claude-SearchBot, PerplexityBot, Applebot) fall
# under "*" and need no group of their own. A token that also switches off
# grounding (Google-Extended governs Gemini's grounding as well as training)
# lists both signals, so it stays open while ai-input is yes.
NAMED_BOTS: tuple[BotRule, ...] = (
    BotRule("GPTBot", "OpenAI model training", ("ai-train",)),
    BotRule("ClaudeBot", "Anthropic model training", ("ai-train",)),
    BotRule("Google-Extended", "Gemini training and grounding", ("ai-train", "ai-input")),
    BotRule("Applebot-Extended", "Apple foundation model training", ("ai-train",)),
    BotRule("CCBot", "Common Crawl corpus, used for training", ("ai-train",)),
    BotRule("Bytespider", "ByteDance model training", ("ai-train",)),
    BotRule("meta-externalagent", "Meta model training and indexing", ("ai-train", "ai-input")),
    BotRule("cohere-ai", "Cohere model training", ("ai-train",)),
)


def content_signal_line() -> str:
    return "Content-Signal: " + ", ".join(
        f"{name}={'yes' if on else 'no'}" for name, on in CONTENT_SIGNALS.items()
    )


def robots_txt(base: str) -> str:
    """The whole file. `base` is the site origin for the Sitemap line (empty
    outside production before PUBLIC_BASE_URL is set, in which case the line
    is left out rather than pointing at a header-derived host)."""
    lines = [
        "# Duty Free Professor: dated duty-free price observations, meant to be read.",
        "# Our own reader is DutyFreeProfessorBot (https://bot.dutyfreeprofessor.com).",
        "",
        "User-agent: *",
        content_signal_line(),
        "Allow: /",
        *[f"Disallow: {path}" for path in DISALLOW_ALL],
    ]
    for bot in NAMED_BOTS:
        allowed = any(CONTENT_SIGNALS.get(signal, False) for signal in bot.signals)
        lines += [
            "",
            f"# {bot.purpose}",
            f"User-agent: {bot.agent}",
            "Allow: /" if allowed else "Disallow: /",
        ]
        if allowed:
            lines += [f"Disallow: {path}" for path in DISALLOW_ALL]
    if base:
        lines += ["", f"Sitemap: {base}/sitemap.xml"]
    return "\n".join(lines) + "\n"