"""The page routes over the built shell: 404s, HEAD, conditional GETs, cache headers.
Until 2026-09-04 an unknown product id or a mistyped path returned 200 with the
default head (so crawlers filed junk addresses as pages), HEAD returned 405, and
nothing carried an ETag or a Cache-Control that would stop a CDN from caching
an owner's edit. No database: `get_db` is overridden and the head builders are
stubbed with the same fixture the renderer tests use.
"""
from datetime import UTC, datetime
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from app import main
from app.db import get_db
from app.routers import catalog
from app.services import catalog_queries, editorial, feeds, seo, urls
from test_crawl_surface import facts
from test_seo_airport import airport, summary
from test_seo_article import article
from test_seo_brand import brand, brand_summary
from test_seo_body import detail
SHELL = (
"t"
''
''
)
NEWEST = datetime(2026, 9, 1, 12, 0, tzinfo=UTC)
@pytest.fixture
def client(tmp_path, monkeypatch):
(tmp_path / "assets").mkdir()
(tmp_path / "medals").mkdir()
(tmp_path / "medals" / "nyisc-double-gold.png").write_bytes(b"png")
(tmp_path / "index.html").write_text(SHELL)
(tmp_path / "logo.png").write_bytes(b"png")
fixture = detail()
def fake_product_head(db, product_id, medal_assets=()):
return seo.head_for_detail(fixture, medal_assets) if product_id == fixture.id else None
def fake_airport_head(db, iata, medal_assets=(), flags=None, *, category=None, sort="featured",
page=1, limit=24, tab=None):
shelf = urls.airport_tab(tab, category, sort, page)
if iata == "JFK":
return seo.head_for_airport(
airport(iata="JFK", path="/airports/jfk-new-york", name="New York JFK", city="New York",
category=category, sort=sort, offset=(page - 1) * limit),
medal_assets, flags, shelf,
)
if iata != "LHR":
return None
return seo.head_for_airport(
airport(category=category, sort=sort, offset=(page - 1) * limit), medal_assets, flags, shelf
)
def fake_brand_head(db, slug, medal_assets=(), flags=None, *, category=None, sort="featured", page=1, limit=24):
# "jw" stands in for an alias row: it resolves to the house, whose path differs.
if slug not in ("johnnie-walker", "jw"):
return None
return seo.head_for_brand(
brand(category=category, sort=sort, offset=(page - 1) * limit), medal_assets, flags
)
def fake_article_head(db, slug, flags=None):
return seo.head_for_article(article(), flags) if slug == "how-to-read-a-duty-free-price" else None
monkeypatch.setattr(seo, "product_head", fake_product_head)
monkeypatch.setattr(seo, "airport_head", fake_airport_head)
monkeypatch.setattr(seo, "brand_head", fake_brand_head)
monkeypatch.setattr(seo, "article_head", fake_article_head)
monkeypatch.setattr(catalog_queries, "dataset_facts", lambda db: facts())
monkeypatch.setattr(feeds, "feed_items", lambda db: [feeds.product_item(summary(), NEWEST)])
monkeypatch.setattr(main.settings, "indexnow_key", "abc123key")
monkeypatch.setattr(seo, "sitemap_entries", lambda db, base, include_my_airports=True: (
f"{base}/", NEWEST))
monkeypatch.setattr(main.settings, "public_base_url", "https://s.example")
monkeypatch.setattr(main.settings, "site_access", "public") # the crawl surface as the public sees it
monkeypatch.setattr(main.settings, "site_access", "public") # the crawl surface as the public sees it
app = FastAPI()
app.middleware("http")(main.cache_headers)
app.dependency_overrides[get_db] = lambda: iter([None])
main.mount_site(app, tmp_path)
return TestClient(app)
class TestNotFound:
def test_unknown_product_id_is_404_with_noindex(self, client):
r = client.get("/products/some-bottle-999")
assert r.status_code == 404
assert 'content="noindex, nofollow"' in r.text
assert '' in r.text # the shell, so the SPA draws its own not-found page
def test_slug_without_an_id_is_404(self, client):
assert client.get("/products/just-words").status_code == 404
def test_unknown_route_is_404(self, client):
assert client.get("/nope").status_code == 404
assert client.get("/feature/").status_code == 404
def test_known_routes_are_200(self, client):
for path in ("/", "/products", "/airports", "/trip", "/feature/alerts", "/settings/"):
assert client.get(path).status_code == 200, path
def test_missing_file_is_a_plain_404(self, client):
assert client.get("/missing.xml").status_code == 404
def test_real_file_is_served(self, client):
assert client.get("/logo.png").status_code == 200
class TestProductPage:
def test_stale_slug_redirects_to_canonical(self, client):
r = client.get("/products/1156", follow_redirects=False)
assert r.status_code == 301
assert r.headers["location"] == "/products/santa-teresa-1796-ron-antiguo-de-solera-70cl-1156"
def test_body_and_absolute_canonical(self, client):
r = client.get("/products/santa-teresa-1796-ron-antiguo-de-solera-70cl-1156")
assert r.status_code == 200
assert "All products' in r.text
assert '' in r.text
# The filtered view canonicalises to the clean page; garbage paging is page one.
assert 'href="https://s.example/airports/heathrow-lhr-london"' in r.text
assert '' in client.get("/airports/heathrow-lhr-london?page=abc&sort=bogus").text
def test_head_and_conditional_get(self, client):
r = client.head("/airports/heathrow-lhr-london")
assert r.status_code == 200 and r.content == b"" and r.headers["etag"].startswith('W/"')
first = client.get("/airports/heathrow-lhr-london")
assert client.get("/airports/heathrow-lhr-london", headers={"If-None-Match": first.headers["etag"]}).status_code == 304
def test_api_404s_a_code_that_is_not_an_airport(self, monkeypatch):
monkeypatch.setattr(catalog_queries, "airport_detail", lambda db, iata, **kw: None)
api = FastAPI()
api.include_router(catalog.router)
api.dependency_overrides[get_db] = lambda: iter([None])
client = TestClient(api)
assert client.get("/api/airports/LHR").status_code == 404
assert client.get("/api/airports/london").status_code == 404
assert client.get("/api/airports/LHR?sort=bogus").status_code == 422
class TestBrandPage:
def test_body_canonical_and_seed(self, client):
r = client.get("/brands/johnnie-walker")
assert r.status_code == 200
assert "Johnnie Walker in duty free" in r.text
assert 'href="https://s.example/brands/johnnie-walker"' in r.text
assert '"@id": "https://s.example/brands/johnnie-walker#brand"' in r.text
assert "window.__DFP_BRAND__ = " in r.text
assert r.headers["last-modified"] == "Tue, 25 Aug 2026 23:51:00 GMT"
assert client.head("/brands/johnnie-walker").status_code == 200
def test_alias_and_case_redirect_to_the_house_keeping_the_query(self, client):
r = client.get("/brands/jw?sort=price", follow_redirects=False)
assert r.status_code == 301 and r.headers["location"] == "/brands/johnnie-walker?sort=price"
r = client.get("/brands/Johnnie-Walker", follow_redirects=False)
assert r.status_code == 301 and r.headers["location"] == "/brands/johnnie-walker"
def test_unknown_or_thin_brand_is_a_404(self, client):
r = client.get("/brands/campbeltown")
assert r.status_code == 404 and 'content="noindex, nofollow"' in r.text
assert client.get("/brands/johnnie-walker/").status_code == 404
def test_filters_and_paging(self, client):
r = client.get("/brands/johnnie-walker?category=Whisky&page=2&sort=bogus")
assert r.status_code == 200
assert "Johnnie Walker whisky" in r.text
assert '' not in r.text # 21 fit on one page: no pager
assert 'href="https://s.example/brands/johnnie-walker"' in r.text
def test_api_404s_an_unknown_brand(self, monkeypatch):
monkeypatch.setattr(catalog_queries, "brand_detail", lambda db, slug, **kw: None)
api = FastAPI()
api.include_router(catalog.router)
api.dependency_overrides[get_db] = lambda: iter([None])
client = TestClient(api)
assert client.get("/api/brands/nope").status_code == 404
assert client.get("/api/brands/nope?sort=bogus").status_code == 422
class TestArticlePage:
"""Stream D's pages behind B's routes: the index is a static head over the SPA,
the piece has a route of its own, and a draft or unknown slug is a real 404."""
def test_shell_carries_the_social_profiles_beside_the_flags(self, client):
r = client.get("/articles")
assert "window.__DFP_SOCIAL__ = {}" in r.text # nothing configured in tests: the SPA draws no icon
def test_index_is_a_known_route_with_its_head(self, client):
r = client.get("/articles")
assert r.status_code == 200
assert "Articles | Duty Free Professor" in r.text
assert 'href="https://s.example/articles"' in r.text
assert '' in r.text # the list itself is the SPA's
def test_piece_body_canonical_seed_and_head(self, client):
r = client.get("/articles/how-to-read-a-duty-free-price")
assert r.status_code == 200
assert 'How to read a duty-free price
' in r.text
assert "destination" in r.text
assert 'href="https://s.example/articles/how-to-read-a-duty-free-price"' in r.text
assert "window.__DFP_ARTICLE__ = " in r.text
assert '"@type": "Article"' in r.text
assert r.headers["last-modified"] == "Thu, 03 Sep 2026 16:00:00 GMT"
assert r.headers["cache-control"] == "no-store"
assert client.head("/articles/how-to-read-a-duty-free-price").status_code == 200
assert client.get("/articles/how-to-read-a-duty-free-price", headers={"If-None-Match": r.headers["etag"]}).status_code == 304
def test_draft_or_unknown_slug_is_a_404(self, client):
r = client.get("/articles/not-published-yet")
assert r.status_code == 404 and 'content="noindex, nofollow"' in r.text
assert client.get("/articles/how-to-read-a-duty-free-price/").status_code == 404
assert client.get("/articles/" + "x" * 161).status_code == 404
class TestCrawlSurface:
"""robots.txt by bot, the feed, llms.txt, the IndexNow key file and the data
page, each with HEAD and conditional GET like every other machine-read URL."""
def test_robots_carries_the_per_bot_policy_and_signals(self, client):
r = client.get("/robots.txt")
assert r.status_code == 200 and r.headers["content-type"].startswith("text/plain")
assert "Content-Signal: search=yes, ai-input=yes, ai-train=no" in r.text
assert "User-agent: GPTBot\nDisallow: /" in r.text
assert "Disallow: /api/" in r.text
assert "Sitemap: https://s.example/sitemap.xml" in r.text
def test_feed_is_rss_with_absolute_links_and_the_newest_item_date(self, client):
r = client.get("/feed.xml")
assert r.status_code == 200 and r.headers["content-type"] == "application/rss+xml"
assert "https://s.example/products/santa-teresa-1796-solera-rum-1l-1156" in r.text
assert r.headers["last-modified"] == "Tue, 01 Sep 2026 12:00:00 GMT"
assert client.head("/feed.xml").status_code == 200
assert client.get("/feed.xml", headers={"If-None-Match": r.headers["etag"]}).status_code == 304
def test_llms_txt_is_markdown_from_the_facts(self, client):
r = client.get("/llms.txt")
assert r.status_code == 200 and r.headers["content-type"].startswith("text/markdown")
assert r.text.startswith("# Duty Free Professor")
assert "(https://s.example/airports/heathrow-lhr-london): 266 products" in r.text
assert r.headers["last-modified"] == "Tue, 25 Aug 2026 23:51:00 GMT"
def test_indexnow_key_file_is_served_only_for_the_configured_key(self, client):
r = client.get("/abc123key.txt")
assert r.status_code == 200 and r.text == "abc123key"
assert client.get("/otherkey.txt").status_code == 404
def test_data_page_has_the_body_and_the_dataset_markup(self, client):
r = client.get("/data")
assert r.status_code == 200
assert "Price observations2,315" in r.text
assert 'href="https://s.example/data"' in r.text
assert "window.__DFP_DATASET__ = " in r.text
assert r.headers["cache-control"] == "no-store"
assert client.head("/data").status_code == 200
def test_every_head_links_the_feed(self, client):
assert 'type="application/rss+xml" title="Duty Free Professor: new in duty free" href="https://s.example/feed.xml"' in client.get("/").text
class TestSitemap:
def test_hub_pages_and_products_carry_their_newest_observation(self, monkeypatch):
lhr = airport()
monkeypatch.setattr(catalog_queries, "list_airports", lambda db: [lhr])
monkeypatch.setattr(catalog_queries, "list_brands", lambda db: [brand_summary()])
monkeypatch.setattr(seo, "_product_sitemap_rows", lambda db: [(1156, "Santa Teresa 1796", NEWEST)])
monkeypatch.setattr(editorial, "sitemap_rows", lambda db: []) # articles: test_seo_article.py
xml, newest = seo.sitemap_entries(None, "https://s.example", include_my_airports=False)
assert "https://s.example/airports/heathrow-lhr-london2026-08-25" in xml
assert "https://s.example/brands/johnnie-walker2026-08-25" in xml
assert "https://s.example/products/santa-teresa-1796-11562026-09-01" in xml
assert "https://s.example/airports" in xml
assert "https://s.example/data" in xml
assert "/savings" not in xml
assert newest == NEWEST
class TestHeadAndConditional:
def test_head_returns_headers_only(self, client):
r = client.head("/products/santa-teresa-1796-ron-antiguo-de-solera-70cl-1156")
assert r.status_code == 200
assert r.headers["etag"].startswith('W/"')
assert r.content == b""
assert client.head("/robots.txt").status_code == 200
assert client.head("/sitemap.xml").status_code == 200
assert client.head("/").status_code == 200
def test_if_none_match_gets_304(self, client):
first = client.get("/")
r = client.get("/", headers={"If-None-Match": first.headers["etag"]})
assert r.status_code == 304
assert r.content == b""
assert r.headers["etag"] == first.headers["etag"]
def test_if_modified_since_gets_304_only_when_current(self, client):
path = "/sitemap.xml"
first = client.get(path)
assert first.headers["last-modified"] == "Tue, 01 Sep 2026 12:00:00 GMT"
assert client.get(path, headers={"If-Modified-Since": first.headers["last-modified"]}).status_code == 304
assert client.get(path, headers={"If-Modified-Since": "Mon, 31 Aug 2026 12:00:00 GMT"}).status_code == 200
def test_a_404_is_never_a_304(self, client):
first = client.get("/nope")
assert client.get("/nope", headers={"If-None-Match": first.headers["etag"]}).status_code == 404
class TestCacheControl:
def test_html_is_no_store(self, client):
assert client.get("/").headers["cache-control"] == "no-store"
assert client.get("/products/santa-teresa-1796-ron-antiguo-de-solera-70cl-1156").headers["cache-control"] == "no-store"
def test_json_is_no_store(self, client):
client.app.get("/api/ping")(lambda: {"ok": True})
assert client.get("/api/ping").headers["cache-control"] == "no-store"
def test_assets_are_immutable(self, client):
assert client.get("/logo.png").headers.get("cache-control") is None
client.app.get("/assets/x.js")(lambda: "x")
assert "immutable" in client.get("/assets/x.js").headers["cache-control"]
def test_robots_names_the_configured_sitemap(self, client):
r = client.get("/robots.txt")
assert "Sitemap: https://s.example/sitemap.xml" in r.text
assert "etag" in r.headers and "last-modified" in r.headers
class TestMergedProducts:
"""Every merged product 301s to its survivor (Decision 6). Until migration #3
the resolver is the identity; these pin the consumers so A only has to fill
in the lookup."""
def test_page_route_redirects_a_merged_id_to_the_survivor(self, client, monkeypatch):
monkeypatch.setattr(catalog_queries, "resolve_product_id", lambda db, pid: 1156 if pid == 42 else pid)
r = client.get("/products/old-bottle-42", follow_redirects=False)
assert r.status_code == 301
assert r.headers["location"] == "/products/santa-teresa-1796-ron-antiguo-de-solera-70cl-1156"
def test_api_redirects_a_merged_id_and_keeps_the_query(self, monkeypatch):
monkeypatch.setattr(catalog_queries, "resolve_product_id", lambda db, pid: 1156 if pid == 42 else pid)
api = FastAPI()
api.include_router(catalog.router)
api.dependency_overrides[get_db] = lambda: iter([None])
client = TestClient(api)
r = client.get("/api/products/42", follow_redirects=False)
assert r.status_code == 301 and r.headers["location"] == "/api/products/1156"
r = client.get("/api/products/42/similar?limit=4&at=LHR", follow_redirects=False)
assert r.status_code == 301 and r.headers["location"] == "/api/products/1156/similar?limit=4&at=LHR"
def test_identity_resolver_is_a_pass_through_until_migration_3(self):
assert catalog_queries.resolve_product_id(None, 7) == 7