"""What we cover, what we do not, and how good each source's data is. Being straight about the gaps is a feature: several of the world's busiest airports publish no prices at all, and a shopper (or a client) is better served by seeing that plainly than by wondering why a search came back empty. """ from sqlalchemy import distinct, func, select from sqlalchemy.orm import Session from app.models import Listing, Shop, PriceObservation, ProductVariant, Retailer from app.models.schemas import CoverageShop, UncoveredAirport from app.services import place_guides from app.services.catalog_queries import publishable from app.services.urls import airport_path # Airports we cannot collect, and why. Kept here rather than in the page so the # reason travels with the data and stays reviewable. UNCOVERED: list[UncoveredAirport] = [ UncoveredAirport( name="Doha Hamad", iata="DOH", operator="Qatar Duty Free", reason="No public product catalogue at all; the site's checkout sells raffle tickets.", ), UncoveredAirport( name="Amsterdam Schiphol", iata="AMS", operator="Schiphol / See Buy Fly", reason="Schiphol's own site carries a product catalogue, but its bot protection turns automated readers away, and we treat a locked door as a no. One for a conversation, not a crawl.", ), UncoveredAirport( name="Istanbul", iata="IST", operator="Unifree / Heinemann", reason="Publishes how large its catalogue is, but withholds the product_variants.", ), UncoveredAirport( name="Seoul Incheon", iata="ICN", operator="Lotte / Shilla", reason="Lotte's crawling rules ask everyone but the big search engines to stay out, and we respect that. Shilla's store is open to crawlers and shows prices, so a collector there is under assessment.", ), UncoveredAirport( name="Singapore Changi (iShopChangi)", iata="SIN", operator="Changi Airport Group", reason="Welcomes crawlers (permissive robots, full product sitemaps with 30,000+ items including wine, spirits and beauty), but every page is built in the browser, so there is nothing in the page itself to read. A rendering step or a partnership would open one of the world's biggest airports.", ), UncoveredAirport( name="Duty Free Americas (a second operator at JFK, Miami and other US airports)", iata="MIA", operator="Duty Free Americas", reason="Their robots.txt asks crawlers not to browse any part of the site, so we do not. The Avolta shops at the same airports are open to us, which is how JFK is covered at all.", ), UncoveredAirport( name="Heinemann: its catalogue, Sydney, Keflavik and the border shop", iata="FRA", operator="Gebr. Heinemann", reason="Every site on Heinemann's platform publishes a crawling rule that covers its product listings, so we do not read them. This is the clearest partnership ask in the project: their catalogue carries the barcodes that make cross-shop matching work.", ), ] # Airports we could add without new code — the collector already handles their # platform and their robots.txt permits it. Kept separate from UNCOVERED because # "not configured yet" is a very different thing from "cannot be collected". AVAILABLE_NOT_YET_ADDED = ( "Avolta runs one platform across 168 airports and we collect a handful of them. " "The rest are a configuration list rather than new work." ) # A category-at-airport page exists when the pairing holds at least this many published # product variants (the structure proposal's threshold: "enough published product variants to be a real list", # "around a hundred" such pages). Measured on the 11 Sep nightly copy: 15 gives 96 pairs, 12 # gives 112, 20 gives 81. Read at request time by `catalog_queries.airport_category_detail` # and the sitemap, so a pair switches on the moment its coverage crosses the bar, beauty # included, with no code change. The number is rian's to move (a decide on the running list). CATEGORY_AT_AIRPORT_MIN_PRODUCTS = 15 def category_at_airport_on() -> bool: """Built in full, held back by decision: rian (13 Sep) is saving these pages for a later quote rather than delivering them unasked. Off, no pair qualifies, no page answers, the sitemap and the airport page's rail carry none; on is one environment line.""" from app.config import settings return bool(settings.feature_category_at_airport) def qualifying_pairs(db: Session, iata: str | None = None) -> list[tuple[str, str, int]]: """`(iata, category, product variants)` for every pairing at or over the bar, biggest first.""" from app.services.catalog_queries import category_pair_counts # lazy: that module imports this one's bar if not category_at_airport_on(): return [] return [ (code, category, n) for code, category, n in category_pair_counts(db, iata) if n >= CATEGORY_AT_AIRPORT_MIN_PRODUCTS ] def coverage(db: Session) -> list[CoverageShop]: # Hidden shops are absent here too: the coverage page describes the # site the visitor is on, and every card links to the airport's page, which # answers 404 for a shop verify has blocked. The fuller collected-but-not- # shown story belongs to the client review page, not this one. rows = db.execute( select( Shop, Retailer.name, func.count(distinct(Listing.variant_id)).label("product_variants"), func.count(distinct(ProductVariant.id)) .filter(ProductVariant.gtin.isnot(None)) .label("with_barcode"), func.max(PriceObservation.observed_at).label("last_collected_at"), ) .join(Retailer, Retailer.id == Shop.retailer_id) .outerjoin(Listing, Listing.shop_id == Shop.id) .outerjoin(ProductVariant, ProductVariant.id == Listing.variant_id) .outerjoin(PriceObservation, PriceObservation.listing_id == Listing.id) .where(publishable(db)) .group_by(Shop.id, Retailer.name) .order_by(func.count(distinct(Listing.variant_id)).desc()) ).all() return [ CoverageShop( code=shop.code, name=shop.name, iata=shop.iata, path=( airport_path(shop.iata, shop.city, shop.name) if shop.iata and not shop.is_catalogue_only else None ), city=shop.city, country=shop.country, currency=shop.currency, retailer_name=retailer_name, is_catalogue_only=shop.is_catalogue_only, product_variants=product_variants or 0, with_barcode=with_barcode or 0, last_collected_at=last_collected_at, has_guide=place_guides.has_guide(db, shop.iata), ) for shop, retailer_name, product_variants, with_barcode, last_collected_at in rows ]