from documents.models import Document, Category
from collections import Counter, defaultdict
import re

print(f"=== {SITE} ===")
docs = Document.objects.select_related("category").all()
print(f"  total: {docs.count()} | with secondary file: {docs.exclude(secondary_file='').exclude(secondary_file=None).count()}")

print("  categories:")
for c in Category.objects.all():
    print(f"    {c.name:28} {c.documents.count():4}")
print(f"    {'(uncategorised)':28} {docs.filter(category=None).count():4}")

# same-day clusters = natural batches
byday = defaultdict(list)
for d in docs:
    if d.published_date:
        byday[d.published_date].append(d.title)
clusters = {k: v for k, v in byday.items() if len(v) > 1}
print(f"  same-date clusters (candidate batches): {len(clusters)} covering {sum(len(v) for v in clusters.values())} docs")
for day in sorted(clusters, reverse=True)[:6]:
    print(f"    {day} ({len(clusters[day])}):")
    for t in clusters[day][:5]:
        print(f"       - {t[:64]}")

# keyword shape of titles
kw = Counter()
for d in docs:
    t = d.title.lower()
    for k in ["agm","minute","financial","budget","insurance","bio","resume","cv","newsletter",
              "bylaw","capex","statement","meeting","presentation","report","proxy","strata",
              "board","committee","policy","certificate","audit","reserve","notice","agenda"]:
        if k in t:
            kw[k] += 1
print("  title keywords:", dict(kw.most_common(12)))