# robots.txt — Olympia Food Co-op
# Created 2026-08-05. Replaces the Yoast virtual robots.txt (sitemap line preserved below).
#
# WHY THIS FILE EXISTS
# AI training crawlers were repeatedly downloading very large PDFs from
# /wp-content/uploads/ — several are 40-81 MB each, and ~333 MB/day was being
# served to them. Ordinary search engines are deliberately left fully allowed;
# only the bulk-download of media by AI scrapers is restricted.
#
# NOTE: robots.txt is voluntary. Well-behaved crawlers (GPTBot, ClaudeBot,
# Applebot, Bytespider) honour it. Bare HTTP clients such as axios/ do not,
# and must be handled at the server level if they persist.

# --- Search engines: fully allowed, nothing changed ---
User-agent: *
Disallow:

# --- AI training / dataset crawlers: keep off the heavy media library ---
User-agent: GPTBot
Disallow: /wp-content/uploads/

User-agent: ChatGPT-User
Disallow: /wp-content/uploads/

User-agent: OAI-SearchBot
Disallow: /wp-content/uploads/

User-agent: ClaudeBot
Disallow: /wp-content/uploads/

User-agent: Claude-Web
Disallow: /wp-content/uploads/

User-agent: anthropic-ai
Disallow: /wp-content/uploads/

User-agent: CCBot
Disallow: /wp-content/uploads/

User-agent: Bytespider
Disallow: /wp-content/uploads/

User-agent: Amazonbot
Disallow: /wp-content/uploads/

User-agent: Applebot-Extended
Disallow: /wp-content/uploads/

User-agent: PerplexityBot
Disallow: /wp-content/uploads/

User-agent: Google-Extended
Disallow: /wp-content/uploads/

User-agent: meta-externalagent
Disallow: /wp-content/uploads/

User-agent: Diffbot
Disallow: /wp-content/uploads/

User-agent: omgili
Disallow: /wp-content/uploads/

User-agent: Timpibot
Disallow: /wp-content/uploads/

# --- Admin and internals: no crawler needs these ---
User-agent: *
Disallow: /wp-admin/
Disallow: /dashlogin/
Disallow: /wp-content/cache/
Disallow: /wp-content/updraft/
Allow: /wp-admin/admin-ajax.php

Sitemap: https://olympiafood.coop/sitemap_index.xml