The Architecture of Web Scraping & Bot Governance: Robots Exclusion Protocol, Crawl-Delay & AI Agents

+---------------------------------------------------------------------------------------------------+
|                        CHRONOLOGICAL EVOLUTION OF THE ROBOTS EXCLUSION PROTOCOL                   |
+---------------------------------------------------------------------------------------------------+
|  1994: Martijn Koster Consensus Draft ("A Method for Web Robots Control")                         |
|  * Disallow directives | User-agent matching | Informal convention with zero formal RFC status    |
+---------------------------------------------------------------------------------------------------+
                                                  |
                                                  v
+---------------------------------------------------------------------------------------------------+
|  1996: Internet Draft (Koster) Expired without ratification                                       |
|  * Crawlers implement custom extensions (Allow, Crawl-delay, Sitemap)                             |
+---------------------------------------------------------------------------------------------------+
                                                  |
                                                  v
+---------------------------------------------------------------------------------------------------+
|  2008: Google & Major Search Engines Standardize Wildcards (* and $)                              |
|  * Path prefix matching and end-of-URL anchors introduced                                         |
+---------------------------------------------------------------------------------------------------+
                                                  |
                                                  v
+---------------------------------------------------------------------------------------------------+
|  2022: IETF Ratifies RFC 9309 (Robots Exclusion Protocol - Proposed Standard)                      |
|  * Authored by Martijn Koster, Gary Illyes, Henner Zeller, and Lizzi Sassman                      |
|  * Formally defines ABNF syntax, UTF-8 parsing, 500 KiB file limits, and 24-hour cache TTLs       |
+---------------------------------------------------------------------------------------------------+

+---------------------------------------------------------------------------------------------------+
|                        RFC 9309 ROBOTS.TXT FILE STRUCTURAL ANATOMY                                |
+---------------------------------------------------------------------------------------------------+
|  # Group 1: General search engines and fallback crawlers                                          |
|  User-agent: *                                                                                    |
|  Disallow: /admin/                                                                                |
|  Disallow: /api/private/                                                                          |
|  Allow: /api/private/public-feed.json                                                             |
|                                                                                                   |
|  # Group 2: AI Foundation Model Scrapers (Strict Disallow)                                       |
|  User-agent: GPTBot                                                                               |
|  User-agent: ClaudeBot                                                                            |
|  User-agent: CCBot                                                                                |
|  Disallow: /                                                                                      |
|                                                                                                   |
|  # Global Directives (Not tied to record groups)                                                  |
|  Sitemap: https://innoxtools.com/sitemap.xml                                                      |
+---------------------------------------------------------------------------------------------------+

+---------------------------------------------------------------------------------------------------+
|                        LONGEST-MATCH PRECEDENCE RESOLUTION EXAMPLES                               |
+---------------------------------------------------------------------------------------------------+
|  Rule Set:                                                                                        |
|  User-agent: *                                                                                    |
|  Disallow: /catalog/                                 [Length: 9 characters]                       |
|  Allow: /catalog/public/                             [Length: 16 characters]                      |
|  Disallow: /catalog/public/confidential.pdf          [Length: 32 characters]                      |
+---------------------------------------------------------------------------------------------------+
|  Target URL                                 Matching Rules               Winning Rule   Verdict   |
+---------------------------------------------------------------------------------------------------+
|  /catalog/items.html                        Disallow: /catalog/          Length: 9      BLOCKED   |
|  /catalog/public/item.html                  Disallow (9), Allow (16)     Allow (16)     ALLOWED   |
|  /catalog/public/confidential.pdf           All 3 rules match            Disallow (32)  BLOCKED   |
|                                                                                                   |
|  Equal Length Conflict Case:                                                                      |
|  Disallow: /data                                     [Length: 5 characters]                       |
|  Allow: /data                                        [Length: 5 characters]                       |
|  Target: /data/file.csv                     Equal length (5 vs 5)        ALLOW WINS     ALLOWED   |
+---------------------------------------------------------------------------------------------------+

+---------------------------------------------------------------------------------------------------+
|                        ROBOTS.TXT PATH MATCHING SPECIFICATION TABLE                               |
+---------------------------------------------------------------------------------------------------+
|  Directive Pattern          Target URL Path                          Match Result   Access State  |
+---------------------------------------------------------------------------------------------------+
|  Disallow: /finance         https://example.com/finance              MATCH          BLOCKED       |
|  Disallow: /finance         https://example.com/finances             MATCH          BLOCKED       |
|  Disallow: /finance/        https://example.com/finances             NO MATCH       ALLOWED       |
|  Disallow: /finance/        https://example.com/finance/ledger.xlsx  MATCH          BLOCKED       |
|  Disallow: /*?sort=         https://example.com/shop?sort=price      MATCH          BLOCKED       |
|  Disallow: /*?sort=         https://example.com/shop?color=red       NO MATCH       ALLOWED       |
|  Disallow: /*.gif$          https://example.com/img/cat.gif          MATCH          BLOCKED       |
|  Disallow: /*.gif$          https://example.com/img/cat.gif?v=2      NO MATCH       ALLOWED       |
+---------------------------------------------------------------------------------------------------+

+---------------------------------------------------------------------------------------------------+
|                        RFC 9309 USER-AGENT DISPATCH RESOLUTION                                    |
+---------------------------------------------------------------------------------------------------+
|  Crawler Identity: "Googlebot-Image/2.1"                                                          |
+---------------------------------------------------------------------------------------------------+
                                                  |
                                                  v
+---------------------------------------------------------------------------------------------------+
|  Step 1: Specific Token Search                                                                    |
|  * Does robots.txt contain a group with "User-agent: Googlebot-Image"?                            |
|    -> IF YES: Select this group exclusively! IGNORE all other groups (including Googlebot & *).    |
+---------------------------------------------------------------------------------------------------+
                                                  | (If No)
                                                  v
+---------------------------------------------------------------------------------------------------+
|  Step 2: Parent Token Search                                                                      |
|  * Does robots.txt contain a group with "User-agent: Googlebot"?                                  |
|    -> IF YES: Select this group exclusively! IGNORE the wildcard (*) group.                       |
+---------------------------------------------------------------------------------------------------+
                                                  | (If No)
                                                  v
+---------------------------------------------------------------------------------------------------+
|  Step 3: Wildcard Fallback Search                                                                 |
|  * Does robots.txt contain a group with "User-agent: *"?                                          |
|    -> IF YES: Apply the wildcard group rules.                                                     |
|    -> IF NO: Complete unrestricted access is granted (Allow All)!                                 |
+---------------------------------------------------------------------------------------------------+

+---------------------------------------------------------------------------------------------------+
|                        THE DUAL REALITY OF MODERN BOT SEGMENTATION                                |
+---------------------------------------------------------------------------------------------------+
|                                  INCOMING WEB CRAWLERS                                            |
+---------------------------------------------------------------------------------------------------+
                                                  |
                         +------------------------+------------------------+
                         |                                                 |
                         v                                                 v
+---------------------------------------------------+     +-----------------------------------------+
|            ORGANIC SEARCH SPIDERS                 |     |           AI TRAINING HARVESTERS        |
|  * Googlebot, Bingbot, YandexBot, Baiduspider     |     |  * GPTBot, ClaudeBot, CCBot, Bytespider |
|  * Value Exchange: Server load for traffic        |     |  * Value Exchange: Zero traffic return  |
|  * Policy: PERMIT / OPTIMIZE                      |     |  * Policy: SELECTIVELY BLOCK            |
+---------------------------------------------------+     +-----------------------------------------+

# ==============================================================================
# INNOXTOOLS ENTERPRISE BOT GOVERNANCE POLICY (RFC 9309 COMPLIANT)
# ==============================================================================

# SECTION 1: Permitted Search Engine Spiders (Full Access for SEO)
User-agent: Googlebot
User-agent: Bingbot
User-agent: Slurp
Disallow: /admin/
Disallow: /api/internal/
Disallow: /profile/
Allow: /

# SECTION 2: AI Foundation Training Harvesters (Complete Block)
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: CCBot
User-agent: Bytespider
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Diffbot
Disallow: /

# SECTION 3: Real-Time Interactive AI Search Engines (Permit for Citations)
User-agent: ChatGPT-User
User-agent: PerplexityBot
Disallow: /admin/
Disallow: /api/internal/
Allow: /

# SECTION 4: Default Fallback for All Other Crawlers
User-agent: *
Disallow: /admin/
Disallow: /api/internal/
Disallow: /search?*
Allow: /

# SECTION 5: Global XML Sitemap Index
Sitemap: https://innoxtools.com/sitemap.xml

+---------------------------------------------------------------------------------------------------+
|                        CRAWL-DELAY INTERPRETATION IN BINGBOT VS GOOGLEBOT                         |
+---------------------------------------------------------------------------------------------------+
|  Directive in robots.txt:                                                                         |
|  User-agent: Bingbot                                                                              |
|  Crawl-delay: 5                                                                                   |
+---------------------------------------------------------------------------------------------------+
|  * Bingbot Response: Pauses 5 seconds between request N and request N+1 -> CRAWL RATE REDUCED!   |
|                                                                                                   |
|  User-agent: Googlebot                                                                            |
|  Crawl-delay: 5                                                                                   |
+---------------------------------------------------------------------------------------------------+
|  * Googlebot Response: Ignores line completely! Crawls at server capacity -> ZERO EFFECT!        |
+---------------------------------------------------------------------------------------------------+

   Sitemap: https://innoxtools.com/sitemaps/pages.xml
   Sitemap: https://innoxtools.com/sitemaps/tools.xml
   Sitemap: https://innoxtools.com/sitemaps/blog.xml
   

+---------------------------------------------------------------------------------------------------+
|                        CRAWL BUDGET DYNAMICS & LIMITING FACTORS                                   |
+---------------------------------------------------------------------------------------------------+
|  1. Crawl Demand (How much Google WANT to crawl):                                                 |
|     * Driven by URL popularity, PageRank authority, update frequency, and index staleness.        |
+---------------------------------------------------------------------------------------------------+
                                                  x
+---------------------------------------------------------------------------------------------------+
|  2. Crawl Capacity Rate (How much your server CAN handle):                                        |
|     * Driven by server response latency (TTFB), HTTP error rates (5xx/429), and hardware limits.  |
+---------------------------------------------------------------------------------------------------+
                                                  |
                                                  v
+---------------------------------------------------------------------------------------------------+
|  NET EFFECTIVE CRAWL BUDGET (Pages indexed per day without degrading human user experience)       |
+---------------------------------------------------------------------------------------------------+

HTTP/1.1 429 Too Many Requests
Content-Type: text/plain; charset=utf-8
Retry-After: 120
Connection: close

Rate limit exceeded. Crawlers must back off for 120 seconds.

+---------------------------------------------------------------------------------------------------+
|                        ROBOTS.TXT HTTP STATUS CODE DECISION MATRIX                                |
+---------------------------------------------------------------------------------------------------+
|  HTTP Status Code                   RFC 9309 Crawler Interpretation          Resulting Access     |
+---------------------------------------------------------------------------------------------------+
|  200 OK                             Parse file contents normally             Follow Directives    |
|  3xx Redirection (up to 5 hops)     Follow redirect to target URI            Parse Final File     |
|  404 Not Found / 410 Gone           No robots.txt exists                     COMPLETE ACCESS (ALL)|
|  401 Unauthorized / 403 Forbidden   Access to robots.txt explicitly denied   COMPLETE BLOCK (NONE)|
|  5xx Server Errors (500, 502, 503)  Server is unhealthy                      COMPLETE BLOCK (NONE)|
|  Unreachable / Connection Timeout   Network failure                          COMPLETE BLOCK (NONE)|
+---------------------------------------------------------------------------------------------------+

+---------------------------------------------------------------------------------------------------+
|                        MULTI-LAYER BOT DEFENSE INFRASTRUCTURE                                     |
+---------------------------------------------------------------------------------------------------+
|  Layer 1: Advisory Layer (robots.txt)                                                             |
|  * Voluntary governance for compliant crawlers (Googlebot, GPTBot, ClaudeBot)                     |
+---------------------------------------------------------------------------------------------------+
                                                  |
                                                  v
+---------------------------------------------------------------------------------------------------+
|  Layer 2: Edge CDN / WAF Enforcement (Cloudflare, AWS WAF, Fastly)                                |
|  * Verified Bot Signatures (Reverse DNS lookup & cryptographic TLS fingerprinting)                |
|  * ASN & Datacenter IP Range Blocking (OVH, Hetzner, DigitalOcean scraping farms)                 |
+---------------------------------------------------------------------------------------------------+
                                                  |
                                                  v
+---------------------------------------------------------------------------------------------------+
|  Layer 3: Reverse Proxy Rate Limiting (Nginx / Envoy)                                             |
|  * Token bucket rate limiters per IP / subnet: limit_req_zone $binary_remote_addr                 |
+---------------------------------------------------------------------------------------------------+
                                                  |
                                                  v
+---------------------------------------------------------------------------------------------------+
|  Layer 4: In-App Honeypot Traps & Dynamic IP Blacklisting                                         |
|  * Invisible hidden links (display:none) disallowed in robots.txt                                 |
|  * Any client requesting the honeypot is instantaneously firewalled!                              |
+---------------------------------------------------------------------------------------------------+

   User-agent: *
   Disallow: /trap/crawler-honeypot.html
   

   <!-- Invisible to human users, visible only to DOM scrapers -->
   <a href="/trap/crawler-honeypot.html" style="display:none;" rel="nofollow" aria-hidden="true">Index</a>
   

export interface RobotsRule {
  type: 'allow' | 'disallow';
  pattern: string;
  regex: RegExp;
  length: number;
}

export interface UserAgentGroup {
  userAgents: string[];
  rules: RobotsRule[];
}

export interface EvaluationResult {
  allowed: boolean;
  matchedRule: RobotsRule | null;
  matchedUserAgent: string;
  reason: string;
}

export class Rfc9309RobotsParser {
  private groups: UserAgentGroup[] = [];
  private sitemaps: string[] = [];

  constructor(robotsTxtContent: string) {
    this.parse(robotsTxtContent);
  }

  private parse(content: string): void {
    // RFC 9309: Truncate at 500 KiB (512,000 bytes)
    const truncatedContent = content.slice(0, 512000);
    const lines = truncatedContent.split(/
|
|
/);

    let currentUserAgents: string[] = [];
    let currentRules: RobotsRule[] = [];

    for (let rawLine of lines) {
      // 1. Strip comments (#) and trim whitespace
      const commentIndex = rawLine.indexOf('#');
      if (commentIndex !== -1) {
        rawLine = rawLine.slice(0, commentIndex);
      }
      const line = rawLine.trim();
      if (!line) continue;

      const colonIndex = line.indexOf(':');
      if (colonIndex === -1) continue;

      const field = line.slice(0, colonIndex).trim().toLowerCase();
      const value = line.slice(colonIndex + 1).trim();

      if (field === 'user-agent') {
        // If we transition from rules back to user-agents, flush previous group
        if (currentRules.length > 0) {
          if (currentUserAgents.length > 0) {
            this.groups.push({ userAgents: currentUserAgents, rules: currentRules });
          }
          currentUserAgents = [];
          currentRules = [];
        }
        currentUserAgents.push(value.toLowerCase());
      } else if (field === 'allow' || field === 'disallow') {
        if (!value && field === 'disallow') {
          // Empty Disallow: equals allow all
          continue;
        }
        currentRules.push({
          type: field as 'allow' | 'disallow',
          pattern: value,
          regex: this.compilePatternToRegex(value),
          length: value.length
        });
      } else if (field === 'sitemap') {
        this.sitemaps.push(value);
      }
    }

    // Flush final group
    if (currentUserAgents.length > 0) {
      this.groups.push({ userAgents: currentUserAgents, rules: currentRules });
    }
  }

  /**
   * Compiles RFC 9309 glob wildcards (*) and end anchors ($) into standard RegExp.
   */
  private compilePatternToRegex(pattern: string): RegExp {
    let escaped = pattern.replace(/[-[\]{}()+?.,\^$|#\s]/g, '\$&');
    escaped = escaped.replace(/\\*/g, '.*');
    if (escaped.endsWith('\$')) {
      escaped = escaped.slice(0, -2) + '$';
    }
    return new RegExp('^' + escaped);
  }

  /**
   * Evaluates whether a given userAgent is permitted to crawl targetPath.
   */
  public isAllowed(userAgent: string, targetPath: string): EvaluationResult {
    const ua = userAgent.toLowerCase();

    // 1. Find matching group via RFC 9309 specificity hierarchy
    let targetGroup: UserAgentGroup | null = null;
    let matchedUA = '*';

    // Check specific user agent tokens first
    for (const group of this.groups) {
      for (const groupUA of group.userAgents) {
        if (groupUA !== '*' && ua.includes(groupUA)) {
          targetGroup = group;
          matchedUA = groupUA;
          break;
        }
      }
      if (targetGroup) break;
    }

    // Fallback to wildcard (*) group if no specific match found
    if (!targetGroup) {
      targetGroup = this.groups.find(g => g.userAgents.includes('*')) ?? null;
    }

    // If no group matches, default access is ALLOW
    if (!targetGroup || targetGroup.rules.length === 0) {
      return { allowed: true, matchedRule: null, matchedUserAgent: matchedUA, reason: 'No matching disallow rules.' };
    }

    // 2. Evaluate all matching rules according to LONGEST MATCH WINS
    let bestMatch: RobotsRule | null = null;

    for (const rule of targetGroup.rules) {
      if (rule.regex.test(targetPath)) {
        if (!bestMatch || rule.length > bestMatch.length) {
          bestMatch = rule;
        } else if (rule.length === bestMatch.length) {
          // RFC 9309: Equal length conflict -> ALLOW takes precedence!
          if (rule.type === 'allow') {
            bestMatch = rule;
          }
        }
      }
    }

    if (!bestMatch) {
      return { allowed: true, matchedRule: null, matchedUserAgent: matchedUA, reason: 'Path matched no explicit rules.' };
    }

    return {
      allowed: bestMatch.type === 'allow',
      matchedRule: bestMatch,
      matchedUserAgent: matchedUA,
      reason: Matched ${bestMatch.type.toUpperCase()} directive: "${bestMatch.pattern}" (Length: ${bestMatch.length})
    };
  }

  public getSitemaps(): string[] {
    return this.sitemaps;
  }
}