Developer Utilities Tier 2 Ready POST /v1/robots-llms-analyzer

robots.txt & llms.txt Analyzer API — Audit AI Bot Access & Generate llms.txt

Audit which AI crawlers your robots.txt allows and generate a valid llms.txt file. Analyze robots.txt for AI crawler permissions (GPTBot, ClaudeBot, PerplexityBot, Google-Extended), lint existing llms.txt files, and generate valid llms.txt.

Try It Live in Your Browser · No API Key Required

1. Interactive Sandbox & Sample JSON Output

200 OK · Ready

Editable Request JSON (`POST /v1/robots-llms-analyzer`)

Live 200 OK Response Envelope

{
  "ok": true,
  "data": {
    "test_path": "/",
    "ai_crawlers": [
      {
        "bot": "GPTBot",
        "vendor": "OpenAI",
        "purpose": "ai_training",
        "status": "blocked",
        "matched_agent": "GPTBot",
        "matched_rule": "Disallow: /"
      },
      {
        "bot": "OAI-SearchBot",
        "vendor": "OpenAI",
        "purpose": "ai_search",
        "status": "allowed",
        "matched_agent": "OAI-SearchBot",
        "matched_rule": "Allow: /"
      },
      {
        "bot": "ChatGPT-User",
        "vendor": "OpenAI",
        "purpose": "ai_search",
        "status": "allowed",
        "matched_agent": "*",
        "matched_rule": "Allow: /"
      },
      {
        "bot": "ClaudeBot",
        "vendor": "Anthropic",
        "purpose": "ai_training",
        "status": "allowed",
        "matched_agent": "*",
        "matched_rule": "Allow: /"
      },
      {
        "bot": "Claude-Web",
        "vendor": "Anthropic",
        "purpose": "ai_search",
        "status": "allowed",
        "matched_agent": "*",
        "matched_rule": "Allow: /"
      },
      {
        "bot": "Google-Extended",
        "vendor": "Google",
        "purpose": "ai_training",
        "status": "allowed",
        "matched_agent": "*",
        "matched_rule": "Allow: /"
      },
      {
        "bot": "PerplexityBot",
        "vendor": "Perplexity",
        "purpose": "ai_search",
        "status": "allowed",
        "matched_agent": "*",
        "matched_rule": "Allow: /"
      },
      {
        "bot": "CCBot",
        "vendor": "Common Crawl",
        "purpose": "ai_training",
        "status": "allowed",
        "matched_agent": "*",
        "matched_rule": "Allow: /"
      },
      {
        "bot": "Bytespider",
        "vendor": "ByteDance",
        "purpose": "ai_training",
        "status": "allowed",
        "matched_agent": "*",
        "matched_rule": "Allow: /"
      },
      {
        "bot": "Applebot-Extended",
        "vendor": "Apple",
        "purpose": "ai_training",
        "status": "allowed",
        "matched_agent": "*",
        "matched_rule": "Allow: /"
      },
      {
        "bot": "Meta-ExternalAgent",
        "vendor": "Meta",
        "purpose": "ai_training",
        "status": "allowed",
        "matched_agent": "*",
        "matched_rule": "Allow: /"
      },
      {
        "bot": "Amazonbot",
        "vendor": "Amazon",
        "purpose": "ai_search",
        "status": "allowed",
        "matched_agent": "*",
        "matched_rule": "Allow: /"
      }
    ],
    "summary": {
      "training_bots_allowed": 6,
      "training_bots_blocked": 1,
      "search_bots_allowed": 5,
      "search_bots_blocked": 0,
      "sitemaps": []
    },
    "llms_txt_analysis": {
      "present": false,
      "valid": false,
      "title": null,
      "summary": null,
      "section_count": 0,
      "link_count": 0,
      "issues": [
        "No llms.txt content provided."
      ]
    },
    "generated_robots_ai_snippet": "# --- AI Crawler Policy (Generated by Tinplate) ---\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: OAI-SearchBot\nAllow: /\n\nUser-agent: ChatGPT-User\nAllow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: Claude-Web\nAllow: /\n\nUser-agent: Google-Extended\nDisallow: /\n\nUser-agent: PerplexityBot\nAllow: /\n\nUser-agent: CCBot\nDisallow: /\n\nUser-agent: Bytespider\nDisallow: /\n\nUser-agent: Applebot-Extended\nDisallow: /\n\nUser-agent: Meta-ExternalAgent\nDisallow: /\n\nUser-agent: Amazonbot\nAllow: /",
    "generated_llms_txt": null
  },
  "meta": {
    "request_id": "req_rla_01",
    "ms": 1
  }
}

Get API Key on RapidAPI Get API Key on Zyla Hub Download OpenAPI 3.0.3 Spec

2. Quickstart Code Examples

Full TypeScript & Python SDK →

curl -X POST "https://api.tinplate.dev/v1/robots-llms-analyzer" \
  -H "Content-Type: application/json" \
  -H "X-RapidAPI-Key: YOUR_RAPIDAPI_KEY" \
  -d '{"robots_txt":"User-agent: *\nAllow: /\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: OAI-SearchBot\nAllow: /"}'

import { TinplateClient } from "@tinplate/sdk"; // npm install @tinplate/sdk

const tinplate = new TinplateClient({
  rapidApiKey: process.env.RAPIDAPI_KEY!,
});

const { data, meta } = await tinplate.call("/v1/robots-llms-analyzer", {
  "robots_txt": "User-agent: *\nAllow: /\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: OAI-SearchBot\nAllow: /"
});
console.log(`[${meta.ms} ms]`, data);

import requests

response = requests.post(
    "https://api.tinplate.dev/v1/robots-llms-analyzer",
    headers={
        "Content-Type": "application/json",
        "X-RapidAPI-Key": "YOUR_RAPIDAPI_KEY",
    },
    json={
  "robots_txt": "User-agent: *\nAllow: /\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: OAI-SearchBot\nAllow: /"
}
)
print(response.json())

const response = await fetch("https://api.tinplate.dev/v1/robots-llms-analyzer", {
  method: "POST",
  headers: {
    "Content-Type": "application/json",
    "X-RapidAPI-Key": "YOUR_RAPIDAPI_KEY",
  },
  body: JSON.stringify({
  "robots_txt": "User-agent: *\nAllow: /\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: OAI-SearchBot\nAllow: /"
}),
});
const result = await response.json();
console.log(result);

{
  "jsonrpc": "2.0",
  "id": 1,
  "method": "tools/call",
  "params": {
    "name": "tinplate_robots_llms_analyzer",
    "arguments": {
      "robots_txt": "User-agent: *\nAllow: /\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: OAI-SearchBot\nAllow: /"
    }
  }
}

OpenAPI 3.0.3 Schema Contract

Request Parameters & Response Schema Reference

Request Body Parameters (`POST /v1/robots-llms-analyzer`)

Parameter	Type & Constraints	Description
`robots_txt`optional	`string` maxLen: 131072	Raw robots.txt content to analyze.
`llms_txt`optional	`string` maxLen: 131072	Optional raw llms.txt content to validate against the llmstxt.org specification.
`test_path`optional	`string` maxLen: 512, default: "/"	URL path to test crawler permissions against (default '/').
`desired_ai_policy`optional	`"allow_all" \| "allow_search_block_training" \| "block_all_ai"` default: "allow_search_block_training"	Policy used to generate the recommended robots.txt AI crawler snippet.
`site_info`optional	`object`	Optional site metadata used to generate a valid llms.txt file.

Response `data` Object Fields (`200 OK`)

Field	Type	Description
`test_path`	`string`	Field 'test_path'
`ai_crawlers`	`object[]`	Field 'ai_crawlers'
`summary`	`object`	Field 'summary'
`llms_txt_analysis`	`object`	Field 'llms_txt_analysis'
`generated_robots_ai_snippet`	`string`	Field 'generated_robots_ai_snippet'
`generated_llms_txt`	`string \| null`	Field 'generated_llms_txt'

3. What robots.txt & llms.txt AI Crawler Analyzer & Generator API Detects & Validates

Evaluates robots.txt permissions across 18+ AI crawlers (GPTBot, OAI-SearchBot, ChatGPT-User, ClaudeBot, Google-Extended, PerplexityBot, CCBot, Applebot-Extended, Bytespider)
Separates AI model training bots from live AI search/citation bots
Generates ready-to-paste robots.txt blocks for 3 policies: allow_all, allow_search_block_training, or block_all_ai
Lints and generates standard llms.txt Markdown index files for LLM documentation discovery

Honest Engineering Boundaries

4. Known Limitations & Test Verification

Release Version: v1.0.7 · Last Verified / Benchmarked: 14 November 2026 · Latency Note: Sub-millisecond figures measure in-Worker V8 compute time (typical client HTTPS network RTT to the nearest Cloudflare PoP is 15–65 ms) · See Full Portfolio Benchmarks & Corpora →

Evaluates RFC 9309 robots.txt rules; respects robots.txt only for compliant crawlers (malicious scrapers require WAF rules).
Tested v1.0.4 (14 November 2026).

5. Marketplace Plans & Payload Quotas

Plan	Monthly Price	Included Requests	Rate Limit	Max Request Payload	Best For
BASIC	$0.00 / month	500 / month (hard limit)	10 req / min	64 KB	Evaluation, local development & CI smoke tests
PRO	$9.99 / month	15,000 / month	60 req / min	128 KB	Solo developers & production apps
ULTRA	$29.99 / month	100,000 / month	300 req / min	256 KB	Growing SaaS & multi-agent pipelines
MEGA	$79.99 / month	500,000 / month	1,000 req / min	256 KB	High-volume enterprise workloads

Subscribe on RapidAPI Subscribe on Zyla Hub Full Pricing Details

6. Frequently Asked Questions

How can I allow ChatGPT Search citations while blocking OpenAI model training?

Use policy preset 'allow_search_block_training', which disallows GPTBot and CCBot while allowing OAI-SearchBot and ChatGPT-User.

What is llms.txt?

llms.txt is a Markdown file at the root of a website (/llms.txt) that gives LLMs and coding agents a concise index of your documentation and APIs.

Does it validate existing llms.txt files?

Yes, pass llms_txt to lint the H1 title, summary blockquote, and Markdown link sections.

Zero Payload Logging

7. Data Handling, Privacy & Marketplace Proxy Disclosure

Stateless in-memory analysis.

Worker Zero-Logging Guarantee: Cloudflare Workers Observability and invocation logging are disabled in configuration (observability.enabled = false, head_sampling_rate = 0, invocation_logs = false). Payloads are processed strictly in volatile V8 memory and never logged or stored by Tinplate.

Marketplace Proxies vs. Direct Bearer Key: Requests sent via RapidAPI or Zyla API Hub pass through those marketplaces' proxy infrastructure (subject to their respective privacy policies). Security-sensitive workloads requiring end-to-end zero-intermediary processing can connect directly to https://api.tinplate.dev using a Direct Bearer Key (Authorization: Bearer <API_KEY>). Read our full Privacy Notice and OpenAPI 3.0.3 JSON Schema.