Bot View API

For one URL, how every known crawler is treated by the site's robots.txt: search engines, AI training and AI search crawlers, assistants, AI agents, SEO tools, link previews and archives (274 today). Each verdict gives the rule and line that decided it. Around it: llms.txt, robots meta tags and X-Robots-Tag, TDMRep, firewall and what the raw HTML gives a crawler that runs no JavaScript.

Need the page's content? Use Scrape.

GEThttps://rankfield.dev/v1/bot-view?url={url}

Send Authorization: Bearer YOUR_API_KEY. Get a free key; no account needed.

curl "https://rankfield.dev/v1/bot-view?url=https://www.airbnb.com/&agents=GPTBot,ClaudeBot" \
  -H "Authorization: Bearer $RANKFIELD_API_KEY"

import os
import requests

response = requests.get(
    "https://rankfield.dev/v1/bot-view?url=https://www.airbnb.com/&agents=GPTBot,ClaudeBot",
    headers={"Authorization": f"Bearer {os.environ['RANKFIELD_API_KEY']}"},
)
print(response.json())

const response = await fetch("https://rankfield.dev/v1/bot-view?url=https://www.airbnb.com/&agents=GPTBot,ClaudeBot", {
  headers: {
    Authorization: `Bearer ${process.env.RANKFIELD_API_KEY}`,
  },
});
console.log(await response.json());

Response

{
  "url": "https://www.airbnb.com/",
  "agents": [
    "GPTBot",
    "ClaudeBot"
  ],
  "finalUrl": "https://www.airbnb.com/",
  "status": 200,
  "redirects": [],
  "summary": {
    "agents": 2,
    "allowed": 0,
    "blocked": 2,
    "blockedSite": 2,
    "purposes": {
      "aiTraining": {
        "agents": 2,
        "allowed": 0,
        "blocked": 2,
        "blockedSite": 2
      }
    },
    "robotsTxt": "found",
    "pageChecked": true
  },
  "companies": [
    {
      "company": "Anthropic",
      "agents": [
        {
          "token": "ClaudeBot",
          "matchedAs": "ClaudeBot",
          "purpose": "aiTraining",
          "controlToken": false,
          "respectsRobots": "yes",
          "known": true,
          "allowed": false,
          "page": {
            "allowed": false,
            "group": "ClaudeBot",
            "groupKind": "named",
            "rule": "Disallow: /",
            "line": 695,
            "note": null
          },
          "site": {
            "allowed": false,
            "group": "ClaudeBot",
            "groupKind": "named",
            "rule": "Disallow: /",
            "line": 695,
            "note": null
          },
          "finalUrl": null,
          "crawlDelay": null,
          "directives": [],
          "docs": "https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
          "note": null
        }
      ]
    },
    {
      "company": "OpenAI",
      "agents": [
        {
          "token": "GPTBot",
          "matchedAs": "GPTBot",
          "purpose": "aiTraining",
          "controlToken": false,
          "respectsRobots": "yes",
          "known": true,
          "allowed": false,
          "page": {
            "allowed": false,
            "group": "GPTBot",
            "groupKind": "named",
            "rule": "Disallow: /",
            "line": 691,
            "note": null
          },
          "site": {
            "allowed": false,
            "group": "GPTBot",
            "groupKind": "named",
            "rule": "Disallow: /",
            "line": 691,
            "note": null
          },
          "finalUrl": null,
          "crawlDelay": null,
          "directives": [],
          "docs": "https://developers.openai.com/api/docs/bots",
          "note": null
        }
      ]
    }
  ],
  "robotsTxt": {
    "url": "https://www.airbnb.com/robots.txt",
    "status": 200,
    "outcome": "found",
    "reason": null,
    "fetchedAt": "2026-10-25T14:51:40Z",
    "bytes": 20930,
    "truncated": false,
    "groups": 18,
    "rules": 711,
    "sitemaps": [
      "https://www.airbnb.com/sitemap-master-index.xml.gz"
    ],
    "contentSignals": [],
    "problems": [],
    "final": null
  },
  "llmsTxt": {
    "url": "https://www.airbnb.com/llms.txt",
    "finalUrl": "https://www.airbnb.com/404",
    "status": 404,
    "present": false,
    "valid": null,
    "servedHtml": true,
    "contentType": "text/html;charset=utf-8",
    "bytes": 2947,
    "truncated": false,
    "title": null,
    "summary": null,
    "sections": [],
    "links": 0,
    "problems": [],
    "note": null
  },
  "llmsFullTxt": {
    "url": "https://www.airbnb.com/llms-full.txt",
    "finalUrl": "https://www.airbnb.com/404",
    "status": 404,
    "present": false,
    "valid": null,
    "servedHtml": true,
    "contentType": "text/html;charset=utf-8",
    "bytes": 2947,
    "truncated": false,
    "title": null,
    "words": null,
    "problems": [],
    "note": null
  },
  "pageDirectives": {
    "checked": true,
    "note": null,
    "metaRobots": [],
    "botMetas": [
      {
        "name": "google",
        "content": "notranslate",
        "directives": [
          "notranslate"
        ],
        "agents": []
      }
    ],
    "xRobotsTag": [],
    "all": {
      "noindex": false,
      "nofollow": false,
      "noarchive": false,
      "nosnippet": false,
      "noimageindex": false,
      "notranslate": false,
      "noai": false,
      "noimageai": false,
      "maxSnippet": null,
      "maxImagePreview": null,
      "maxVideoPreview": null,
      "unavailableAfter": null
    }
  },
  "tdm": {
    "reserved": null,
    "source": null,
    "policy": null,
    "policySource": null,
    "problems": [],
    "headers": {
      "checked": true,
      "reservation": null,
      "policy": null
    },
    "meta": {
      "checked": true,
      "reservation": null,
      "policy": null
    },
    "tdmrepJson": {
      "url": "https://www.airbnb.com/.well-known/tdmrep.json",
      "status": 404,
      "present": false,
      "valid": null,
      "rules": null,
      "matchedRule": null,
      "problems": [],
      "note": null
    },
    "note": null
  },
  "firewall": {
    "vendor": null,
    "challenged": false,
    "kind": null,
    "signals": []
  },
  "rawHtml": {
    "checked": true,
    "note": null,
    "bytes": 664490,
    "truncated": false,
    "visibleWords": 510,
    "mainTextWords": 426,
    "mainTextMode": "article",
    "noscriptWords": 14,
    "scripts": {
      "inline": 13,
      "external": 40,
      "inlineBytes": 322856
    },
    "jsShell": false,
    "jsShellSignals": [
      "a <noscript> message asks for JavaScript",
      "322 KB of inline script against 3582 bytes of text"
    ],
    "frameworks": [],
    "generator": null
  },
  "registry": {
    "version": "2026-10-24",
    "agents": 274
  },
  "seenAs": {
    "agent": "Rankfield-User",
    "userAgent": "Mozilla/5.0 (compatible; Rankfield-User/1.0; +https://rankfield.dev/bot)",
    "page": {
      "allowed": true,
      "group": "*",
      "groupKind": "wildcard",
      "rule": null,
      "line": null,
      "note": null
    },
    "pageFetched": true
  },
  "cached": false,
  "ageSeconds": 0
}

Parameters#

Name	Type	Description
`url`query, required	string	The page, a full `http(s)` URL.
`agents`query, optional	stringdefault `all`	Comma-separated: `all`, `ai`, a purpose (`aiTraining`, `aiSearch`, `userFetch`, `aiAgent`, `search`, `seo`, `linkPreview`, `archive` …), a company (`google`, `openai` …) or crawler names (`GPTBot`, `bingbot` …). Up to 50 unknown names, checked as given.
`rendered`query, optional	booleandefault `false`	`true` also renders the page and compares it with the raw HTML: what changes when a crawler runs its JavaScript. It's slower.
`maxAge`query, optional	integerdefault `600`	Seconds a stored answer may be old (default 10 minutes), up to 2,592,000. `0` reads the site fresh.
`timeout`query, optional	integerdefault `20`	Seconds the call may take, 1 to 60.

Response#

Field	Type	Description
`url` `agents` `finalUrl` `status` `redirects`	mixed	What you asked, and the page that answered (`null` when we did not fetch it).
`summary`	object	Agents checked, `allowed`, `blocked` on this page, `blockedSite` on `/`, the same per purpose, whether robots.txt was `found`, `missing` or `unreachable`, and `pageChecked`.
`companies`	array	`[{company, agents}]`. Each agent: `token`, `matchedAs`, `purpose`, `controlToken`, `respectsRobots`, `known`, `allowed`, the `page`, `site` and `finalUrl` verdicts `{allowed, group, groupKind, rule, line, note}`, `crawlDelay`, `directives`, `docs`, `note`.
`robotsTxt`	object	The file: status, groups, rules, sitemaps, `contentSignals` (e.g. `ai-train: no`) and `problems` by line.
`llmsTxt` `llmsFullTxt`	object	`present`, `valid` (llmstxt.org), sections, links and problems.
`pageDirectives` `tdm`	object	Robots meta and X-Robots-Tag per crawler and for all (`noindex`, `noai` …); TDMRep reservation and its source.
`firewall` `rawHtml`	object	How the site answered us (e.g. Cloudflare `challenge`); visible words without JavaScript, `jsShell`, frameworks.
`registry` `seenAs`	object	The crawler list's version and size; our own verdict as `Rankfield-User`.
`rendered`	object	With `rendered=true`: `headline` (what changes, in one sentence), `diff` (words, links, title, canonical, description, meta robots, hreflang, structured data, H1) and `robotsBlockedResources`.
`cached` `ageSeconds`	boolean, integer	Whether the answer came from our store, and its age.

Errors#

Errors come as {"error":{"code":"…","message":"…","details":{…}}} with the HTTP status.

Status	Code	When
`400`	`invalid_input`	A parameter is missing, unknown or malformed.
`401`	`unauthorized`	The key is missing, unknown or revoked.
`403`	`blocked`	The website's bot protection refused us. Reported at once and never worked around; `details` says which protection. Not counted as usage.
`403`	`site_excluded`	The site is on the exclusion list: Reddit, LinkedIn, Facebook, Instagram, X/Twitter and TikTok (subdomains included), Google search result pages, and sites whose owners asked us.
`422`	`address_not_public`	The host resolves to a private or internal address.
`429`	`busy`	Our fetch capacity is busy: the call was refused at once, waited 4 seconds without a turn, or its time limit ran out while it waited (then `details.reason` is `capacity`). A new job also gets it when the shared job queue is full (`Retry-After` 60). Not counted as usage. Retry after `Retry-After` seconds.
`500`	`internal`	Our bug. Retry.
`502`	`fetch_failed`	The website could not be reached (DNS, connection, TLS, more than 5 redirects or a bad one, a malformed answer); `details.reason` says which.
`503`	`unavailable`	The service is restarting or briefly unavailable. Retry after `Retry-After`. (A full fetch capacity answers `429` `busy`.)
`504`	`timeout`	The website did not answer within `timeout`. With `details.reason: "sitePace"`, the call waited its turn for this website instead, which we fetch gently, shared by all customers. Retry after `Retry-After`, or allow a longer `timeout`.

Price and limits#

Free; no per-call charge.
No per-key limits. When the server is busy you get 429 busy with Retry-After.
Only answered calls count as usage (plus the website errors listed below); our own errors (500, 503), 429 busy and a 504 with details.reason sitePace don't.

Up to 10,000 URLs in one job#

POST /v1/bot-view/jobs with an Idempotency-Key header and {"items":[{"url":"https://example.com/"}, …]}. Each item takes the same parameters as one call, except timeout: items use the default, 20 seconds. Every item is checked first: if any is invalid, the whole request is a 400 that lists the first 20. A new job answers 202; the same Idempotency-Key with a byte-identical body returns the same job (200, Idempotency-Replayed: true), and with any other body is a 409.

Read GET /v1/bot-view/jobs/{id}/results?cursor=0&limit=100: finished items as they complete, each {index, ok, result} or {index, ok: false, error}. Pass cursor back until hasMore is false. POST /v1/bot-view/jobs/{id}/cancel stops a job. A job runs 16 items at a time; an item that meets busy waits and retries by itself. Results are kept 24 hours.

Send as many jobs as you like. Jobs from all keys share one queue: when 200,000 items are already waiting, a new job answers 429 busy with Retry-After (60 seconds).

Reference: https://rankfield.dev/docs/get-started/batch-jobs

Notes#

The verdicts are what the site's robots.txt tells each crawler. We fetch only as ourselves, so a firewall that blocks one bot but not another is not visible here; firewall shows how the site answered us. A robots.txt that disallows us is part of the answer, not an error.
The crawler list comes from ai.robots.txt and the vendors' own crawler pages; registry.version dates it.
We fetch as Rankfield-User, obey robots.txt on every request and redirect, run no JavaScript unless the call asks for it, use no cookies or proxies, and never work around a firewall. We fetch each site gently, shared by all customers. More: rankfield.dev/bot.