> ## Documentation index
> Fetch the complete documentation index at: https://rankfield.dev/llms.txt
> Use this file to discover all available pages before exploring further.

# Check how crawlers see a page

> See which of 274 search, AI and SEO crawlers a page's robots.txt allows, with the deciding rule, plus llms.txt, meta robots and TDM signals.

`GET https://rankfield.dev/v1/bot-view?url={url}`

```bash cURL
curl 'https://rankfield.dev/v1/bot-view?url=https://www.airbnb.com/' \
  -H 'Authorization: Bearer YOUR_API_KEY'
```

```python Python
import requests

response = requests.get(
    "https://rankfield.dev/v1/bot-view?url=https://www.airbnb.com/",
    headers={"Authorization": "Bearer YOUR_API_KEY"},
)
print(response.json())
```

```javascript JavaScript
const response = await fetch("https://rankfield.dev/v1/bot-view?url=https://www.airbnb.com/", {
  headers: {
    "Authorization": "Bearer YOUR_API_KEY",
  },
});
console.log(await response.json());
```

```json 200
{
  "url": "https://www.airbnb.com/",
  "agents": [
    "all"
  ],
  "finalUrl": "https://www.airbnb.com/",
  "status": 200,
  "redirects": [],
  "summary": {
    "agents": 274,
    "allowed": 265,
    "blocked": 9,
    "blockedSite": 9,
    "purposes": {
      "aiTraining": {
        "agents": 44,
        "allowed": 38,
        "blocked": 6,
        "blockedSite": 6
      },
      "aiSearch": {
        "agents": 30,
        "allowed": 30,
        "blocked": 0,
        "blockedSite": 0
      },
      "search": {
        "agents": 29,
        "allowed": 29,
        "blocked": 0,
        "blockedSite": 0
      }
    },
    "robotsTxt": "found",
    "pageChecked": true
  },
  "companies": [
    {
      "company": "OpenAI",
      "agents": [
        {
          "token": "GPTBot",
          "matchedAs": "GPTBot",
          "purpose": "aiTraining",
          "controlToken": false,
          "respectsRobots": "yes",
          "known": true,
          "allowed": false,
          "page": {
            "allowed": false,
            "group": "GPTBot",
            "groupKind": "named",
            "rule": "Disallow: /",
            "line": 691,
            "note": null
          },
          "site": {
            "allowed": false,
            "group": "GPTBot",
            "groupKind": "named",
            "rule": "Disallow: /",
            "line": 691,
            "note": null
          },
          "finalUrl": null,
          "crawlDelay": null,
          "directives": [],
          "docs": "https://developers.openai.com/api/docs/bots",
          "note": null
        },
        {
          "token": "OAI-SearchBot",
          "matchedAs": "OAI-SearchBot",
          "purpose": "aiSearch",
          "controlToken": false,
          "respectsRobots": "yes",
          "known": true,
          "allowed": true,
          "page": {
            "allowed": true,
            "group": "OAI-SearchBot",
            "groupKind": "named",
            "rule": null,
            "line": null,
            "note": null
          },
          "site": {
            "allowed": true,
            "group": "OAI-SearchBot",
            "groupKind": "named",
            "rule": null,
            "line": null,
            "note": null
          },
          "finalUrl": null,
          "crawlDelay": null,
          "directives": [],
          "docs": "https://developers.openai.com/api/docs/bots",
          "note": null
        }
      ]
    }
  ],
  "robotsTxt": {
    "url": "https://www.airbnb.com/robots.txt",
    "status": 200,
    "outcome": "found",
    "reason": null,
    "fetchedAt": "2026-10-24T17:15:30Z",
    "bytes": 20930,
    "truncated": false,
    "groups": 18,
    "rules": 711,
    "sitemaps": [
      "https://www.airbnb.com/sitemap-master-index.xml.gz"
    ],
    "contentSignals": [],
    "problems": [],
    "final": null
  },
  "llmsTxt": {
    "url": "https://www.airbnb.com/llms.txt",
    "finalUrl": "https://www.airbnb.com/404",
    "status": 404,
    "present": false,
    "valid": null,
    "servedHtml": true,
    "contentType": "text/html;charset=utf-8",
    "bytes": 2947,
    "truncated": false,
    "title": null,
    "summary": null,
    "sections": [],
    "links": 0,
    "problems": [],
    "note": null
  },
  "llmsFullTxt": {
    "url": "https://www.airbnb.com/llms-full.txt",
    "finalUrl": "https://www.airbnb.com/404",
    "status": 404,
    "present": false,
    "valid": null,
    "servedHtml": true,
    "contentType": "text/html;charset=utf-8",
    "bytes": 2947,
    "truncated": false,
    "title": null,
    "words": null,
    "problems": [],
    "note": null
  },
  "pageDirectives": {
    "checked": true,
    "note": null,
    "metaRobots": [],
    "botMetas": [
      {
        "name": "google",
        "content": "notranslate",
        "directives": [
          "notranslate"
        ],
        "agents": []
      }
    ],
    "xRobotsTag": [],
    "all": {
      "noindex": false,
      "nofollow": false,
      "noarchive": false,
      "nosnippet": false,
      "noimageindex": false,
      "notranslate": false,
      "noai": false,
      "noimageai": false,
      "maxSnippet": null,
      "maxImagePreview": null,
      "maxVideoPreview": null,
      "unavailableAfter": null
    }
  },
  "tdm": {
    "reserved": null,
    "source": null,
    "policy": null,
    "policySource": null,
    "problems": [],
    "headers": {
      "checked": true,
      "reservation": null,
      "policy": null
    },
    "meta": {
      "checked": true,
      "reservation": null,
      "policy": null
    },
    "tdmrepJson": {
      "url": "https://www.airbnb.com/.well-known/tdmrep.json",
      "status": 404,
      "present": false,
      "valid": null,
      "rules": null,
      "matchedRule": null,
      "problems": [],
      "note": null
    },
    "note": null
  },
  "firewall": {
    "vendor": null,
    "challenged": false,
    "kind": null,
    "signals": []
  },
  "rawHtml": {
    "checked": true,
    "note": null,
    "bytes": 664934,
    "truncated": false,
    "visibleWords": 511,
    "mainTextWords": 426,
    "mainTextMode": "article",
    "noscriptWords": 14,
    "scripts": {
      "inline": 13,
      "external": 40,
      "inlineBytes": 321685
    },
    "jsShell": false,
    "jsShellSignals": [
      "a <noscript> message asks for JavaScript",
      "321 KB of inline script against 3583 bytes of text"
    ],
    "frameworks": [],
    "generator": null
  },
  "registry": {
    "version": "2026-10-24",
    "agents": 274
  },
  "seenAs": {
    "agent": "Rankfield-User",
    "userAgent": "Mozilla/5.0 (compatible; Rankfield-User/1.0; +https://rankfield.dev/bot)",
    "page": {
      "allowed": true,
      "group": "*",
      "groupKind": "wildcard",
      "rule": null,
      "line": null,
      "note": null
    },
    "pageFetched": true
  },
  "cached": false,
  "ageSeconds": 0
}
```

```json 401
{
  "error": {
    "code": "unauthorized",
    "message": "A valid API key is required"
  }
}
```

For one URL, the robots.txt verdict for every known crawler (search engines, AI training and AI search crawlers, assistants that fetch for a user, AI agents, SEO tools, link previews, archives and ad checkers), with the rule and line that decided it, grouped by company and summed up per purpose. Around it: robots.txt itself, llms.txt and llms-full.txt, the page's robots meta tags and X-Robots-Tag headers, TDMRep reservations, whether a firewall answered, and what the raw HTML gives a crawler that runs no JavaScript.

Send the parameters as a query string with `GET`, or as a JSON object (up to 1 MB) with `POST` to the same path. For many URLs, [start a job](/docs/bot-view/start-a-job).

## Parameters

### Headers

| Name | Type | Required | Default | Description |
|---|---|---|---|---|
| `Authorization` | string | Yes | – | `Bearer` followed by your API key. See [Authentication](/docs/get-started/authentication). |

### Query parameters

| Name | Type | Required | Default | Description |
|---|---|---|---|---|
| `url` | string | Yes | – | The page, a full `http(s)` URL. |
| `agents` | string | No | `all` | Which crawlers to report, comma-separated: `all`; `ai` (every agent on the ai.robots.txt list); a purpose (`aiTraining`, `aiSearch`, `userFetch`, `aiAgent`, `dataProvider`, `search`, `ads`, `seo`, `linkPreview`, `archive`, `undocumented`); a company (`google`, `openai`…); or crawler names (`GPTBot`, `bingbot`…). Other robots.txt names are checked as given with `known: false`, up to 50. Case is ignored. |
| `rendered` | boolean | No | `false` | `true` also renders the page and compares it with the raw HTML. It's slower. See [Before and after JavaScript](#before-and-after-javascript). |
| `maxAge` | integer | No | `600` | How old a stored answer may be, in seconds, up to 2,592,000 (30 days). `0` fetches fresh. |
| `timeout` | integer | No | `20` | How long the call may take, in seconds, 1–60. |

## Response

| Field | Type | Description |
|---|---|---|
| `url`, `agents` | string, string[] | What you asked (`url` without its `#fragment`). |
| `finalUrl`, `status`, `redirects` |  | The page that answered. `null` and `[]` when the page was not fetched. |
| `summary` | object | `agents`, `allowed`, `blocked` (this page), `blockedSite` (on `/`), the same per purpose in `purposes`, and `robotsTxt`: `found`, `missing` or `unreachable`. |
| `companies` | array | `{company, agents}`. Each agent: `token`, `matchedAs`, `purpose`, `respectsRobots` (what the vendor says), `known`, `allowed`, the `page` and `site` verdicts, `crawlDelay`, `directives`, `docs` and `note`. |
| `page`, `site`, `finalUrl (verdicts)` | object | `{allowed, group, groupKind, rule, line, note}`. `groupKind`: `named`, `fallback` (such as Googlebot for Googlebot-Image), `wildcard` or `none`. |
| `robotsTxt` | object | `url`, `status`, `outcome`, `bytes`, how many `groups` and `rules`, `sitemaps`, `contentSignals` (such as `ai-train: no`) and `problems` by line. |
| `llmsTxt`, `llmsFullTxt` | object | Whether each file is there and valid by llmstxt.org: `present`, `valid`, `servedHtml` (the path answers a web page), `title`, `sections`, `links` or `words`, `problems`. |
| `pageDirectives` | object | `metaRobots`, `botMetas` (metas named after a crawler), `xRobotsTag`, and `all`: what applies to every crawler (`noindex`, `nofollow`, `noai`, `maxSnippet`…). |
| `tdm` | object | TDMRep: `reserved` (true, false or `null`), `source`, `policy`, and what each place said. |
| `firewall` | object \| null | `{vendor, challenged, kind, signals}` for the page we fetched. |
| `rawHtml` | object | `visibleWords` and `mainTextWords` without JavaScript, `jsShell` (an empty app shell) with `jsShellSignals`, `frameworks` and `generator`. |
| `registry` | object | `version` (date) and `agents`: the crawler list in use. |
| `seenAs` | object | Our own user agent and verdict, and whether we fetched the page. |
| `rendered` | object | With `rendered=true`: `headline`, `diff` and `robotsBlockedResources`, as in [Before and after JavaScript](#before-and-after-javascript). |
| `cached`, `ageSeconds` | boolean, integer | Whether this answer came from our store, and how old it is. |

## What the verdicts can and cannot tell you

The verdicts are what the site's robots.txt tells each crawler. We fetch only as ourselves, so we cannot see how a firewall or CDN treats each crawler: a site can block GPTBot at its firewall while its robots.txt allows it. `firewall` shows only how the site answered us.

robots.txt names may only hold letters, `-` and `_`, and parsers (Google's too) read `User-agent: MJ12bot` as `MJ`. Crawlers with digits, dots or spaces in their names are matched by that prefix; `matchedAs` shows it.

The crawler list is built from ai.robots.txt (MIT) and the vendors' own crawler pages; each agent links its vendor's docs.

## Before and after JavaScript

Googlebot runs a page's JavaScript; most AI crawlers don't. With `rendered=true`:

- `rendered.headline` says in one sentence what changes. For example: "Before rendering a crawler sees 0 words and 0 links; after rendering, 1,240 words and 38 links."
- `rendered.diff` has the details: words, links, title, canonical, description, meta robots, hreflang, structured data and H1.
- `rendered.robotsBlockedResources` lists the scripts and files robots.txt keeps from crawlers.

## How we fetch

We fetch as `Rankfield-User` (`Mozilla/5.0 (compatible; Rankfield-User/1.0; +https://rankfield.dev/bot)`), as plain HTML without running JavaScript, unless you ask for a rendered page or a screenshot.

- **robots.txt:** obeyed on every request and redirect, following the rules for `Rankfield-User`, or for `RankfieldBot` when the file has none for it, or `*` when it has neither. Requests: robots.txt, the page, then `/llms.txt`, `/llms-full.txt` and `/.well-known/tdmrep.json`: 5, or 6 with a redirect's new robots.txt; redirects add their hops. A robots.txt that answers with a server error counts as closed to us, as the standard says.
- **Firewalls:** a challenge, CAPTCHA or block page is reported in the answer, never worked around and never retried.
- **Never:** cookies, logins, proxies or other bots' names. JavaScript runs only when you ask for it.
- **Excluded sites:** Reddit, LinkedIn, Facebook, Instagram, X/Twitter and TikTok (subdomains included), Google search result pages, and sites whose owners asked us are never fetched.
- **Pace:** each website is fetched gently, shared by all customers, slower when its robots.txt sets a `Crawl-delay` or after a slow answer. Many URLs on one site go at that site's pace.

Repeat calls to the same website within about a minute can wait their turn for that site's slot. If the wait runs past the call's time limit (20 seconds by default), the call ends with `504` `timeout` and `details.reason` `sitePace`. It doesn't count as usage: retry after `Retry-After`, or allow a longer `timeout`.

More: [Rankfield crawlers](/bot).

## Errors

Errors are `{"error": {"code", "message", "details"}}`. Branch on `code`; `message` is for people.

| Status | Code | When |
|---|---|---|
| 400 | `invalid_input` | A parameter is missing, unknown or malformed. |
| 401 | `unauthorized` | No valid API key. |
| 403 | `site_excluded` | The site is on the exclusion list. |
| 403 | `blocked` | The website's bot protection refused us. Reported at once and never worked around; `details` says which protection. Not counted as usage. |
| 422 | `address_not_public` | The host resolves to a private or internal address. |
| 429 | `busy` | Our fetch capacity is busy: the call was refused at once, waited 4 seconds without a turn, or its time limit ran out while it waited (then `details.reason` is `capacity`). Not counted as usage. Retry after `Retry-After` seconds. |
| 500 | `internal` | Our bug. Retry. |
| 502 | `fetch_failed` | The site could not be reached at all (its robots.txt did not answer). A page that fails after robots.txt answered still gives the verdicts. |
| 503 | `unavailable` | The service is restarting or briefly unavailable. Retry after `Retry-After`. (A full fetch capacity answers `429` `busy`.) |
| 504 | `timeout` | The website didn't answer within the call's time limit. When `details.reason` is `sitePace`, the call waited its turn for this website instead, which we fetch gently, shared by all customers: retry after `Retry-After`, or allow a longer `timeout`. |

A robots.txt that disallows us is not an error here: the verdicts still come back, and the page checks say they were not checked. Only answered calls and the `502` and website-side `504` errors count as usage; refusals, our own errors (`500`, `503`), `429` `busy` and a `504` with `details.reason` `sitePace` don't.

## Limits

No per-key limits. When the server is busy you get `429` `busy` with `Retry-After`. Waiting calls are served in turn across keys, so a key with a few calls is never stuck behind one with thousands.

## MCP

On the [MCP server](/docs/agents/mcp-server), the tool for this is `bot_view`.