{"data":{"name":"dev.workers.quietworklabs.robots-check/robots-check","title":"Robots.txt & AI Crawler Checker","description":"robots.txt tester (RFC 9309), AI crawler access (GPTBot, ClaudeBot...) and sitemap check.","version":"1.0.0","websiteUrl":"https://robots-check.quietworklabs.workers.dev/","repositoryUrl":null,"remotes":[{"url":"https://robots-check.quietworklabs.workers.dev/mcp","type":"streamable-http"}],"packages":[],"status":"active","isLatest":true,"publishedAt":"2026-10-18T03:32:52.238Z","updatedAt":"2026-10-18T03:32:52.238Z","source":"official-registry","verification":"official","confidence":100,"stars":null,"sources":[{"stars":null,"title":"Robots.txt & AI Crawler Checker","source":"official-registry","sourceId":"dev.workers.quietworklabs.robots-check/robots-check","updatedAt":"2026-10-18T03:32:52.238Z","confidence":100,"websiteUrl":"https://robots-check.quietworklabs.workers.dev/","description":"robots.txt tester (RFC 9309), AI crawler access (GPTBot, ClaudeBot...) and sitemap check.","verification":"official","repositoryUrl":null}],"toolIndex":{"tools":[{"serverName":"dev.workers.quietworklabs.robots-check/robots-check","endpointFingerprint":"843e9264326464638f2c4c8f522ab80c9cd9184a6cc768d8c810ba299165e8e2","name":"ai_crawler_access","title":"Which AI crawlers a site allows or blocks","description":"AI crawler access report for a URL: robots.txt verdict for 20 tokens (GPTBot, OAI-SearchBot, ChatGPT-User, ClaudeBot, Claude-SearchBot, Claude-User, Google-Extended, Applebot-Extended, PerplexityBot, Meta-ExternalAgent, CCBot, Bytespider, Amazonbot and more) grouped by training/search/user purpose, plus page meta robots, X-Robots-Tag (noindex/noai) and llms.txt presence. Deterministic, no AI. Price: $0.002 USDC per call via x402 (Base); the first 5 calls per day per client are free.","inputSchema":{"type":"object","properties":{"url":{"type":"string","description":"Site or page URL","examples":["https://www.theguardian.com/"]}},"required":["url"]},"annotations":{"readOnlyHint":true,"openWorldHint":true},"definitionHash":"5833e227300b273e87927258c63ae620d0f8a7eff5be7f92e54963553e0f93cd","schemaStatus":"valid","origin":"official-registry","observedAt":"2026-10-31T08:46:18.639Z","metadataTrust":"untrusted"},{"serverName":"dev.workers.quietworklabs.robots-check/robots-check","endpointFingerprint":"843e9264326464638f2c4c8f522ab80c9cd9184a6cc768d8c810ba299165e8e2","name":"robots_check","title":"May this user-agent crawl this URL? (RFC 9309)","description":"Crawl-permission preflight for agents and scrapers: fetches the site's robots.txt and evaluates the URL for a user-agent token per RFC 9309 (group by product token else *, longest match, Allow wins ties, * and $ wildcards, 4xx=allow all, 5xx=disallow all). Returns allowed, the deciding rule and line, crawl-delay and sitemaps. Deterministic, no AI. Price: $0.001 USDC per call via x402 (Base); the first 5 calls per day per client are free.","inputSchema":{"type":"object","properties":{"url":{"type":"string","description":"Page URL to test","examples":["https://www.nytimes.com/section/technology"]},"user_agent":{"type":"string","description":"Crawler product token, e.g. GPTBot or MyAgent. Default *","examples":["GPTBot"]}},"required":["url"]},"annotations":{"readOnlyHint":true,"openWorldHint":true},"definitionHash":"ab2142b4c89a3464db1556b99c572e064468fff0d48b05bf7041cc098b9ee930","schemaStatus":"valid","origin":"official-registry","observedAt":"2026-10-31T08:46:18.639Z","metadataTrust":"untrusted"},{"serverName":"dev.workers.quietworklabs.robots-check/robots-check","endpointFingerprint":"843e9264326464638f2c4c8f522ab80c9cd9184a6cc768d8c810ba299165e8e2","name":"sitemap_check","title":"Find and validate an XML sitemap","description":"Finds a site's sitemap (from robots.txt, else /sitemap.xml) or checks the sitemap URL given, and validates it: urlset vs sitemapindex, entry count against the 50,000 limit, lastmod format and range, relative/off-host/duplicate/http URLs, gzip support. Returns sample URLs or child sitemaps. Deterministic, no AI. Price: $0.002 USDC per call via x402 (Base); the first 5 calls per day per client are free.","inputSchema":{"type":"object","properties":{"url":{"type":"string","description":"Site URL (discovers the sitemap) or a sitemap URL","examples":["https://www.python.org/"]}},"required":["url"]},"annotations":{"readOnlyHint":true,"openWorldHint":true},"definitionHash":"376d54c4379e8d105949f27be860d6ae34578a02781e19a7b225dac7fcb717bd","schemaStatus":"valid","origin":"official-registry","observedAt":"2026-10-31T08:46:18.639Z","metadataTrust":"untrusted"}],"endpoints":[{"endpointFingerprint":"843e9264326464638f2c4c8f522ab80c9cd9184a6cc768d8c810ba299165e8e2","protocol":"streamable-http","status":"indexed","toolCount":3,"observedAt":"2026-10-31T08:46:18.639Z"}],"authentication":"unknown"},"connectionConfigurations":{"status":"ready","authentication":"not embedded","clients":{"Claude Desktop":{"mcpServers":{"dev.workers.quietworklabs.robots-check/robots-check":{"url":"https://robots-check.quietworklabs.workers.dev/mcp"}}},"Cursor":{"mcpServers":{"dev.workers.quietworklabs.robots-check/robots-check":{"url":"https://robots-check.quietworklabs.workers.dev/mcp"}}},"VS Code":{"servers":{"dev.workers.quietworklabs.robots-check/robots-check":{"type":"http","url":"https://robots-check.quietworklabs.workers.dev/mcp"}}}}},"resourceType":"mcp-server","callable":true,"runtime":{"callable":true,"protocols":["mcp"]},"interfaces":[{"protocol":"mcp","transport":"streamable-http","url":"https://robots-check.quietworklabs.workers.dev/mcp"}],"profile":{"apiVersion":"1","schemaVersion":"1","id":"mcp-server:dev.workers.quietworklabs.robots-check%2Frobots-check","resourceType":"mcp-server","canonicalUrl":"https://www.agentsjunction.com/api/v1/servers/dev.workers.quietworklabs.robots-check%2Frobots-check","name":"dev.workers.quietworklabs.robots-check/robots-check","description":"robots.txt tester (RFC 9309), AI crawler access (GPTBot, ClaudeBot...) and sitemap check.","interfaces":[{"protocol":"mcp","transport":"streamable-http","url":"https://robots-check.quietworklabs.workers.dev/mcp"}],"capabilities":[],"identity":{"source":"official-registry","websiteUrl":"https://robots-check.quietworklabs.workers.dev/","repositoryUrl":null,"version":"1.0.0"},"verification":{"state":"official","confidence":100},"operations":{"resourceType":"mcp-server","name":"dev.workers.quietworklabs.robots-check/robots-check","status":"online","lastCheckedAt":"2026-11-03T17:19:18.574Z","lastSuccessfulCheck":"2026-11-03T17:19:18.574Z","latencyMs":438,"consecutiveFailures":0,"isStale":false,"checks":{"endpointReachable":true,"protocolValid":true,"agentCardValid":null,"mcpHandshakeValid":true},"lastSeen":null,"lastSuccessfulInteraction":null,"lastHealthCheck":"2026-11-03T17:19:18.574Z","source":{"type":"health-check","url":null},"observedAt":"2026-11-03T17:19:18.574Z"},"security":null,"timestamps":{"createdAt":"2026-10-18T03:32:52.238Z","updatedAt":"2026-10-18T03:32:52.238Z","observedAt":"2026-11-03T17:19:18.574Z"},"compatibility":{"serverVersion":"1.0.0","protocolVersions":[{"protocol":"mcp","version":null}]}}}}