{"data":{"name":"dev.workers.mike-tusa.robotsgate/robotsgate","title":"RobotsGate","description":"Generate, validate and check robots.txt rules for AI crawlers. Free; optional Pro key.","version":"0.3.1","websiteUrl":"https://robotsgate.mike-tusa.workers.dev/","repositoryUrl":null,"remotes":[{"url":"https://robotsgate.mike-tusa.workers.dev/mcp","type":"streamable-http","headers":[{"name":"Authorization","isSecret":true,"description":"Optional. 'Bearer <RobotsGate Pro license key>' for higher limits. Works without a key."}]}],"packages":[],"status":"active","isLatest":true,"publishedAt":"2026-10-16T13:19:07.848Z","updatedAt":"2026-10-16T13:19:07.848Z","source":"official-registry","verification":"official","confidence":100,"stars":null,"sources":[{"stars":null,"title":"RobotsGate","source":"official-registry","sourceId":"dev.workers.mike-tusa.robotsgate/robotsgate","updatedAt":"2026-10-16T13:19:07.848Z","confidence":100,"websiteUrl":"https://robotsgate.mike-tusa.workers.dev/","description":"Generate, validate and check robots.txt rules for AI crawlers. Free; optional Pro key.","verification":"official","repositoryUrl":null}],"toolIndex":{"tools":[{"serverName":"dev.workers.mike-tusa.robotsgate/robotsgate","endpointFingerprint":"59c4b6e3af0b6c9c2361ebfb3bcd0be5401f1fcfa443b65b7e31a309ad4c0935","name":"check_site","title":"Check which AI crawlers a live site allows","description":"Fetch a site's /robots.txt (http/https on the default port only; private, loopback and internal addresses are refused; up to 512,000 bytes; results cached for 5 minutes) and report, for one path, which registry AI crawlers are allowed or blocked, plus validation errors and warnings. Read-only: one GET of /robots.txt with a RobotsGate user-agent. Same engine as GET /api/check.","inputSchema":{"type":"object","properties":{"url":{"type":"string","minLength":1,"maxLength":2048,"description":"Site URL or bare hostname, e.g. example.com. Only the scheme and host are used."},"path":{"type":"string","maxLength":2048,"description":"URL path to test, starting with \"/\" (default \"/\")."}},"required":["url"],"additionalProperties":false},"annotations":{"title":"Check a live site","readOnlyHint":true,"destructiveHint":false,"idempotentHint":true,"openWorldHint":true},"definitionHash":"8fbd55130baefce8327c2bafb16fa7bbb12e3caffbb56c546a98acf712913cb1","schemaStatus":"valid","origin":"official-registry","observedAt":"2026-10-31T08:45:57.471Z","metadataTrust":"untrusted"},{"serverName":"dev.workers.mike-tusa.robotsgate/robotsgate","endpointFingerprint":"59c4b6e3af0b6c9c2361ebfb3bcd0be5401f1fcfa443b65b7e31a309ad4c0935","name":"generate_robots","title":"Generate a robots.txt for AI crawlers","description":"Build a robots.txt from per-category choices (allow, block or omit for training, dataset, search, user_fetch or other), optional per-token overrides, a default policy for other bots, paths, sitemaps and custom rules. Returns the file text, a per-crawler summary and notes. Read-only: it only returns text and changes nothing. Same engine as POST /api/generate.","inputSchema":{"type":"object","properties":{"categories":{"type":"object","description":"Action per crawler category. Omitted categories get no group of their own.","properties":{"training":{"type":"string","enum":["allow","block","omit"]},"dataset":{"type":"string","enum":["allow","block","omit"]},"search":{"type":"string","enum":["allow","block","omit"]},"user_fetch":{"type":"string","enum":["allow","block","omit"]},"other":{"type":"string","enum":["allow","block","omit"]}},"additionalProperties":false},"agents":{"type":"object","description":"Per-token overrides, e.g. {\"GPTBot\": \"block\"}. Tokens: letters, digits, \"_\", \".\" and \"-\", up to 64 characters (not \"*\"). At most 100.","maxProperties":100,"propertyNames":{"pattern":"^[A-Za-z0-9_.-]{1,64}$"},"additionalProperties":{"type":"string","enum":["allow","block","omit"]}},"default":{"type":"string","enum":["allow","block"],"description":"Policy for \"User-agent: *\" (default allow)."},"disallow_paths":{"type":"array","maxItems":100,"items":{"type":"string","minLength":1,"maxLength":512},"description":"Paths to disallow for \"*\" and for allowed AI groups. Each path starts with \"/\" or \"*\" and has no whitespace or \"#\". At most 100."},"allow_paths":{"type":"array","maxItems":100,"items":{"type":"string","minLength":1,"maxLength":512},"description":"Paths to allow explicitly. Each path starts with \"/\" or \"*\" and has no whitespace or \"#\". At most 100."},"sitemaps":{"type":"array","maxItems":20,"items":{"type":"string","maxLength":2048},"description":"Absolute http(s) sitemap URLs. At most 20."},"sitemap":{"type":"string","maxLength":2048,"description":"A single sitemap URL (alternative to sitemaps)."},"custom_rules":{"type":"string","maxLength":20000,"description":"Extra robots.txt lines appended after validation (max 20000 bytes)."},"date":{"type":"string","maxLength":10,"pattern":"^\\d{4}-\\d{2}-\\d{2}$","description":"Date for the header comment (YYYY-MM-DD). Default: today (UTC)."}},"additionalProperties":false},"annotations":{"title":"Generate robots.txt","readOnlyHint":true,"destructiveHint":false,"idempotentHint":true,"openWorldHint":false},"definitionHash":"033fd27404d7ae5807588de6a3e06491af1d9b5f4804aa5c6223d783cdcc8998","schemaStatus":"valid","origin":"official-registry","observedAt":"2026-10-31T08:45:57.471Z","metadataTrust":"untrusted"},{"serverName":"dev.workers.mike-tusa.robotsgate/robotsgate","endpointFingerprint":"59c4b6e3af0b6c9c2361ebfb3bcd0be5401f1fcfa443b65b7e31a309ad4c0935","name":"list_crawlers","title":"List the AI crawler registry","description":"Return RobotsGate's AI crawler registry: each crawler's robots.txt token, operator, category (training, dataset, search, user_fetch or other), whether the operator says it respects robots.txt, and a link to the operator's documentation, plus unverified entries and notes. Read-only, no network access. Same data as GET /api/crawlers.","inputSchema":{"type":"object","properties":{"category":{"type":"string","enum":["training","dataset","search","user_fetch","other"],"description":"Optional: only crawlers in this category."}},"additionalProperties":false},"annotations":{"title":"List AI crawlers","readOnlyHint":true,"destructiveHint":false,"idempotentHint":true,"openWorldHint":false},"definitionHash":"dbcc349b44251e34ff19cca27c19f1594740a5ff257ef6d9e697f882fab1c738","schemaStatus":"valid","origin":"official-registry","observedAt":"2026-10-31T08:45:57.471Z","metadataTrust":"untrusted"},{"serverName":"dev.workers.mike-tusa.robotsgate/robotsgate","endpointFingerprint":"59c4b6e3af0b6c9c2361ebfb3bcd0be5401f1fcfa443b65b7e31a309ad4c0935","name":"validate_robots","title":"Validate robots.txt text","description":"Validate robots.txt text against RFC 9309 and report errors, warnings and, for one URL path, which registry AI crawlers are allowed or blocked and by which rule. Read-only, no network access. Same engine as POST /api/validate. Up to 50,000 characters over MCP (the whole JSON-RPC request must also fit in 64 KB); for larger files use POST /api/validate or check_site.","inputSchema":{"type":"object","properties":{"robots_txt":{"type":"string","maxLength":50000,"description":"The robots.txt file content."},"path":{"type":"string","maxLength":2048,"description":"URL path to test, starting with \"/\" (default \"/\")."}},"required":["robots_txt"],"additionalProperties":false},"annotations":{"title":"Validate robots.txt","readOnlyHint":true,"destructiveHint":false,"idempotentHint":true,"openWorldHint":false},"definitionHash":"3235b2d684a514e30d13af6273a630bb8071426726a8a8bcebe2ab45dea2efc4","schemaStatus":"valid","origin":"official-registry","observedAt":"2026-10-31T08:45:57.471Z","metadataTrust":"untrusted"}],"endpoints":[{"endpointFingerprint":"59c4b6e3af0b6c9c2361ebfb3bcd0be5401f1fcfa443b65b7e31a309ad4c0935","protocol":"streamable-http","status":"indexed","toolCount":4,"observedAt":"2026-10-31T08:45:57.471Z"}],"authentication":"unknown"},"connectionConfigurations":{"status":"ready","authentication":"not embedded","clients":{"Claude Desktop":{"mcpServers":{"dev.workers.mike-tusa.robotsgate/robotsgate":{"url":"https://robotsgate.mike-tusa.workers.dev/mcp"}}},"Cursor":{"mcpServers":{"dev.workers.mike-tusa.robotsgate/robotsgate":{"url":"https://robotsgate.mike-tusa.workers.dev/mcp"}}},"VS Code":{"servers":{"dev.workers.mike-tusa.robotsgate/robotsgate":{"type":"http","url":"https://robotsgate.mike-tusa.workers.dev/mcp"}}}}},"resourceType":"mcp-server","callable":true,"runtime":{"callable":true,"protocols":["mcp"]},"interfaces":[{"protocol":"mcp","transport":"streamable-http","url":"https://robotsgate.mike-tusa.workers.dev/mcp"}],"profile":{"apiVersion":"1","schemaVersion":"1","id":"mcp-server:dev.workers.mike-tusa.robotsgate%2Frobotsgate","resourceType":"mcp-server","canonicalUrl":"https://www.agentsjunction.com/api/v1/servers/dev.workers.mike-tusa.robotsgate%2Frobotsgate","name":"dev.workers.mike-tusa.robotsgate/robotsgate","description":"Generate, validate and check robots.txt rules for AI crawlers. Free; optional Pro key.","interfaces":[{"protocol":"mcp","transport":"streamable-http","url":"https://robotsgate.mike-tusa.workers.dev/mcp"}],"capabilities":[],"identity":{"source":"official-registry","websiteUrl":"https://robotsgate.mike-tusa.workers.dev/","repositoryUrl":null,"version":"0.3.1"},"verification":{"state":"official","confidence":100},"operations":{"resourceType":"mcp-server","name":"dev.workers.mike-tusa.robotsgate/robotsgate","status":"online","lastCheckedAt":"2026-11-03T17:19:12.267Z","lastSuccessfulCheck":"2026-11-03T17:19:12.267Z","latencyMs":417,"consecutiveFailures":0,"isStale":false,"checks":{"endpointReachable":true,"protocolValid":true,"agentCardValid":null,"mcpHandshakeValid":true},"lastSeen":null,"lastSuccessfulInteraction":null,"lastHealthCheck":"2026-11-03T17:19:12.267Z","source":{"type":"health-check","url":null},"observedAt":"2026-11-03T17:19:12.267Z"},"security":null,"timestamps":{"createdAt":"2026-10-16T13:19:07.848Z","updatedAt":"2026-10-16T13:19:07.848Z","observedAt":"2026-11-03T17:19:12.267Z"},"compatibility":{"serverVersion":"0.3.1","protocolVersions":[{"protocol":"mcp","version":null}]}}}}