{"resource":"https://netintel.dev/robots-txt/analyze","service":"NetIntel","areas":["web_search"],"answering":true,"http_status":402,"latency_ms":185,"price_usdc":0.01,"buyers_30d":3,"calls_30d":4,"description":"Fetch and parse a domain's robots.txt file — returns all crawl rules by user-agent, sitemap URLs, crawl delay settings, and checks whether a specific path is allowed or blocked for any bot — so agents can respect crawl policies and locate s","id":"da46c1b0d125","category":"DNS and domains","accuracy":null,"tool_calls":null,"checked_day":"2026-10-11","page":"https://velliumandco.com/sellers/da46c1b0d125"}