GET /robots.txt

A robots.txt that contradicts itself, sets impossible crawl delays, or returns malformed directives. Crawlers that parse strictly should reject; lenient crawlers will produce unpredictable behaviour.

mode contradictory (default; Allow and Disallow for the same path), tarpit (50 narrow Allow paths that all loop), malformed (missing colons, invalid directives), infinite-crawl-delay (Crawl-delay: 999999999).

build a request:

chaos mode

expect: 200 OK with Content-Type: text/plain. Body content depends on the chosen mode. X-Chaos-Robots-Mode reflects the selection.

bash

curl -i 'https://bots.catastrophic.io/robots.txt?mode=contradictory'

import urllib.request
resp = urllib.request.urlopen("https://bots.catastrophic.io/robots.txt?mode=contradictory")
print(resp.status, resp.headers["X-Chaos-Robots-Mode"])
print(resp.read().decode())

const res = await fetch("https://bots.catastrophic.io/robots.txt?mode=contradictory");
console.log(res.status, res.headers.get("x-chaos-robots-mode"));
console.log(await res.text());

package main

import (
    "fmt"
    "io"
    "net/http"
)

func main() {
    resp, _ := http.Get("https://bots.catastrophic.io/robots.txt?mode=contradictory")
    defer resp.Body.Close()
    body, _ := io.ReadAll(resp.Body)
    fmt.Println(resp.StatusCode, resp.Header.Get("X-Chaos-Robots-Mode"))
    fmt.Println(string(body))
}

// Cargo.toml: reqwest = { version = "0.12", features = ["blocking"] }
fn main() -> Result<(), Box> {
    let resp = reqwest::blocking::get("https://bots.catastrophic.io/robots.txt?mode=contradictory")?;
    println!("{} {:?}", resp.status(), resp.headers().get("x-chaos-robots-mode"));
    println!("{}", resp.text()?);
    Ok(())
}

import java.net.URI;
import java.net.http.*;

public class BotsRobots {
    public static void main(String[] args) throws Exception {
        var client = HttpClient.newHttpClient();
        var req = HttpRequest.newBuilder(URI.create("https://bots.catastrophic.io/robots.txt?mode=contradictory")).build();
        var resp = client.send(req, HttpResponse.BodyHandlers.ofString());
        System.out.println(resp.statusCode() + " " +
            resp.headers().firstValue("X-Chaos-Robots-Mode").orElse(""));
        System.out.println(resp.body());
    }
}

using var client = new HttpClient();
var resp = await client.GetAsync("https://bots.catastrophic.io/robots.txt?mode=contradictory");
resp.Headers.TryGetValues("X-Chaos-Robots-Mode", out var mode);
Console.WriteLine($"{(int)resp.StatusCode} {mode?.FirstOrDefault()}");
Console.WriteLine(await resp.Content.ReadAsStringAsync());

require "net/http"
res = Net::HTTP.get_response(URI("https://bots.catastrophic.io/robots.txt?mode=contradictory"))
puts "#{res.code} #{res['X-Chaos-Robots-Mode']}"
puts res.body

$r = Invoke-WebRequest -Uri 'https://bots.catastrophic.io/robots.txt?mode=contradictory'
$r.Headers['X-Chaos-Robots-Mode']
$r.Content