#!/usr/bin/env bash
# SquareRanked AI Crawler Check
# Part of the free SquareRanked AI Crawler Unblock Kit · squareranked.com
#
# What it does:
#   1. Fetches your robots.txt and reads the rules for each bot below.
#   2. Requests your homepage as each bot's user-agent and records the HTTP status.
#   3. Prints PASS / BLOCKED / CHECK for 10 search and AI-retrieval bots.
#
# Usage:
#   bash verify-bots.sh https://yoursite.com
#
# Runs on macOS and Linux Terminal. On Windows, use Git Bash or WSL.
# Needs: bash, curl, awk (preinstalled on macOS and most Linux).
#
# PASS     robots.txt does not disallow the bot (by name or via "User-agent: *")
#          AND the homepage answered 2xx/3xx to that user-agent.
# BLOCKED  robots.txt has "Disallow: /" for the bot, or the server answered 401/403.
# CHECK    Anything else (timeouts, 5xx). Run it again in a few minutes.
#
# Limits: this checks the rules your site publishes. It cannot see firewall rules
# that only trigger on a bot's real IP address, and a PASS does not guarantee an
# AI engine will cite you. It tells you the door is open.

set -u

if [ -z "${1:-}" ]; then
  echo "Usage: bash verify-bots.sh https://yoursite.com"
  exit 1
fi

SITE_URL="${1%/}"
case "${SITE_URL}" in
  http://*|https://*) ;;
  *) SITE_URL="https://${SITE_URL}" ;;
esac
ROBOTS_URL="${SITE_URL}/robots.txt"

echo ""
echo "================================================================"
echo "  SquareRanked AI Crawler Check"
echo "  Site: ${SITE_URL}"
echo "  Date: $(date -u +'%Y-%m-%d %H:%M UTC')"
echo "================================================================"
echo ""

echo "STEP 1 / 3  Fetching robots.txt ..."
ROBOTS=$(curl -sS -L --max-time 10 "${ROBOTS_URL}" 2>/dev/null | tr -d '\r' || true)
if [ -z "${ROBOTS}" ]; then
  echo "  Could not fetch ${ROBOTS_URL}. Continuing with HTTP checks only."
else
  echo "  Fetched $(printf '%s\n' "${ROBOTS}" | wc -l | tr -d ' ') lines."
fi
echo ""

# Prints: none | open | disallowed  for the group that names this token exactly.
# Handles stacked "User-agent:" lines that share one rule block.
robots_verdict() {
  printf '%s\n' "${ROBOTS}" | awk -v tok="$1" '
    BEGIN { tok = tolower(tok); inrules = 0; m = 0; found = 0; res = "none" }
    {
      line = $0
      sub(/#.*/, "", line)
      low = tolower(line)
      if (low ~ /^[[:space:]]*user-agent[[:space:]]*:/) {
        ua = low
        sub(/^[^:]*:[[:space:]]*/, "", ua)
        gsub(/[[:space:]]+$/, "", ua)
        if (inrules) { m = 0; inrules = 0 }
        if (ua == tok) { m = 1; found = 1; if (res == "none") res = "open" }
        next
      }
      if (low ~ /^[[:space:]]*(dis)?allow[[:space:]]*:/) {
        inrules = 1
        if (m && low ~ /^[[:space:]]*disallow[[:space:]]*:/) {
          val = low
          sub(/^[^:]*:[[:space:]]*/, "", val)
          gsub(/[[:space:]]+$/, "", val)
          if (val == "/") res = "disallowed"
        }
      }
    }
    END { print res }
  '
}

WILDCARD=$(robots_verdict "*")

echo "STEP 2 / 3  Checking each bot ..."
echo ""

# label|user-agent sent|robots.txt token
BOTS=(
  "Googlebot|Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)|Googlebot"
  "Bingbot|Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)|bingbot"
  "OAI-SearchBot|Mozilla/5.0 (compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot)|OAI-SearchBot"
  "ChatGPT-User|Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ChatGPT-User/1.0; +https://openai.com/bot)|ChatGPT-User"
  "PerplexityBot|Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)|PerplexityBot"
  "Perplexity-User|Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user)|Perplexity-User"
  "Claude-SearchBot|Claude-SearchBot/1.0|Claude-SearchBot"
  "Claude-User|Claude-User/1.0|Claude-User"
  "Applebot|Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)|Applebot"
  "DuckDuckBot|DuckDuckBot/1.1; (+http://duckduckgo.com/duckduckbot.html)|DuckDuckBot"
)

PASS_COUNT=0
BLOCKED_COUNT=0
CHECK_COUNT=0

for entry in "${BOTS[@]}"; do
  LABEL=$(printf '%s' "${entry}" | awk -F'|' '{print $1}')
  UA=$(printf '%s' "${entry}" | awk -F'|' '{print $2}')
  TOKEN=$(printf '%s' "${entry}" | awk -F'|' '{print $3}')

  RV="none"
  if [ -n "${ROBOTS}" ]; then
    RV=$(robots_verdict "${TOKEN}")
    # A bot with no group of its own falls back to the "User-agent: *" group.
    if [ "${RV}" = "none" ] && [ "${WILDCARD}" = "disallowed" ]; then
      RV="disallowed-wildcard"
    fi
  fi

  HTTP_CODE=$(curl -sS -o /dev/null -w "%{http_code}" --max-time 12 -A "${UA}" -L "${SITE_URL}/" 2>/dev/null || true)
  [ -z "${HTTP_CODE}" ] && HTTP_CODE="000"

  if [ "${RV}" = "disallowed" ]; then
    VERDICT="BLOCKED  robots.txt disallows ${TOKEN}"
    BLOCKED_COUNT=$((BLOCKED_COUNT + 1))
  elif [ "${RV}" = "disallowed-wildcard" ]; then
    VERDICT="BLOCKED  robots.txt disallows all bots (User-agent: *)"
    BLOCKED_COUNT=$((BLOCKED_COUNT + 1))
  elif [ "${HTTP_CODE}" = "401" ] || [ "${HTTP_CODE}" = "403" ]; then
    VERDICT="BLOCKED  server answered HTTP ${HTTP_CODE} to this user-agent"
    BLOCKED_COUNT=$((BLOCKED_COUNT + 1))
  elif printf '%s' "${HTTP_CODE}" | grep -Eq '^[23][0-9][0-9]$'; then
    if [ "${RV}" = "open" ]; then
      VERDICT="PASS     named in robots.txt, not disallowed (HTTP ${HTTP_CODE})"
    else
      VERDICT="PASS     not disallowed in robots.txt (HTTP ${HTTP_CODE})"
    fi
    PASS_COUNT=$((PASS_COUNT + 1))
  else
    VERDICT="CHECK    unexpected HTTP ${HTTP_CODE}, run again later"
    CHECK_COUNT=$((CHECK_COUNT + 1))
  fi

  printf "  %-18s %s\n" "${LABEL}" "${VERDICT}"
done

echo ""
echo "STEP 3 / 3  Summary"
echo ""
echo "  PASS:    ${PASS_COUNT} / 10"
echo "  BLOCKED: ${BLOCKED_COUNT} / 10"
echo "  CHECK:   ${CHECK_COUNT} / 10"
echo ""

if [ "${PASS_COUNT}" = "10" ]; then
  echo "  All 10 bots can reach your site. The door is open."
  echo "  Whether AI engines cite you now depends on what they find:"
  echo "  schema, clear answers, and who stands behind the site."
elif [ "${BLOCKED_COUNT}" -gt 0 ]; then
  echo "  ${BLOCKED_COUNT} bot(s) blocked. On Squarespace, open Settings > Crawlers."
  echo "  If 'Block known artificial intelligence crawlers' is checked, uncheck it."
  echo "  If 'Block search engine crawlers' is checked, uncheck it."
  echo "  Save, wait a few minutes, then run this script again."
else
  echo "  Some checks were inconclusive. Run the script again in a few minutes."
fi

echo ""
echo "  Questions: hello@billyreiner.com"
echo "================================================================"
echo ""