#!/usr/bin/env bash
# Refresh Google's published crawler/AdsBot IP ranges (special-crawlers.json:
# AdsBot-Google, AdsBot-Google-Mobile + other special crawlers; common-
# crawlers.json: Googlebot + other common crawlers) into a flat CIDR list,
# then regen every app's Caddy config so geo-blocking's exemption (bin/paas,
# emit_geo_directives) picks up the new ranges. Runs daily via systemd timer
# as root (idle priority) — Google documents these ranges as the
# authoritative way to identify its crawlers by IP:
# https://developers.google.com/search/docs/crawling-indexing/verifying-googlebot
set -euo pipefail
DIR=/srv/paas/geoip
OUT="$DIR/google-crawlers.cidr"
mkdir -p "$DIR"
t1="$(mktemp)" t2="$(mktemp)" tmp="$(mktemp)"
trap 'rm -f "$t1" "$t2" "$tmp"' EXIT

if curl -fsSL "https://developers.google.com/static/crawling/ipranges/special-crawlers.json" -o "$t1" \
    && curl -fsSL "https://developers.google.com/static/crawling/ipranges/common-crawlers.json" -o "$t2" \
    && python3 - "$t1" "$t2" >"$tmp" <<'PY'
import json, sys
prefixes = set()
for path in sys.argv[1:]:
    with open(path) as f:
        data = json.load(f)
    for p in data.get("prefixes", []):
        cidr = p.get("ipv4Prefix") or p.get("ipv6Prefix")
        if cidr:
            prefixes.add(cidr)
for cidr in sorted(prefixes):
    print(cidr)
PY
then
    if [ -s "$tmp" ]; then
        install -m 644 "$tmp" "$OUT"
        # the list is baked into every app's generated Caddyfile (Caddy has
        # no "load IP ranges from file" matcher) — a plain reload wouldn't
        # pick up new ranges, so regen everything, same as a config change.
        sudo -u deploy -H bash -c 'cd /srv/paas && /usr/local/bin/paas regen' >/dev/null
        echo "[google-crawlers] refreshed $OUT ($(wc -l <"$OUT" | tr -d ' ') ranges)"
        exit 0
    fi
fi
echo "[google-crawlers] refresh failed — keeping existing list" >&2
[ -f "$OUT" ]   # exit non-zero only if we have no list at all