# Is the daemon running?
systemctl status gslbd          # systemd
docker compose ps               # Docker
kubectl get pods -l app=gslbd   # Kubernetes

# Is it healthy?
curl -s http://localhost:8080/api/v1/health | jq .

# Recent log lines (structured JSON)
journalctl -u gslbd -n 50 --output=json | jq -r '[.SYSLOG_TIMESTAMP, .MESSAGE] | @tsv'

# Is the listener up?
dig @127.0.0.1 -p 5353 A example.com +time=2

# Is port 5353 open?
ss -ulnp | grep 5353   # UDP
ss -tlnp | grep 5353   # TCP

# Any bind errors in logs?
journalctl -u gslbd | grep -i "listen\|bind\|address in use"

# Enable query logging in config: dns.queryLog: true, then:
journalctl -u gslbd | grep "dns query"

# Check the service exists in DB
curl -s http://localhost:8080/api/v1/services | jq '.[] | select(.domain == "app.example.com.")'

# Check pool members exist and are healthy
curl -s http://localhost:8080/api/v1/pools/<pool-id>/status | jq .

# Pool health overview
curl -s http://localhost:8080/api/v1/pools | jq '.[] | {id, name}'
curl -s http://localhost:8080/api/v1/pools/<pool-id>/status

# Metrics (if enabled)
curl -s http://localhost:9090/metrics | grep gslbd_health_endpoints_healthy

# Health checker logs
journalctl -u gslbd | grep -E "health|probe|unhealthy"

# Check the pool's current health status
curl -s http://localhost:8080/api/v1/pools/<pool-id>/status | jq .

# Is the detection latency in line with your checkInterval?
curl -s http://localhost:9090/metrics | grep failover_detection
# gslbd_health_failover_detection_seconds_bucket{direction="down",...}

# What is the configured check interval and timeout?
curl -s http://localhost:8080/api/v1/pools/<pool-id>/health-check | jq '{intervalMs, timeoutMs}'

# Is NATS connected?
curl -s http://localhost:9090/metrics | grep gslbd_state_nats_connected

# What is the NATS transit lag?
curl -s http://localhost:9090/metrics | grep merge_lag_ms

# What TTL are you serving?
dig +noall +answer @<gslbd-host> -p <port> <service-domain> A | awk '{print $2}'

# Latency breakdown from metrics
curl -s http://localhost:9090/metrics | grep gslbd_dns_query_duration

# Database slow queries (SQLite WAL mode should be fast, but check)
# Add -v to see timing:
time curl -s http://localhost:8080/api/v1/services > /dev/null

# Is the DB index present? (query the rqlite datastore over its HTTP API)
curl -sG http://localhost:4001/db/query --data-urlencode \
  "q=SELECT name FROM sqlite_master WHERE type='index' AND tbl_name='services'"
# Should include: idx_services_domain

# Check NATS connection status
curl -s http://localhost:9090/metrics | grep gslbd_state_nats

# Logs
journalctl -u gslbd | grep -i "nats\|jetstream\|state"

# Can the node reach NATS?
nc -zv <nats-host> 4222

curl -s http://localhost:9090/metrics | grep gslbd_gitops
journalctl -u gslbd | grep -i "gitops\|reconcile\|signature\|verify"

# Test DNSSEC validation end-to-end
dig @127.0.0.1 -p 5353 +dnssec A app.example.com.

# Check key expiry
curl -s http://localhost:9090/metrics | grep gslbd_dnssec_key_days_remaining

# Verify DS record is published at registrar
dig DS example.com. @8.8.8.8

# Test the system API key directly
curl -s -o /dev/null -w '%{http_code}' \
  -H "Authorization: Bearer $GSLB_API_KEY" \
  http://localhost:8080/api/v1/pools

# Test a session token
curl -s -H "Authorization: Bearer <token>" http://localhost:8080/api/v1/auth/me

# Check whether the daemon has GSLB_API_KEY set (dev mode check)
sudo cat /etc/gslb/env | grep GSLB_API_KEY

# Query the rqlite datastore over its HTTP API
curl -sG http://localhost:4001/db/query --data-urlencode \
  "q=SELECT email, tenant_id, password_hash IS NOT NULL FROM users"
curl -sG http://localhost:4001/db/query --data-urlencode \
  "q=SELECT id, name FROM tenants"

curl -s -X POST http://localhost:8080/api/v1/auth/login \
  -H 'Content-Type: application/json' \
  -d '{"email": "you@example.com", "password": "...", "totpCode": "123456"}'

# Install Python bcrypt if needed
pip3 install bcrypt

python3 <<'EOF'
import sqlite3, bcrypt

db = '/var/lib/gslbd/gslbd.db'
email = 'admin@example.com'
new_password = 'temporary-reset-password'

pw_hash = bcrypt.hashpw(new_password.encode(), bcrypt.gensalt(rounds=12)).decode()

conn = sqlite3.connect(db)
conn.execute('UPDATE users SET password_hash=?, totp_secret=NULL, updated_at=strftime("%s","now") WHERE email=?',
             (pw_hash, email))
conn.commit()
rows = conn.execute('SELECT email, role, tenant_id FROM users WHERE email=?', (email,)).fetchall()
print('Updated:', rows)
conn.close()
EOF

openssl rand -hex 32
# Add to /etc/gslb/env:
# GSLB_SECRET_KEY=<output>

python3 -c "
import sqlite3
c = sqlite3.connect('/var/lib/gslbd/gslbd.db')
c.execute('UPDATE users SET totp_secret=NULL')
c.commit()
print('Cleared TOTP secrets for', c.execute('SELECT COUNT(*) FROM users WHERE totp_secret IS NOT NULL').fetchone()[0], 'users')
"

journalctl -u gslbd | grep -i "license\|rps\|rate"
curl -s http://localhost:8080/api/v1/health | jq .license

# Check DB file integrity
sqlite3 /var/lib/gslbd/gslbd.db "PRAGMA integrity_check;"

# Is the DB owned by gslbd?
ls -la /var/lib/gslbd/

# Check available disk space
df -h /var/lib/gslbd

sqlite3 /var/lib/gslbd/gslbd.db "VACUUM INTO '/var/backups/gslbd/gslbd-$(date +%Y%m%d-%H%M).db';"

gslbd -config /etc/gslb/config.yaml -validate

# Is the API port listening?
curl -v http://localhost:8080/api/v1/health

# gslbctl uses GSLB_API_URL and GSLB_API_KEY env vars
export GSLB_API_URL=http://localhost:8080
export GSLB_API_KEY=your-key
gslbctl pools list

#!/usr/bin/env bash
OUT="gslbd-diag-$(date +%Y%m%d-%H%M)"
mkdir "$OUT"

# Logs (last 500 lines)
journalctl -u gslbd -n 500 --output=json > "$OUT/journal.json" 2>/dev/null

# Version
gslbd -version > "$OUT/version.txt" 2>&1

# Config (redact secrets first)
cp /etc/gslb/config.yaml "$OUT/config.yaml"
sed -i 's/key:.*/key: REDACTED/g; s/secret:.*/secret: REDACTED/g; s/password:.*/password: REDACTED/g' "$OUT/config.yaml"

# Metrics snapshot
curl -s http://localhost:9090/metrics > "$OUT/metrics.txt" 2>/dev/null

# Health API
curl -s http://localhost:8080/api/v1/health > "$OUT/health.json" 2>/dev/null

# DB integrity (no data, just check)
sqlite3 /var/lib/gslbd/gslbd.db "PRAGMA integrity_check;" > "$OUT/db-integrity.txt" 2>/dev/null

tar czf "${OUT}.tar.gz" "$OUT" && rm -rf "$OUT"
echo "Bundle: ${OUT}.tar.gz"