Clawd BotandClaude Opus 4.6
ca9b510922
- Reset master to upstream/main (16,697 commits) - Overlay 2,271 local-only files (skills, tools, workspace, configs, apps) - Restore IDENTITY.md and USER.md templates - Build verified, gateway running, Discord working Co-Authored-By: Claude Opus 4.6 <[email protected]>
245 lines
8.5 KiB
Bash
245 lines
8.5 KiB
Bash
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
# monitor.sh
|
|
# Poll one or more HTTP endpoints once per minute for 30 minutes (default).
|
|
# Captures status, latency, key rate-limit headers, and optional JSON fields.
|
|
# Requires: curl, jq (jq used only when response is JSON and JQ_FILTER is set).
|
|
|
|
DURATION_MINUTES="${DURATION_MINUTES:-30}"
|
|
INTERVAL_SECONDS="${INTERVAL_SECONDS:-60}"
|
|
ENDPOINTS_FILE="${ENDPOINTS_FILE:-}"
|
|
OUT_DIR="${OUT_DIR:-}"
|
|
|
|
# Optional curl args/auth
|
|
# Examples:
|
|
# export CURL_HEADERS=$'Authorization: Bearer ...\nX-Api-Key: ...'
|
|
# export CURL_EXTRA_ARGS='--http1.1'
|
|
CURL_HEADERS="${CURL_HEADERS:-}"
|
|
CURL_EXTRA_ARGS="${CURL_EXTRA_ARGS:-}"
|
|
|
|
# Optional: if you want to extract specific JSON fields into the CSV
|
|
# Example: export JQ_FILTER='.usage as $u | {prompt_tokens:$u.prompt_tokens, completion_tokens:$u.completion_tokens, total_tokens:$u.total_tokens}'
|
|
JQ_FILTER="${JQ_FILTER:-}"
|
|
|
|
usage() {
|
|
cat <<'EOF'
|
|
Usage:
|
|
monitor.sh [--out <dir>] [--minutes <n>] [--interval <sec>] <url1> [url2 ...]
|
|
monitor.sh [--endpoints-file <file>]
|
|
|
|
Environment variables:
|
|
DURATION_MINUTES (default: 30)
|
|
INTERVAL_SECONDS (default: 60)
|
|
ENDPOINTS_FILE (alternative to positional urls)
|
|
OUT_DIR (default: ./artifacts/<timestamp>)
|
|
CURL_HEADERS (newline-separated header lines)
|
|
CURL_EXTRA_ARGS (extra args appended to curl)
|
|
JQ_FILTER (jq filter to extract JSON fields into a compact object)
|
|
|
|
Artifacts:
|
|
<out>/samples.csv (one line per endpoint per tick)
|
|
<out>/responses/<tick>_<idx>.json (when content-type is json)
|
|
EOF
|
|
}
|
|
|
|
# Parse args
|
|
urls=()
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
-h|--help) usage; exit 0 ;;
|
|
--out) OUT_DIR="$2"; shift 2 ;;
|
|
--minutes) DURATION_MINUTES="$2"; shift 2 ;;
|
|
--interval) INTERVAL_SECONDS="$2"; shift 2 ;;
|
|
--endpoints-file) ENDPOINTS_FILE="$2"; shift 2 ;;
|
|
--) shift; break ;;
|
|
-*) echo "Unknown flag: $1" >&2; usage; exit 2 ;;
|
|
*) urls+=("$1"); shift ;;
|
|
esac
|
|
done
|
|
|
|
if [[ -n "$ENDPOINTS_FILE" ]]; then
|
|
if [[ ! -f "$ENDPOINTS_FILE" ]]; then
|
|
echo "ENDPOINTS_FILE not found: $ENDPOINTS_FILE" >&2
|
|
exit 2
|
|
fi
|
|
while IFS= read -r line; do
|
|
[[ -z "$line" ]] && continue
|
|
[[ "$line" =~ ^\s*# ]] && continue
|
|
urls+=("$line")
|
|
done < "$ENDPOINTS_FILE"
|
|
fi
|
|
|
|
if [[ ${#urls[@]} -eq 0 ]]; then
|
|
echo "No endpoints provided." >&2
|
|
usage
|
|
exit 2
|
|
fi
|
|
|
|
stamp="$(date -u +%Y%m%dT%H%M%SZ)"
|
|
if [[ -z "$OUT_DIR" ]]; then
|
|
OUT_DIR="$(pwd)/artifacts/$stamp"
|
|
fi
|
|
mkdir -p "$OUT_DIR/responses"
|
|
|
|
csv="$OUT_DIR/samples.csv"
|
|
# Base columns are stable. JSON fields (if any) are stored as a compact JSON object in json_extract.
|
|
echo "ts_utc,tick,endpoint_idx,url,http_status,total_time_ms,namelookup_ms,connect_ms,appconnect_ms,pretransfer_ms,starttransfer_ms,size_download,content_type,ratelimit_limit,ratelimit_remaining,ratelimit_reset,retry_after,x_ratelimit_limit,x_ratelimit_remaining,x_ratelimit_reset,rate_limit_limit,rate_limit_remaining,rate_limit_reset,other_rl_headers,json_extract" > "$csv"
|
|
|
|
# Build curl header args
|
|
header_args=()
|
|
if [[ -n "$CURL_HEADERS" ]]; then
|
|
while IFS= read -r h; do
|
|
[[ -z "$h" ]] && continue
|
|
header_args+=( -H "$h" )
|
|
done <<< "$CURL_HEADERS"
|
|
fi
|
|
|
|
# shellcheck disable=SC2206
|
|
extra_args=( $CURL_EXTRA_ARGS )
|
|
|
|
get_header() {
|
|
# $1=header block, $2=case-insensitive header name
|
|
awk -v key="$2" 'BEGIN{IGNORECASE=1} $0 ~ "^"key":" {sub(/\r$/, ""); sub("^[^:]*: ?", ""); print; exit}' <<< "$1"
|
|
}
|
|
|
|
collect_other_rl_headers() {
|
|
# Extract any headers containing ratelimit/limit/remaining/reset (best-effort) other than the explicit ones.
|
|
awk 'BEGIN{IGNORECASE=1}
|
|
/^[^:]+:/{
|
|
k=tolower($1);
|
|
gsub(/:$/, "", k);
|
|
if (k ~ /rate/ || k ~ /limit/ || k ~ /remaining/ || k ~ /reset/) {
|
|
line=$0; sub(/\r$/, "", line);
|
|
print line;
|
|
}
|
|
}' <<< "$1" | jq -R -s -c 'split("\n") | map(select(length>0))'
|
|
}
|
|
|
|
minutes="$DURATION_MINUTES"
|
|
interval="$INTERVAL_SECONDS"
|
|
ticks=$minutes
|
|
|
|
echo "Writing artifacts to: $OUT_DIR" >&2
|
|
echo "Polling ${#urls[@]} endpoint(s) every ${interval}s for ${minutes} minute(s) (ticks=${ticks})." >&2
|
|
|
|
for ((tick=1; tick<=ticks; tick++)); do
|
|
ts="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
for i in "${!urls[@]}"; do
|
|
url="${urls[$i]}"
|
|
|
|
# Use curl to capture headers + body. We also capture timing in a single line via -w.
|
|
# We intentionally do NOT send any modifying verbs; caller may include them via CURL_EXTRA_ARGS at their own risk.
|
|
tmp_headers="$OUT_DIR/.headers_${tick}_$i.txt"
|
|
tmp_body="$OUT_DIR/.body_${tick}_$i.bin"
|
|
tmp_meta="$OUT_DIR/.meta_${tick}_$i.txt"
|
|
|
|
curl_exit=0
|
|
curl -sS -D "$tmp_headers" -o "$tmp_body" \
|
|
-w "%{http_code} %{time_total} %{time_namelookup} %{time_connect} %{time_appconnect} %{time_pretransfer} %{time_starttransfer} %{size_download} %{content_type}" \
|
|
"${header_args[@]}" "${extra_args[@]}" \
|
|
"$url" > "$tmp_meta" || curl_exit=$?
|
|
|
|
if [[ $curl_exit -ne 0 ]]; then
|
|
# curl failures still get recorded
|
|
http_status="0"
|
|
total_ms="0"
|
|
name_ms="0"
|
|
conn_ms="0"
|
|
app_ms="0"
|
|
pre_ms="0"
|
|
start_ms="0"
|
|
size_dl="0"
|
|
ctype=""
|
|
headers_block="$(cat "$tmp_headers" 2>/dev/null || true)"
|
|
else
|
|
read -r http_status total name conn app pre start size_dl ctype < "$tmp_meta"
|
|
total_ms="$(awk -v t="$total" 'BEGIN{printf("%.0f", t*1000)}')"
|
|
name_ms="$(awk -v t="$name" 'BEGIN{printf("%.0f", t*1000)}')"
|
|
conn_ms="$(awk -v t="$conn" 'BEGIN{printf("%.0f", t*1000)}')"
|
|
app_ms="$(awk -v t="$app" 'BEGIN{printf("%.0f", t*1000)}')"
|
|
pre_ms="$(awk -v t="$pre" 'BEGIN{printf("%.0f", t*1000)}')"
|
|
start_ms="$(awk -v t="$start" 'BEGIN{printf("%.0f", t*1000)}')"
|
|
headers_block="$(cat "$tmp_headers" 2>/dev/null || true)"
|
|
fi
|
|
|
|
# Extract common rate-limit headers across ecosystems
|
|
rl_limit="$(get_header "$headers_block" "ratelimit-limit")"
|
|
rl_remaining="$(get_header "$headers_block" "ratelimit-remaining")"
|
|
rl_reset="$(get_header "$headers_block" "ratelimit-reset")"
|
|
retry_after="$(get_header "$headers_block" "retry-after")"
|
|
x_rl_limit="$(get_header "$headers_block" "x-ratelimit-limit")"
|
|
x_rl_remaining="$(get_header "$headers_block" "x-ratelimit-remaining")"
|
|
x_rl_reset="$(get_header "$headers_block" "x-ratelimit-reset")"
|
|
rate_limit_limit="$(get_header "$headers_block" "rate-limit-limit")"
|
|
rate_limit_remaining="$(get_header "$headers_block" "rate-limit-remaining")"
|
|
rate_limit_reset="$(get_header "$headers_block" "rate-limit-reset")"
|
|
|
|
other_rl="$(collect_other_rl_headers "$headers_block" 2>/dev/null || echo '[]')"
|
|
|
|
json_extract=""
|
|
if [[ -n "$ctype" && "$ctype" == *"json"* ]]; then
|
|
resp_path="$OUT_DIR/responses/${tick}_${i}.json"
|
|
# Best-effort: ensure body is valid UTF-8-ish before saving
|
|
cp "$tmp_body" "$resp_path" 2>/dev/null || true
|
|
if [[ -n "$JQ_FILTER" ]]; then
|
|
json_extract="$(jq -c "$JQ_FILTER" "$resp_path" 2>/dev/null || echo '')"
|
|
fi
|
|
fi
|
|
|
|
# CSV escaping: wrap potentially complex fields in JSON strings
|
|
url_json="$(jq -R -s -c . <<< "$url")"
|
|
ctype_json="$(jq -R -s -c . <<< "$ctype")"
|
|
|
|
rl_limit_json="$(jq -R -s -c . <<< "${rl_limit:-}")"
|
|
rl_remaining_json="$(jq -R -s -c . <<< "${rl_remaining:-}")"
|
|
rl_reset_json="$(jq -R -s -c . <<< "${rl_reset:-}")"
|
|
retry_after_json="$(jq -R -s -c . <<< "${retry_after:-}")"
|
|
|
|
x_rl_limit_json="$(jq -R -s -c . <<< "${x_rl_limit:-}")"
|
|
x_rl_remaining_json="$(jq -R -s -c . <<< "${x_rl_remaining:-}")"
|
|
x_rl_reset_json="$(jq -R -s -c . <<< "${x_rl_reset:-}")"
|
|
|
|
rate_limit_limit_json="$(jq -R -s -c . <<< "${rate_limit_limit:-}")"
|
|
rate_limit_remaining_json="$(jq -R -s -c . <<< "${rate_limit_remaining:-}")"
|
|
rate_limit_reset_json="$(jq -R -s -c . <<< "${rate_limit_reset:-}")"
|
|
|
|
json_extract_json="$(jq -R -s -c . <<< "${json_extract:-}")"
|
|
|
|
printf '%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n' \
|
|
"$ts" \
|
|
"$tick" \
|
|
"$i" \
|
|
"${url_json}" \
|
|
"$http_status" \
|
|
"$total_ms" \
|
|
"$name_ms" \
|
|
"$conn_ms" \
|
|
"$app_ms" \
|
|
"$pre_ms" \
|
|
"$start_ms" \
|
|
"$size_dl" \
|
|
"${ctype_json}" \
|
|
"${rl_limit_json}" \
|
|
"${rl_remaining_json}" \
|
|
"${rl_reset_json}" \
|
|
"${retry_after_json}" \
|
|
"${x_rl_limit_json}" \
|
|
"${x_rl_remaining_json}" \
|
|
"${x_rl_reset_json}" \
|
|
"${rate_limit_limit_json}" \
|
|
"${rate_limit_remaining_json}" \
|
|
"${rate_limit_reset_json}" \
|
|
"${other_rl}" \
|
|
"${json_extract_json}" >> "$csv"
|
|
|
|
rm -f "$tmp_headers" "$tmp_body" "$tmp_meta" 2>/dev/null || true
|
|
done
|
|
|
|
if [[ $tick -lt $ticks ]]; then
|
|
sleep "$interval"
|
|
fi
|
|
done
|
|
|
|
echo "Done. CSV: $csv" >&2
|