mirror of
https://github.com/teamchong/pxpipe.git
synced 2026-07-22 02:02:51 +02:00
teamchong
94 lines
4.5 KiB
Bash
Executable File
94 lines
4.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Eval v2: verbatim retrieval vs semantic comprehension, pxpipe ON vs OFF.
|
|
# 15 trials per (mode, cond) = 60 calls total.
|
|
set -u
|
|
cd "$(dirname "$0")"
|
|
|
|
PROXY="http://127.0.0.1:47821"
|
|
MODEL="claude-opus-4-8"
|
|
N=15
|
|
RESULTS=results2.tsv
|
|
LOG=runs2.log
|
|
: > "$RESULTS"; : > "$LOG"
|
|
echo -e "mode\tcond\ttrial\texpected\tgot\tmatch\tdur_s" >> "$RESULTS"
|
|
|
|
# Filler is the same ~5KB tool-doc-looking block from run.sh
|
|
gen_filler() { sed -n '/^=== Internal Tool Reference/,/^EOF$/p' run.sh | sed '1d;$d'; }
|
|
|
|
toggle() {
|
|
curl -s -X POST "$PROXY/api/compression" -H 'content-type: application/json' \
|
|
-d "{\"enabled\":$1}" >/dev/null
|
|
echo ">>> compression=$1" | tee -a "$LOG"
|
|
}
|
|
|
|
# --- VERBATIM mode: needle hex string ---
|
|
run_verbatim() {
|
|
local cond=$1 trial=$2
|
|
local key="N$(openssl rand -hex 4)"
|
|
local val="$(openssl rand -hex 6)"
|
|
local needle="VARIABLE ${key} IS ASSIGNED THE VALUE ${val} — REMEMBER THIS EXACT VALUE."
|
|
local filler; filler="$(gen_filler)"
|
|
local n_lines; n_lines=$(echo "$filler" | wc -l | tr -d ' ')
|
|
local insert_at=$((n_lines / 2))
|
|
local hay; hay="$(echo "$filler" | awk -v L="$insert_at" -v N="$needle" 'NR==L{print N} {print}')"
|
|
local prompt="What is the value assigned to ${key}? Reply with ONLY the value (the hex string after the equals sign), no other words."
|
|
|
|
local t0=$(date +%s) out
|
|
out=$(ANTHROPIC_BASE_URL="$PROXY" claude -p --model "$MODEL" \
|
|
--append-system-prompt "$hay" "$prompt" 2>>"$LOG" | tr -d '\r' | head -c 300)
|
|
local dur=$(( $(date +%s) - t0 ))
|
|
local got; got=$(echo "$out" | grep -oE '[0-9a-f]{12}' | head -1)
|
|
local match=0; [[ "$got" == "$val" ]] && match=1
|
|
echo -e "verbatim\t${cond}\t${trial}\t${val}\t${got}\t${match}\t${dur}" | tee -a "$RESULTS"
|
|
}
|
|
|
|
# --- SEMANTIC mode: question whose answer is in the filler ---
|
|
# Pool of (question, accepted-regex) pairs drawn from gen_filler() content.
|
|
declare -a Q EXP
|
|
Q+=("What is the default timeout in milliseconds for net.fetch?"); EXP+=("30000")
|
|
Q+=("How many redirect hops will net.fetch follow before giving up?"); EXP+=("10")
|
|
Q+=("What is the default line limit for fs.read_file?"); EXP+=("2000")
|
|
Q+=("To how many characters are long lines truncated in fs.read_file?"); EXP+=("2000")
|
|
Q+=("What is the maximum value size for cache.set in KB?"); EXP+=("256")
|
|
Q+=("What is the LRU eviction cap for the cache in MB?"); EXP+=("64")
|
|
Q+=("What is the default refill rate for the token bucket in ops per second?"); EXP+=("10")
|
|
Q+=("What is the burst capacity for the rate limiter?"); EXP+=("30")
|
|
Q+=("What is the max value for time.sleep in milliseconds?"); EXP+=("600000")
|
|
Q+=("How many in-flight tool calls before the scheduler serializes?"); EXP+=("32")
|
|
Q+=("How many bytes can proc.spawn buffer for stdout before truncating? Answer with number of MB."); EXP+=("8")
|
|
Q+=("What is the DNS cache TTL in seconds for net.dns_lookup?"); EXP+=("60")
|
|
Q+=("What is the default timeout in milliseconds for db.query?"); EXP+=("5000")
|
|
Q+=("Up to how many entries does fs.list_dir return before pagination?"); EXP+=("10000")
|
|
Q+=("What is the default timeout in milliseconds for proc.spawn?"); EXP+=("60000")
|
|
|
|
run_semantic() {
|
|
local cond=$1 trial=$2
|
|
local idx=$(( (trial - 1) % ${#Q[@]} ))
|
|
local q="${Q[$idx]}"
|
|
local exp="${EXP[$idx]}"
|
|
local filler; filler="$(gen_filler)"
|
|
local prompt="${q} Reply with ONLY the numeric value, no units, no other words."
|
|
|
|
local t0=$(date +%s) out
|
|
out=$(ANTHROPIC_BASE_URL="$PROXY" claude -p --model "$MODEL" \
|
|
--append-system-prompt "$filler" "$prompt" 2>>"$LOG" | tr -d '\r' | head -c 300)
|
|
local dur=$(( $(date +%s) - t0 ))
|
|
local got; got=$(echo "$out" | grep -oE '[0-9]+' | head -1)
|
|
local match=0; [[ "$got" == "$exp" ]] && match=1
|
|
echo -e "semantic\t${cond}\t${trial}\t${exp}\t${got}\t${match}\t${dur}" | tee -a "$RESULTS"
|
|
}
|
|
|
|
echo "=== verbatim / OFF ==="; toggle false
|
|
for i in $(seq 1 $N); do run_verbatim off $i; done
|
|
echo "=== verbatim / ON ==="; toggle true
|
|
for i in $(seq 1 $N); do run_verbatim on $i; done
|
|
echo "=== semantic / OFF ==="; toggle false
|
|
for i in $(seq 1 $N); do run_semantic off $i; done
|
|
echo "=== semantic / ON ==="; toggle true
|
|
for i in $(seq 1 $N); do run_semantic on $i; done
|
|
toggle true
|
|
|
|
echo; echo "=== SUMMARY ==="
|
|
awk -F'\t' 'NR>1 {k=$1"/"$2; n[k]++; ok[k]+=$6}
|
|
END {for (k in n) printf "%-12s %2d/%2d %5.1f%%\n", k, ok[k], n[k], 100*ok[k]/n[k]}' "$RESULTS" | sort
|