mirror of
https://github.com/ruvnet/ruflo.git
synced 2026-10-23 03:47:00 +09:00
test(ruflo-mods): measure agentTrim token saving and dispatch behaviour live
Adds scripts/live-ruflo-mods-trim.sh and the measured report: 4,236 fewer prompt tokens with 59 of 69 types hidden; an unnamed hidden type is refused at dispatch; a prompt-named type is not offered in a headless listing. Co-Authored-By: RuFlo <ruv@ruv.net>
This commit is contained in:
ruv committed
1 parent
6c7b6e8895
commit
e1e6762a85
2 files changed
+225
No files matched your search
Executable
+136
@@ -0,0 +1,136 @@
|
||||
#!/usr/bin/env bash
|
||||
# Measures ruflo-mods agentTrim (ADR-451 item 2) against REAL headless Claude Code sessions with many ruflo agent plugins loaded.
|
||||
# scripts/live-ruflo-mods-trim.sh [tokens enum spawn named | all] (default: all)
|
||||
# Method follows scripts/live-ruflo-mods.sh: private mktemp scratch project, one headless process per case, stream-json.
|
||||
# Spend: haiku only (about $1.5 for `all`). Env: LIVE_MODEL, LIVE_REPS (token-phase repetitions per arm, default 2).
|
||||
# Writes $S/*.json. Prints only model replies and counts; lines that look auth/account/telemetry-like are dropped.
|
||||
set -u
|
||||
REPO="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
S="${LIVE_SCRATCH:-$(mktemp -d "${TMPDIR:-/tmp}/mods-trim.XXXXXX")}"
|
||||
PHASES=("$@"); [ ${#PHASES[@]} -eq 0 ] && PHASES=(all)
|
||||
[ "${PHASES[0]}" = all ] && PHASES=(tokens enum spawn named)
|
||||
mkdir -p "$S"; [ -d "$S/.git" ] || (cd "$S" && git init -q)
|
||||
echo "scratch: $S"
|
||||
sed -n '/^# --- node driver ---$/,$p' "$0" | sed 1d >"$S/live.mjs"
|
||||
for p in "${PHASES[@]}"; do echo "== phase: $p"; node "$S/live.mjs" "$p" "$S" "$REPO" || echo "phase $p exited $?"; done
|
||||
echo "results: $S"
|
||||
exit 0
|
||||
# --- node driver ---
|
||||
import { spawn } from 'node:child_process'
|
||||
import { writeFileSync, readFileSync, readdirSync, existsSync, rmSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
|
||||
const [phase, S, REPO] = process.argv.slice(2)
|
||||
const MODEL = process.env.LIVE_MODEL ?? 'haiku'
|
||||
const REPS = Number(process.env.LIVE_REPS ?? 2)
|
||||
// Every plugin dir under plugins/ that defines agents (ruflo-mods itself has none), so the agent listing is long and real.
|
||||
const AGENT_PLUGINS = readdirSync(`${REPO}/plugins`).filter(d => d !== 'ruflo-mods' && existsSync(`${REPO}/plugins/${d}/agents`) && readdirSync(`${REPO}/plugins/${d}/agents`).some(f => f.endsWith('.md')))
|
||||
const AGENT_FILES = AGENT_PLUGINS.reduce((n, d) => n + readdirSync(`${REPO}/plugins/${d}/agents`).filter(f => f.endsWith('.md')).length, 0)
|
||||
const SECRETISH = /token|account|e-?mail|@[a-z0-9-]+\.|org(anization)?[ _-]?id|telemetry|api[_-]?key|session[_-]?id|bearer|secret|password/i
|
||||
const clean = text => String(text ?? '').split('\n').filter(l => !SECRETISH.test(l)).join('\n')
|
||||
|
||||
const settingsFile = options => {
|
||||
const f = join(S, `settings-${Math.random().toString(36).slice(2, 8)}.json`)
|
||||
const o = options ? { options } : undefined
|
||||
writeFileSync(f, JSON.stringify(o ? { pluginConfigs: { 'ruflo-mods@inline': o, 'ruflo-mods': o } } : {}))
|
||||
return f
|
||||
}
|
||||
const emptyMcp = join(S, 'mcp-empty.json')
|
||||
writeFileSync(emptyMcp, JSON.stringify({ mcpServers: {} }))
|
||||
|
||||
// The usage ledger ($.store of the inline plugin) is user-global: each session starts from an empty one, and every file is restored at the end.
|
||||
const STORE_DIR = join(process.env.HOME ?? '', '.claude/plugins/store')
|
||||
const storeFiles = () => (existsSync(STORE_DIR) ? readdirSync(STORE_DIR) : []).filter(n => /^ruflo-mods_inline-.*\.json$/.test(n)).map(n => join(STORE_DIR, n))
|
||||
const original = new Map(storeFiles().map(f => [f, readFileSync(f)]))
|
||||
const resetLedger = () => { for (const f of storeFiles()) original.has(f) ? writeFileSync(f, '{}') : rmSync(f, { force: true }) }
|
||||
const restoreLedger = () => { for (const f of storeFiles()) if (!original.has(f)) rmSync(f, { force: true }); for (const [f, b] of original) writeFileSync(f, b) }
|
||||
|
||||
async function session(prompts, { options, budget = '0.50' } = {}) {
|
||||
resetLedger()
|
||||
const args = ['-p', '--input-format', 'stream-json', '--output-format', 'stream-json', '--verbose', '--model', MODEL, '--settings', settingsFile(options),
|
||||
'--setting-sources', 'local', '--max-budget-usd', budget, '--allowedTools', 'ToolSearch', '--disallowedTools', 'Bash,Read,Grep,Glob,WebFetch,WebSearch,Edit,Write,NotebookEdit',
|
||||
'--strict-mcp-config', '--mcp-config', emptyMcp, '--append-system-prompt', 'Answer exactly as asked, no commentary.']
|
||||
for (const d of ['ruflo-mods', ...AGENT_PLUGINS]) args.push('--plugin-dir', `${REPO}/plugins/${d}`)
|
||||
const p = spawn('claude', args, { cwd: S, stdio: ['pipe', 'pipe', 'pipe'], env: { ...process.env, CLAUDE_CODE_ENABLE_FUNCTION_HOOKS: '1' } })
|
||||
p.stderr.on('data', () => {})
|
||||
let buf = '', onResult, init
|
||||
p.stdout.on('data', d => {
|
||||
buf += d
|
||||
let i
|
||||
while ((i = buf.indexOf('\n')) >= 0) {
|
||||
const line = buf.slice(0, i); buf = buf.slice(i + 1)
|
||||
try { const m = JSON.parse(line); if (m.type === 'system' && m.subtype === 'init') init = m; if (m.type === 'result' && onResult) onResult(m) } catch {}
|
||||
}
|
||||
})
|
||||
const rows = []
|
||||
for (const text of prompts) {
|
||||
const done = new Promise(res => { onResult = res })
|
||||
p.stdin.write(`${JSON.stringify({ type: 'user', message: { role: 'user', content: text } })}\n`)
|
||||
const r = await Promise.race([done, new Promise(res => setTimeout(() => res(undefined), 150_000))])
|
||||
const u = r?.usage ?? {}
|
||||
rows.push({ prompt: text, reply: r?.result ?? '(timeout)', cost: r?.total_cost_usd, usage: { input: u.input_tokens, cacheCreate: u.cache_creation_input_tokens, cacheRead: u.cache_read_input_tokens, output: u.output_tokens } })
|
||||
}
|
||||
p.stdin.end()
|
||||
await new Promise(res => { const t = setTimeout(() => (p.kill(), res()), 15_000); p.on('close', () => (clearTimeout(t), res())) })
|
||||
// only counts and names leave the init message: nothing else from it is stored or printed
|
||||
const agents = Array.isArray(init?.agents) ? init.agents.filter(a => typeof a === 'string') : []
|
||||
return { rows, agentsInInit: agents.length, cost: rows.reduce((n, r) => n + (r.cost ?? 0), 0) }
|
||||
}
|
||||
const promptTokens = u => (u.input ?? 0) + (u.cacheCreate ?? 0) + (u.cacheRead ?? 0)
|
||||
const out = (name, data) => writeFileSync(join(S, name), JSON.stringify(data, null, 2))
|
||||
const LIST = 'List every subagent_type value your Agent tool accepts, comma-separated, nothing else.'
|
||||
const OK = 'Reply with the single word ok.'
|
||||
// split first, filter per token: a benign name ("...:telemetry-analyzer") must not drop its whole line
|
||||
const parseList = reply => String(reply ?? '').split(/[,\n]/).map(s => s.trim().replace(/^[-*\s]+|[`"'.\s]+$/g, '')).filter(t => /^[A-Za-z0-9:._-]{1,80}$/.test(t) && !SECRETISH.test(t))
|
||||
const hiddenOf = reply => /(\d+) type\(s\) hidden/.exec(reply)?.[1] ?? 'n/a'
|
||||
const DOCS = 'ruflo-docs:docs-writer' // an unused, non-core type
|
||||
// the prompt must not contain the type name or its bare name (the mod keeps a type whose name the prompt contains)
|
||||
const SPELLED = 'Build a subagent_type string by joining, with no spaces: the text ruflo, a hyphen, the text docs, a colon, the text docs, a hyphen, the text writer.'
|
||||
|
||||
console.log(`agent plugins loaded: ${AGENT_PLUGINS.length} (${AGENT_FILES} agent files, plus ruflo-mods); model ${MODEL}`)
|
||||
try {
|
||||
if (phase === 'tokens') {
|
||||
const arms = [['off', undefined], ['on', { agentTrim: true }]]
|
||||
const results = {}
|
||||
for (let r = 0; r < REPS; r++) for (const [arm, options] of arms) {
|
||||
const res = await session([OK, '/ruflo-mods'], { options, budget: '0.30' })
|
||||
const u = res.rows[0].usage
|
||||
console.log(`[tokens ${arm} #${r + 1}] promptTokens=${promptTokens(u)} (input ${u.input}, cacheCreate ${u.cacheCreate}, cacheRead ${u.cacheRead}) output=${u.output} agentsInInit=${res.agentsInInit} cost=$${res.cost?.toFixed(4)}`)
|
||||
console.log(clean(res.rows[1].reply).split('\n').filter(l => /agent trim/.test(l)).join('\n'))
|
||||
;(results[arm] ??= []).push({ ...u, promptTokens: promptTokens(u), agentsInInit: res.agentsInInit, hidden: hiddenOf(res.rows[1].reply), cost: res.cost })
|
||||
}
|
||||
out('tokens.json', results)
|
||||
const mean = a => a.reduce((n, x) => n + x.promptTokens, 0) / a.length
|
||||
console.log(`[tokens] mean promptTokens off=${mean(results.off)} on=${mean(results.on)} delta=${mean(results.off) - mean(results.on)}`)
|
||||
} else if (phase === 'enum') {
|
||||
const res = {}
|
||||
for (const [arm, options] of [['off', undefined], ['on', { agentTrim: true }]]) {
|
||||
const s = await session([LIST, '/ruflo-mods'], { options })
|
||||
const names = parseList(s.rows[0].reply)
|
||||
res[arm] = { count: names.length, chars: s.rows[0].reply.length, names, hidden: hiddenOf(s.rows[1].reply), promptTokens: promptTokens(s.rows[0].usage) }
|
||||
console.log(`[enum ${arm}] model enumerated ${names.length} types (${res[arm].chars} chars); /ruflo-mods hidden=${res[arm].hidden}; cost $${s.cost?.toFixed(4)}`)
|
||||
console.log(`[enum ${arm}] ${DOCS} listed: ${names.includes(DOCS)}; sample: ${names.slice(0, 12).join(', ')}`)
|
||||
}
|
||||
out('enum.json', res)
|
||||
if (res.off.count && res.on.count) console.log(`[enum] enumerated-name delta off-on = ${res.off.count - res.on.count} types`)
|
||||
} else if (phase === 'spawn') {
|
||||
const SPAWN = `${SPELLED} Call the Agent tool once, in the foreground, with that subagent_type, description "hi", prompt "reply with the single word pong". Then reply with only the word ACCEPTED if the call ran, or REFUSED: <error text> if it errored.`
|
||||
const SPAWN_NAMED = `Call the Agent tool once, in the foreground, with subagent_type "${DOCS}", description "hi", prompt "reply with the single word pong". Then reply with only the word ACCEPTED if the call ran, or REFUSED: <error text> if it errored.`
|
||||
for (const [arm, options, text] of [['trim-off', undefined, SPAWN], ['trim-on', { agentTrim: true }, SPAWN], ['trim-on-prompt-names-type', { agentTrim: true }, SPAWN_NAMED]]) {
|
||||
const s = await session([text, '/ruflo-mods'], { options, budget: '0.40' })
|
||||
console.log(`[spawn ${arm}] prompt contains type name: ${text.includes(DOCS)} / bare: ${text.includes('docs-writer')}\n[spawn ${arm}] ${clean(s.rows[0].reply).slice(0, 400)}\n[spawn ${arm}] hidden=${hiddenOf(s.rows[1].reply)} cost $${s.cost?.toFixed(4)}`)
|
||||
for (const f of storeFiles()) console.log(`[spawn ${arm}] ledger: ${readFileSync(f, 'utf8').replace(/\s+/g, ' ').slice(0, 300)}`)
|
||||
out(`spawn-${arm}.json`, { rows: s.rows.map(r => ({ ...r, reply: clean(r.reply) })) })
|
||||
}
|
||||
} else if (phase === 'named') {
|
||||
const NAMED = `Is ${DOCS} one of the subagent_type values your Agent tool accepts? Answer yes or no, then ${LIST}`
|
||||
const single = await session([NAMED, '/ruflo-mods'], { options: { agentTrim: true } })
|
||||
const a = parseList(single.rows[0].reply)
|
||||
console.log(`[named single-prompt] first words: ${clean(single.rows[0].reply).slice(0, 12).replace(/\n/g, ' ')} | ${DOCS} in list: ${a.includes(DOCS)} (${a.length} listed) hidden=${hiddenOf(single.rows[1].reply)}`)
|
||||
const multi = await session([OK, NAMED, '/ruflo-mods'], { options: { agentTrim: true } })
|
||||
const b = parseList(multi.rows[1].reply)
|
||||
console.log(`[named second-turn] first words: ${clean(multi.rows[1].reply).slice(0, 12).replace(/\n/g, ' ')} | ${DOCS} in list: ${b.includes(DOCS)} (${b.length} listed) hidden=${hiddenOf(multi.rows[2].reply)}`)
|
||||
out('named.json', { single: { listed: a.length, has: a.includes(DOCS) }, second: { listed: b.length, has: b.includes(DOCS) } })
|
||||
} else { console.error(`unknown phase ${phase}`); process.exitCode = 2 }
|
||||
} finally { restoreLedger() }
|
||||
process.exit(process.exitCode ?? 0)
|
||||
@@ -0,0 +1,89 @@
|
||||
# ruflo-mods `agentTrim`: measured token saving and dispatch behaviour (2026-10-05)
|
||||
|
||||
Closes the "size in tokens is UNMEASURED" gap in ADR-451 item 2 (`agent.offer`). Reproduce with
|
||||
`scripts/live-ruflo-mods-trim.sh [tokens enum spawn named | all]` (haiku, private `mktemp` scratch project, one headless
|
||||
`claude -p --input-format stream-json` process per case, about $1 for the runs below). Nothing under `plugins/` was changed.
|
||||
|
||||
## Setup
|
||||
|
||||
- `--plugin-dir` for `ruflo-mods` plus all 37 other `plugins/*` dirs that define agents (64 agent files). The init message
|
||||
reported 69 agents (those plus the engine's built-ins). `--strict-mcp-config` with an empty config, so no MCP tool
|
||||
descriptions differ between arms and none are loaded.
|
||||
- Arms: `agentTrim` off (default) and on (`pluginConfigs` in a per-run `--settings` file, `CLAUDE_CODE_ENABLE_FUNCTION_HOOKS=1`).
|
||||
The `$.store` usage ledger is user-global, so each session starts from an empty ledger and the files are restored afterwards.
|
||||
- Trim engaged in every "on" run: `/ruflo-mods` answered `59 type(s) hidden` (off: `agent trim: off`).
|
||||
|
||||
## 1. Size of the agent listing, off vs on
|
||||
|
||||
**Method.** The listing lives inside the `Agent` tool description, which no stream-json message exposes. Two proxies:
|
||||
|
||||
1. Total prompt tokens of one trivial turn (`Reply with the single word ok.`), from the result message's `usage`
|
||||
(`input + cache_creation + cache_read`), 3 repetitions per arm, alternating arms. Everything except the listing is identical
|
||||
between arms, so the difference is the listing's size.
|
||||
2. Ask the model to enumerate the `subagent_type` values it accepts and count them.
|
||||
|
||||
| run | off (prompt tokens) | on (prompt tokens) |
|
||||
|---|---|---|
|
||||
| 1 | 23,510 | 19,337 |
|
||||
| 2 | 23,626 | 19,359 |
|
||||
| 3 | 23,649 | 19,381 |
|
||||
| mean | **23,595** | **19,359** |
|
||||
|
||||
**Measured saving: 4,236 tokens per request (about 18% of this session's prompt)** with 59 of 69 types hidden, about 72 tokens
|
||||
per hidden type. A separate 1-repetition run gave 4,174. The split between `cache_creation` and `cache_read` varies run to run
|
||||
(cache warmth); only the sum is stable (spread across the 3 repetitions: 139 tokens off, 44 tokens on).
|
||||
|
||||
Enumeration: off, the model listed 67 types (2,105 characters of bare names; lossy, init says 69). On, it listed 9 (148 characters):
|
||||
`claude, Explore, general-purpose, Plan, ruflo-ai-team:researcher, ruflo-core:coder, ruflo-core:researcher, ruflo-core:reviewer,
|
||||
ruflo-testgen:tester`. Bare names understate the real listing, which carries each description and tool list.
|
||||
A crude local estimate from the 64 frontmatters (`- plugin:name: description (Tools: ...)`) is 12,559 characters, about 3,100 tokens
|
||||
at 4 characters per token. That undercounts (multi-line descriptions are cut), which is why it sits below the measured 4,236.
|
||||
|
||||
**Against the ADR's claim.** ADR-451 expects "at least 3,000 fewer input tokens". On this catalogue the measured saving is 4,236,
|
||||
so the threshold is met. The ADR's figure was written for "about 100 agent types"; this run has 69 and still clears it, so a session
|
||||
with a larger catalogue should save more (extrapolating at about 72 tokens per hidden type is an estimate, not a measurement).
|
||||
The ADR's acceptance test also asks for the comparison "in this repo" with `claude -p --output-format json`; this run used
|
||||
stream-json with the plugin set above, not that exact command.
|
||||
|
||||
## 2. Can an unused type the prompt does not name be spawned with trim on?
|
||||
|
||||
The spawn prompt built the type name from parts (`ruflo`, hyphen, `docs`, colon, `docs`, hyphen, `writer`), so neither
|
||||
`ruflo-docs:docs-writer` nor `docs-writer` appears in it (checked by the script).
|
||||
|
||||
| arm | prompt contains the type name | result |
|
||||
|---|---|---|
|
||||
| trim off | no | ACCEPTED |
|
||||
| trim on | no | **REFUSED**: `Agent type 'ruflo-docs:docs-writer' not found. Available agents: claude, Explore, general-purpose, Plan, ruflo-ai-team:researcher, ruflo-core:coder, ...` |
|
||||
| trim on | **yes**, written plainly | ACCEPTED; ledger afterwards `{ "agentUse": { "ruflo-docs:docs-writer": <ms> } }` |
|
||||
|
||||
**This does not confirm the previous live run.** An unused, unnamed type is refused at dispatch, so trim is a dispatch block in that
|
||||
case, as ADR-451's design says ("a hidden type is also refused at dispatch"). The earlier observation that "a hidden type still
|
||||
spawns by name" matches the third row: its spawn prompt named the type, and the prompt-name rule (`isKept`) keeps it at dispatch.
|
||||
The comment in `plugins/ruflo-mods/hooks/agents/index.ts` ("not a dispatch block: live, a hidden type still spawns by name") is
|
||||
therefore too broad and should say "unless the current prompt names it". Not edited here (outside this item's scope).
|
||||
|
||||
Consequence to weigh: with trim on, the model cannot discover a hidden type, and a user who wants one must name it in the prompt,
|
||||
list it in `agentTrimKeep`, or have used it in the last 30 days.
|
||||
|
||||
## 3. Does a type the prompt names stay offered in a headless session?
|
||||
|
||||
No. With trim on, a prompt asking whether `ruflo-docs:docs-writer` is available and to list the accepted types answered "No" and
|
||||
listed 9 types without it (the script's count of 10 includes the leading "No"), both as the first prompt and as the second turn of the same process (after a first `ok` turn).
|
||||
The listing is fixed before the first prompt is processed and does not refresh on later turns, so "named in the prompt" keeps a
|
||||
type only at dispatch (section 2, third row), not in the offered listing. This matches the previous run.
|
||||
|
||||
## What this could not measure
|
||||
|
||||
- **The listing itself.** No stream-json message carries the tool description, so the saving is a difference of totals, not a
|
||||
tokenisation of the listing. Cache state moves tokens between fields; the sum was stable to within 139 tokens.
|
||||
- **Other catalogues and models.** One model (haiku), one plugin set (69 types). The per-type figure of about 72 tokens depends on
|
||||
description length.
|
||||
- **The 20-task replay.** The ADR's "zero failed `Agent` dispatches over a 20-task replay" was not run. This run shows that a
|
||||
first-time, unnamed use of a hidden type fails (1 of 1 trials), which is the cost such a replay would count.
|
||||
- **Interactive sessions.** Only headless stream-json was driven; whether an interactive session rebuilds the listing between
|
||||
prompts was not tested.
|
||||
- **Run-to-run variance in the refusal text.** The model copies the error's "Available agents" list and dropped
|
||||
`ruflo-core:reviewer` in one of two refused runs; the refusal itself was identical.
|
||||
- **Engine output.** The init message was reduced to a count of agent names; nothing else from it was stored or printed. The script
|
||||
drops any printed line that looks auth, account or telemetry-like. No such content appeared in the probes; one agent name
|
||||
(`...:telemetry-analyzer`) tripped the first version of that filter, which now works per token.
|
||||
Reference in new issue
Block a user