darwin: iter 2 proposal — trec-covid retrieval — adaptive candidate pool for large corpus

docs/darwin/log.jsonl

@@ -1,2 +1,3 @@
 {"kind":"baseline","iter":0,"ts":"2026-06-26T19:56:22.136Z","dimensions":[{"dim":"beir-scifact","score":0.5694,"sotaTarget":0.74,"observedTopBaseline":0.722,"gap":0.1706,"ourRank":10},{"dim":"beir-trec-covid","score":0.6319,"sotaTarget":0.78,"observedTopBaseline":0.778,"gap":0.1481,"ourRank":6},{"dim":"beir-arguana","score":0.4324,"sotaTarget":0.55,"observedTopBaseline":0.636,"gap":0.1176,"ourRank":5},{"dim":"beir-nfcorpus","score":0.3282,"sotaTarget":0.38,"observedTopBaseline":0.38,"gap":0.0518,"ourRank":4}]}
 {"iter":1,"ts":"2026-06-26T20:34:00.000Z","dimension":"beir-scifact","change":"adaptive-rrfk-minmax-norm","rationale":"Score fusion requires normalized ranges: BGE cosine is [0,1] while BM25 varies [0,20+]. Min-max normalization ensures equal contribution. Adaptive RRF k (40 for small corpora <20k) tightens early-result weighting—critical for scifact (5.2k docs) where precision matters more than broad recall. Reciprocal Rank Fusion is standard in retrieval (Robertson et al., Cormack et al.); smaller k parameter shifts weight distribution toward top-ranked results, improving nDCG for focused datasets. Theory predicts +0.03–0.05 nDCG@10 (13% of remaining 0.17 gap).","deltaScore":null,"action":"proposed","commit":"50c684ffa"}
+{"iter":2,"ts":"2026-06-26T21:14:47.000Z","dimension":"beir-trec-covid","change":"adaptive-top-k-candidate-pool","rationale":"Large corpus (171k docs) requires broader candidate pool for fusion. Recall@10=0.0189 indicates relevant docs are ranked <500 in individual systems. Candidate pool is now adaptive: >150k corpus uses top-2000 (vs fixed top-500 in iter 1), 50k–150k uses top-1000. This expands the intersection for RRF fusion without breaking small-corpus gains (scifact 5.2k still uses 500). Medical/scientific domains benefit from wider pooling: TREC COVID queries are verbose/technical, relevant docs spread across ranking. Maintains iter 1 min-max norm + adaptive RRF k. Theory predicts +0.02–0.04 nDCG@10 (13–27% of remaining 0.148 gap) by lifting more relevant docs into top-100 fusion before reranking.","deltaScore":null,"action":"proposed","commit":null}

v3/@claude-flow/cli/scripts/run-beir-hybrid.mjs

@@ -56,6 +56,7 @@ function mrr(retrieved, qrels, k) { for (let i = 0; i < Math.min(retrieved.lengt
 function recall(retrieved, qrels, k) { const tot = [...qrels.values()].filter((v) => v > 0).length; if (tot === 0) return 0; let h = 0; for (let i = 0; i < Math.min(retrieved.length, k); i++) if ((qrels.get(retrieved[i]) ?? 0) > 0) h++; return h / tot; }
 function cosine(a, b) { let s = 0; for (let i = 0; i < a.length; i++) s += a[i] * b[i]; return s; }
 function adaptiveRrfK(corpusSize) { return corpusSize < 20000 ? 40 : 60; }  // tighter weighting for small corpora
+function adaptiveTopK(corpusSize) { return corpusSize > 150000 ? 2000 : corpusSize > 50000 ? 1000 : 500; }  // pool more candidates for large corpora (iter 2)
 function minMaxNorm(scores) { const [min, max] = [Math.min(...scores), Math.max(...scores)]; return scores.map((s) => max === min ? 0.5 : (s - min) / (max - min)); }
 
 function loadJsonl(path) { return readFileSync(path, 'utf-8').split('\n').filter(Boolean).map((l) => JSON.parse(l)); }
@@ -78,11 +79,12 @@ async function main() {
   const corpus = loadJsonl(join(DATA_DIR, 'corpus.jsonl'));
   const RRF_K = adaptiveRrfK(corpus.length);  // adaptive k based on corpus size (iter 1: normalize scores + tighter k for small corpora)
 
-  console.log(`# BEIR ${dataset} — hybrid RRF${RERANK ? ' + cross-encoder rerank' : ''} (ADR-087 + iter1)`);
+  console.log(`# BEIR ${dataset} — hybrid RRF${RERANK ? ' + cross-encoder rerank' : ''} (ADR-087 + iter1 + iter2)`);
   console.log(`Data:  ${DATA_DIR}`);
   console.log(`Dense: ${BGE_MODEL}`);
   console.log(`RRF k: ${RRF_K} (adaptive for corpus size ${corpus.length})${RERANK ? `, rerank top-${RERANK_TOP_K}` : ''}`);
   console.log(`Normalization: min-max before RRF fusion`);
+  console.log(`Candidate pool: top-${adaptiveTopK(corpus.length)} per system (iter 2: scaled for large corpora)`);
   const queries = loadJsonl(join(DATA_DIR, 'queries.jsonl'));
   const qrels = loadQrels(join(DATA_DIR, 'qrels/test.tsv'));
   console.log(`Corpus: ${corpus.length} docs · Test qrels: ${qrels.size}`);
@@ -168,7 +170,8 @@ async function main() {
 
     // §3 — RRF fusion: normalize scores first, then score = sum over systems of 1/(k + rank).
     // Min-max normalize each system's scores to [0,1] for fair fusion (iter 1 optimization).
-    const TOP_PER_SYSTEM = 500;
+    // For large corpora, pull more candidates before fusion (iter 2 optimization).
+    const TOP_PER_SYSTEM = adaptiveTopK(corpus.length);
     const denseTopK = denseScored.slice(0, Math.min(TOP_PER_SYSTEM, denseScored.length));
     const bm25TopK = bm25Scored.slice(0, Math.min(TOP_PER_SYSTEM, bm25Scored.length)).filter((s) => s.score > 0);
     const denseScoresNorm = minMaxNorm(denseTopK.map((s) => s.score));