mirror of
https://github.com/ruvnet/ruflo.git
synced 2026-11-09 14:04:06 +09:00
dream(intelligence): wire real Fisher-information estimation into SONAManager's EWC++ path
SONAManager's EWC++ catastrophic-forgetting mechanism was completely inert: ewcState.fisher/.means were initialized empty and never written anywhere in the core @claude-flow/neural package; consolidateEWC() (the only method touching fisher post-init) only decayed an always-empty map and had zero callers. Every computeEWCPenalty/computeEWCLoss call across all 5 learning modes therefore always evaluated to exactly 0, in production (SONAManager backs NeuralLearningSystem, loaded by @claude-flow/memory's learning-bridge). Scoped fix (default 'balanced' mode only): BalancedMode now exposes getConsolidationSnapshot(), deriving Fisher as squared-gradient and means as the raw gradient from its existing gradientAccumulator (mirroring a working reference pattern already in this monorepo's plugins/cognitive-kernel/src/bridges/sona-bridge.ts). SONAManager's triggerLearning() folds this into ewcState via the existing (unchanged) consolidateEWC() decay step, then additive accumulation - online-EWC style. Also removes 6 stale, never-gitignored .js files under v3/@claude-flow/neural/src/modes/ that shadowed the current .ts sources for any explicit .js-extension import (Vite/Vitest resolve a literal file on disk over transforming a sibling .ts) and were themselves circularly broken - this was blocking any vitest test of SONAManager/mode classes from ever exercising real logic, confirmed via git-stash to reproduce identically on the pre-candidate baseline and to be a necessary precondition for tonight's evaluation, not part of the EWC logic change itself. Disclosed scope limit: this makes the EWC *state* real and measurable; the computed penalty is not yet wired into the gradient update itself (same gap already disclosed as a stub in the reference plugin's own consolidate()). Co-Authored-By: RuFlo <ruv@ruv.net> Claude-Session: https://claude.ai/code/session_01ReiDH3LBmCGHLADvSaqWs2
This commit is contained in:
10 files changed
+218
-1528
No files matched your search
@@ -0,0 +1,129 @@
|
||||
/**
|
||||
* Dream Cycle 2026-10-27 (DEEP=intelligence) — discriminating tests for
|
||||
* SONAManager's EWC++ consolidation path.
|
||||
*
|
||||
* Before this fix: `ewcState.fisher`/`.means` were initialized empty in
|
||||
* `SONAManager.initialize()` and nothing in the core package ever wrote to
|
||||
* them; `consolidateEWC()` (the only method that touches `fisher` post-init)
|
||||
* only decayed existing entries and had zero callers anywhere in the repo.
|
||||
* Every one of these tests fails against that baseline and passes against
|
||||
* the candidate (verified via `git stash` isolation on `sona-manager.ts`,
|
||||
* `modes/base.ts`, `modes/balanced.ts`).
|
||||
*
|
||||
* Scope disclosure: these tests verify the EWC *state* (fisher/means/decay)
|
||||
* becomes real and data-dependent. They do NOT assert that the computed
|
||||
* penalty changes `learn()`'s returned `improvementDelta` — it doesn't;
|
||||
* `computeEWCPenalty()`'s result feeds a local `totalGradientNorm` that
|
||||
* `BalancedMode.learn()` already discarded before this change, a separate,
|
||||
* deeper, deliberately-not-fixed-tonight gap (matching the equally-stubbed
|
||||
* `consolidate()` placeholder in this monorepo's own reference pattern at
|
||||
* `plugins/cognitive-kernel/src/bridges/sona-bridge.ts`).
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { createSONAManager, type SONAManager } from '../src/sona-manager.js';
|
||||
import { deepDecode } from '../src/utils/serialize.js';
|
||||
import type { EWCState, Trajectory } from '../src/types.js';
|
||||
|
||||
function embedding(seed: number, len = 16): Float32Array {
|
||||
const arr = new Float32Array(len);
|
||||
for (let i = 0; i < len; i++) arr[i] = Math.sin(seed * 0.37 + i * 0.11);
|
||||
return arr;
|
||||
}
|
||||
|
||||
async function runLearningCycle(
|
||||
manager: SONAManager,
|
||||
domain: Trajectory['domain'],
|
||||
seed: number,
|
||||
quality: number
|
||||
): Promise<void> {
|
||||
const id = manager.beginTrajectory(`ctx-${seed}`, domain);
|
||||
manager.recordStep(id, 'act-1', quality, embedding(seed));
|
||||
manager.recordStep(id, 'act-2', quality, embedding(seed + 1));
|
||||
manager.completeTrajectory(id, quality);
|
||||
await manager.triggerLearning('test');
|
||||
}
|
||||
|
||||
function decodeEwcState(manager: SONAManager): EWCState {
|
||||
const decoded = deepDecode(manager.serialize()) as { ewcState: EWCState };
|
||||
return decoded.ewcState;
|
||||
}
|
||||
|
||||
describe('SONAManager EWC++ consolidation', () => {
|
||||
it('populates ewcState.fisher/means with real, non-zero values after a learning cycle (was permanently empty)', async () => {
|
||||
const manager = createSONAManager('balanced');
|
||||
await manager.initialize();
|
||||
|
||||
await runLearningCycle(manager, 'code', 1, 0.9);
|
||||
|
||||
const ewcState = decodeEwcState(manager);
|
||||
expect(ewcState.fisher.size).toBeGreaterThan(0);
|
||||
expect(ewcState.means.size).toBeGreaterThan(0);
|
||||
|
||||
let sawNonZero = false;
|
||||
for (const fisher of ewcState.fisher.values()) {
|
||||
for (const value of fisher) {
|
||||
expect(Number.isFinite(value)).toBe(true);
|
||||
expect(value).toBeGreaterThanOrEqual(0); // Fisher (squared gradient) is never negative
|
||||
if (value > 0) sawNonZero = true;
|
||||
}
|
||||
}
|
||||
expect(sawNonZero).toBe(true);
|
||||
expect(ewcState.taskCount).toBeGreaterThan(0);
|
||||
expect(ewcState.lastConsolidation).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('decays existing Fisher importance before folding in fresh evidence (online EWC), rather than overwriting it', async () => {
|
||||
const manager = createSONAManager('balanced');
|
||||
await manager.initialize();
|
||||
|
||||
await runLearningCycle(manager, 'code', 1, 0.9);
|
||||
const after1 = decodeEwcState(manager);
|
||||
const key = after1.fisher.keys().next().value as string;
|
||||
const valueAfter1 = after1.fisher.get(key)![0];
|
||||
expect(valueAfter1).toBeGreaterThan(0);
|
||||
|
||||
await runLearningCycle(manager, 'code', 1, 0.9); // identical seed -> identical fresh-Fisher magnitude
|
||||
const after2 = decodeEwcState(manager);
|
||||
const valueAfter2 = after2.fisher.get(key)![0];
|
||||
|
||||
// decay(0.9) * valueAfter1 + freshFisher(== valueAfter1's own fresh contribution):
|
||||
// strictly greater than a pure decay (0.9x) and strictly less than a naive
|
||||
// sum-with-no-decay (2x) would be if decay were skipped entirely in the
|
||||
// other direction — the real assertion that matters is monotonic growth
|
||||
// under repeated same-magnitude evidence, which a permanently-empty map
|
||||
// (the baseline) could never produce.
|
||||
expect(valueAfter2).toBeGreaterThan(valueAfter1 * 0.9);
|
||||
expect(after2.taskCount).toBe(after1.taskCount + 1);
|
||||
});
|
||||
|
||||
it('control: a trigger with no completed trajectories leaves ewcState untouched (no fabricated state)', async () => {
|
||||
const manager = createSONAManager('balanced');
|
||||
await manager.initialize();
|
||||
|
||||
await manager.triggerLearning('test'); // nothing completed yet -> learn() short-circuits
|
||||
|
||||
const ewcState = decodeEwcState(manager);
|
||||
expect(ewcState.fisher.size).toBe(0);
|
||||
expect(ewcState.means.size).toBe(0);
|
||||
expect(ewcState.taskCount).toBe(0);
|
||||
});
|
||||
|
||||
it('Fisher values equal means squared (diagonal empirical-Fisher construction), for every populated key', async () => {
|
||||
const manager = createSONAManager('balanced');
|
||||
await manager.initialize();
|
||||
|
||||
await runLearningCycle(manager, 'code', 3, 0.85);
|
||||
|
||||
const ewcState = decodeEwcState(manager);
|
||||
expect(ewcState.fisher.size).toBeGreaterThan(0);
|
||||
|
||||
for (const [key, fisher] of ewcState.fisher) {
|
||||
const means = ewcState.means.get(key)!;
|
||||
expect(means).toBeDefined();
|
||||
for (let i = 0; i < fisher.length; i++) {
|
||||
expect(fisher[i]).toBeCloseTo(means[i] * means[i], 5);
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -1,234 +0,0 @@
|
||||
/**
|
||||
* Balanced Mode Implementation
|
||||
*
|
||||
* General-purpose mode with:
|
||||
* - +25% quality improvement
|
||||
* - 18ms overhead
|
||||
* - Rank-4 LoRA
|
||||
* - Pattern caching
|
||||
* - Standard learning pipeline
|
||||
*/
|
||||
import { BaseModeImplementation } from './index.js';
|
||||
/**
|
||||
* Balanced mode for general-purpose learning
|
||||
*/
|
||||
export class BalancedMode extends BaseModeImplementation {
|
||||
mode = 'balanced';
|
||||
// Pattern cache
|
||||
patternCache = new Map();
|
||||
cacheHits = 0;
|
||||
cacheMisses = 0;
|
||||
// Learning state
|
||||
gradientAccumulator = new Map();
|
||||
momentumBuffers = new Map();
|
||||
// Stats
|
||||
totalPatternMatches = 0;
|
||||
totalPatternTime = 0;
|
||||
totalLearnTime = 0;
|
||||
learnIterations = 0;
|
||||
qualityImprovements = [];
|
||||
async initialize() {
|
||||
await super.initialize();
|
||||
this.patternCache.clear();
|
||||
this.gradientAccumulator.clear();
|
||||
this.momentumBuffers.clear();
|
||||
}
|
||||
async cleanup() {
|
||||
this.patternCache.clear();
|
||||
this.gradientAccumulator.clear();
|
||||
this.momentumBuffers.clear();
|
||||
await super.cleanup();
|
||||
}
|
||||
/**
|
||||
* Find patterns using similarity search with caching
|
||||
*/
|
||||
async findPatterns(embedding, k, patterns) {
|
||||
const startTime = performance.now();
|
||||
// Check cache
|
||||
const cacheKey = this.computeCacheKey(embedding);
|
||||
const cached = this.patternCache.get(cacheKey);
|
||||
if (cached && cached.length >= k) {
|
||||
this.cacheHits++;
|
||||
this.totalPatternTime += performance.now() - startTime;
|
||||
this.totalPatternMatches++;
|
||||
return cached.slice(0, k);
|
||||
}
|
||||
this.cacheMisses++;
|
||||
// Compute similarities for all patterns
|
||||
const matches = [];
|
||||
for (const pattern of patterns) {
|
||||
const similarity = this.cosineSimilarity(embedding, pattern.embedding);
|
||||
matches.push({
|
||||
pattern,
|
||||
similarity,
|
||||
confidence: similarity * pattern.successRate,
|
||||
latencyMs: 0,
|
||||
});
|
||||
}
|
||||
// Sort by similarity descending
|
||||
matches.sort((a, b) => b.similarity - a.similarity);
|
||||
const topK = matches.slice(0, k);
|
||||
// Cache result
|
||||
if (this.patternCache.size > 500) {
|
||||
const firstKey = this.patternCache.keys().next().value;
|
||||
if (firstKey)
|
||||
this.patternCache.delete(firstKey);
|
||||
}
|
||||
this.patternCache.set(cacheKey, topK);
|
||||
this.totalPatternTime += performance.now() - startTime;
|
||||
this.totalPatternMatches++;
|
||||
return topK;
|
||||
}
|
||||
/**
|
||||
* Learn from trajectories using standard gradient descent
|
||||
*/
|
||||
async learn(trajectories, config, ewcState) {
|
||||
const startTime = performance.now();
|
||||
if (trajectories.length === 0)
|
||||
return 0;
|
||||
const qualityThreshold = config.qualityThreshold;
|
||||
const learningRate = config.learningRate;
|
||||
// Separate positive and negative examples
|
||||
const goodTrajectories = trajectories.filter(t => t.qualityScore >= qualityThreshold);
|
||||
const badTrajectories = trajectories.filter(t => t.qualityScore < qualityThreshold);
|
||||
if (goodTrajectories.length === 0)
|
||||
return 0;
|
||||
// Compute gradients from trajectory pairs
|
||||
let totalGradientNorm = 0;
|
||||
for (const good of goodTrajectories) {
|
||||
// Use last step embedding as "goal state"
|
||||
if (good.steps.length === 0)
|
||||
continue;
|
||||
const goalState = good.steps[good.steps.length - 1].stateAfter;
|
||||
// Positive gradient: move toward good outcomes
|
||||
const posGradient = this.computeGradient(goalState, good.qualityScore);
|
||||
totalGradientNorm += this.accumulateGradient('positive', posGradient, learningRate);
|
||||
// Negative gradient: move away from bad outcomes (contrastive)
|
||||
for (const bad of badTrajectories.slice(0, 3)) {
|
||||
if (bad.steps.length === 0)
|
||||
continue;
|
||||
const badState = bad.steps[bad.steps.length - 1].stateAfter;
|
||||
const negGradient = this.computeGradient(badState, -bad.qualityScore);
|
||||
totalGradientNorm += this.accumulateGradient('negative', negGradient, learningRate * 0.5);
|
||||
}
|
||||
}
|
||||
// Apply EWC regularization
|
||||
const ewcPenalty = this.computeEWCPenalty(ewcState, config.ewcLambda);
|
||||
totalGradientNorm += ewcPenalty;
|
||||
// Compute improvement delta
|
||||
const avgGoodQuality = goodTrajectories.reduce((s, t) => s + t.qualityScore, 0) / goodTrajectories.length;
|
||||
const baselineQuality = 0.5;
|
||||
const improvementDelta = avgGoodQuality - baselineQuality;
|
||||
this.qualityImprovements.push(improvementDelta);
|
||||
if (this.qualityImprovements.length > 100) {
|
||||
this.qualityImprovements = this.qualityImprovements.slice(-100);
|
||||
}
|
||||
this.totalLearnTime += performance.now() - startTime;
|
||||
this.learnIterations++;
|
||||
return Math.max(0, improvementDelta);
|
||||
}
|
||||
/**
|
||||
* Apply LoRA adaptations with rank-4
|
||||
*/
|
||||
async applyLoRA(input, weights) {
|
||||
if (!weights) {
|
||||
return input;
|
||||
}
|
||||
const output = new Float32Array(input.length);
|
||||
output.set(input);
|
||||
const rank = this.config.loraRank;
|
||||
// Apply to all target modules
|
||||
for (const module of ['q_proj', 'v_proj', 'k_proj', 'o_proj']) {
|
||||
const A = weights.A.get(module);
|
||||
const B = weights.B.get(module);
|
||||
if (A && B) {
|
||||
const adapted = this.applyLoRATransform(input, A, B, rank);
|
||||
const alpha = 0.2; // Moderate blending
|
||||
for (let i = 0; i < output.length; i++) {
|
||||
output[i] = output[i] * (1 - alpha) + adapted[i] * alpha;
|
||||
}
|
||||
}
|
||||
}
|
||||
return output;
|
||||
}
|
||||
getStats() {
|
||||
const avgImprovement = this.qualityImprovements.length > 0
|
||||
? this.qualityImprovements.reduce((a, b) => a + b, 0) / this.qualityImprovements.length
|
||||
: 0;
|
||||
return {
|
||||
cacheHitRate: this.cacheHits + this.cacheMisses > 0
|
||||
? this.cacheHits / (this.cacheHits + this.cacheMisses)
|
||||
: 0,
|
||||
avgPatternMatchMs: this.totalPatternMatches > 0
|
||||
? this.totalPatternTime / this.totalPatternMatches
|
||||
: 0,
|
||||
avgLearnMs: this.learnIterations > 0
|
||||
? this.totalLearnTime / this.learnIterations
|
||||
: 0,
|
||||
avgImprovement,
|
||||
patternCacheSize: this.patternCache.size,
|
||||
learnIterations: this.learnIterations,
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Compute cache key from embedding
|
||||
*/
|
||||
computeCacheKey(embedding) {
|
||||
const keyParts = [];
|
||||
for (let i = 0; i < Math.min(16, embedding.length); i++) {
|
||||
keyParts.push(embedding[i].toFixed(2));
|
||||
}
|
||||
return keyParts.join(',');
|
||||
}
|
||||
/**
|
||||
* Compute gradient from state and reward
|
||||
*/
|
||||
computeGradient(state, reward) {
|
||||
const gradient = new Float32Array(state.length);
|
||||
for (let i = 0; i < state.length; i++) {
|
||||
gradient[i] = state[i] * reward;
|
||||
}
|
||||
return gradient;
|
||||
}
|
||||
/**
|
||||
* Accumulate gradient with momentum
|
||||
*/
|
||||
accumulateGradient(key, gradient, lr) {
|
||||
let momentum = this.momentumBuffers.get(key);
|
||||
if (!momentum) {
|
||||
momentum = new Float32Array(gradient.length);
|
||||
this.momentumBuffers.set(key, momentum);
|
||||
}
|
||||
let accumulator = this.gradientAccumulator.get(key);
|
||||
if (!accumulator) {
|
||||
accumulator = new Float32Array(gradient.length);
|
||||
this.gradientAccumulator.set(key, accumulator);
|
||||
}
|
||||
const beta = 0.9; // Momentum coefficient
|
||||
let norm = 0;
|
||||
for (let i = 0; i < gradient.length; i++) {
|
||||
momentum[i] = beta * momentum[i] + (1 - beta) * gradient[i];
|
||||
accumulator[i] += lr * momentum[i];
|
||||
norm += momentum[i] * momentum[i];
|
||||
}
|
||||
return Math.sqrt(norm);
|
||||
}
|
||||
/**
|
||||
* Compute EWC penalty for continual learning
|
||||
*/
|
||||
computeEWCPenalty(ewcState, lambda) {
|
||||
let penalty = 0;
|
||||
for (const [key, fisher] of ewcState.fisher) {
|
||||
const means = ewcState.means.get(key);
|
||||
const current = this.gradientAccumulator.get(key);
|
||||
if (means && current) {
|
||||
for (let i = 0; i < Math.min(fisher.length, means.length, current.length); i++) {
|
||||
const diff = current[i] - means[i];
|
||||
penalty += fisher[i] * diff * diff;
|
||||
}
|
||||
}
|
||||
}
|
||||
return lambda * penalty * 0.5;
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=balanced.js.map
|
||||
@@ -296,4 +296,40 @@ export class BalancedMode extends BaseModeImplementation {
|
||||
|
||||
return lambda * penalty * 0.5;
|
||||
}
|
||||
|
||||
/**
|
||||
* Derive a Fisher-information/means snapshot from this mode's own
|
||||
* accumulated gradients, for EWC consolidation.
|
||||
*
|
||||
* Uses the diagonal empirical-Fisher approximation (Fisher_i ~= gradient_i^2),
|
||||
* the same style of estimator already proven out — for this exact
|
||||
* squared-gradient-of-reward update rule — in this monorepo's own
|
||||
* `plugins/cognitive-kernel/src/bridges/sona-bridge.ts::computeFisher()`.
|
||||
* Non-finite gradient components are dropped (treated as zero) so a bad
|
||||
* sample can't poison consolidated state.
|
||||
*/
|
||||
getConsolidationSnapshot(): {
|
||||
fisher: Map<string, Float32Array>;
|
||||
means: Map<string, Float32Array>;
|
||||
} {
|
||||
const fisher = new Map<string, Float32Array>();
|
||||
const means = new Map<string, Float32Array>();
|
||||
|
||||
for (const [key, gradient] of this.gradientAccumulator) {
|
||||
const f = new Float32Array(gradient.length);
|
||||
const m = new Float32Array(gradient.length);
|
||||
|
||||
for (let i = 0; i < gradient.length; i++) {
|
||||
const g = gradient[i];
|
||||
const safe = Number.isFinite(g) ? g : 0;
|
||||
f[i] = safe * safe;
|
||||
m[i] = safe;
|
||||
}
|
||||
|
||||
fisher.set(key, f);
|
||||
means.set(key, m);
|
||||
}
|
||||
|
||||
return { fisher, means };
|
||||
}
|
||||
}
|
||||
@@ -49,6 +49,18 @@ export interface ModeImplementation {
|
||||
|
||||
/** Get mode-specific stats */
|
||||
getStats(): Record<string, number>;
|
||||
|
||||
/**
|
||||
* Optional: return a Fisher-information/means snapshot derived from this
|
||||
* mode's own internal gradient state, for the manager to fold into EWC
|
||||
* consolidation after a learning cycle. Modes that don't track gradients
|
||||
* internally (or haven't wired this yet) simply omit it; the manager
|
||||
* treats a missing implementation (or an empty snapshot) as a no-op.
|
||||
*/
|
||||
getConsolidationSnapshot?(): {
|
||||
fisher: Map<string, Float32Array>;
|
||||
means: Map<string, Float32Array>;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,316 +0,0 @@
|
||||
/**
|
||||
* Batch Mode Implementation
|
||||
*
|
||||
* Optimized for high-throughput processing with:
|
||||
* - Large batch sizes (128)
|
||||
* - Rank-8 LoRA
|
||||
* - Gradient accumulation
|
||||
* - Async batch processing
|
||||
* - 50ms latency budget
|
||||
*/
|
||||
import { BaseModeImplementation } from './index.js';
|
||||
/**
|
||||
* Batch mode for high-throughput processing
|
||||
*/
|
||||
export class BatchMode extends BaseModeImplementation {
|
||||
mode = 'batch';
|
||||
// Batch processing queues
|
||||
patternQueue = [];
|
||||
learningQueue = [];
|
||||
// Batch buffers
|
||||
embeddingBuffer = null;
|
||||
batchEmbeddings = [];
|
||||
// Gradient accumulation
|
||||
accumulatedGradients = new Map();
|
||||
gradientSteps = 0;
|
||||
// Batch processing state
|
||||
isBatchProcessing = false;
|
||||
batchTimer = null;
|
||||
// Stats
|
||||
totalBatches = 0;
|
||||
totalItems = 0;
|
||||
totalBatchTime = 0;
|
||||
learnIterations = 0;
|
||||
async initialize() {
|
||||
await super.initialize();
|
||||
this.patternQueue = [];
|
||||
this.learningQueue = [];
|
||||
this.accumulatedGradients.clear();
|
||||
this.gradientSteps = 0;
|
||||
}
|
||||
async cleanup() {
|
||||
if (this.batchTimer) {
|
||||
clearTimeout(this.batchTimer);
|
||||
}
|
||||
this.patternQueue = [];
|
||||
this.learningQueue = [];
|
||||
this.accumulatedGradients.clear();
|
||||
await super.cleanup();
|
||||
}
|
||||
/**
|
||||
* Find patterns - queues for batch processing
|
||||
*/
|
||||
async findPatterns(embedding, k, patterns) {
|
||||
// For immediate needs, process synchronously
|
||||
if (patterns.length < 100) {
|
||||
return this.findPatternsDirect(embedding, k, patterns);
|
||||
}
|
||||
// Queue for batch processing
|
||||
return new Promise(resolve => {
|
||||
this.patternQueue.push({ embedding, k, resolve });
|
||||
this.scheduleBatchProcessing(patterns);
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Learn from trajectories - accumulates for batch
|
||||
*/
|
||||
async learn(trajectories, config, ewcState) {
|
||||
const startTime = performance.now();
|
||||
if (trajectories.length === 0)
|
||||
return 0;
|
||||
// Add to learning queue
|
||||
this.learningQueue.push(...trajectories);
|
||||
// Process when queue is full
|
||||
if (this.learningQueue.length >= config.batchSize) {
|
||||
return this.processBatchLearning(config, ewcState);
|
||||
}
|
||||
// Return estimated improvement
|
||||
const avgQuality = trajectories.reduce((s, t) => s + t.qualityScore, 0) / trajectories.length;
|
||||
this.totalBatchTime += performance.now() - startTime;
|
||||
return Math.max(0, avgQuality - 0.5) * 0.5; // Partial estimate
|
||||
}
|
||||
/**
|
||||
* Apply LoRA with rank-8
|
||||
*/
|
||||
async applyLoRA(input, weights) {
|
||||
if (!weights) {
|
||||
return input;
|
||||
}
|
||||
// Batch mode can process multiple inputs efficiently
|
||||
this.batchEmbeddings.push(new Float32Array(input));
|
||||
// Process immediately for single requests
|
||||
if (this.batchEmbeddings.length === 1) {
|
||||
const output = await this.applyLoRADirect(input, weights);
|
||||
this.batchEmbeddings = [];
|
||||
return output;
|
||||
}
|
||||
// For multiple inputs, process as batch
|
||||
const outputs = await this.applyLoRABatch(this.batchEmbeddings, weights);
|
||||
this.batchEmbeddings = [];
|
||||
return outputs[outputs.length - 1];
|
||||
}
|
||||
getStats() {
|
||||
return {
|
||||
totalBatches: this.totalBatches,
|
||||
avgItemsPerBatch: this.totalBatches > 0 ? this.totalItems / this.totalBatches : 0,
|
||||
avgBatchTimeMs: this.totalBatches > 0 ? this.totalBatchTime / this.totalBatches : 0,
|
||||
pendingPatternRequests: this.patternQueue.length,
|
||||
pendingTrajectories: this.learningQueue.length,
|
||||
accumulatedGradientSteps: this.gradientSteps,
|
||||
learnIterations: this.learnIterations,
|
||||
};
|
||||
}
|
||||
// ========================================================================
|
||||
// Direct processing (for small batches)
|
||||
// ========================================================================
|
||||
/**
|
||||
* Direct pattern matching without batching
|
||||
*/
|
||||
findPatternsDirect(embedding, k, patterns) {
|
||||
const matches = [];
|
||||
for (const pattern of patterns) {
|
||||
const similarity = this.cosineSimilarity(embedding, pattern.embedding);
|
||||
matches.push({
|
||||
pattern,
|
||||
similarity,
|
||||
confidence: similarity * pattern.successRate,
|
||||
latencyMs: 0,
|
||||
});
|
||||
}
|
||||
matches.sort((a, b) => b.similarity - a.similarity);
|
||||
return matches.slice(0, k);
|
||||
}
|
||||
/**
|
||||
* Direct LoRA application
|
||||
*/
|
||||
async applyLoRADirect(input, weights) {
|
||||
const output = new Float32Array(input.length);
|
||||
output.set(input);
|
||||
const rank = this.config.loraRank;
|
||||
for (const module of ['q_proj', 'v_proj', 'k_proj', 'o_proj']) {
|
||||
const A = weights.A.get(module);
|
||||
const B = weights.B.get(module);
|
||||
if (A && B) {
|
||||
const adapted = this.applyLoRATransform(input, A, B, rank);
|
||||
const alpha = 0.25;
|
||||
for (let i = 0; i < output.length; i++) {
|
||||
output[i] = output[i] * (1 - alpha) + adapted[i] * alpha;
|
||||
}
|
||||
}
|
||||
}
|
||||
return output;
|
||||
}
|
||||
// ========================================================================
|
||||
// Batch processing
|
||||
// ========================================================================
|
||||
/**
|
||||
* Schedule batch processing
|
||||
*/
|
||||
scheduleBatchProcessing(patterns) {
|
||||
if (this.batchTimer)
|
||||
return;
|
||||
this.batchTimer = setTimeout(() => {
|
||||
this.processBatchPatterns(patterns);
|
||||
}, 10); // Wait 10ms to accumulate requests
|
||||
}
|
||||
/**
|
||||
* Process pattern requests in batch
|
||||
*/
|
||||
async processBatchPatterns(patterns) {
|
||||
this.batchTimer = null;
|
||||
if (this.patternQueue.length === 0)
|
||||
return;
|
||||
const startTime = performance.now();
|
||||
this.isBatchProcessing = true;
|
||||
const batch = this.patternQueue;
|
||||
this.patternQueue = [];
|
||||
// Pre-compute pattern embeddings matrix
|
||||
const patternMatrix = patterns.map(p => p.embedding);
|
||||
// Process all queries in batch
|
||||
for (const request of batch) {
|
||||
const matches = this.batchSimilaritySearch(request.embedding, request.k, patterns, patternMatrix);
|
||||
request.resolve(matches);
|
||||
}
|
||||
this.totalBatches++;
|
||||
this.totalItems += batch.length;
|
||||
this.totalBatchTime += performance.now() - startTime;
|
||||
this.isBatchProcessing = false;
|
||||
}
|
||||
/**
|
||||
* Batch similarity search
|
||||
*/
|
||||
batchSimilaritySearch(query, k, patterns, patternMatrix) {
|
||||
const similarities = [];
|
||||
for (let i = 0; i < patternMatrix.length; i++) {
|
||||
const sim = this.cosineSimilarity(query, patternMatrix[i]);
|
||||
similarities.push({ idx: i, sim });
|
||||
}
|
||||
similarities.sort((a, b) => b.sim - a.sim);
|
||||
const topK = similarities.slice(0, k);
|
||||
return topK.map(s => ({
|
||||
pattern: patterns[s.idx],
|
||||
similarity: s.sim,
|
||||
confidence: s.sim * patterns[s.idx].successRate,
|
||||
latencyMs: 0,
|
||||
}));
|
||||
}
|
||||
/**
|
||||
* Process batch learning
|
||||
*/
|
||||
async processBatchLearning(config, ewcState) {
|
||||
const startTime = performance.now();
|
||||
const batch = this.learningQueue.slice(0, config.batchSize);
|
||||
this.learningQueue = this.learningQueue.slice(config.batchSize);
|
||||
const qualityThreshold = config.qualityThreshold;
|
||||
const learningRate = config.learningRate;
|
||||
// Separate by quality
|
||||
const good = batch.filter(t => t.qualityScore >= qualityThreshold);
|
||||
const bad = batch.filter(t => t.qualityScore < qualityThreshold);
|
||||
if (good.length === 0) {
|
||||
this.totalBatchTime += performance.now() - startTime;
|
||||
return 0;
|
||||
}
|
||||
// Accumulate gradients
|
||||
for (const trajectory of good) {
|
||||
this.accumulateTrajectoryGradient(trajectory, learningRate);
|
||||
}
|
||||
// Contrastive learning from bad examples
|
||||
for (const trajectory of bad.slice(0, good.length)) {
|
||||
this.accumulateTrajectoryGradient(trajectory, -learningRate * 0.3);
|
||||
}
|
||||
this.gradientSteps++;
|
||||
// Apply accumulated gradients every N steps
|
||||
if (this.gradientSteps >= 4) {
|
||||
await this.applyAccumulatedGradients(ewcState, config.ewcLambda);
|
||||
this.gradientSteps = 0;
|
||||
}
|
||||
// Compute improvement
|
||||
const avgQuality = good.reduce((s, t) => s + t.qualityScore, 0) / good.length;
|
||||
const improvement = avgQuality - 0.5;
|
||||
this.learnIterations++;
|
||||
this.totalBatchTime += performance.now() - startTime;
|
||||
return Math.max(0, improvement);
|
||||
}
|
||||
/**
|
||||
* Accumulate gradient from trajectory
|
||||
*/
|
||||
accumulateTrajectoryGradient(trajectory, scale) {
|
||||
if (trajectory.steps.length === 0)
|
||||
return;
|
||||
const key = trajectory.domain;
|
||||
let gradient = this.accumulatedGradients.get(key);
|
||||
if (!gradient) {
|
||||
const dim = trajectory.steps[0].stateAfter.length;
|
||||
gradient = new Float32Array(dim);
|
||||
this.accumulatedGradients.set(key, gradient);
|
||||
}
|
||||
// Add trajectory contribution
|
||||
const weight = trajectory.qualityScore * scale;
|
||||
for (const step of trajectory.steps) {
|
||||
for (let i = 0; i < Math.min(gradient.length, step.stateAfter.length); i++) {
|
||||
gradient[i] += step.stateAfter[i] * weight * step.reward;
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Apply accumulated gradients with EWC
|
||||
*/
|
||||
async applyAccumulatedGradients(ewcState, ewcLambda) {
|
||||
for (const [key, gradient] of this.accumulatedGradients) {
|
||||
// Normalize gradient
|
||||
const norm = Math.sqrt(gradient.reduce((s, v) => s + v * v, 0));
|
||||
if (norm > 0) {
|
||||
for (let i = 0; i < gradient.length; i++) {
|
||||
gradient[i] /= norm;
|
||||
}
|
||||
}
|
||||
// Apply EWC penalty
|
||||
const fisher = ewcState.fisher.get(key);
|
||||
const means = ewcState.means.get(key);
|
||||
if (fisher && means) {
|
||||
for (let i = 0; i < gradient.length; i++) {
|
||||
const penalty = ewcLambda * fisher[i] * (gradient[i] - means[i]);
|
||||
gradient[i] -= penalty;
|
||||
}
|
||||
}
|
||||
// Clear gradient for next accumulation
|
||||
gradient.fill(0);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Apply LoRA to batch of inputs
|
||||
*/
|
||||
async applyLoRABatch(inputs, weights) {
|
||||
const outputs = [];
|
||||
const rank = this.config.loraRank;
|
||||
// Process all inputs together for cache efficiency
|
||||
for (const input of inputs) {
|
||||
const output = new Float32Array(input.length);
|
||||
output.set(input);
|
||||
for (const module of ['q_proj', 'v_proj', 'k_proj', 'o_proj']) {
|
||||
const A = weights.A.get(module);
|
||||
const B = weights.B.get(module);
|
||||
if (A && B) {
|
||||
const adapted = this.applyLoRATransform(input, A, B, rank);
|
||||
const alpha = 0.25;
|
||||
for (let i = 0; i < output.length; i++) {
|
||||
output[i] = output[i] * (1 - alpha) + adapted[i] * alpha;
|
||||
}
|
||||
}
|
||||
}
|
||||
outputs.push(output);
|
||||
}
|
||||
return outputs;
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=batch.js.map
|
||||
@@ -1,310 +0,0 @@
|
||||
/**
|
||||
* Edge Mode Implementation
|
||||
*
|
||||
* Optimized for resource-constrained environments with:
|
||||
* - <5MB memory footprint
|
||||
* - Minimal latency (<1ms)
|
||||
* - Micro-LoRA (rank-1)
|
||||
* - Aggressive pruning
|
||||
* - Async updates
|
||||
*/
|
||||
import { BaseModeImplementation } from './index.js';
|
||||
/**
|
||||
* Edge mode for resource-constrained devices
|
||||
*/
|
||||
export class EdgeMode extends BaseModeImplementation {
|
||||
mode = 'edge';
|
||||
// Minimal pattern storage (compressed)
|
||||
compressedPatterns = new Map();
|
||||
// Quantized LoRA weights (int8)
|
||||
quantizedWeights = new Map();
|
||||
quantizationScale = 1.0;
|
||||
// Pending async updates
|
||||
pendingUpdates = [];
|
||||
updateTimer = null;
|
||||
// Stats
|
||||
totalOps = 0;
|
||||
totalTime = 0;
|
||||
async initialize() {
|
||||
await super.initialize();
|
||||
this.compressedPatterns.clear();
|
||||
this.quantizedWeights.clear();
|
||||
this.pendingUpdates = [];
|
||||
}
|
||||
async cleanup() {
|
||||
if (this.updateTimer) {
|
||||
clearTimeout(this.updateTimer);
|
||||
}
|
||||
this.compressedPatterns.clear();
|
||||
this.quantizedWeights.clear();
|
||||
this.pendingUpdates = [];
|
||||
await super.cleanup();
|
||||
}
|
||||
/**
|
||||
* Find patterns using compressed embeddings
|
||||
*/
|
||||
async findPatterns(embedding, k, patterns) {
|
||||
const startTime = performance.now();
|
||||
// Compress input embedding for comparison
|
||||
const compressedInput = this.compressEmbedding(embedding);
|
||||
// Fast similarity on compressed representations
|
||||
const matches = [];
|
||||
for (const pattern of patterns) {
|
||||
let compressed = this.compressedPatterns.get(pattern.patternId);
|
||||
if (!compressed) {
|
||||
compressed = this.createCompressedPattern(pattern);
|
||||
this.compressedPatterns.set(pattern.patternId, compressed);
|
||||
}
|
||||
// Use compressed similarity (faster but less accurate)
|
||||
const score = this.compressedSimilarity(compressedInput, compressed.embedding);
|
||||
matches.push({ pattern, score });
|
||||
}
|
||||
// Quick partial sort
|
||||
matches.sort((a, b) => b.score - a.score);
|
||||
const topK = matches.slice(0, k);
|
||||
this.totalOps++;
|
||||
this.totalTime += performance.now() - startTime;
|
||||
return topK.map(m => ({
|
||||
pattern: m.pattern,
|
||||
similarity: m.score,
|
||||
confidence: m.score * m.pattern.successRate,
|
||||
latencyMs: 0,
|
||||
}));
|
||||
}
|
||||
/**
|
||||
* Lightweight learning with async updates
|
||||
*/
|
||||
async learn(trajectories, config, ewcState) {
|
||||
const startTime = performance.now();
|
||||
if (trajectories.length === 0)
|
||||
return 0;
|
||||
// Edge mode: only learn from high-quality trajectories
|
||||
const threshold = config.qualityThreshold;
|
||||
const good = trajectories.filter(t => t.qualityScore >= threshold);
|
||||
if (good.length === 0)
|
||||
return 0;
|
||||
const avgQuality = good.reduce((s, t) => s + t.qualityScore, 0) / good.length;
|
||||
// Queue async update for later processing
|
||||
this.queueAsyncUpdate(async () => {
|
||||
await this.performLightweightUpdate(good, config.learningRate);
|
||||
});
|
||||
this.totalOps++;
|
||||
this.totalTime += performance.now() - startTime;
|
||||
return Math.max(0, avgQuality - 0.5);
|
||||
}
|
||||
/**
|
||||
* Apply quantized LoRA
|
||||
*/
|
||||
async applyLoRA(input, weights) {
|
||||
if (!weights) {
|
||||
return input;
|
||||
}
|
||||
const output = new Float32Array(input.length);
|
||||
output.set(input);
|
||||
// Quantize weights if not already done
|
||||
for (const module of ['q_proj', 'v_proj']) {
|
||||
const A = weights.A.get(module);
|
||||
const B = weights.B.get(module);
|
||||
if (A && B) {
|
||||
const qA = this.getOrQuantize(`A_${module}`, A);
|
||||
const qB = this.getOrQuantize(`B_${module}`, B);
|
||||
// Apply quantized LoRA
|
||||
const adapted = this.applyQuantizedLoRA(input, qA, qB);
|
||||
const alpha = 0.05; // Very small blending for edge
|
||||
for (let i = 0; i < output.length; i++) {
|
||||
output[i] = output[i] * (1 - alpha) + adapted[i] * alpha;
|
||||
}
|
||||
}
|
||||
}
|
||||
return output;
|
||||
}
|
||||
getStats() {
|
||||
return {
|
||||
avgLatencyMs: this.totalOps > 0 ? this.totalTime / this.totalOps : 0,
|
||||
compressedPatterns: this.compressedPatterns.size,
|
||||
quantizedWeights: this.quantizedWeights.size,
|
||||
pendingUpdates: this.pendingUpdates.length,
|
||||
memoryEstimateMb: this.estimateMemoryUsage(),
|
||||
};
|
||||
}
|
||||
// ========================================================================
|
||||
// Compression utilities
|
||||
// ========================================================================
|
||||
/**
|
||||
* Compress embedding to 8-bit representation
|
||||
*/
|
||||
compressEmbedding(embedding) {
|
||||
const compressed = new Int8Array(embedding.length);
|
||||
const max = Math.max(...embedding.map(Math.abs));
|
||||
const scale = max > 0 ? 127 / max : 1;
|
||||
for (let i = 0; i < embedding.length; i++) {
|
||||
compressed[i] = Math.round(embedding[i] * scale);
|
||||
}
|
||||
return compressed;
|
||||
}
|
||||
/**
|
||||
* Create compressed pattern representation
|
||||
*/
|
||||
createCompressedPattern(pattern) {
|
||||
return {
|
||||
id: pattern.patternId,
|
||||
embedding: this.compressEmbedding(pattern.embedding),
|
||||
successRate: Math.round(pattern.successRate * 255),
|
||||
usageCount: Math.min(pattern.usageCount, 255),
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Fast similarity on compressed embeddings
|
||||
*/
|
||||
compressedSimilarity(a, b) {
|
||||
if (a.length !== b.length)
|
||||
return 0;
|
||||
let dot = 0;
|
||||
let normA = 0;
|
||||
let normB = 0;
|
||||
// Process in chunks of 8 for better cache utilization
|
||||
const len = a.length;
|
||||
for (let i = 0; i < len; i++) {
|
||||
dot += a[i] * b[i];
|
||||
normA += a[i] * a[i];
|
||||
normB += b[i] * b[i];
|
||||
}
|
||||
const denom = Math.sqrt(normA * normB);
|
||||
return denom > 0 ? dot / denom : 0;
|
||||
}
|
||||
/**
|
||||
* Get or create quantized weights
|
||||
*/
|
||||
getOrQuantize(key, weights) {
|
||||
let quantized = this.quantizedWeights.get(key);
|
||||
if (!quantized) {
|
||||
quantized = this.quantizeWeights(weights);
|
||||
this.quantizedWeights.set(key, quantized);
|
||||
}
|
||||
return quantized;
|
||||
}
|
||||
/**
|
||||
* Quantize float weights to int8
|
||||
*/
|
||||
quantizeWeights(weights) {
|
||||
const max = Math.max(...weights.map(Math.abs));
|
||||
this.quantizationScale = max > 0 ? 127 / max : 1;
|
||||
const quantized = new Int8Array(weights.length);
|
||||
for (let i = 0; i < weights.length; i++) {
|
||||
quantized[i] = Math.round(weights[i] * this.quantizationScale);
|
||||
}
|
||||
return quantized;
|
||||
}
|
||||
/**
|
||||
* Apply LoRA with quantized weights
|
||||
*/
|
||||
applyQuantizedLoRA(input, qA, qB) {
|
||||
const dim = input.length;
|
||||
const rank = 1; // Edge mode uses rank-1
|
||||
const output = new Float32Array(dim);
|
||||
const dequantScale = 1 / this.quantizationScale;
|
||||
// A * input -> intermediate (scalar for rank-1)
|
||||
let intermediate = 0;
|
||||
for (let d = 0; d < dim; d++) {
|
||||
intermediate += (qA[d] * dequantScale) * input[d];
|
||||
}
|
||||
// B * intermediate -> output
|
||||
for (let d = 0; d < dim; d++) {
|
||||
output[d] = (qB[d] * dequantScale) * intermediate;
|
||||
}
|
||||
return output;
|
||||
}
|
||||
// ========================================================================
|
||||
// Async updates
|
||||
// ========================================================================
|
||||
/**
|
||||
* Queue an async update
|
||||
*/
|
||||
queueAsyncUpdate(update) {
|
||||
this.pendingUpdates.push(update);
|
||||
// Schedule processing if not already scheduled
|
||||
if (!this.updateTimer) {
|
||||
this.updateTimer = setTimeout(() => {
|
||||
this.processAsyncUpdates();
|
||||
}, 100); // Process updates every 100ms
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Process pending async updates
|
||||
*/
|
||||
async processAsyncUpdates() {
|
||||
this.updateTimer = null;
|
||||
const updates = this.pendingUpdates;
|
||||
this.pendingUpdates = [];
|
||||
// Process up to 5 updates at a time
|
||||
for (const update of updates.slice(0, 5)) {
|
||||
try {
|
||||
await update();
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Edge mode async update failed:', error);
|
||||
}
|
||||
}
|
||||
// Re-queue remaining updates
|
||||
if (updates.length > 5) {
|
||||
this.pendingUpdates = updates.slice(5);
|
||||
this.updateTimer = setTimeout(() => {
|
||||
this.processAsyncUpdates();
|
||||
}, 100);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Perform lightweight parameter update
|
||||
*/
|
||||
async performLightweightUpdate(trajectories, learningRate) {
|
||||
// Simple exponential moving average update
|
||||
const alpha = learningRate;
|
||||
for (const trajectory of trajectories) {
|
||||
if (trajectory.steps.length === 0)
|
||||
continue;
|
||||
// Update compressed patterns based on trajectory success
|
||||
const lastStep = trajectory.steps[trajectory.steps.length - 1];
|
||||
const pattern = this.findSimilarCompressedPattern(lastStep.stateAfter);
|
||||
if (pattern) {
|
||||
// Update success rate with EMA
|
||||
const newRate = alpha * trajectory.qualityScore + (1 - alpha) * (pattern.successRate / 255);
|
||||
pattern.successRate = Math.round(newRate * 255);
|
||||
pattern.usageCount = Math.min(255, pattern.usageCount + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Find most similar compressed pattern
|
||||
*/
|
||||
findSimilarCompressedPattern(embedding) {
|
||||
const compressed = this.compressEmbedding(embedding);
|
||||
let best = null;
|
||||
let bestSim = -1;
|
||||
for (const pattern of this.compressedPatterns.values()) {
|
||||
const sim = this.compressedSimilarity(compressed, pattern.embedding);
|
||||
if (sim > bestSim) {
|
||||
bestSim = sim;
|
||||
best = pattern;
|
||||
}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
/**
|
||||
* Estimate memory usage in MB
|
||||
*/
|
||||
estimateMemoryUsage() {
|
||||
let bytes = 0;
|
||||
// Compressed patterns
|
||||
for (const pattern of this.compressedPatterns.values()) {
|
||||
bytes += pattern.embedding.byteLength + 8; // embedding + overhead
|
||||
}
|
||||
// Quantized weights
|
||||
for (const weights of this.quantizedWeights.values()) {
|
||||
bytes += weights.byteLength;
|
||||
}
|
||||
// Pending updates (minimal)
|
||||
bytes += this.pendingUpdates.length * 100;
|
||||
return bytes / (1024 * 1024);
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=edge.js.map
|
||||
@@ -1,83 +0,0 @@
|
||||
/**
|
||||
* SONA Learning Modes Index
|
||||
*
|
||||
* Exports all learning mode implementations and the common interface.
|
||||
*/
|
||||
/**
|
||||
* Base class for mode implementations
|
||||
*/
|
||||
export class BaseModeImplementation {
|
||||
config;
|
||||
optimizations;
|
||||
isInitialized = false;
|
||||
constructor(config, optimizations) {
|
||||
this.config = config;
|
||||
this.optimizations = optimizations;
|
||||
}
|
||||
async initialize() {
|
||||
this.isInitialized = true;
|
||||
}
|
||||
async cleanup() {
|
||||
this.isInitialized = false;
|
||||
}
|
||||
/**
|
||||
* Compute cosine similarity between two vectors (SIMD-optimized)
|
||||
*/
|
||||
cosineSimilarity(a, b) {
|
||||
if (a.length !== b.length)
|
||||
return 0;
|
||||
let dotProduct = 0;
|
||||
let normA = 0;
|
||||
let normB = 0;
|
||||
// Process 4 elements at a time for SIMD-like behavior
|
||||
const len = a.length;
|
||||
const simdLen = len - (len % 4);
|
||||
for (let i = 0; i < simdLen; i += 4) {
|
||||
dotProduct += a[i] * b[i] + a[i + 1] * b[i + 1] + a[i + 2] * b[i + 2] + a[i + 3] * b[i + 3];
|
||||
normA += a[i] * a[i] + a[i + 1] * a[i + 1] + a[i + 2] * a[i + 2] + a[i + 3] * a[i + 3];
|
||||
normB += b[i] * b[i] + b[i + 1] * b[i + 1] + b[i + 2] * b[i + 2] + b[i + 3] * b[i + 3];
|
||||
}
|
||||
// Handle remaining elements
|
||||
for (let i = simdLen; i < len; i++) {
|
||||
dotProduct += a[i] * b[i];
|
||||
normA += a[i] * a[i];
|
||||
normB += b[i] * b[i];
|
||||
}
|
||||
const denom = Math.sqrt(normA) * Math.sqrt(normB);
|
||||
return denom > 0 ? dotProduct / denom : 0;
|
||||
}
|
||||
/**
|
||||
* Apply LoRA: output = input + BA * input (simplified)
|
||||
*/
|
||||
applyLoRATransform(input, A, B, rank) {
|
||||
const dim = input.length;
|
||||
const output = new Float32Array(dim);
|
||||
// Copy input to output
|
||||
output.set(input);
|
||||
// Compute A * input -> intermediate (rank dimensions)
|
||||
const intermediate = new Float32Array(rank);
|
||||
for (let r = 0; r < rank; r++) {
|
||||
let sum = 0;
|
||||
for (let d = 0; d < dim; d++) {
|
||||
sum += A[d * rank + r] * input[d];
|
||||
}
|
||||
intermediate[r] = sum;
|
||||
}
|
||||
// Compute B * intermediate -> delta (dim dimensions)
|
||||
for (let d = 0; d < dim; d++) {
|
||||
let sum = 0;
|
||||
for (let r = 0; r < rank; r++) {
|
||||
sum += B[r * dim + d] * intermediate[r];
|
||||
}
|
||||
output[d] += sum;
|
||||
}
|
||||
return output;
|
||||
}
|
||||
}
|
||||
// Export mode implementations
|
||||
export { RealTimeMode } from './real-time.js';
|
||||
export { BalancedMode } from './balanced.js';
|
||||
export { ResearchMode } from './research.js';
|
||||
export { EdgeMode } from './edge.js';
|
||||
export { BatchMode } from './batch.js';
|
||||
//# sourceMappingURL=index.js.map
|
||||
@@ -1,196 +0,0 @@
|
||||
/**
|
||||
* Real-Time Mode Implementation
|
||||
*
|
||||
* Optimized for sub-millisecond adaptation with:
|
||||
* - 2200 ops/sec target
|
||||
* - <0.5ms latency
|
||||
* - Micro-LoRA (rank-2)
|
||||
* - SIMD vectorization
|
||||
* - Aggressive caching
|
||||
*/
|
||||
import { BaseModeImplementation } from './index.js';
|
||||
/**
|
||||
* Real-Time mode for sub-millisecond adaptation
|
||||
*/
|
||||
export class RealTimeMode extends BaseModeImplementation {
|
||||
mode = 'real-time';
|
||||
// Pattern cache for fast lookups
|
||||
patternCache = new Map();
|
||||
cacheHits = 0;
|
||||
cacheMisses = 0;
|
||||
// Pre-computed pattern embeddings for fast similarity
|
||||
patternEmbeddings = [];
|
||||
patternIds = [];
|
||||
// Stats
|
||||
totalPatternMatches = 0;
|
||||
totalPatternTime = 0;
|
||||
totalLearnTime = 0;
|
||||
learnIterations = 0;
|
||||
async initialize() {
|
||||
await super.initialize();
|
||||
this.patternCache.clear();
|
||||
this.patternEmbeddings = [];
|
||||
this.patternIds = [];
|
||||
}
|
||||
async cleanup() {
|
||||
this.patternCache.clear();
|
||||
this.patternEmbeddings = [];
|
||||
this.patternIds = [];
|
||||
await super.cleanup();
|
||||
}
|
||||
/**
|
||||
* Find patterns using cached similarity search
|
||||
* Target: <1ms for k=3
|
||||
*/
|
||||
async findPatterns(embedding, k, patterns) {
|
||||
const startTime = performance.now();
|
||||
// Check cache first (hash first 8 floats for cache key)
|
||||
const cacheKey = this.computeCacheKey(embedding);
|
||||
const cached = this.patternCache.get(cacheKey);
|
||||
if (cached && cached.length >= k) {
|
||||
this.cacheHits++;
|
||||
this.totalPatternTime += performance.now() - startTime;
|
||||
this.totalPatternMatches++;
|
||||
return cached.slice(0, k);
|
||||
}
|
||||
this.cacheMisses++;
|
||||
// Update pattern embeddings if patterns changed
|
||||
if (patterns.length !== this.patternIds.length) {
|
||||
this.updatePatternIndex(patterns);
|
||||
}
|
||||
// Fast similarity search using pre-computed embeddings
|
||||
const similarities = [];
|
||||
for (let i = 0; i < this.patternEmbeddings.length; i++) {
|
||||
const similarity = this.cosineSimilarity(embedding, this.patternEmbeddings[i]);
|
||||
similarities.push({ index: i, similarity });
|
||||
}
|
||||
// Partial sort to get top-k (faster than full sort)
|
||||
const topK = this.partialSort(similarities, k);
|
||||
const matches = topK.map(item => ({
|
||||
pattern: patterns[item.index],
|
||||
similarity: item.similarity,
|
||||
confidence: item.similarity * patterns[item.index].successRate,
|
||||
latencyMs: 0, // Will be set by caller
|
||||
}));
|
||||
// Cache the result
|
||||
if (this.patternCache.size > 1000) {
|
||||
// LRU eviction: remove oldest entries
|
||||
const firstKey = this.patternCache.keys().next().value;
|
||||
if (firstKey)
|
||||
this.patternCache.delete(firstKey);
|
||||
}
|
||||
this.patternCache.set(cacheKey, matches);
|
||||
this.totalPatternTime += performance.now() - startTime;
|
||||
this.totalPatternMatches++;
|
||||
return matches;
|
||||
}
|
||||
/**
|
||||
* Fast learning using Micro-LoRA updates
|
||||
* Target: <10ms per batch
|
||||
*/
|
||||
async learn(trajectories, config, ewcState) {
|
||||
const startTime = performance.now();
|
||||
if (trajectories.length === 0)
|
||||
return 0;
|
||||
// Real-time mode uses simplified learning for speed
|
||||
// Only process high-quality trajectories
|
||||
const qualityThreshold = config.qualityThreshold;
|
||||
const goodTrajectories = trajectories.filter(t => t.qualityScore >= qualityThreshold);
|
||||
if (goodTrajectories.length === 0)
|
||||
return 0;
|
||||
// Compute average quality improvement
|
||||
const avgQuality = goodTrajectories.reduce((sum, t) => sum + t.qualityScore, 0) / goodTrajectories.length;
|
||||
// Simplified gradient update (for real-time, we skip full backprop)
|
||||
// Just update EWC means to track what works
|
||||
const improvementDelta = avgQuality - 0.5; // Relative to baseline of 0.5
|
||||
this.totalLearnTime += performance.now() - startTime;
|
||||
this.learnIterations++;
|
||||
return Math.max(0, improvementDelta);
|
||||
}
|
||||
/**
|
||||
* Apply LoRA with minimal overhead
|
||||
* Target: <0.05ms
|
||||
*/
|
||||
async applyLoRA(input, weights) {
|
||||
if (!weights) {
|
||||
// No adaptation, return input as-is
|
||||
return input;
|
||||
}
|
||||
// Micro-LoRA: only apply to key modules
|
||||
const output = new Float32Array(input.length);
|
||||
output.set(input);
|
||||
// Apply rank-2 adaptation (minimal overhead)
|
||||
const rank = this.config.loraRank;
|
||||
for (const module of ['q_proj', 'v_proj']) {
|
||||
const A = weights.A.get(module);
|
||||
const B = weights.B.get(module);
|
||||
if (A && B) {
|
||||
const adapted = this.applyLoRATransform(input, A, B, rank);
|
||||
// Blend with small alpha for stability
|
||||
const alpha = 0.1;
|
||||
for (let i = 0; i < output.length; i++) {
|
||||
output[i] = output[i] * (1 - alpha) + adapted[i] * alpha;
|
||||
}
|
||||
}
|
||||
}
|
||||
return output;
|
||||
}
|
||||
getStats() {
|
||||
return {
|
||||
cacheHitRate: this.cacheHits + this.cacheMisses > 0
|
||||
? this.cacheHits / (this.cacheHits + this.cacheMisses)
|
||||
: 0,
|
||||
avgPatternMatchMs: this.totalPatternMatches > 0
|
||||
? this.totalPatternTime / this.totalPatternMatches
|
||||
: 0,
|
||||
avgLearnMs: this.learnIterations > 0
|
||||
? this.totalLearnTime / this.learnIterations
|
||||
: 0,
|
||||
patternCacheSize: this.patternCache.size,
|
||||
indexedPatterns: this.patternEmbeddings.length,
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Compute cache key from embedding
|
||||
*/
|
||||
computeCacheKey(embedding) {
|
||||
// Use first 8 floats for cache key (fast hash)
|
||||
const keyParts = [];
|
||||
for (let i = 0; i < Math.min(8, embedding.length); i++) {
|
||||
keyParts.push(embedding[i].toFixed(3));
|
||||
}
|
||||
return keyParts.join(',');
|
||||
}
|
||||
/**
|
||||
* Update pattern index for fast similarity search
|
||||
*/
|
||||
updatePatternIndex(patterns) {
|
||||
this.patternEmbeddings = patterns.map(p => p.embedding);
|
||||
this.patternIds = patterns.map(p => p.patternId);
|
||||
this.patternCache.clear();
|
||||
}
|
||||
/**
|
||||
* Partial sort to get top-k elements (faster than full sort)
|
||||
*/
|
||||
partialSort(items, k) {
|
||||
if (items.length <= k) {
|
||||
return items.sort((a, b) => b.similarity - a.similarity);
|
||||
}
|
||||
// Use a simple selection algorithm for small k
|
||||
const result = [];
|
||||
for (let i = 0; i < k; i++) {
|
||||
let maxIdx = 0;
|
||||
let maxVal = -Infinity;
|
||||
for (let j = 0; j < items.length; j++) {
|
||||
if (items[j].similarity > maxVal) {
|
||||
maxVal = items[j].similarity;
|
||||
maxIdx = j;
|
||||
}
|
||||
}
|
||||
result.push(items[maxIdx]);
|
||||
items[maxIdx] = { index: -1, similarity: -Infinity };
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=real-time.js.map
|
||||
@@ -1,389 +0,0 @@
|
||||
/**
|
||||
* Research Mode Implementation
|
||||
*
|
||||
* Optimized for maximum quality with:
|
||||
* - +55% quality improvement target
|
||||
* - Learning rate 0.002 (sweet spot)
|
||||
* - Rank-16 LoRA
|
||||
* - Gradient checkpointing
|
||||
* - Full learning pipeline
|
||||
*/
|
||||
import { BaseModeImplementation } from './index.js';
|
||||
/**
|
||||
* Research mode for maximum quality learning
|
||||
*/
|
||||
export class ResearchMode extends BaseModeImplementation {
|
||||
mode = 'research';
|
||||
// Extended pattern storage
|
||||
patternIndex = new Map();
|
||||
clusterCentroids = [];
|
||||
// Learning state with checkpointing
|
||||
gradientHistory = [];
|
||||
checkpoints = [];
|
||||
// Adam optimizer state
|
||||
adamM = new Map();
|
||||
adamV = new Map();
|
||||
adamStep = 0;
|
||||
// Stats
|
||||
totalPatternMatches = 0;
|
||||
totalPatternTime = 0;
|
||||
totalLearnTime = 0;
|
||||
learnIterations = 0;
|
||||
qualityHistory = [];
|
||||
explorationRewards = [];
|
||||
async initialize() {
|
||||
await super.initialize();
|
||||
this.patternIndex.clear();
|
||||
this.clusterCentroids = [];
|
||||
this.gradientHistory = [];
|
||||
this.checkpoints = [];
|
||||
this.adamM.clear();
|
||||
this.adamV.clear();
|
||||
this.adamStep = 0;
|
||||
}
|
||||
async cleanup() {
|
||||
this.patternIndex.clear();
|
||||
this.clusterCentroids = [];
|
||||
this.gradientHistory = [];
|
||||
this.checkpoints = [];
|
||||
this.adamM.clear();
|
||||
this.adamV.clear();
|
||||
await super.cleanup();
|
||||
}
|
||||
/**
|
||||
* Find patterns using cluster-based search
|
||||
*/
|
||||
async findPatterns(embedding, k, patterns) {
|
||||
const startTime = performance.now();
|
||||
// Update clusters if needed
|
||||
if (this.clusterCentroids.length !== this.config.patternClusters) {
|
||||
await this.rebuildClusters(patterns);
|
||||
}
|
||||
// Find nearest cluster
|
||||
let bestCluster = 0;
|
||||
let bestSim = -1;
|
||||
for (let c = 0; c < this.clusterCentroids.length; c++) {
|
||||
const sim = this.cosineSimilarity(embedding, this.clusterCentroids[c]);
|
||||
if (sim > bestSim) {
|
||||
bestSim = sim;
|
||||
bestCluster = c;
|
||||
}
|
||||
}
|
||||
// Search within cluster + nearby clusters
|
||||
const clustersToSearch = this.getNearestClusters(embedding, 3);
|
||||
const candidates = [];
|
||||
for (const pattern of patterns) {
|
||||
const patternCluster = this.patternIndex.get(pattern.patternId);
|
||||
if (patternCluster !== undefined && clustersToSearch.includes(patternCluster)) {
|
||||
const similarity = this.cosineSimilarity(embedding, pattern.embedding);
|
||||
candidates.push({
|
||||
pattern,
|
||||
similarity,
|
||||
confidence: this.computeConfidence(pattern, similarity),
|
||||
latencyMs: 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
// If not enough candidates, search all patterns
|
||||
if (candidates.length < k) {
|
||||
for (const pattern of patterns) {
|
||||
if (!candidates.find(c => c.pattern.patternId === pattern.patternId)) {
|
||||
const similarity = this.cosineSimilarity(embedding, pattern.embedding);
|
||||
candidates.push({
|
||||
pattern,
|
||||
similarity,
|
||||
confidence: this.computeConfidence(pattern, similarity),
|
||||
latencyMs: 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
// Sort and return top-k
|
||||
candidates.sort((a, b) => b.similarity - a.similarity);
|
||||
this.totalPatternTime += performance.now() - startTime;
|
||||
this.totalPatternMatches++;
|
||||
return candidates.slice(0, k);
|
||||
}
|
||||
/**
|
||||
* Learn using full Adam optimizer with gradient checkpointing
|
||||
*/
|
||||
async learn(trajectories, config, ewcState) {
|
||||
const startTime = performance.now();
|
||||
if (trajectories.length === 0)
|
||||
return 0;
|
||||
// Research mode uses low threshold to learn from all data
|
||||
const learningRate = config.learningRate; // 0.002 sweet spot
|
||||
const batchSize = config.batchSize;
|
||||
// Sort trajectories by quality
|
||||
const sortedTrajectories = [...trajectories].sort((a, b) => b.qualityScore - a.qualityScore);
|
||||
// Create checkpoint before learning
|
||||
if (this.learnIterations % 10 === 0) {
|
||||
this.createCheckpoint();
|
||||
}
|
||||
let totalLoss = 0;
|
||||
let batchCount = 0;
|
||||
// Process in mini-batches
|
||||
for (let i = 0; i < sortedTrajectories.length; i += batchSize) {
|
||||
const batch = sortedTrajectories.slice(i, i + batchSize);
|
||||
const batchLoss = await this.processBatch(batch, learningRate, ewcState, config.ewcLambda);
|
||||
totalLoss += batchLoss;
|
||||
batchCount++;
|
||||
}
|
||||
// Track quality
|
||||
const avgQuality = sortedTrajectories.reduce((s, t) => s + t.qualityScore, 0) / sortedTrajectories.length;
|
||||
this.qualityHistory.push(avgQuality);
|
||||
if (this.qualityHistory.length > 1000) {
|
||||
this.qualityHistory = this.qualityHistory.slice(-1000);
|
||||
}
|
||||
// Compute improvement
|
||||
const recentAvg = this.qualityHistory.slice(-10).reduce((a, b) => a + b, 0) / Math.min(10, this.qualityHistory.length);
|
||||
const oldAvg = this.qualityHistory.slice(0, Math.max(1, this.qualityHistory.length - 10)).reduce((a, b) => a + b, 0) / Math.max(1, this.qualityHistory.length - 10);
|
||||
const improvementDelta = recentAvg - oldAvg;
|
||||
this.totalLearnTime += performance.now() - startTime;
|
||||
this.learnIterations++;
|
||||
return Math.max(0, improvementDelta * 2); // Scale for research mode
|
||||
}
|
||||
/**
|
||||
* Apply LoRA with rank-16 for maximum expressivity
|
||||
*/
|
||||
async applyLoRA(input, weights) {
|
||||
if (!weights) {
|
||||
return input;
|
||||
}
|
||||
const output = new Float32Array(input.length);
|
||||
output.set(input);
|
||||
const rank = this.config.loraRank; // 16 for research mode
|
||||
// Apply to all modules with higher blending
|
||||
for (const module of ['q_proj', 'v_proj', 'k_proj', 'o_proj']) {
|
||||
const A = weights.A.get(module);
|
||||
const B = weights.B.get(module);
|
||||
if (A && B) {
|
||||
const adapted = this.applyLoRATransform(input, A, B, rank);
|
||||
const alpha = 0.3; // Higher blending for research
|
||||
for (let i = 0; i < output.length; i++) {
|
||||
output[i] = output[i] * (1 - alpha) + adapted[i] * alpha;
|
||||
}
|
||||
}
|
||||
}
|
||||
return output;
|
||||
}
|
||||
getStats() {
|
||||
const avgQuality = this.qualityHistory.length > 0
|
||||
? this.qualityHistory.reduce((a, b) => a + b, 0) / this.qualityHistory.length
|
||||
: 0;
|
||||
const recentQuality = this.qualityHistory.slice(-10).reduce((a, b) => a + b, 0) / Math.min(10, this.qualityHistory.length) || 0;
|
||||
return {
|
||||
avgPatternMatchMs: this.totalPatternMatches > 0
|
||||
? this.totalPatternTime / this.totalPatternMatches
|
||||
: 0,
|
||||
avgLearnMs: this.learnIterations > 0
|
||||
? this.totalLearnTime / this.learnIterations
|
||||
: 0,
|
||||
avgQuality,
|
||||
recentQuality,
|
||||
qualityImprovement: recentQuality - avgQuality,
|
||||
clusterCount: this.clusterCentroids.length,
|
||||
checkpointCount: this.checkpoints.length,
|
||||
adamStep: this.adamStep,
|
||||
learnIterations: this.learnIterations,
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Rebuild cluster centroids using k-means
|
||||
*/
|
||||
async rebuildClusters(patterns) {
|
||||
if (patterns.length === 0)
|
||||
return;
|
||||
const k = Math.min(this.config.patternClusters, patterns.length);
|
||||
const dim = patterns[0].embedding.length;
|
||||
// Initialize centroids randomly
|
||||
this.clusterCentroids = [];
|
||||
const indices = new Set();
|
||||
while (indices.size < k) {
|
||||
indices.add(Math.floor(Math.random() * patterns.length));
|
||||
}
|
||||
for (const idx of indices) {
|
||||
this.clusterCentroids.push(new Float32Array(patterns[idx].embedding));
|
||||
}
|
||||
// Run k-means iterations
|
||||
for (let iter = 0; iter < 10; iter++) {
|
||||
// Assign patterns to clusters
|
||||
const clusterAssignments = Array.from({ length: k }, () => []);
|
||||
for (let p = 0; p < patterns.length; p++) {
|
||||
let bestCluster = 0;
|
||||
let bestSim = -1;
|
||||
for (let c = 0; c < k; c++) {
|
||||
const sim = this.cosineSimilarity(patterns[p].embedding, this.clusterCentroids[c]);
|
||||
if (sim > bestSim) {
|
||||
bestSim = sim;
|
||||
bestCluster = c;
|
||||
}
|
||||
}
|
||||
clusterAssignments[bestCluster].push(p);
|
||||
this.patternIndex.set(patterns[p].patternId, bestCluster);
|
||||
}
|
||||
// Update centroids
|
||||
for (let c = 0; c < k; c++) {
|
||||
if (clusterAssignments[c].length > 0) {
|
||||
const newCentroid = new Float32Array(dim);
|
||||
for (const p of clusterAssignments[c]) {
|
||||
for (let d = 0; d < dim; d++) {
|
||||
newCentroid[d] += patterns[p].embedding[d];
|
||||
}
|
||||
}
|
||||
for (let d = 0; d < dim; d++) {
|
||||
newCentroid[d] /= clusterAssignments[c].length;
|
||||
}
|
||||
this.clusterCentroids[c] = newCentroid;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Get nearest clusters to embedding
|
||||
*/
|
||||
getNearestClusters(embedding, n) {
|
||||
const similarities = [];
|
||||
for (let c = 0; c < this.clusterCentroids.length; c++) {
|
||||
similarities.push({ cluster: c, sim: this.cosineSimilarity(embedding, this.clusterCentroids[c]) });
|
||||
}
|
||||
similarities.sort((a, b) => b.sim - a.sim);
|
||||
return similarities.slice(0, n).map(s => s.cluster);
|
||||
}
|
||||
/**
|
||||
* Compute confidence for pattern match
|
||||
*/
|
||||
computeConfidence(pattern, similarity) {
|
||||
// Combine similarity with pattern history
|
||||
const historyWeight = Math.min(pattern.usageCount / 10, 1);
|
||||
return similarity * (1 - historyWeight * 0.3) + pattern.successRate * historyWeight * 0.3;
|
||||
}
|
||||
/**
|
||||
* Create learning checkpoint
|
||||
*/
|
||||
createCheckpoint() {
|
||||
const state = new Map();
|
||||
for (const [key, value] of this.adamM) {
|
||||
state.set(`m_${key}`, new Float32Array(value));
|
||||
}
|
||||
for (const [key, value] of this.adamV) {
|
||||
state.set(`v_${key}`, new Float32Array(value));
|
||||
}
|
||||
this.checkpoints.push({ iteration: this.learnIterations, state });
|
||||
// Keep only last 10 checkpoints
|
||||
if (this.checkpoints.length > 10) {
|
||||
this.checkpoints = this.checkpoints.slice(-10);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Process a mini-batch with Adam optimizer
|
||||
*/
|
||||
async processBatch(batch, learningRate, ewcState, ewcLambda) {
|
||||
const beta1 = 0.9;
|
||||
const beta2 = 0.999;
|
||||
const epsilon = 1e-8;
|
||||
this.adamStep++;
|
||||
let totalLoss = 0;
|
||||
for (const trajectory of batch) {
|
||||
if (trajectory.steps.length === 0)
|
||||
continue;
|
||||
// Compute gradient from trajectory
|
||||
const gradient = this.computeTrajectoryGradient(trajectory);
|
||||
for (const [key, grad] of gradient) {
|
||||
// Get or initialize Adam state
|
||||
let m = this.adamM.get(key);
|
||||
let v = this.adamV.get(key);
|
||||
if (!m) {
|
||||
m = new Float32Array(grad.length);
|
||||
this.adamM.set(key, m);
|
||||
}
|
||||
if (!v) {
|
||||
v = new Float32Array(grad.length);
|
||||
this.adamV.set(key, v);
|
||||
}
|
||||
// Update biased first moment estimate
|
||||
for (let i = 0; i < grad.length; i++) {
|
||||
m[i] = beta1 * m[i] + (1 - beta1) * grad[i];
|
||||
}
|
||||
// Update biased second moment estimate
|
||||
for (let i = 0; i < grad.length; i++) {
|
||||
v[i] = beta2 * v[i] + (1 - beta2) * grad[i] * grad[i];
|
||||
}
|
||||
// Bias correction
|
||||
const mHat = new Float32Array(grad.length);
|
||||
const vHat = new Float32Array(grad.length);
|
||||
for (let i = 0; i < grad.length; i++) {
|
||||
mHat[i] = m[i] / (1 - Math.pow(beta1, this.adamStep));
|
||||
vHat[i] = v[i] / (1 - Math.pow(beta2, this.adamStep));
|
||||
}
|
||||
// Compute loss contribution
|
||||
for (let i = 0; i < grad.length; i++) {
|
||||
totalLoss += grad[i] * grad[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
// Add EWC penalty
|
||||
const ewcPenalty = this.computeEWCLoss(ewcState, ewcLambda);
|
||||
totalLoss += ewcPenalty;
|
||||
return totalLoss / batch.length;
|
||||
}
|
||||
/**
|
||||
* Compute gradient from trajectory
|
||||
*/
|
||||
computeTrajectoryGradient(trajectory) {
|
||||
const gradient = new Map();
|
||||
if (trajectory.steps.length < 2)
|
||||
return gradient;
|
||||
// Use advantage estimation
|
||||
const rewards = trajectory.steps.map(s => s.reward);
|
||||
const advantages = this.computeAdvantages(rewards);
|
||||
for (let i = 0; i < trajectory.steps.length; i++) {
|
||||
const step = trajectory.steps[i];
|
||||
const advantage = advantages[i];
|
||||
// Policy gradient: grad = advantage * grad_log_pi
|
||||
const stateGrad = new Float32Array(step.stateAfter.length);
|
||||
for (let j = 0; j < stateGrad.length; j++) {
|
||||
stateGrad[j] = step.stateAfter[j] * advantage;
|
||||
}
|
||||
gradient.set(`step_${i}`, stateGrad);
|
||||
}
|
||||
return gradient;
|
||||
}
|
||||
/**
|
||||
* Compute advantages using GAE
|
||||
*/
|
||||
computeAdvantages(rewards) {
|
||||
const gamma = 0.99;
|
||||
const lambda = 0.95;
|
||||
const advantages = new Array(rewards.length).fill(0);
|
||||
let lastGae = 0;
|
||||
for (let t = rewards.length - 1; t >= 0; t--) {
|
||||
const nextValue = t < rewards.length - 1 ? rewards[t + 1] : 0;
|
||||
const delta = rewards[t] + gamma * nextValue - rewards[t];
|
||||
lastGae = delta + gamma * lambda * lastGae;
|
||||
advantages[t] = lastGae;
|
||||
}
|
||||
// Normalize advantages
|
||||
const mean = advantages.reduce((a, b) => a + b, 0) / advantages.length;
|
||||
const std = Math.sqrt(advantages.reduce((a, b) => a + (b - mean) ** 2, 0) / advantages.length) + 1e-8;
|
||||
return advantages.map(a => (a - mean) / std);
|
||||
}
|
||||
/**
|
||||
* Compute EWC loss for continual learning
|
||||
*/
|
||||
computeEWCLoss(ewcState, lambda) {
|
||||
let loss = 0;
|
||||
for (const [key, fisher] of ewcState.fisher) {
|
||||
const means = ewcState.means.get(key);
|
||||
const current = this.adamM.get(key);
|
||||
if (means && current) {
|
||||
for (let i = 0; i < Math.min(fisher.length, means.length, current.length); i++) {
|
||||
const diff = current[i] - means[i];
|
||||
loss += fisher[i] * diff * diff;
|
||||
}
|
||||
}
|
||||
}
|
||||
return lambda * loss * 0.5;
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=research.js.map
|
||||
@@ -449,6 +449,8 @@ export class SONAManager {
|
||||
this.ewcState!
|
||||
);
|
||||
|
||||
this.updateEWCConsolidation();
|
||||
|
||||
this.learningCycles++;
|
||||
|
||||
this.emitEvent({
|
||||
@@ -579,6 +581,45 @@ export class SONAManager {
|
||||
this.ewcState.lastConsolidation = Date.now();
|
||||
}
|
||||
|
||||
/**
|
||||
* Fold the active mode's current gradient state into EWC consolidation
|
||||
* after a learning cycle. Decays existing Fisher importance first (via
|
||||
* `consolidateEWC()`, online-EWC style), then additively accumulates the
|
||||
* mode's fresh Fisher snapshot into the decayed map and refreshes `means`
|
||||
* to the latest consolidation point.
|
||||
*
|
||||
* A no-op when the active mode doesn't implement `getConsolidationSnapshot`
|
||||
* (not yet wired for that mode) or reports nothing to consolidate — so
|
||||
* `ewcState.fisher` stays exactly as inert as it is today for any mode
|
||||
* this fix doesn't cover, rather than silently fabricating state for it.
|
||||
*/
|
||||
private updateEWCConsolidation(): void {
|
||||
if (!this.ewcState) return;
|
||||
|
||||
if (typeof this.modeImpl.getConsolidationSnapshot !== 'function') return;
|
||||
|
||||
const snapshot = this.modeImpl.getConsolidationSnapshot();
|
||||
if (snapshot.fisher.size === 0) return;
|
||||
|
||||
// Decay existing importance before folding in fresh evidence.
|
||||
this.consolidateEWC();
|
||||
|
||||
for (const [key, newFisher] of snapshot.fisher) {
|
||||
const existing = this.ewcState.fisher.get(key);
|
||||
if (existing && existing.length === newFisher.length) {
|
||||
for (let i = 0; i < newFisher.length; i++) {
|
||||
existing[i] += newFisher[i];
|
||||
}
|
||||
} else {
|
||||
this.ewcState.fisher.set(key, newFisher);
|
||||
}
|
||||
}
|
||||
|
||||
for (const [key, m] of snapshot.means) {
|
||||
this.ewcState.means.set(key, m);
|
||||
}
|
||||
}
|
||||
|
||||
// ==========================================================================
|
||||
// Statistics
|
||||
// ==========================================================================
|
||||
|
||||
Reference in new issue
Block a user