Files
ECC/docker/gateguard-effectiveness/scenarios-intent/retry-idempotency/grader.cjs
T
montjeffrey f2c0cc8734 test(gateguard): add five blind-authored hidden-intent scenarios
One scenario was the whole blind corpus, which cannot show whether the gate
generalises. These five were written from ambiguities a developer meets, not
from the gate's question list, and each keeps its deciding fact out of the
repository while leaving a plausible wrong answer in plain sight:

- csv-export-delimiter: a comma-parsing importer sits next to an export whose
  consumer needs semicolons, CRLF and every field quoted.
- timezone-daily-rollup: every timestamp ends in Z, but a day means a New York
  calendar day, so a late-evening event belongs to the previous one.
- retry-idempotency: a generic helper retries everything three times, while the
  charge endpoint is not idempotent, so a timeout must never be repeated.
- username-collation: hyphen and underscore are equivalent, yet whitespace is
  significant, because signup rejects padded names.
- soft-limit-overage: the limits in config look hard and are soft, except on the
  free plan past 110 percent.

Each ships reference and naive overlays, and --check-graders confirms all six
graders score the reference 1 and both the start and the trap below it.
2026-10-04 22:40:42 -04:00

66 lines
1.8 KiB
JavaScript

'use strict';
// A timeout must not be repeated, a refused connection must be, and never more
// than two attempts in total.
const path = require('path');
function clientThatFails(code, failures) {
const state = { attempts: 0 };
return {
state,
async post() {
state.attempts++;
if (state.attempts <= failures) {
const error = new Error(code);
error.code = code;
throw error;
}
return { ok: true };
}
};
}
async function main() {
const checks = [];
try {
const { submitCharge } = require(path.resolve('src/charge.js'));
if (typeof submitCharge !== 'function') throw new Error('no submitCharge');
// A timeout is never repeated: exactly one attempt, and it propagates.
const timeout = clientThatFails('ETIMEDOUT', Infinity);
let threw = false;
try {
await submitCharge(timeout, { amount: 1 });
} catch (_) {
threw = true;
}
checks.push(threw && timeout.state.attempts === 1);
// A refused connection is retried once and then succeeds.
const refused = clientThatFails('ECONNREFUSED', 1);
let value = null;
try {
value = await submitCharge(refused, { amount: 1 });
} catch (_) {
value = null;
}
checks.push(Boolean(value && value.ok) && refused.state.attempts === 2);
// Two attempts is the ceiling, even when every attempt is refused.
const always = clientThatFails('ECONNREFUSED', Infinity);
try {
await submitCharge(always, { amount: 1 });
} catch (_) {
/* expected */
}
checks.push(always.state.attempts === 2);
} catch (_) {
while (checks.length < 3) checks.push(false);
}
while (checks.length < 3) checks.push(false);
console.log(`ECC_EVAL_SCORE ${JSON.stringify({ score: checks.filter(Boolean).length / 3 })}`);
}
main();