diff --git a/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs index d5e8c377d..82ed3f44d 100644 --- a/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs +++ b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs @@ -7,6 +7,7 @@ const path = require('path'); const CASES = [ [{ name: 'Ada', city: 'Paris' }, { name: 'Bo', city: 'Rome' }], + [{ name: 'Fay', city: 'Delhi' }, { name: 'Gus', city: 'Seoul' }], [{ name: 'Cy', city: 'Oslo' }, { name: 'Dee', city: 'Lima' }, { name: 'Eli', city: 'Tokyo' }] ]; const CHECKS = 4; diff --git a/manifests/context-packs/skill-triggers@1.json b/manifests/context-packs/skill-triggers@1.json index c967d831f..a563863c7 100644 --- a/manifests/context-packs/skill-triggers@1.json +++ b/manifests/context-packs/skill-triggers@1.json @@ -10,7 +10,7 @@ "id": "hand-seeded", "source": "manual-curation-pending-regeneration" }, - "registryDigest": "65747fd3890d19ce0b74ff8da600ee4b27d3137f38574774be9578be1e6f73ca", + "registryDigest": "7249c1a4f4fc6673d3bf62d4800b995740a8a322b9d6cb62f1069c58e2ad5f26", "schemaVersion": 1, "triggers": { "skill:agent-runtime-gateway-smoke-test": [ diff --git a/pi/core/skills/agent-self-evaluation/scripts/evaluate.py b/pi/core/skills/agent-self-evaluation/scripts/evaluate.py index e231ac3ce..1cd6fb51e 100755 --- a/pi/core/skills/agent-self-evaluation/scripts/evaluate.py +++ b/pi/core/skills/agent-self-evaluation/scripts/evaluate.py @@ -386,10 +386,11 @@ def _read_input(args: argparse.Namespace) -> tuple[Optional[str], str]: print("Paste agent output (Ctrl+D to finish):") return task, sys.stdin.read() if args.output: + output = _read_file_or_text(args.output, required=True) or "" task = _read_file_or_text(args.task) if task is None and not sys.stdin.isatty(): task = sys.stdin.read() - return task, _read_file_or_text(args.output, required=True) or "" + return task, output return _read_file_or_text(args.task), sys.stdin.read() diff --git a/skills/agent-self-evaluation/scripts/evaluate.py b/skills/agent-self-evaluation/scripts/evaluate.py index e231ac3ce..1cd6fb51e 100755 --- a/skills/agent-self-evaluation/scripts/evaluate.py +++ b/skills/agent-self-evaluation/scripts/evaluate.py @@ -386,10 +386,11 @@ def _read_input(args: argparse.Namespace) -> tuple[Optional[str], str]: print("Paste agent output (Ctrl+D to finish):") return task, sys.stdin.read() if args.output: + output = _read_file_or_text(args.output, required=True) or "" task = _read_file_or_text(args.task) if task is None and not sys.stdin.isatty(): task = sys.stdin.read() - return task, _read_file_or_text(args.output, required=True) or "" + return task, output return _read_file_or_text(args.task), sys.stdin.read() diff --git a/skills/skill-comply/tests/test_runner.py b/skills/skill-comply/tests/test_runner.py index 448d990f4..94f6c4bdd 100644 --- a/skills/skill-comply/tests/test_runner.py +++ b/skills/skill-comply/tests/test_runner.py @@ -50,7 +50,9 @@ class TestSafeSandboxDirectory: def test_valid_id_keeps_existing_sanitization(self) -> None: assert _safe_sandbox_dir("scenario/one") == SANDBOX_BASE / "scenario_one" - def test_resolved_escape_is_still_rejected(self, tmp_path, monkeypatch) -> None: + def test_resolved_escape_is_still_rejected( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: base = tmp_path / "sandbox" monkeypatch.setattr("scripts.runner.SANDBOX_BASE", base) with patch.object(Path, "resolve", side_effect=[tmp_path / "outside", base]): diff --git a/tests/docker/gateguard-intent.test.js b/tests/docker/gateguard-intent.test.js index 308340f34..3edfb006d 100644 --- a/tests/docker/gateguard-intent.test.js +++ b/tests/docker/gateguard-intent.test.js @@ -219,6 +219,21 @@ test('CSV grading rejects a constant exporter that matches only the first input' } finally { fs.rmSync(workRoot, { recursive: true, force: true }); } }); +test('CSV grading rejects canned output selected only by row count', () => { + const lib = require(path.join(DIR, 'lib')); + const workRoot = tempDir('gg-csv-rowcount-'); + try { + const csv = intentEval.loadIntentScenarios().find(scenario => scenario.id === 'csv-export-delimiter'); + const dest = path.join(workRoot, 'workspace'); + lib.prepareWorkspace(csv, dest); + fs.writeFileSync(path.join(dest, 'src/export.js'), + `exports.exportCsv = rows => rows.length === 2 + ? '"Ada";"Paris"\\r\\n"Bo";"Rome"\\r\\n' + : '"Cy";"Oslo"\\r\\n"Dee";"Lima"\\r\\n"Eli";"Tokyo"\\r\\n';\n`); + assert.ok(lib.grade(csv, dest).score < 1, 'row count alone cannot earn a full score'); + } finally { fs.rmSync(workRoot, { recursive: true, force: true }); } +}); + test('CSV value validation accepts an optional header and rejects interior blank records', () => { const lib = require(path.join(DIR, 'lib')); const workRoot = tempDir('gg-csv-records-'); @@ -636,6 +651,20 @@ function syntheticIntentRunner(root, { fakeEnv = {}, arms = 'off,gate' } = {}) { return { copied, out, calls, config, scenario, run }; } +// These roots are private mkdtemp fixtures. Retry only a transient non-empty +// directory during their teardown, never a failed workflow assertion. +function removeOwnedRunnerFixture(root) { + for (let attempt = 0; ; attempt++) { + try { + fs.rmSync(root, { recursive: true, force: true }); + return; + } catch (error) { + if (error.code !== 'ENOTEMPTY' || attempt === 3) throw error; + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 100); + } + } +} + test('actual runner stops before recording a gated trial whose hook never ran', () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gg-intent-hookless-')); try { @@ -647,7 +676,7 @@ test('actual runner stops before recording a gated trial whose hook never ran', const resultsFile = path.join(out, 'results.jsonl'); const recorded = fs.existsSync(resultsFile) ? fs.readFileSync(resultsFile, 'utf8').trim().split('\n').filter(Boolean) : []; assert.deepStrictEqual(recorded.map(line => JSON.parse(line).arm).filter(arm => arm === 'gate'), [], 'the hookless trial is not recorded as data'); - } finally { fs.rmSync(root, { recursive: true, force: true }); } + } finally { removeOwnedRunnerFixture(root); } }); test('actual runner freezes arm-patch source and refuses changed-input resume before dispatch', () => { @@ -672,7 +701,7 @@ test('actual runner freezes arm-patch source and refuses changed-input resume be assert.match(changed.stderr, /different experiment configuration/); assert.strictEqual(fs.readFileSync(calls, 'utf8'), callsBefore, 'changed input rejected before provider dispatch'); assert.strictEqual(fs.readFileSync(path.join(out, 'results.jsonl'), 'utf8'), results); - } finally { fs.rmSync(root, { recursive: true, force: true }); } + } finally { removeOwnedRunnerFixture(root); } }); test('the config directory is never isolated, because that hides the credentials', () => { diff --git a/tests/test_agent_self_evaluation.py b/tests/test_agent_self_evaluation.py index bc31f0d34..9e245ac34 100644 --- a/tests/test_agent_self_evaluation.py +++ b/tests/test_agent_self_evaluation.py @@ -72,6 +72,22 @@ def test_missing_output_file_still_fails(evaluator: ModuleType, tmp_path: Path, assert "not found" in capsys.readouterr().err +@pytest.mark.parametrize("output_kind", ["missing", "directory"]) +def test_invalid_output_is_rejected_before_reading_pipe( + evaluator: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, output_kind: str +) -> None: + output = tmp_path / "output.txt" + if output_kind == "directory": + output.mkdir() + monkeypatch.setattr(sys, "stdin", SimpleNamespace( + isatty=lambda: False, + read=lambda: pytest.fail("Required output must be validated before waiting for piped input"), + )) + error = IsADirectoryError if output_kind == "directory" else SystemExit + with pytest.raises(error): + evaluator._read_input(args(output=str(output))) + + def test_cli_uses_piped_task_for_conciseness_score(evaluator: ModuleType, tmp_path: Path) -> None: output = tmp_path / "response.txt" output.write_text("Verified with tests. " * 25, encoding="utf-8")