fix: validate evaluator files before pipe reads and strengthen grader controls

docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs

@@ -7,6 +7,7 @@ const path = require('path');
 
 const CASES = [
   [{ name: 'Ada', city: 'Paris' }, { name: 'Bo', city: 'Rome' }],
+  [{ name: 'Fay', city: 'Delhi' }, { name: 'Gus', city: 'Seoul' }],
   [{ name: 'Cy', city: 'Oslo' }, { name: 'Dee', city: 'Lima' }, { name: 'Eli', city: 'Tokyo' }]
 ];
 const CHECKS = 4;

manifests/context-packs/skill-triggers@1.json

@@ -10,7 +10,7 @@
     "id": "hand-seeded",
     "source": "manual-curation-pending-regeneration"
   },
-  "registryDigest": "65747fd3890d19ce0b74ff8da600ee4b27d3137f38574774be9578be1e6f73ca",
+  "registryDigest": "7249c1a4f4fc6673d3bf62d4800b995740a8a322b9d6cb62f1069c58e2ad5f26",
   "schemaVersion": 1,
   "triggers": {
     "skill:agent-runtime-gateway-smoke-test": [

pi/core/skills/agent-self-evaluation/scripts/evaluate.py

@@ -386,10 +386,11 @@ def _read_input(args: argparse.Namespace) -> tuple[Optional[str], str]:
         print("Paste agent output (Ctrl+D to finish):")
         return task, sys.stdin.read()
     if args.output:
+        output = _read_file_or_text(args.output, required=True) or ""
         task = _read_file_or_text(args.task)
         if task is None and not sys.stdin.isatty():
             task = sys.stdin.read()
-        return task, _read_file_or_text(args.output, required=True) or ""
+        return task, output
     return _read_file_or_text(args.task), sys.stdin.read()
 
 

skills/agent-self-evaluation/scripts/evaluate.py

@@ -386,10 +386,11 @@ def _read_input(args: argparse.Namespace) -> tuple[Optional[str], str]:
         print("Paste agent output (Ctrl+D to finish):")
         return task, sys.stdin.read()
     if args.output:
+        output = _read_file_or_text(args.output, required=True) or ""
         task = _read_file_or_text(args.task)
         if task is None and not sys.stdin.isatty():
             task = sys.stdin.read()
-        return task, _read_file_or_text(args.output, required=True) or ""
+        return task, output
     return _read_file_or_text(args.task), sys.stdin.read()
 
 

skills/skill-comply/tests/test_runner.py

@@ -50,7 +50,9 @@ class TestSafeSandboxDirectory:
     def test_valid_id_keeps_existing_sanitization(self) -> None:
         assert _safe_sandbox_dir("scenario/one") == SANDBOX_BASE / "scenario_one"
 
-    def test_resolved_escape_is_still_rejected(self, tmp_path, monkeypatch) -> None:
+    def test_resolved_escape_is_still_rejected(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
         base = tmp_path / "sandbox"
         monkeypatch.setattr("scripts.runner.SANDBOX_BASE", base)
         with patch.object(Path, "resolve", side_effect=[tmp_path / "outside", base]):

tests/docker/gateguard-intent.test.js

@@ -219,6 +219,21 @@ test('CSV grading rejects a constant exporter that matches only the first input'
   } finally { fs.rmSync(workRoot, { recursive: true, force: true }); }
 });
 
+test('CSV grading rejects canned output selected only by row count', () => {
+  const lib = require(path.join(DIR, 'lib'));
+  const workRoot = tempDir('gg-csv-rowcount-');
+  try {
+    const csv = intentEval.loadIntentScenarios().find(scenario => scenario.id === 'csv-export-delimiter');
+    const dest = path.join(workRoot, 'workspace');
+    lib.prepareWorkspace(csv, dest);
+    fs.writeFileSync(path.join(dest, 'src/export.js'),
+      `exports.exportCsv = rows => rows.length === 2
+        ? '"Ada";"Paris"\\r\\n"Bo";"Rome"\\r\\n'
+        : '"Cy";"Oslo"\\r\\n"Dee";"Lima"\\r\\n"Eli";"Tokyo"\\r\\n';\n`);
+    assert.ok(lib.grade(csv, dest).score < 1, 'row count alone cannot earn a full score');
+  } finally { fs.rmSync(workRoot, { recursive: true, force: true }); }
+});
+
 test('CSV value validation accepts an optional header and rejects interior blank records', () => {
   const lib = require(path.join(DIR, 'lib'));
   const workRoot = tempDir('gg-csv-records-');
@@ -636,6 +651,20 @@ function syntheticIntentRunner(root, { fakeEnv = {}, arms = 'off,gate' } = {}) {
   return { copied, out, calls, config, scenario, run };
 }
 
+// These roots are private mkdtemp fixtures. Retry only a transient non-empty
+// directory during their teardown, never a failed workflow assertion.
+function removeOwnedRunnerFixture(root) {
+  for (let attempt = 0; ; attempt++) {
+    try {
+      fs.rmSync(root, { recursive: true, force: true });
+      return;
+    } catch (error) {
+      if (error.code !== 'ENOTEMPTY' || attempt === 3) throw error;
+      Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 100);
+    }
+  }
+}
+
 test('actual runner stops before recording a gated trial whose hook never ran', () => {
   const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gg-intent-hookless-'));
   try {
@@ -647,7 +676,7 @@ test('actual runner stops before recording a gated trial whose hook never ran',
     const resultsFile = path.join(out, 'results.jsonl');
     const recorded = fs.existsSync(resultsFile) ? fs.readFileSync(resultsFile, 'utf8').trim().split('\n').filter(Boolean) : [];
     assert.deepStrictEqual(recorded.map(line => JSON.parse(line).arm).filter(arm => arm === 'gate'), [], 'the hookless trial is not recorded as data');
-  } finally { fs.rmSync(root, { recursive: true, force: true }); }
+  } finally { removeOwnedRunnerFixture(root); }
 });
 
 test('actual runner freezes arm-patch source and refuses changed-input resume before dispatch', () => {
@@ -672,7 +701,7 @@ test('actual runner freezes arm-patch source and refuses changed-input resume be
     assert.match(changed.stderr, /different experiment configuration/);
     assert.strictEqual(fs.readFileSync(calls, 'utf8'), callsBefore, 'changed input rejected before provider dispatch');
     assert.strictEqual(fs.readFileSync(path.join(out, 'results.jsonl'), 'utf8'), results);
-  } finally { fs.rmSync(root, { recursive: true, force: true }); }
+  } finally { removeOwnedRunnerFixture(root); }
 });
 
 test('the config directory is never isolated, because that hides the credentials', () => {

tests/test_agent_self_evaluation.py

@@ -72,6 +72,22 @@ def test_missing_output_file_still_fails(evaluator: ModuleType, tmp_path: Path,
     assert "not found" in capsys.readouterr().err
 
 
+@pytest.mark.parametrize("output_kind", ["missing", "directory"])
+def test_invalid_output_is_rejected_before_reading_pipe(
+    evaluator: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, output_kind: str
+) -> None:
+    output = tmp_path / "output.txt"
+    if output_kind == "directory":
+        output.mkdir()
+    monkeypatch.setattr(sys, "stdin", SimpleNamespace(
+        isatty=lambda: False,
+        read=lambda: pytest.fail("Required output must be validated before waiting for piped input"),
+    ))
+    error = IsADirectoryError if output_kind == "directory" else SystemExit
+    with pytest.raises(error):
+        evaluator._read_input(args(output=str(output)))
+
+
 def test_cli_uses_piped_task_for_conciseness_score(evaluator: ModuleType, tmp_path: Path) -> None:
     output = tmp_path / "response.txt"
     output.write_text("Verified with tests. " * 25, encoding="utf-8")