ssh/tailssh: add exit-status test harness

Extract the in-process SSH server harness out of tailssh_test.go into
tailssh_exitcodes_test.go so the exit-status tests can share it, and add
the integrationtest-tagged exit-code suite (Go and OpenSSH clients).

Updates #18256

Signed-off-by: Kristoffer Dalby <kristoffer@tailscale.com>
This commit is contained in:
Kristoffer Dalby authored and Kristoffer Dalby committed 2026-07-30 10:18:00 +02:00
1 parent e17e38c290
commit ea3f83216b
3 files changed
+574 -89

No files matched your search

+152
View File
@@ -0,0 +1,152 @@
// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
//go:build linux || darwin
package tailssh
import (
"errors"
"fmt"
"net"
"net/netip"
"os/exec"
"os/user"
"testing"
gliderssh "github.com/tailscale/gliderssh"
"tailscale.com/ipn/ipnlocal"
"tailscale.com/ipn/store/mem"
"tailscale.com/tailcfg"
"tailscale.com/tsd"
"tailscale.com/tstest"
"tailscale.com/types/logid"
"tailscale.com/wgengine"
)
// testSSHHarness is the in-process SSH server harness shared by TestSSH
// and TestExitCodePassthrough: a tailssh.server on a local listener
// with auth bypassed, plus an execSSH closure that talks to it via
// the system ssh client.
type testSSHHarness struct {
user *user.User
execSSH func(args ...string) *exec.Cmd
}
func newTestSSHHarness(t *testing.T) *testSSHHarness {
t.Helper()
logf := tstest.WhileTestRunningLogger(t)
sys := tsd.NewSystem()
eng, err := wgengine.NewFakeUserspaceEngine(logf, sys.Set, sys.HealthTracker.Get(), sys.UserMetricsRegistry(), sys.Bus.Get())
if err != nil {
t.Fatal(err)
}
sys.Set(eng)
sys.Set(new(mem.Store))
lb, err := ipnlocal.NewLocalBackend(logf, logid.PublicID{}, sys, 0)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { lb.Shutdown() })
lb.SetVarRoot(t.TempDir())
srv := &server{lb: lb, logf: logf}
sc, err := srv.newConn()
if err != nil {
t.Fatal(err)
}
sc.insecureSkipTailscaleAuth = true
u, err := user.Current()
if err != nil {
t.Fatal(err)
}
um, err := userLookup(u.Username)
if err != nil {
t.Fatal(err)
}
sc.localUser = um
sc.info = &sshConnInfo{
sshUser: "test",
src: netip.MustParseAddrPort("1.2.3.4:32342"),
dst: netip.MustParseAddrPort("1.2.3.5:22"),
node: (&tailcfg.Node{}).View(),
uprof: tailcfg.UserProfile{},
}
sc.action0 = &tailcfg.SSHAction{Accept: true}
sc.finalAction = sc.action0
sc.authCompleted.Store(true)
sc.Handler = func(s gliderssh.Session) {
sc.newSSHSession(s).run()
}
ln, err := net.Listen("tcp4", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { ln.Close() })
port := ln.Addr().(*net.TCPAddr).Port
go func() {
for {
c, err := ln.Accept()
if err != nil {
if !errors.Is(err, net.ErrClosed) {
t.Errorf("Accept: %v", err)
}
return
}
go sc.HandleConn(c)
}
}()
execSSH := func(args ...string) *exec.Cmd {
cmd := exec.Command("ssh",
"-F", "none",
"-v",
"-p", fmt.Sprint(port),
"-o", "StrictHostKeyChecking=no",
"user@127.0.0.1")
cmd.Args = append(cmd.Args, args...)
return cmd
}
return &testSSHHarness{user: u, execSSH: execSSH}
}
// TestExitCodePassthrough is the in-process, fast-feedback companion
// to TestIntegrationExitCodes: 0 / 42 / 127 via the system ssh client
// to an in-process tailssh.server. 127 = POSIX command-not-found.
// https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_08_02
func TestExitCodePassthrough(t *testing.T) {
h := newTestSSHHarness(t)
tests := []struct {
name string
cmd string
want int
}{
{"zero", "true", 0},
{"passthrough", "exit 42", 42},
{"command_not_found", "/nonexistent/binary", 127},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := h.execSSH(tt.cmd).Run()
if tt.want == 0 {
if err != nil {
t.Fatalf("want exit 0, got error: %v", err)
}
return
}
var ee *exec.ExitError
if !errors.As(err, &ee) {
t.Fatalf("want *exec.ExitError, got %T: %v", err, err)
}
if got := ee.ExitCode(); got != tt.want {
t.Errorf("exit code = %d, want %d", got, tt.want)
}
})
}
}
@@ -0,0 +1,406 @@
// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
//go:build integrationtest
package tailssh
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"log"
"net"
"os"
"os/exec"
"os/user"
"path/filepath"
"runtime"
"testing"
"time"
"golang.org/x/crypto/ssh"
"tailscale.com/tstest"
)
// init fail-fasts any missing invariant (TAILSCALED_PATH, test user,
// login shell) so CI failures point at the broken piece instead of
// cryptic mid-test crashes. Logs the minimum context (GOOS, ssh
// version, resolved user shell) needed to attribute a failure.
func init() {
log.Printf("preflight: GOOS=%s GOARCH=%s euid=%d", runtime.GOOS, runtime.GOARCH, os.Geteuid())
if p := os.Getenv("TAILSCALED_PATH"); p != "" {
fi, err := os.Stat(p)
if err != nil {
log.Fatalf("preflight: TAILSCALED_PATH=%q not usable: %v", p, err)
}
if fi.Mode()&0111 == 0 {
log.Fatalf("preflight: TAILSCALED_PATH=%q is not executable (mode %v)", p, fi.Mode())
}
}
if _, err := exec.LookPath("ssh"); err == nil {
if out, err := exec.Command("ssh", "-V").CombinedOutput(); err == nil {
log.Printf("preflight: ssh -V: %s", bytes.TrimSpace(out))
}
}
username := exitCodeTestUser()
if _, err := user.Lookup(username); err != nil {
log.Fatalf("preflight: user.Lookup(%q) failed: %v", username, err)
}
um, err := userLookup(username)
if err != nil {
log.Fatalf("preflight: userLookup(%q) failed: %v", username, err)
}
shell := um.LoginShell()
if shell == "" {
log.Fatalf("preflight: empty login shell for %q", username)
}
if _, err := os.Stat(shell); err != nil {
log.Fatalf("preflight: login shell %q for user %q not usable: %v", shell, username, err)
}
log.Printf("preflight: user=%q shell=%q", username, shell)
}
// exitCodeTestUser is the local OS user the exit-code tests run as,
// overridable via TS_SSH_INTEGRATION_TEST_USER (testuser on Linux
// docker, runner on macOS CI).
func exitCodeTestUser() string {
if u := os.Getenv("TS_SSH_INTEGRATION_TEST_USER"); u != "" {
return u
}
return "testuser"
}
// dialTestClientForUser returns the dial error rather than t.Fatal'ing,
// so retry-aware tests can distinguish transport noise from assertion
// failure.
func dialTestClientForUser(t *testing.T, username string, forceV1Behavior, allowSendEnv bool, authMethods ...ssh.AuthMethod) (*ssh.Client, error) {
t.Helper()
addr := testServer(t, username, forceV1Behavior, allowSendEnv)
return ssh.Dial("tcp", addr, &ssh.ClientConfig{
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
Auth: authMethods,
Timeout: 15 * time.Second,
})
}
// dumpIncubatorLogOnFail prints /tmp/tailscalessh.log on subtest
// failure. The incubator runs in its own process; its log doesn't
// reach t.Log otherwise.
func dumpIncubatorLogOnFail(t *testing.T) {
t.Helper()
if !t.Failed() {
return
}
b, err := os.ReadFile("/tmp/tailscalessh.log")
if err != nil {
t.Logf("incubator log unreadable: %v", err)
return
}
if len(b) == 0 {
t.Logf("incubator log empty (no incubator launched, or log rotated)")
return
}
t.Logf("---- /tmp/tailscalessh.log (%d bytes) ----\n%s\n---- end ----", len(b), b)
}
// TestIntegrationExitCodes pins the SSH exit-status frame end-to-end
// through the real server stack (gliderssh + tailssh + incubator) with
// a Go x/crypto/ssh client. Transport noise (dial, pre-exec) is retried
// via tstest.WaitFor; an exit-code mismatch is the assertion and never
// retries.
func TestIntegrationExitCodes(t *testing.T) {
username := exitCodeTestUser()
tests := []struct {
name string
cmd string
wantCode int
}{
{"success", "true", 0},
{"exit_code_passthrough", "exit 42", 42},
// 127 = command-not-found, POSIX shell convention.
// https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_08_02
{"command_not_found", "/nonexistent/binary", 127},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
defer dumpIncubatorLogOnFail(t)
runOnce := func() (gotCode int, transportErr error, out []byte) {
cl, dialErr := dialTestClientForUser(t, username, false, false)
if dialErr != nil {
return -1, dialErr, nil
}
defer cl.Close()
s, err := cl.NewSession()
if err != nil {
return -1, fmt.Errorf("NewSession: %w", err), nil
}
defer s.Close()
type result struct {
out []byte
err error
}
done := make(chan result, 1)
go func() {
o, e := s.CombinedOutput(tt.cmd)
done <- result{o, e}
}()
var res result
select {
case res = <-done:
case <-time.After(20 * time.Second):
return -1, errors.New("ssh command timed out"), nil
}
if res.err == nil {
return 0, nil, res.out
}
var ee *ssh.ExitError
if errors.As(res.err, &ee) {
return ee.ExitStatus(), nil, res.out
}
// EOF before exit-status, channel teardown, etc. — treat as
// transport noise so the retry loop can act. The bug we're
// catching only surfaces as a wrong ExitStatus().
return -1, fmt.Errorf("non-exit ssh error: %w", res.err), res.out
}
// tstest.WaitFor retries on transport noise; a definitive
// exit-code observation returns nil so the assertion runs
// once, after WaitFor.
var gotCode int
var lastOut []byte
err := tstest.WaitFor(5*time.Second, func() error {
code, transportErr, out := runOnce()
lastOut = out
if transportErr != nil {
t.Logf("transport failure: %v; output:\n%s", transportErr, out)
return transportErr
}
gotCode = code
return nil
})
if err != nil {
t.Fatalf("ssh command %q never completed cleanly: %v; last output:\n%s",
tt.cmd, err, lastOut)
}
if gotCode != tt.wantCode {
t.Fatalf("exit code = %d, want %d; output:\n%s", gotCode, tt.wantCode, lastOut)
}
})
}
}
// TestOpenSSHExitCodes is TestIntegrationExitCodes against the system
// ssh binary, which is what users of #18256 are actually running
// (macOS ships a LibreSSL fork). Auth pinned to "none" with every
// other method explicitly disabled so OpenSSH can't fall back to
// a different path on different versions.
func TestOpenSSHExitCodes(t *testing.T) {
sshPath, err := exec.LookPath("ssh")
if err != nil {
t.Skipf("skipping without OpenSSH client: %v", err)
}
username := exitCodeTestUser()
if out, err := exec.Command(sshPath, "-V").CombinedOutput(); err == nil {
t.Logf("OpenSSH version: %s", bytes.TrimSpace(out))
}
t.Logf("OpenSSH test user: %s", username)
addr := testServer(t, username, false, false)
host, port, err := net.SplitHostPort(addr)
if err != nil {
t.Fatal(err)
}
t.Logf("tailssh server listening on %s", addr)
exitStatus := func(t *testing.T, err error) int {
t.Helper()
if err == nil {
return 0
}
var ee *exec.ExitError
if !errors.As(err, &ee) {
t.Fatalf("want *exec.ExitError, got %T: %v", err, err)
}
return ee.ExitCode()
}
// OpenSSH rc=255 is "ssh internal error" (connect/auth fail before
// the remote command runs); treat as transport, not the assertion.
// https://man.openbsd.org/ssh.1#EXIT_STATUS
isTransport := func(rc int) bool { return rc == 255 }
tests := []struct {
name string
cmd string
wantCode int
}{
{"success", "true", 0},
{"exit_code_passthrough", "exit 42", 42},
{"command_not_found", "/nonexistent/binary", 127},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
defer dumpIncubatorLogOnFail(t)
runOnce := func() (rc int, out []byte) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
cmd := exec.CommandContext(ctx, sshPath,
"-vvv",
"-F", "/dev/null",
"-T",
"-o", "BatchMode=yes",
"-o", "ConnectTimeout=15",
"-o", "GSSAPIAuthentication=no",
"-o", "GlobalKnownHostsFile=/dev/null",
"-o", "HostbasedAuthentication=no",
"-o", "IdentityAgent=none",
"-o", "KbdInteractiveAuthentication=no",
"-o", "NumberOfPasswordPrompts=0",
"-o", "PasswordAuthentication=no",
"-o", "PreferredAuthentications=none",
"-o", "PubkeyAuthentication=no",
"-o", "StrictHostKeyChecking=no",
"-o", "UserKnownHostsFile=/dev/null",
"-p", port,
username+"@"+host,
tt.cmd,
)
o, err := cmd.CombinedOutput()
if ctx.Err() == context.DeadlineExceeded {
t.Logf("ssh command timed out; output:\n%s", o)
return 255, o
}
return exitStatus(t, err), o
}
var gotRC int
var lastOut []byte
err := tstest.WaitFor(5*time.Second, func() error {
rc, out := runOnce()
lastOut = out
if isTransport(rc) && rc != tt.wantCode {
t.Logf("transport failure (rc=255); output:\n%s", out)
return fmt.Errorf("ssh rc=%d (transport)", rc)
}
gotRC = rc
return nil
})
if err != nil {
t.Fatalf("ssh command %q never returned a non-transport exit status: %v; last output:\n%s",
tt.cmd, err, lastOut)
}
if gotRC != tt.wantCode {
t.Fatalf("ssh exit code = %d, want %d; output:\n%s", gotRC, tt.wantCode, lastOut)
}
})
}
}
// TestLocalUnixForwardingHalfClose: after the client closes its write
// side, the server's still-in-flight response must arrive. The old
// cancel-on-first-direction bicopy tore the channel down too early.
func TestLocalUnixForwardingHalfClose(t *testing.T) {
debugTest.Store(true)
t.Cleanup(func() { debugTest.Store(false) })
socketDir, err := os.MkdirTemp("", "tailssh-test-")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { os.RemoveAll(socketDir) })
socketPath := filepath.Join(socketDir, "halfclose.sock")
ul, err := net.Listen("unix", socketPath)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { ul.Close() })
// Delayed-response service: read everything, sleep, then write.
const response = "delayed-response-after-client-closes-write"
go func() {
for {
conn, err := ul.Accept()
if err != nil {
return
}
go func() {
defer conn.Close()
io.ReadAll(conn)
time.Sleep(100 * time.Millisecond)
io.WriteString(conn, response)
}()
}
}()
addr := testServerWithOpts(t, testServerOpts{
username: "testuser",
allowLocalPortForwarding: true,
})
cl, err := ssh.Dial("tcp", addr, &ssh.ClientConfig{
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { cl.Close() })
conn, err := cl.Dial("unix", socketPath)
if err != nil {
t.Fatalf("failed to dial unix socket through SSH: %s", err)
}
defer conn.Close()
// (*ssh.Client).Dial("unix", ...) returns a *chanConn that embeds
// ssh.Channel; ssh.Channel exposes CloseWrite (RFC 4254 §5.3 EOF).
// Assert to that capability, not *net.TCPConn.
if _, err := io.WriteString(conn, "request data"); err != nil {
t.Fatalf("failed to write: %s", err)
}
cw, ok := conn.(interface{ CloseWrite() error })
if !ok {
t.Fatalf("conn %T does not implement CloseWrite; cannot test half-close", conn)
}
if err := cw.CloseWrite(); err != nil {
t.Fatalf("CloseWrite: %v", err)
}
// *chanConn.SetReadDeadline returns an error, so bound the read in
// a goroutine: a bicopy regression must fail fast, not hang CI.
type readResult struct {
data []byte
err error
}
done := make(chan readResult, 1)
go func() {
got, err := io.ReadAll(conn)
done <- readResult{got, err}
}()
select {
case res := <-done:
if res.err != nil {
t.Fatalf("failed to read response: %s", res.err)
}
if string(res.data) != response {
t.Errorf("got %q, want %q", res.data, response)
}
case <-time.After(15 * time.Second):
t.Fatalf("timed out waiting for response after half-close; bicopy may be tearing down the channel prematurely")
}
}
+16 -89
View File
@@ -21,7 +21,6 @@ import (
"net/http/httptest"
"net/netip"
"os"
"os/exec"
"os/user"
"reflect"
"runtime"
@@ -34,27 +33,21 @@ import (
"testing/synctest"
"time"
gliderssh "github.com/tailscale/gliderssh"
"golang.org/x/net/http2"
"golang.org/x/net/http2/h2c"
"tailscale.com/cmd/testwrapper/flakytest"
"tailscale.com/ipn/ipnlocal"
"tailscale.com/ipn/store/mem"
"tailscale.com/net/memnet"
"tailscale.com/net/tsdial"
"tailscale.com/sessionrecording"
"tailscale.com/tailcfg"
testssh "tailscale.com/tempfork/sshtest/ssh"
"tailscale.com/tsd"
"tailscale.com/tstest"
"tailscale.com/types/key"
"tailscale.com/types/logid"
"tailscale.com/types/netmap"
"tailscale.com/util/cibuild"
"tailscale.com/util/lineiter"
"tailscale.com/util/must"
"tailscale.com/version/distro"
"tailscale.com/wgengine"
)
func TestMatchRule(t *testing.T) {
@@ -562,6 +555,7 @@ func TestSSHRecordingCancelsSessionsOnUploadFailure(t *testing.T) {
handler func(w http.ResponseWriter, r *http.Request)
sshCommand string
wantClientOutput string
wantExitCode int // SSH exit status; 0 = don't check. 254 = recording infra failure.
clientOutputMustNotContain []string
}{
@@ -572,6 +566,9 @@ func TestSSHRecordingCancelsSessionsOnUploadFailure(t *testing.T) {
},
sshCommand: "echo hello",
wantClientOutput: "session rejected\r\n",
// 254 once recording failures get their own exit code; the
// current code exits 1 here.
wantExitCode: 0,
clientOutputMustNotContain: []string{"hello"},
},
@@ -619,6 +616,16 @@ func TestSSHRecordingCancelsSessionsOnUploadFailure(t *testing.T) {
} else {
t.Errorf("client did not get kicked out: %q", got)
}
if tt.wantExitCode != 0 {
var exitErr *testssh.ExitError
if errors.As(err, &exitErr) {
if got := exitErr.ExitStatus(); got != tt.wantExitCode {
t.Errorf("exit code = %d, want %d", got, tt.wantExitCode)
}
} else {
t.Errorf("want *ssh.ExitError, got %T: %v", err, err)
}
}
gotStr := string(got)
if !strings.HasSuffix(gotStr, tt.wantClientOutput) {
t.Errorf("client got %q, want %q", got, tt.wantClientOutput)
@@ -1171,88 +1178,8 @@ func TestSSHAuthFlow(t *testing.T) {
}
func TestSSH(t *testing.T) {
logf := tstest.WhileTestRunningLogger(t)
sys := tsd.NewSystem()
eng, err := wgengine.NewFakeUserspaceEngine(logf, sys.Set, sys.HealthTracker.Get(), sys.UserMetricsRegistry(), sys.Bus.Get())
if err != nil {
t.Fatal(err)
}
sys.Set(eng)
sys.Set(new(mem.Store))
lb, err := ipnlocal.NewLocalBackend(logf, logid.PublicID{}, sys, 0)
if err != nil {
t.Fatal(err)
}
defer lb.Shutdown()
dir := t.TempDir()
lb.SetVarRoot(dir)
srv := &server{
lb: lb,
logf: logf,
}
sc, err := srv.newConn()
if err != nil {
t.Fatal(err)
}
// Remove the auth checks for the test
sc.insecureSkipTailscaleAuth = true
u, err := user.Current()
if err != nil {
t.Fatal(err)
}
um, err := userLookup(u.Username)
if err != nil {
t.Fatal(err)
}
sc.localUser = um
sc.info = &sshConnInfo{
sshUser: "test",
src: netip.MustParseAddrPort("1.2.3.4:32342"),
dst: netip.MustParseAddrPort("1.2.3.5:22"),
node: (&tailcfg.Node{}).View(),
uprof: tailcfg.UserProfile{},
}
sc.action0 = &tailcfg.SSHAction{Accept: true}
sc.finalAction = sc.action0
sc.authCompleted.Store(true)
sc.Handler = func(s gliderssh.Session) {
sc.newSSHSession(s).run()
}
ln, err := net.Listen("tcp4", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer ln.Close()
port := ln.Addr().(*net.TCPAddr).Port
go func() {
for {
c, err := ln.Accept()
if err != nil {
if !errors.Is(err, net.ErrClosed) {
t.Errorf("Accept: %v", err)
}
return
}
go sc.HandleConn(c)
}
}()
execSSH := func(args ...string) *exec.Cmd {
cmd := exec.Command("ssh",
"-F",
"none",
"-v",
"-p", fmt.Sprint(port),
"-o", "StrictHostKeyChecking=no",
"user@127.0.0.1")
cmd.Args = append(cmd.Args, args...)
return cmd
}
h := newTestSSHHarness(t)
u, execSSH := h.user, h.execSSH
t.Run("env", func(t *testing.T) {
if cibuild.On() {