diff --git a/.github/workflows/natlab-test.yml b/.github/workflows/natlab-test.yml index 2cd6941bd..24b61529f 100644 --- a/.github/workflows/natlab-test.yml +++ b/.github/workflows/natlab-test.yml @@ -150,7 +150,7 @@ jobs: # single-test-per-matrix-job model. They stay runnable locally. run: | set -euo pipefail - exclude='^(TestGrid|TestPair|TestVnetPerf.*)$' + exclude='^(TestGrid|TestPair|TestMatrix|TestVnetPerf.*)$' tmp=$(mktemp) for pkg_dir in tstest/natlab/vmtest; do pkg="./${pkg_dir}/" diff --git a/tstest/natlab/README.md b/tstest/natlab/README.md index 7a6305582..74ff9bb5e 100644 --- a/tstest/natlab/README.md +++ b/tstest/natlab/README.md @@ -26,3 +26,14 @@ package. You may need to raise it further or disable the timeout entirely (`-timeout=0`) when running the full suite. Alternatively, select individual tests with the usual `-run`. + +## Matrix testing (early stages) + +Many tests can effectively be run across multiple variations. To wire up all +these tests programatically and not manually, a tool exists in +`./cmd/natlab-test-matrix` that will parse the test files in the folder +`./tstest/natlab/vmtest` and generate a full set of tests to run. + +A `VariationMap` defines variations to be tested. Right now the only variation +used is OS image variation, but this can be expanded to NAT, portmapper and so +on. diff --git a/tstest/natlab/vmtest/cloudinit.go b/tstest/natlab/vmtest/cloudinit.go index d61ce8b0c..ea48d92a5 100644 --- a/tstest/natlab/vmtest/cloudinit.go +++ b/tstest/natlab/vmtest/cloudinit.go @@ -39,6 +39,8 @@ func (e *Env) createCloudInitISO(n *Node) (string, error) { // Linux cloud-init needs network-config to configure interfaces before // systemd-networkd-wait-online blocks boot. + // TODO(cmol): Disable DNS on the SLIRP interface as older debian versions + // does not handle this correctly. if n.os.GOOS() == "linux" { files["network-config"] = `version: 2 ethernets: diff --git a/tstest/natlab/vmtest/cmd/natlab-test-matrix/natlab-test-matrix.go b/tstest/natlab/vmtest/cmd/natlab-test-matrix/natlab-test-matrix.go new file mode 100644 index 000000000..6c05e6403 --- /dev/null +++ b/tstest/natlab/vmtest/cmd/natlab-test-matrix/natlab-test-matrix.go @@ -0,0 +1,213 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +// The natlab-matrix-generator command is an experimental package for +// iterating over existing tests for natlab/vmtest, and creating a list of +// test runs based on a set of variations for each test parameter. +package main + +import ( + "bytes" + "fmt" + "go/ast" + "go/types" + "html/template" + "log" + "maps" + "os" + "path/filepath" + "slices" + "strings" + + "golang.org/x/tools/go/packages" +) + +type candidate struct { + name string + params []string +} + +// testVariation is the input given to the generator template +// for each test variation found in the package. +type testVariation struct { + FuncName string // test function name + TestName string // name given to t.Run + JoinedParams string // variant parameters (eg. vmtest.Ubuntu2404) join with ", " +} + +// VariationMap is a map from base variation to a list of variations that should +// be applied to that test. +var VariationMap = map[string][]string{ + "vmtest.OSImageLinux": { + "vmtest.Ubuntu2404", + "vmtest.Fedora43", + "vmtest.Debian13", // debian12 takes 10x the time as debian13 + }, +} + +// main parses all packages in the natlab test dir and finds candidates that +// can be used for matrix testing. See the README.md in tstest/natlab for more. +func main() { + if len(os.Args) < 2 { + fmt.Printf("usage: %s [path/to/vmtest/package]\n", os.Args[0]) + return + } + + cfg := &packages.Config{ + Mode: packages.NeedName | packages.NeedFiles | packages.NeedSyntax, + Dir: "", // $PWD + Tests: true, + } + pkgs, err := packages.Load(cfg, os.Args[1]) + if err != nil { + log.Fatal(err.Error()) + } + packages.PrintErrors(pkgs) + candidates := []candidate{} + for _, pkg := range pkgs { + if strings.HasSuffix(pkg.ID, ".test") { + continue + } + for _, f := range pkg.Syntax { + for _, decl := range f.Decls { + // Select top-level functions starting with "test". + fn, ok := decl.(*ast.FuncDecl) + if !ok { + continue + } + if !strings.HasPrefix(strings.ToLower(fn.Name.Name), "test") { + continue + } + // We only care about functions, not methods. + if fn.Recv != nil { + continue + } + + var params []string + for _, arg := range fn.Type.Params.List { + typ := types.ExprString(arg.Type) + for range len(arg.Names) { + params = append(params, typ) + } + } + candidates = append(candidates, candidate{ + name: fn.Name.Name, + params: params, + }) + } + } + } + + keys := slices.Collect(maps.Keys(VariationMap)) + + // Filter out candidates that are not relevant, having an unrecognizable + // signature. + candidates = slices.DeleteFunc(candidates, func(c candidate) bool { + if len(c.params) < 2 { + return true + } + // First param should be "*testing.T". + if !strings.Contains(c.params[0], "testing.T") { + return true + } + + for _, param := range c.params[1:] { + if !slices.Contains(keys, param) { + return true + } + } + return false + }) + + if len(candidates) < 1 { + log.Fatal("no candidates found") + } + log.Println("====== CANDIDATES FOUND ======") + for _, c := range candidates { + log.Printf("func %s(%v)\n", c.name, c.params) + } + + var variations []testVariation + + for _, candidate := range candidates { + results := generateTree(nil, candidate.params[1:]) + for _, res := range results { + + strippedParams := make([]string, len(res)) + // Clean up names for the test name. + for i, par := range res { + strippedParams[i] = strings.TrimPrefix(par, "vmtest.") + } + variations = append(variations, testVariation{ + FuncName: candidate.name, + JoinedParams: strings.Join(res, ", "), + TestName: fmt.Sprintf("%s_%s", candidate.name, strings.Join(strippedParams, "_")), + }) + } + } + + log.Println("====== VARIATIONS FOUND ======") + + for _, variation := range variations { + log.Printf("%s: func %s(t, %s)\n", variation.TestName, variation.FuncName, variation.JoinedParams) + } + + var buf bytes.Buffer + if err := tmpl.Execute(&buf, struct { + Package string + Tests []testVariation + }{"vmtest_test", variations}); err != nil { + log.Fatalf("Generate output: %s", err.Error()) + } + + if err := os.WriteFile(filepath.Join(os.Args[1], "matrix_generated_test.go"), + buf.Bytes(), 0o644); err != nil { + log.Fatalf("Failed writing result to disk: %s", err.Error()) + } +} + +func generateTree(selected, params []string) [][]string { + if len(params) == 0 { + return [][]string{selected} + } + + variations, ok := VariationMap[params[0]] + if !ok { + panic(fmt.Sprintf("variation for %s not found", params[0])) + } + + var result [][]string + for _, variation := range variations { + withVariation := append(slices.Clip(selected), variation) + result = append(result, generateTree(withVariation, params[1:])...) + } + return result +} + +var tmpl = template.Must(template.New("matrix").Parse(`// Code generated by cmd/natlab-test-matrix. DO NOT EDIT. +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package {{.Package}} + +import ( + "testing" + + "tailscale.com/tstest/natlab/vmtest" +) + +var testMatrix = []struct { + name string + fn func(t *testing.T) +}{ +{{- range .Tests}} + {name: "{{.TestName}}", fn: func(t *testing.T) { {{.FuncName}}(t, {{.JoinedParams}}) }}, +{{- end}} +} + +func TestMatrix(t *testing.T) { + for _, tt := range testMatrix { + t.Run(tt.name, tt.fn) + } +} +`)) diff --git a/tstest/natlab/vmtest/cmd/natlab-test-matrix/natlab-test-matrix_test.go b/tstest/natlab/vmtest/cmd/natlab-test-matrix/natlab-test-matrix_test.go new file mode 100644 index 000000000..77951cd88 --- /dev/null +++ b/tstest/natlab/vmtest/cmd/natlab-test-matrix/natlab-test-matrix_test.go @@ -0,0 +1,70 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package main + +import ( + "slices" + "testing" + + "tailscale.com/tstest" +) + +func TestGenerateTree(t *testing.T) { + tstest.Replace(t, &VariationMap, map[string][]string{ + "vmtest.OSImage": { + "vmtest.Gokrazy", + "vmtest.Ubuntu2404", + "vmtest.Debian12", + }, + }) + + tests := []struct { + name string + inputSelected []string + inputParams []string + want [][]string + }{ + { + name: "single_param", + inputSelected: []string{}, + inputParams: []string{"vmtest.OSImage"}, + want: [][]string{ + {"vmtest.Gokrazy"}, + {"vmtest.Ubuntu2404"}, + {"vmtest.Debian12"}, + }, + }, + { + name: "dual_param", + inputSelected: []string{}, + inputParams: []string{"vmtest.OSImage", "vmtest.OSImage"}, + want: [][]string{ + {"vmtest.Gokrazy", "vmtest.Gokrazy"}, + {"vmtest.Gokrazy", "vmtest.Ubuntu2404"}, + {"vmtest.Gokrazy", "vmtest.Debian12"}, + {"vmtest.Ubuntu2404", "vmtest.Gokrazy"}, + {"vmtest.Ubuntu2404", "vmtest.Ubuntu2404"}, + {"vmtest.Ubuntu2404", "vmtest.Debian12"}, + {"vmtest.Debian12", "vmtest.Gokrazy"}, + {"vmtest.Debian12", "vmtest.Ubuntu2404"}, + {"vmtest.Debian12", "vmtest.Debian12"}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := generateTree(tt.inputSelected, tt.inputParams) + if len(got) != len(tt.want) { + t.Fatalf("wrong response length, +got len(%d), -want len(%d)\ngot: %+v\nwant: %+v", + len(got), len(tt.want), got, tt.want) + } + for i, test := range got { + if slices.Compare(test, tt.want[i]) != 0 { + t.Errorf("bad param response, +got %v, -want %v", test, tt.want[i]) + } + } + }) + } +} diff --git a/tstest/natlab/vmtest/images.go b/tstest/natlab/vmtest/images.go index 79cc31218..00ea85d21 100644 --- a/tstest/natlab/vmtest/images.go +++ b/tstest/natlab/vmtest/images.go @@ -53,6 +53,10 @@ type OSImage struct { gokrazyKeep []featuretags.FeatureTag } +// OSImageLinux is a type alias used by tests to signal to the the matrix text +// code generator (natlab-test-matrix) that Linux variants should be generated. +type OSImageLinux = OSImage + // gokrazyMakeArgs returns the arguments to make in the gokrazy directory // that build the gokrazy image img, and the name of the qcow2 file that // writes there. @@ -148,6 +152,14 @@ var ( Family: LinuxDebian, } + // Debian13 is Debian 13 (Trixie) generic cloud image. + Debian13 = OSImage{ + Name: "debian-13", + URL: "https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2", + MemoryMB: 1024, + Family: LinuxDebian, + } + // FreeBSD150 is FreeBSD 15.0-RELEASE with BASIC-CLOUDINIT (nuageinit) support. // The image is distributed as xz-compressed qcow2. FreeBSD150 = OSImage{ diff --git a/tstest/natlab/vmtest/matrix_generated_test.go b/tstest/natlab/vmtest/matrix_generated_test.go new file mode 100644 index 000000000..9276de681 --- /dev/null +++ b/tstest/natlab/vmtest/matrix_generated_test.go @@ -0,0 +1,38 @@ +// Code generated by cmd/natlab-test-matrix. DO NOT EDIT. +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package vmtest_test + +import ( + "testing" + + "tailscale.com/tstest/natlab/vmtest" +) + +var testMatrix = []struct { + name string + fn func(t *testing.T) +}{ + {name: "testPeerAPINotReachableFromLAN_Ubuntu2404", fn: func(t *testing.T) { testPeerAPINotReachableFromLAN(t, vmtest.Ubuntu2404) }}, + {name: "testPeerAPINotReachableFromLAN_Fedora43", fn: func(t *testing.T) { testPeerAPINotReachableFromLAN(t, vmtest.Fedora43) }}, + {name: "testPeerAPINotReachableFromLAN_Debian13", fn: func(t *testing.T) { testPeerAPINotReachableFromLAN(t, vmtest.Debian13) }}, + {name: "testSubnetRouterForOS_Ubuntu2404", fn: func(t *testing.T) { testSubnetRouterForOS(t, vmtest.Ubuntu2404) }}, + {name: "testSubnetRouterForOS_Fedora43", fn: func(t *testing.T) { testSubnetRouterForOS(t, vmtest.Fedora43) }}, + {name: "testSubnetRouterForOS_Debian13", fn: func(t *testing.T) { testSubnetRouterForOS(t, vmtest.Debian13) }}, + {name: "testSiteToSite_Ubuntu2404_Ubuntu2404", fn: func(t *testing.T) { testSiteToSite(t, vmtest.Ubuntu2404, vmtest.Ubuntu2404) }}, + {name: "testSiteToSite_Ubuntu2404_Fedora43", fn: func(t *testing.T) { testSiteToSite(t, vmtest.Ubuntu2404, vmtest.Fedora43) }}, + {name: "testSiteToSite_Ubuntu2404_Debian13", fn: func(t *testing.T) { testSiteToSite(t, vmtest.Ubuntu2404, vmtest.Debian13) }}, + {name: "testSiteToSite_Fedora43_Ubuntu2404", fn: func(t *testing.T) { testSiteToSite(t, vmtest.Fedora43, vmtest.Ubuntu2404) }}, + {name: "testSiteToSite_Fedora43_Fedora43", fn: func(t *testing.T) { testSiteToSite(t, vmtest.Fedora43, vmtest.Fedora43) }}, + {name: "testSiteToSite_Fedora43_Debian13", fn: func(t *testing.T) { testSiteToSite(t, vmtest.Fedora43, vmtest.Debian13) }}, + {name: "testSiteToSite_Debian13_Ubuntu2404", fn: func(t *testing.T) { testSiteToSite(t, vmtest.Debian13, vmtest.Ubuntu2404) }}, + {name: "testSiteToSite_Debian13_Fedora43", fn: func(t *testing.T) { testSiteToSite(t, vmtest.Debian13, vmtest.Fedora43) }}, + {name: "testSiteToSite_Debian13_Debian13", fn: func(t *testing.T) { testSiteToSite(t, vmtest.Debian13, vmtest.Debian13) }}, +} + +func TestMatrix(t *testing.T) { + for _, tt := range testMatrix { + t.Run(tt.name, tt.fn) + } +} diff --git a/tstest/natlab/vmtest/peerapi_lan_test.go b/tstest/natlab/vmtest/peerapi_lan_test.go index 8460c59bd..fc9c5db27 100644 --- a/tstest/natlab/vmtest/peerapi_lan_test.go +++ b/tstest/natlab/vmtest/peerapi_lan_test.go @@ -55,7 +55,7 @@ func TestPeerAPINotReachableFromLAN_FreeBSD(t *testing.T) { testPeerAPINotReachableFromLAN(t, vmtest.FreeBSD150) } -func testPeerAPINotReachableFromLAN(t *testing.T, targetOS vmtest.OSImage) { +func testPeerAPINotReachableFromLAN(t *testing.T, targetOS vmtest.OSImageLinux) { env := vmtest.New(t, vmtest.SameTailnetUser(), vmtest.AllOnline()) // The shared LAN. It needs a WAN IP so the target can reach the diff --git a/tstest/natlab/vmtest/vmtest.go b/tstest/natlab/vmtest/vmtest.go index 8397b8401..e91f44ca4 100644 --- a/tstest/natlab/vmtest/vmtest.go +++ b/tstest/natlab/vmtest/vmtest.go @@ -9,6 +9,8 @@ // See tstest/natlab/README.md for prerequisites and how to run. package vmtest +//go:generate go run tailscale.com/tstest/natlab/vmtest/cmd/natlab-test-matrix . + import ( "bytes" "context" diff --git a/tstest/natlab/vmtest/vmtest_test.go b/tstest/natlab/vmtest/vmtest_test.go index 818a7e57d..4e4f5659e 100644 --- a/tstest/natlab/vmtest/vmtest_test.go +++ b/tstest/natlab/vmtest/vmtest_test.go @@ -107,7 +107,7 @@ func TestFedoraDNSDirect(t *testing.T) { env.AssertDNSBackend(node, "direct") } -func testSubnetRouterForOS(t testing.TB, srOS vmtest.OSImage) { +func testSubnetRouterForOS(t testing.TB, srOS vmtest.OSImageLinux) { t.Helper() env := vmtest.New(t) @@ -164,7 +164,7 @@ func testSubnetRouterForOS(t testing.TB, srOS vmtest.OSImage) { } func TestSiteToSite(t *testing.T) { - testSiteToSite(t, vmtest.Ubuntu2404) + testSiteToSite(t, vmtest.Ubuntu2404, vmtest.Ubuntu2404) } // testSiteToSite runs a site-to-site subnet routing test with @@ -179,7 +179,7 @@ func TestSiteToSite(t *testing.T) { // Both subnet routers are on Tailscale with --snat-subnet-routes=false. // The test sends HTTP from backend-a to backend-b through the subnet routers // and verifies that backend-b sees backend-a's LAN IP (not the subnet router's). -func testSiteToSite(t *testing.T, srOS vmtest.OSImage) { +func testSiteToSite(t *testing.T, srOS, backendOS vmtest.OSImageLinux) { env := vmtest.New(t) // WAN networks for each site (each behind NAT). @@ -204,11 +204,11 @@ func testSiteToSite(t *testing.T, srOS vmtest.OSImage) { // Backend servers on each site's LAN (not on Tailscale). // Use Ubuntu so we can SSH in to add static routes. backendA := env.AddNode("backend-a", lanA, - vmtest.OS(vmtest.Ubuntu2404), + vmtest.OS(backendOS), vmtest.DontJoinTailnet(), vmtest.WebServer(8080)) backendB := env.AddNode("backend-b", lanB, - vmtest.OS(vmtest.Ubuntu2404), + vmtest.OS(backendOS), vmtest.DontJoinTailnet(), vmtest.WebServer(8080))