From 41d25e0c8f124a8faa82d701bd5324ee69e78dca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Claus=20Lensb=C3=B8l?= Date: Wed, 28 Oct 2026 14:05:30 -0400 Subject: [PATCH] cmd/natlab-test-matrix,tstest/natlab/vmtest: add test matrix tool (#21818) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add experimental tool for generating a matrix of tests to be run by CI. This tool will parse the test files in the vmtest folder, and find candidates that can be with with a set of variations. For now that is expressed as different flavours of Linux, but it can be expanded across OS'es but also other things like NAT configurations and portmapper extensions available. Updates #21817 Signed-off-by: Claus Lensbøl --- .github/workflows/natlab-test.yml | 2 +- tstest/natlab/README.md | 11 + tstest/natlab/vmtest/cloudinit.go | 2 + .../natlab-test-matrix/natlab-test-matrix.go | 213 ++++++++++++++++++ .../natlab-test-matrix_test.go | 70 ++++++ tstest/natlab/vmtest/images.go | 12 + tstest/natlab/vmtest/matrix_generated_test.go | 38 ++++ tstest/natlab/vmtest/peerapi_lan_test.go | 2 +- tstest/natlab/vmtest/vmtest.go | 2 + tstest/natlab/vmtest/vmtest_test.go | 10 +- 10 files changed, 355 insertions(+), 7 deletions(-) create mode 100644 tstest/natlab/vmtest/cmd/natlab-test-matrix/natlab-test-matrix.go create mode 100644 tstest/natlab/vmtest/cmd/natlab-test-matrix/natlab-test-matrix_test.go create mode 100644 tstest/natlab/vmtest/matrix_generated_test.go diff --git a/.github/workflows/natlab-test.yml b/.github/workflows/natlab-test.yml index 2cd6941bd..24b61529f 100644 --- a/.github/workflows/natlab-test.yml +++ b/.github/workflows/natlab-test.yml @@ -150,7 +150,7 @@ jobs: # single-test-per-matrix-job model. They stay runnable locally. run: | set -euo pipefail - exclude='^(TestGrid|TestPair|TestVnetPerf.*)$' + exclude='^(TestGrid|TestPair|TestMatrix|TestVnetPerf.*)$' tmp=$(mktemp) for pkg_dir in tstest/natlab/vmtest; do pkg="./${pkg_dir}/" diff --git a/tstest/natlab/README.md b/tstest/natlab/README.md index 7a6305582..74ff9bb5e 100644 --- a/tstest/natlab/README.md +++ b/tstest/natlab/README.md @@ -26,3 +26,14 @@ package. You may need to raise it further or disable the timeout entirely (`-timeout=0`) when running the full suite. Alternatively, select individual tests with the usual `-run`. + +## Matrix testing (early stages) + +Many tests can effectively be run across multiple variations. To wire up all +these tests programatically and not manually, a tool exists in +`./cmd/natlab-test-matrix` that will parse the test files in the folder +`./tstest/natlab/vmtest` and generate a full set of tests to run. + +A `VariationMap` defines variations to be tested. Right now the only variation +used is OS image variation, but this can be expanded to NAT, portmapper and so +on. diff --git a/tstest/natlab/vmtest/cloudinit.go b/tstest/natlab/vmtest/cloudinit.go index d61ce8b0c..ea48d92a5 100644 --- a/tstest/natlab/vmtest/cloudinit.go +++ b/tstest/natlab/vmtest/cloudinit.go @@ -39,6 +39,8 @@ func (e *Env) createCloudInitISO(n *Node) (string, error) { // Linux cloud-init needs network-config to configure interfaces before // systemd-networkd-wait-online blocks boot. + // TODO(cmol): Disable DNS on the SLIRP interface as older debian versions + // does not handle this correctly. if n.os.GOOS() == "linux" { files["network-config"] = `version: 2 ethernets: diff --git a/tstest/natlab/vmtest/cmd/natlab-test-matrix/natlab-test-matrix.go b/tstest/natlab/vmtest/cmd/natlab-test-matrix/natlab-test-matrix.go new file mode 100644 index 000000000..6c05e6403 --- /dev/null +++ b/tstest/natlab/vmtest/cmd/natlab-test-matrix/natlab-test-matrix.go @@ -0,0 +1,213 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +// The natlab-matrix-generator command is an experimental package for +// iterating over existing tests for natlab/vmtest, and creating a list of +// test runs based on a set of variations for each test parameter. +package main + +import ( + "bytes" + "fmt" + "go/ast" + "go/types" + "html/template" + "log" + "maps" + "os" + "path/filepath" + "slices" + "strings" + + "golang.org/x/tools/go/packages" +) + +type candidate struct { + name string + params []string +} + +// testVariation is the input given to the generator template +// for each test variation found in the package. +type testVariation struct { + FuncName string // test function name + TestName string // name given to t.Run + JoinedParams string // variant parameters (eg. vmtest.Ubuntu2404) join with ", " +} + +// VariationMap is a map from base variation to a list of variations that should +// be applied to that test. +var VariationMap = map[string][]string{ + "vmtest.OSImageLinux": { + "vmtest.Ubuntu2404", + "vmtest.Fedora43", + "vmtest.Debian13", // debian12 takes 10x the time as debian13 + }, +} + +// main parses all packages in the natlab test dir and finds candidates that +// can be used for matrix testing. See the README.md in tstest/natlab for more. +func main() { + if len(os.Args) < 2 { + fmt.Printf("usage: %s [path/to/vmtest/package]\n", os.Args[0]) + return + } + + cfg := &packages.Config{ + Mode: packages.NeedName | packages.NeedFiles | packages.NeedSyntax, + Dir: "", // $PWD + Tests: true, + } + pkgs, err := packages.Load(cfg, os.Args[1]) + if err != nil { + log.Fatal(err.Error()) + } + packages.PrintErrors(pkgs) + candidates := []candidate{} + for _, pkg := range pkgs { + if strings.HasSuffix(pkg.ID, ".test") { + continue + } + for _, f := range pkg.Syntax { + for _, decl := range f.Decls { + // Select top-level functions starting with "test". + fn, ok := decl.(*ast.FuncDecl) + if !ok { + continue + } + if !strings.HasPrefix(strings.ToLower(fn.Name.Name), "test") { + continue + } + // We only care about functions, not methods. + if fn.Recv != nil { + continue + } + + var params []string + for _, arg := range fn.Type.Params.List { + typ := types.ExprString(arg.Type) + for range len(arg.Names) { + params = append(params, typ) + } + } + candidates = append(candidates, candidate{ + name: fn.Name.Name, + params: params, + }) + } + } + } + + keys := slices.Collect(maps.Keys(VariationMap)) + + // Filter out candidates that are not relevant, having an unrecognizable + // signature. + candidates = slices.DeleteFunc(candidates, func(c candidate) bool { + if len(c.params) < 2 { + return true + } + // First param should be "*testing.T". + if !strings.Contains(c.params[0], "testing.T") { + return true + } + + for _, param := range c.params[1:] { + if !slices.Contains(keys, param) { + return true + } + } + return false + }) + + if len(candidates) < 1 { + log.Fatal("no candidates found") + } + log.Println("====== CANDIDATES FOUND ======") + for _, c := range candidates { + log.Printf("func %s(%v)\n", c.name, c.params) + } + + var variations []testVariation + + for _, candidate := range candidates { + results := generateTree(nil, candidate.params[1:]) + for _, res := range results { + + strippedParams := make([]string, len(res)) + // Clean up names for the test name. + for i, par := range res { + strippedParams[i] = strings.TrimPrefix(par, "vmtest.") + } + variations = append(variations, testVariation{ + FuncName: candidate.name, + JoinedParams: strings.Join(res, ", "), + TestName: fmt.Sprintf("%s_%s", candidate.name, strings.Join(strippedParams, "_")), + }) + } + } + + log.Println("====== VARIATIONS FOUND ======") + + for _, variation := range variations { + log.Printf("%s: func %s(t, %s)\n", variation.TestName, variation.FuncName, variation.JoinedParams) + } + + var buf bytes.Buffer + if err := tmpl.Execute(&buf, struct { + Package string + Tests []testVariation + }{"vmtest_test", variations}); err != nil { + log.Fatalf("Generate output: %s", err.Error()) + } + + if err := os.WriteFile(filepath.Join(os.Args[1], "matrix_generated_test.go"), + buf.Bytes(), 0o644); err != nil { + log.Fatalf("Failed writing result to disk: %s", err.Error()) + } +} + +func generateTree(selected, params []string) [][]string { + if len(params) == 0 { + return [][]string{selected} + } + + variations, ok := VariationMap[params[0]] + if !ok { + panic(fmt.Sprintf("variation for %s not found", params[0])) + } + + var result [][]string + for _, variation := range variations { + withVariation := append(slices.Clip(selected), variation) + result = append(result, generateTree(withVariation, params[1:])...) + } + return result +} + +var tmpl = template.Must(template.New("matrix").Parse(`// Code generated by cmd/natlab-test-matrix. DO NOT EDIT. +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package {{.Package}} + +import ( + "testing" + + "tailscale.com/tstest/natlab/vmtest" +) + +var testMatrix = []struct { + name string + fn func(t *testing.T) +}{ +{{- range .Tests}} + {name: "{{.TestName}}", fn: func(t *testing.T) { {{.FuncName}}(t, {{.JoinedParams}}) }}, +{{- end}} +} + +func TestMatrix(t *testing.T) { + for _, tt := range testMatrix { + t.Run(tt.name, tt.fn) + } +} +`)) diff --git a/tstest/natlab/vmtest/cmd/natlab-test-matrix/natlab-test-matrix_test.go b/tstest/natlab/vmtest/cmd/natlab-test-matrix/natlab-test-matrix_test.go new file mode 100644 index 000000000..77951cd88 --- /dev/null +++ b/tstest/natlab/vmtest/cmd/natlab-test-matrix/natlab-test-matrix_test.go @@ -0,0 +1,70 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package main + +import ( + "slices" + "testing" + + "tailscale.com/tstest" +) + +func TestGenerateTree(t *testing.T) { + tstest.Replace(t, &VariationMap, map[string][]string{ + "vmtest.OSImage": { + "vmtest.Gokrazy", + "vmtest.Ubuntu2404", + "vmtest.Debian12", + }, + }) + + tests := []struct { + name string + inputSelected []string + inputParams []string + want [][]string + }{ + { + name: "single_param", + inputSelected: []string{}, + inputParams: []string{"vmtest.OSImage"}, + want: [][]string{ + {"vmtest.Gokrazy"}, + {"vmtest.Ubuntu2404"}, + {"vmtest.Debian12"}, + }, + }, + { + name: "dual_param", + inputSelected: []string{}, + inputParams: []string{"vmtest.OSImage", "vmtest.OSImage"}, + want: [][]string{ + {"vmtest.Gokrazy", "vmtest.Gokrazy"}, + {"vmtest.Gokrazy", "vmtest.Ubuntu2404"}, + {"vmtest.Gokrazy", "vmtest.Debian12"}, + {"vmtest.Ubuntu2404", "vmtest.Gokrazy"}, + {"vmtest.Ubuntu2404", "vmtest.Ubuntu2404"}, + {"vmtest.Ubuntu2404", "vmtest.Debian12"}, + {"vmtest.Debian12", "vmtest.Gokrazy"}, + {"vmtest.Debian12", "vmtest.Ubuntu2404"}, + {"vmtest.Debian12", "vmtest.Debian12"}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := generateTree(tt.inputSelected, tt.inputParams) + if len(got) != len(tt.want) { + t.Fatalf("wrong response length, +got len(%d), -want len(%d)\ngot: %+v\nwant: %+v", + len(got), len(tt.want), got, tt.want) + } + for i, test := range got { + if slices.Compare(test, tt.want[i]) != 0 { + t.Errorf("bad param response, +got %v, -want %v", test, tt.want[i]) + } + } + }) + } +} diff --git a/tstest/natlab/vmtest/images.go b/tstest/natlab/vmtest/images.go index 79cc31218..00ea85d21 100644 --- a/tstest/natlab/vmtest/images.go +++ b/tstest/natlab/vmtest/images.go @@ -53,6 +53,10 @@ type OSImage struct { gokrazyKeep []featuretags.FeatureTag } +// OSImageLinux is a type alias used by tests to signal to the the matrix text +// code generator (natlab-test-matrix) that Linux variants should be generated. +type OSImageLinux = OSImage + // gokrazyMakeArgs returns the arguments to make in the gokrazy directory // that build the gokrazy image img, and the name of the qcow2 file that // writes there. @@ -148,6 +152,14 @@ var ( Family: LinuxDebian, } + // Debian13 is Debian 13 (Trixie) generic cloud image. + Debian13 = OSImage{ + Name: "debian-13", + URL: "https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2", + MemoryMB: 1024, + Family: LinuxDebian, + } + // FreeBSD150 is FreeBSD 15.0-RELEASE with BASIC-CLOUDINIT (nuageinit) support. // The image is distributed as xz-compressed qcow2. FreeBSD150 = OSImage{ diff --git a/tstest/natlab/vmtest/matrix_generated_test.go b/tstest/natlab/vmtest/matrix_generated_test.go new file mode 100644 index 000000000..9276de681 --- /dev/null +++ b/tstest/natlab/vmtest/matrix_generated_test.go @@ -0,0 +1,38 @@ +// Code generated by cmd/natlab-test-matrix. DO NOT EDIT. +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +package vmtest_test + +import ( + "testing" + + "tailscale.com/tstest/natlab/vmtest" +) + +var testMatrix = []struct { + name string + fn func(t *testing.T) +}{ + {name: "testPeerAPINotReachableFromLAN_Ubuntu2404", fn: func(t *testing.T) { testPeerAPINotReachableFromLAN(t, vmtest.Ubuntu2404) }}, + {name: "testPeerAPINotReachableFromLAN_Fedora43", fn: func(t *testing.T) { testPeerAPINotReachableFromLAN(t, vmtest.Fedora43) }}, + {name: "testPeerAPINotReachableFromLAN_Debian13", fn: func(t *testing.T) { testPeerAPINotReachableFromLAN(t, vmtest.Debian13) }}, + {name: "testSubnetRouterForOS_Ubuntu2404", fn: func(t *testing.T) { testSubnetRouterForOS(t, vmtest.Ubuntu2404) }}, + {name: "testSubnetRouterForOS_Fedora43", fn: func(t *testing.T) { testSubnetRouterForOS(t, vmtest.Fedora43) }}, + {name: "testSubnetRouterForOS_Debian13", fn: func(t *testing.T) { testSubnetRouterForOS(t, vmtest.Debian13) }}, + {name: "testSiteToSite_Ubuntu2404_Ubuntu2404", fn: func(t *testing.T) { testSiteToSite(t, vmtest.Ubuntu2404, vmtest.Ubuntu2404) }}, + {name: "testSiteToSite_Ubuntu2404_Fedora43", fn: func(t *testing.T) { testSiteToSite(t, vmtest.Ubuntu2404, vmtest.Fedora43) }}, + {name: "testSiteToSite_Ubuntu2404_Debian13", fn: func(t *testing.T) { testSiteToSite(t, vmtest.Ubuntu2404, vmtest.Debian13) }}, + {name: "testSiteToSite_Fedora43_Ubuntu2404", fn: func(t *testing.T) { testSiteToSite(t, vmtest.Fedora43, vmtest.Ubuntu2404) }}, + {name: "testSiteToSite_Fedora43_Fedora43", fn: func(t *testing.T) { testSiteToSite(t, vmtest.Fedora43, vmtest.Fedora43) }}, + {name: "testSiteToSite_Fedora43_Debian13", fn: func(t *testing.T) { testSiteToSite(t, vmtest.Fedora43, vmtest.Debian13) }}, + {name: "testSiteToSite_Debian13_Ubuntu2404", fn: func(t *testing.T) { testSiteToSite(t, vmtest.Debian13, vmtest.Ubuntu2404) }}, + {name: "testSiteToSite_Debian13_Fedora43", fn: func(t *testing.T) { testSiteToSite(t, vmtest.Debian13, vmtest.Fedora43) }}, + {name: "testSiteToSite_Debian13_Debian13", fn: func(t *testing.T) { testSiteToSite(t, vmtest.Debian13, vmtest.Debian13) }}, +} + +func TestMatrix(t *testing.T) { + for _, tt := range testMatrix { + t.Run(tt.name, tt.fn) + } +} diff --git a/tstest/natlab/vmtest/peerapi_lan_test.go b/tstest/natlab/vmtest/peerapi_lan_test.go index 8460c59bd..fc9c5db27 100644 --- a/tstest/natlab/vmtest/peerapi_lan_test.go +++ b/tstest/natlab/vmtest/peerapi_lan_test.go @@ -55,7 +55,7 @@ func TestPeerAPINotReachableFromLAN_FreeBSD(t *testing.T) { testPeerAPINotReachableFromLAN(t, vmtest.FreeBSD150) } -func testPeerAPINotReachableFromLAN(t *testing.T, targetOS vmtest.OSImage) { +func testPeerAPINotReachableFromLAN(t *testing.T, targetOS vmtest.OSImageLinux) { env := vmtest.New(t, vmtest.SameTailnetUser(), vmtest.AllOnline()) // The shared LAN. It needs a WAN IP so the target can reach the diff --git a/tstest/natlab/vmtest/vmtest.go b/tstest/natlab/vmtest/vmtest.go index 8397b8401..e91f44ca4 100644 --- a/tstest/natlab/vmtest/vmtest.go +++ b/tstest/natlab/vmtest/vmtest.go @@ -9,6 +9,8 @@ // See tstest/natlab/README.md for prerequisites and how to run. package vmtest +//go:generate go run tailscale.com/tstest/natlab/vmtest/cmd/natlab-test-matrix . + import ( "bytes" "context" diff --git a/tstest/natlab/vmtest/vmtest_test.go b/tstest/natlab/vmtest/vmtest_test.go index 818a7e57d..4e4f5659e 100644 --- a/tstest/natlab/vmtest/vmtest_test.go +++ b/tstest/natlab/vmtest/vmtest_test.go @@ -107,7 +107,7 @@ func TestFedoraDNSDirect(t *testing.T) { env.AssertDNSBackend(node, "direct") } -func testSubnetRouterForOS(t testing.TB, srOS vmtest.OSImage) { +func testSubnetRouterForOS(t testing.TB, srOS vmtest.OSImageLinux) { t.Helper() env := vmtest.New(t) @@ -164,7 +164,7 @@ func testSubnetRouterForOS(t testing.TB, srOS vmtest.OSImage) { } func TestSiteToSite(t *testing.T) { - testSiteToSite(t, vmtest.Ubuntu2404) + testSiteToSite(t, vmtest.Ubuntu2404, vmtest.Ubuntu2404) } // testSiteToSite runs a site-to-site subnet routing test with @@ -179,7 +179,7 @@ func TestSiteToSite(t *testing.T) { // Both subnet routers are on Tailscale with --snat-subnet-routes=false. // The test sends HTTP from backend-a to backend-b through the subnet routers // and verifies that backend-b sees backend-a's LAN IP (not the subnet router's). -func testSiteToSite(t *testing.T, srOS vmtest.OSImage) { +func testSiteToSite(t *testing.T, srOS, backendOS vmtest.OSImageLinux) { env := vmtest.New(t) // WAN networks for each site (each behind NAT). @@ -204,11 +204,11 @@ func testSiteToSite(t *testing.T, srOS vmtest.OSImage) { // Backend servers on each site's LAN (not on Tailscale). // Use Ubuntu so we can SSH in to add static routes. backendA := env.AddNode("backend-a", lanA, - vmtest.OS(vmtest.Ubuntu2404), + vmtest.OS(backendOS), vmtest.DontJoinTailnet(), vmtest.WebServer(8080)) backendB := env.AddNode("backend-b", lanB, - vmtest.OS(vmtest.Ubuntu2404), + vmtest.OS(backendOS), vmtest.DontJoinTailnet(), vmtest.WebServer(8080))