mirror of
https://github.com/tailscale/tailscale.git
synced 2026-10-23 03:37:19 +09:00
tsweb/compserve, client/web: add zstd, remove brotli for precompressed assets
Adds tsweb/compserve: content negotiation for precompressed static variants, with a transcode-to-identity fallback for clients that do not accept an encoding (including when the raw file is absent), and CompressWriter, which live-compresses dynamic responses with zstd in its fastest mode, streamed incrementally with no buffering. Negotiation is q-value and wildcard aware (gzip;q=0 previously matched gzip). client/web serves its prebuilt embedded assets through compserve, replacing brotli with zstd; the embedded FS is wrapped in tsweb/vcstime for conditional-request mod times. tsweb/compserve/gzip.go keeps transitional serving of gzip variants from pre-zstd file systems (such as the currently published web-client-prebuilt module): passthrough to gzip-accepting clients, transcoded to identity otherwise; it becomes inert once a zstd-only module is published. util/zstdframe gains pooled GetDecoder and GetStreamingEncoder (concurrency=1). util/precompress is now a build-time tool, generating zstd variants only. cmd/tsconnect and cmd/build-webclient consume the new precompress/compserve split. tsweb.AcceptsEncoding and tsweb/tswebutil are removed; negotiation lives in compserve and the deprecated shim had no callers. go.mod bumps web-client-prebuilt. Also fixes a transcoding bug where http.ServeContent's size probe via the promoted zstd.Decoder.WriteTo could report a zero length, serving empty bodies. Updates tailscale/corp#20099 Signed-off-by: James Tucker <james@tailscale.com>
This commit is contained in:
James Tucker authored and James Tucker committed
1 parent
178ef3db08
commit
6608b9a387
25 files changed
+2029
-175
No files matched your search
+25
-37
@@ -4,7 +4,7 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"io"
|
||||
"errors"
|
||||
"io/fs"
|
||||
"log"
|
||||
"net/http"
|
||||
@@ -14,13 +14,12 @@ import (
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
prebuilt "github.com/tailscale/web-client-prebuilt"
|
||||
"tailscale.com/tsweb/compserve"
|
||||
"tailscale.com/tsweb/vcstime"
|
||||
)
|
||||
|
||||
var start = time.Now()
|
||||
|
||||
func assetsHandler(devMode bool) (_ http.Handler, cleanup func()) {
|
||||
if devMode {
|
||||
// When in dev mode, proxy asset requests to the Vite dev server.
|
||||
@@ -28,49 +27,38 @@ func assetsHandler(devMode bool) (_ http.Handler, cleanup func()) {
|
||||
return devServerProxy(), cleanup
|
||||
}
|
||||
|
||||
fsys := prebuilt.FS()
|
||||
// vcstime supplies the VCS commit time as mod time for the embedded
|
||||
// assets, enabling conditional requests.
|
||||
fsys := vcstime.FS(prebuilt.FS())
|
||||
opts := compserve.Options{}
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
path := strings.TrimPrefix(r.URL.Path, "/")
|
||||
f, err := openPrecompressedFile(w, r, path, fsys)
|
||||
if err != nil {
|
||||
// Rewrite request to just fetch index.html and let
|
||||
// the frontend router handle it.
|
||||
r = r.Clone(r.Context())
|
||||
path = "index.html"
|
||||
f, err = openPrecompressedFile(w, r, path, fsys)
|
||||
}
|
||||
if f == nil {
|
||||
http.Error(w, err.Error(), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
// fs.File does not claim to implement Seeker, but in practice it does.
|
||||
fSeeker, ok := f.(io.ReadSeeker)
|
||||
if !ok {
|
||||
http.Error(w, "Not seekable", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if strings.HasPrefix(path, "assets/") {
|
||||
// Aggressively cache static assets, since we cache-bust our assets with
|
||||
// hashed filenames.
|
||||
w.Header().Set("Cache-Control", "public, max-age=31535996")
|
||||
w.Header().Set("Vary", "Accept-Encoding")
|
||||
}
|
||||
|
||||
http.ServeContent(w, r, path, start, fSeeker)
|
||||
err := compserve.ServeFile(w, r, fsys, path, opts)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
// Rewrite request to just fetch index.html and let
|
||||
// the frontend router handle it. The fallback must not inherit
|
||||
// cache headers of the original path; nothing has been written
|
||||
// yet, so they can still be adjusted.
|
||||
w.Header().Del("Cache-Control")
|
||||
err = compserve.ServeFile(w, r, fsys, "index.html", opts)
|
||||
}
|
||||
if err != nil {
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
// Errors may embed the request path; log rather than reflect it.
|
||||
log.Printf("web client: serving %q: %v", path, err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
}
|
||||
}), nil
|
||||
}
|
||||
|
||||
func openPrecompressedFile(w http.ResponseWriter, r *http.Request, path string, fs fs.FS) (fs.File, error) {
|
||||
if f, err := fs.Open(path + ".gz"); err == nil {
|
||||
w.Header().Set("Content-Encoding", "gzip")
|
||||
return f, nil
|
||||
}
|
||||
return fs.Open(path) // fallback
|
||||
}
|
||||
|
||||
// startDevServer starts the JS dev server that does on-demand rebuilding
|
||||
// and serving of web client JS and CSS resources.
|
||||
func startDevServer() (cleanup func()) {
|
||||
|
||||
@@ -68,15 +68,12 @@ func build(toolDir, appDir string) error {
|
||||
return fmt.Errorf("Cannot precompress: %w", err)
|
||||
}
|
||||
|
||||
// Cleanup pre-compressed files.
|
||||
// Cleanup: the original (uncompressed) files are not embedded; only the
|
||||
// ".zst" variants are served.
|
||||
for _, f := range compressedFiles {
|
||||
if err := os.Remove(f); err != nil {
|
||||
log.Printf("Failed to cleanup %q: %v", f, err)
|
||||
}
|
||||
// Removing intermediate ".br" version, we use ".gz" asset.
|
||||
if err := os.Remove(f + ".br"); err != nil {
|
||||
log.Printf("Failed to cleanup %q: %v", f+".gz", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
@@ -84,7 +84,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
|
||||
github.com/klauspost/compress/internal/cpuinfo from github.com/klauspost/compress/huff0+
|
||||
💣 github.com/klauspost/compress/internal/le from github.com/klauspost/compress/huff0+
|
||||
github.com/klauspost/compress/internal/snapref from github.com/klauspost/compress/zstd
|
||||
github.com/klauspost/compress/zstd from tailscale.com/util/zstdframe
|
||||
github.com/klauspost/compress/zstd from tailscale.com/util/zstdframe+
|
||||
github.com/klauspost/compress/zstd/internal/xxhash from github.com/klauspost/compress/zstd
|
||||
L 💣 github.com/mdlayher/netlink from github.com/jsimonetti/rtnetlink+
|
||||
L github.com/mdlayher/netlink/nlenc from github.com/jsimonetti/rtnetlink+
|
||||
@@ -856,7 +856,9 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
|
||||
tailscale.com/tstime/mono from tailscale.com/net/tstun+
|
||||
tailscale.com/tstime/rate from tailscale.com/wgengine/filter
|
||||
tailscale.com/tsweb from tailscale.com/util/eventbus+
|
||||
tailscale.com/tsweb/compserve from tailscale.com/client/web
|
||||
tailscale.com/tsweb/varz from tailscale.com/util/usermetric+
|
||||
tailscale.com/tsweb/vcstime from tailscale.com/client/web
|
||||
tailscale.com/types/appctype from tailscale.com/ipn/ipnlocal+
|
||||
tailscale.com/types/bools from tailscale.com/tsnet+
|
||||
tailscale.com/types/dnstype from tailscale.com/ipn/ipnlocal+
|
||||
|
||||
@@ -142,6 +142,14 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
|
||||
L 💣 github.com/jsimonetti/rtnetlink from tailscale.com/net/netmon
|
||||
L github.com/jsimonetti/rtnetlink/internal/unix from github.com/jsimonetti/rtnetlink
|
||||
github.com/kballard/go-shellquote from tailscale.com/cmd/tailscale/cli
|
||||
github.com/klauspost/compress from github.com/klauspost/compress/zstd
|
||||
github.com/klauspost/compress/fse from github.com/klauspost/compress/huff0
|
||||
github.com/klauspost/compress/huff0 from github.com/klauspost/compress/zstd
|
||||
github.com/klauspost/compress/internal/cpuinfo from github.com/klauspost/compress/huff0+
|
||||
💣 github.com/klauspost/compress/internal/le from github.com/klauspost/compress/huff0+
|
||||
github.com/klauspost/compress/internal/snapref from github.com/klauspost/compress/zstd
|
||||
github.com/klauspost/compress/zstd from tailscale.com/tsweb/compserve+
|
||||
github.com/klauspost/compress/zstd/internal/xxhash from github.com/klauspost/compress/zstd
|
||||
💣 github.com/mattn/go-colorable from tailscale.com/cmd/tailscale/cli
|
||||
💣 github.com/mattn/go-isatty from tailscale.com/cmd/tailscale/cli+
|
||||
L 💣 github.com/mdlayher/netlink from github.com/jsimonetti/rtnetlink+
|
||||
@@ -268,7 +276,9 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
|
||||
tailscale.com/tstime/mono from tailscale.com/tstime/rate
|
||||
tailscale.com/tstime/rate from tailscale.com/cmd/tailscale/cli
|
||||
tailscale.com/tsweb from tailscale.com/util/eventbus+
|
||||
tailscale.com/tsweb/compserve from tailscale.com/client/web
|
||||
tailscale.com/tsweb/varz from tailscale.com/util/usermetric+
|
||||
tailscale.com/tsweb/vcstime from tailscale.com/client/web
|
||||
tailscale.com/types/appctype from tailscale.com/client/local+
|
||||
tailscale.com/types/bools from tailscale.com/net/netcheck
|
||||
tailscale.com/types/dnstype from tailscale.com/tailcfg+
|
||||
@@ -332,6 +342,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep
|
||||
W 💣 tailscale.com/util/winutil/authenticode from tailscale.com/clientupdate
|
||||
W 💣 tailscale.com/util/winutil/gp from tailscale.com/util/syspolicy/source
|
||||
W 💣 tailscale.com/util/winutil/winenv from tailscale.com/hostinfo+
|
||||
tailscale.com/util/zstdframe from tailscale.com/tsweb/compserve
|
||||
tailscale.com/version from tailscale.com/client/web+
|
||||
tailscale.com/version/distro from tailscale.com/client/web+
|
||||
tailscale.com/wgengine/filter/filtertype from tailscale.com/types/netmap
|
||||
|
||||
@@ -161,7 +161,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
|
||||
github.com/klauspost/compress/internal/cpuinfo from github.com/klauspost/compress/zstd+
|
||||
💣 github.com/klauspost/compress/internal/le from github.com/klauspost/compress/huff0+
|
||||
github.com/klauspost/compress/internal/snapref from github.com/klauspost/compress/zstd
|
||||
github.com/klauspost/compress/zstd from tailscale.com/util/zstdframe
|
||||
github.com/klauspost/compress/zstd from tailscale.com/util/zstdframe+
|
||||
github.com/klauspost/compress/zstd/internal/xxhash from github.com/klauspost/compress/zstd
|
||||
github.com/kortschak/wol from tailscale.com/feature/wakeonlan
|
||||
LD github.com/kr/fs from github.com/pkg/sftp
|
||||
@@ -441,7 +441,9 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de
|
||||
tailscale.com/tstime/mono from tailscale.com/net/tstun+
|
||||
tailscale.com/tstime/rate from tailscale.com/wgengine/filter
|
||||
tailscale.com/tsweb from tailscale.com/util/eventbus+
|
||||
tailscale.com/tsweb/compserve from tailscale.com/client/web
|
||||
tailscale.com/tsweb/varz from tailscale.com/cmd/tailscaled+
|
||||
tailscale.com/tsweb/vcstime from tailscale.com/client/web
|
||||
tailscale.com/types/appctype from tailscale.com/ipn/ipnlocal+
|
||||
tailscale.com/types/bools from tailscale.com/wgengine/netlog+
|
||||
tailscale.com/types/dnstype from tailscale.com/ipn/ipnlocal+
|
||||
|
||||
+9
-20
@@ -19,7 +19,8 @@ import (
|
||||
"time"
|
||||
|
||||
"tailscale.com/tsweb"
|
||||
"tailscale.com/util/precompress"
|
||||
"tailscale.com/tsweb/compserve"
|
||||
"tailscale.com/tsweb/vcstime"
|
||||
)
|
||||
|
||||
//go:embed index.html
|
||||
@@ -36,7 +37,7 @@ func runServe() {
|
||||
var distFS fs.FS
|
||||
if *distDir == "./dist" {
|
||||
var err error
|
||||
distFS, err = fs.Sub(embeddedDistFS, "dist")
|
||||
distFS, err = fs.Sub(vcstime.FS(embeddedDistFS), "dist")
|
||||
if err != nil {
|
||||
log.Fatalf("Could not drop dist/ prefix from embedded FS: %v", err)
|
||||
}
|
||||
@@ -120,25 +121,13 @@ var entryPointsToDefaultDistPaths = map[string]string{
|
||||
}
|
||||
|
||||
func handleServeDist(w http.ResponseWriter, r *http.Request, distFS fs.FS) {
|
||||
path := r.URL.Path
|
||||
f, err := precompress.OpenPrecompressedFile(w, r, path, distFS)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
// fs.File does not claim to implement Seeker, but in practice it does.
|
||||
fSeeker, ok := f.(io.ReadSeeker)
|
||||
if !ok {
|
||||
http.Error(w, "Not seekable", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// Aggressively cache static assets, since we cache-bust our assets with
|
||||
// hashed filenames.
|
||||
w.Header().Set("Cache-Control", "public, max-age=31535996")
|
||||
w.Header().Set("Vary", "Accept-Encoding")
|
||||
|
||||
http.ServeContent(w, r, path, serveStartTime, fSeeker)
|
||||
err := compserve.ServeFile(w, r, distFS, r.URL.Path, compserve.Options{})
|
||||
if err != nil {
|
||||
// Do not reflect the error (and with it the request path) to the
|
||||
// client.
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}
|
||||
@@ -45,7 +45,7 @@ tailscale.com/cmd/tsidp dependencies: (generated by github.com/tailscale/depawar
|
||||
github.com/klauspost/compress/internal/cpuinfo from github.com/klauspost/compress/huff0+
|
||||
💣 github.com/klauspost/compress/internal/le from github.com/klauspost/compress/huff0+
|
||||
github.com/klauspost/compress/internal/snapref from github.com/klauspost/compress/zstd
|
||||
github.com/klauspost/compress/zstd from tailscale.com/util/zstdframe
|
||||
github.com/klauspost/compress/zstd from tailscale.com/util/zstdframe+
|
||||
github.com/klauspost/compress/zstd/internal/xxhash from github.com/klauspost/compress/zstd
|
||||
L 💣 github.com/mdlayher/netlink from github.com/jsimonetti/rtnetlink+
|
||||
L github.com/mdlayher/netlink/nlenc from github.com/jsimonetti/rtnetlink+
|
||||
@@ -242,7 +242,9 @@ tailscale.com/cmd/tsidp dependencies: (generated by github.com/tailscale/depawar
|
||||
tailscale.com/tstime/mono from tailscale.com/net/tstun+
|
||||
tailscale.com/tstime/rate from tailscale.com/wgengine/filter
|
||||
tailscale.com/tsweb from tailscale.com/util/eventbus+
|
||||
tailscale.com/tsweb/compserve from tailscale.com/client/web
|
||||
tailscale.com/tsweb/varz from tailscale.com/tsweb+
|
||||
tailscale.com/tsweb/vcstime from tailscale.com/client/web
|
||||
tailscale.com/types/appctype from tailscale.com/ipn/ipnlocal+
|
||||
tailscale.com/types/bools from tailscale.com/tsnet+
|
||||
tailscale.com/types/dnstype from tailscale.com/client/local+
|
||||
|
||||
@@ -173,4 +173,4 @@
|
||||
});
|
||||
};
|
||||
}
|
||||
# nix-direnv cache busting line: sha256-aUbAZpPW0yrNVaDn4NRM4OSw521EIcD5nudXo5pTWpY=
|
||||
# nix-direnv cache busting line: sha256-O4Q7kad5olQbg86+asGF4/kVXqWECsMYjQsGq75BY9M=
|
||||
+2
-2
@@ -4,7 +4,7 @@
|
||||
"sri": "sha256-6Bv2BX2mMiWO8gVkqHOq4MmRD1IvWrogYgZ5xuTODQI="
|
||||
},
|
||||
"vendor": {
|
||||
"goModSum": "sha256-C1+FuDLoOkeU1bZjY+vmZsEH29jJheLSvc9L3qi0QGg=",
|
||||
"sri": "sha256-aUbAZpPW0yrNVaDn4NRM4OSw521EIcD5nudXo5pTWpY="
|
||||
"goModSum": "sha256-7PhCdPleOCh4ZexxOQxVSmbXUHCvBfbUGZuWLKoQV/M=",
|
||||
"sri": "sha256-O4Q7kad5olQbg86+asGF4/kVXqWECsMYjQsGq75BY9M="
|
||||
}
|
||||
}
|
||||
@@ -9,7 +9,6 @@ require (
|
||||
github.com/Kodeworks/golang-image-ico v0.0.0-20141118225523-73f0f4cfade9
|
||||
github.com/akutz/memconn v0.1.0
|
||||
github.com/alexbrainman/sspi v0.0.0-20250919150558-7d374ff0d59e
|
||||
github.com/andybalholm/brotli v1.2.3
|
||||
github.com/atotto/clipboard v0.1.4
|
||||
github.com/aws/aws-sdk-go-v2 v1.46.0
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.3
|
||||
|
||||
@@ -93,8 +93,6 @@ github.com/alingse/nilnesserr v0.1.2 h1:Yf8Iwm3z2hUUrP4muWfW83DF4nE3r1xZ26fGWUKC
|
||||
github.com/alingse/nilnesserr v0.1.2/go.mod h1:1xJPrXonEtX7wyTq8Dytns5P2hNzoWymVUIaKm4HNFg=
|
||||
github.com/anchore/go-lzo v0.1.0 h1:NgAacnzqPeGH49Ky19QKLBZEuFRqtTG9cdaucc3Vncs=
|
||||
github.com/anchore/go-lzo v0.1.0/go.mod h1:3kLx0bve2oN1iDwgM1U5zGku1Tfbdb0No5qp1eL1fIk=
|
||||
github.com/andybalholm/brotli v1.2.3 h1:8H1qwOkl2LPfjf3YezB90JnCliZb6SInJ/OJkEbA5NQ=
|
||||
github.com/andybalholm/brotli v1.2.3/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
|
||||
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
|
||||
github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg=
|
||||
@@ -1164,8 +1162,6 @@ github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofm
|
||||
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
|
||||
github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ=
|
||||
github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0=
|
||||
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
|
||||
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
|
||||
github.com/yagipy/maintidx v1.0.0 h1:h5NvIsCz+nRDapQ0exNv4aJ0yXSI0420omVANTv3GJM=
|
||||
github.com/yagipy/maintidx v1.0.0/go.mod h1:0qNf/I/CCZXSMhsRsrEPDZ+DkekpKLXAJfsTACwgXLk=
|
||||
github.com/yeya24/promlinter v0.3.0 h1:JVDbMp08lVCP7Y6NP3qHroGAO6z2yGKQtS5JsjqtoFs=
|
||||
|
||||
@@ -16,4 +16,4 @@
|
||||
) {
|
||||
src = ./.;
|
||||
}).shellNix
|
||||
# nix-direnv cache busting line: sha256-aUbAZpPW0yrNVaDn4NRM4OSw521EIcD5nudXo5pTWpY=
|
||||
# nix-direnv cache busting line: sha256-O4Q7kad5olQbg86+asGF4/kVXqWECsMYjQsGq75BY9M=
|
||||
+3
-1
@@ -45,7 +45,7 @@ tailscale.com/tsnet dependencies: (generated by github.com/tailscale/depaware)
|
||||
github.com/klauspost/compress/internal/cpuinfo from github.com/klauspost/compress/huff0+
|
||||
💣 github.com/klauspost/compress/internal/le from github.com/klauspost/compress/huff0+
|
||||
github.com/klauspost/compress/internal/snapref from github.com/klauspost/compress/zstd
|
||||
github.com/klauspost/compress/zstd from tailscale.com/util/zstdframe
|
||||
github.com/klauspost/compress/zstd from tailscale.com/util/zstdframe+
|
||||
github.com/klauspost/compress/zstd/internal/xxhash from github.com/klauspost/compress/zstd
|
||||
L 💣 github.com/mdlayher/netlink from github.com/jsimonetti/rtnetlink+
|
||||
L github.com/mdlayher/netlink/nlenc from github.com/jsimonetti/rtnetlink+
|
||||
@@ -237,7 +237,9 @@ tailscale.com/tsnet dependencies: (generated by github.com/tailscale/depaware)
|
||||
tailscale.com/tstime/mono from tailscale.com/net/tstun+
|
||||
tailscale.com/tstime/rate from tailscale.com/wgengine/filter
|
||||
tailscale.com/tsweb from tailscale.com/util/eventbus+
|
||||
LDWI tailscale.com/tsweb/compserve from tailscale.com/client/web
|
||||
tailscale.com/tsweb/varz from tailscale.com/tsweb+
|
||||
LDWI tailscale.com/tsweb/vcstime from tailscale.com/client/web
|
||||
tailscale.com/types/appctype from tailscale.com/ipn/ipnlocal+
|
||||
tailscale.com/types/bools from tailscale.com/tsnet+
|
||||
tailscale.com/types/dnstype from tailscale.com/client/local+
|
||||
|
||||
@@ -0,0 +1,329 @@
|
||||
// Copyright (c) Tailscale Inc & contributors
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
package compserve
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"tailscale.com/util/httpm"
|
||||
)
|
||||
|
||||
// CompressWriter wraps w so that its response body is transparently
|
||||
// compressed, incrementally as the handler writes it, with the first
|
||||
// encoding from encodings (default: [DefaultEncodings]) that offers live
|
||||
// compression and that r accepts.
|
||||
//
|
||||
// The caller must call Close exactly once after the wrapped handler
|
||||
// returns, to finish any compressed stream:
|
||||
//
|
||||
// cw := compserve.CompressWriter(w, r)
|
||||
// defer cw.Close()
|
||||
// handler.ServeHTTP(cw, r)
|
||||
func CompressWriter(w http.ResponseWriter, r *http.Request, encodings ...Encoding) *compressWriter {
|
||||
if len(encodings) == 0 {
|
||||
encodings = DefaultEncodings
|
||||
}
|
||||
var encs []Encoding
|
||||
for _, e := range encodings {
|
||||
if e.Compress != nil {
|
||||
encs = append(encs, e)
|
||||
}
|
||||
}
|
||||
// Compression applies to plain responses of standard body-carrying
|
||||
// requests: not HEAD (no body), not CONNECT (tunnel), not protocol
|
||||
// upgrades, not range requests (range semantics apply to the
|
||||
// uncompressed representation).
|
||||
safe := (r.Method == httpm.GET || r.Method == httpm.POST) &&
|
||||
r.Header.Get("Upgrade") == "" &&
|
||||
r.Header.Get("Range") == ""
|
||||
return &compressWriter{
|
||||
ResponseWriter: w,
|
||||
encs: encs,
|
||||
ae: r.Header.Get("Accept-Encoding"),
|
||||
safe: safe,
|
||||
}
|
||||
}
|
||||
|
||||
// compressWriter is a ResponseWriter wrapper that compresses the response
|
||||
// body of dynamic handlers. See [CompressWriter].
|
||||
//
|
||||
// The body is compressed as it is written, with no buffering. The compress
|
||||
// decision is made once from the response headers, sniffing the first
|
||||
// write if the handler sets no Content-Type.
|
||||
//
|
||||
// Compressed responses are chunked: the compressed size is not known up
|
||||
// front, so a handler-set Content-Length is removed. zstd framing may add
|
||||
// a few bytes to incompressible data.
|
||||
//
|
||||
// Responses pass through uncompressed when they already have a
|
||||
// Content-Encoding, on non-GET/POST requests (HEAD, CONNECT, upgrades,
|
||||
// range requests), non-200 statuses, or incompressible content types (see
|
||||
// compressibleContentType).
|
||||
//
|
||||
// Every response gets Vary: Accept-Encoding. Strong ETags are weakened:
|
||||
// validators are representation-specific.
|
||||
type compressWriter struct {
|
||||
http.ResponseWriter
|
||||
encs []Encoding
|
||||
|
||||
ae string // Accept-Encoding header of the request
|
||||
safe bool // request is safe to compress the response of
|
||||
|
||||
code int // stashed status code; 0 until the first WriteHeader or Write
|
||||
decided bool // whether the compress-or-passthrough decision was made
|
||||
compress bool // decision outcome: stream through the compressor
|
||||
hijacked bool // connection hijacked; the wrapper is out of the path
|
||||
cw io.WriteCloser
|
||||
}
|
||||
|
||||
// Unwrap exposes the wrapped ResponseWriter to http.ResponseController for
|
||||
// deadline support; Flush and Hijack stay handled by the wrapper.
|
||||
func (c *compressWriter) Unwrap() http.ResponseWriter {
|
||||
return c.ResponseWriter
|
||||
}
|
||||
|
||||
// Hijack switches the connection out from under the wrapper: after a
|
||||
// successful hijack the connection is raw, no compression applies, and
|
||||
// wrapper writes fail. Handlers reach it via http.NewResponseController or
|
||||
// a direct type assertion, so hijacked paths carry no io overhead from
|
||||
// this wrapper. ResponseController.Hijack unwraps through further
|
||||
// middleware below this wrapper.
|
||||
func (c *compressWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
|
||||
if !c.decided {
|
||||
// No response was produced; the handler will speak for itself on
|
||||
// the hijacked connection. Do not attach a compressor or send
|
||||
// response headers.
|
||||
c.decided = true
|
||||
} else if c.compress {
|
||||
// Compressed bytes were already streamed; finish the frame and
|
||||
// return the encoder to its pool before the connection leaves our
|
||||
// control.
|
||||
err := c.cw.Close()
|
||||
c.compress, c.cw = false, nil
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
c.hijacked = true
|
||||
return http.NewResponseController(c.ResponseWriter).Hijack()
|
||||
}
|
||||
|
||||
// WriteHeader stashes the status code, deciding early if the response is
|
||||
// already disqualified from compression. See [CompressWriter].
|
||||
func (c *compressWriter) WriteHeader(code int) {
|
||||
if c.hijacked {
|
||||
return // the handler owns the connection
|
||||
}
|
||||
if c.code != 0 {
|
||||
return // ignore superfluous calls, like net/http does
|
||||
}
|
||||
c.code = code
|
||||
if !c.mayCompress() {
|
||||
c.decide(nil)
|
||||
}
|
||||
}
|
||||
|
||||
// mayCompress reports whether the stashed response could still be
|
||||
// compressed based on the request and the handler's headers alone.
|
||||
func (c *compressWriter) mayCompress() bool {
|
||||
if len(c.encs) == 0 || c.code != http.StatusOK || !c.safe {
|
||||
return false
|
||||
}
|
||||
// Never double-encode (e.g. precompressed variants served by
|
||||
// [ServeFile] through this wrapper).
|
||||
return c.Header().Get("Content-Encoding") == ""
|
||||
}
|
||||
|
||||
// Write passes the body through the streaming compressor or the wrapped
|
||||
// writer, deciding on the first write. See [CompressWriter].
|
||||
func (c *compressWriter) Write(p []byte) (int, error) {
|
||||
if c.hijacked {
|
||||
return 0, http.ErrHijacked
|
||||
}
|
||||
if !c.decided {
|
||||
c.decide(p)
|
||||
}
|
||||
if c.compress {
|
||||
return c.cw.Write(p)
|
||||
}
|
||||
return c.ResponseWriter.Write(p)
|
||||
}
|
||||
|
||||
// Flush flushes buffered compressed data and the underlying writer. See
|
||||
// [CompressWriter].
|
||||
func (c *compressWriter) Flush() {
|
||||
if !c.decided {
|
||||
c.decide(nil)
|
||||
}
|
||||
if c.compress {
|
||||
type flusher interface{ Flush() }
|
||||
if f, ok := c.cw.(flusher); ok {
|
||||
f.Flush()
|
||||
}
|
||||
}
|
||||
if f, ok := c.ResponseWriter.(http.Flusher); ok {
|
||||
f.Flush()
|
||||
}
|
||||
}
|
||||
|
||||
// Close finishes any compressed stream and must be called exactly once
|
||||
// after the wrapped handler returns. See [CompressWriter].
|
||||
func (c *compressWriter) Close() error {
|
||||
if c.hijacked {
|
||||
return nil // the connection left our control in Hijack
|
||||
}
|
||||
if !c.decided {
|
||||
c.decide(nil)
|
||||
}
|
||||
if c.compress {
|
||||
return c.cw.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// decide makes the compress-or-passthrough decision from the request, the
|
||||
// response headers, and (when no Content-Type is set) the first body
|
||||
// bytes. It forwards the response headers.
|
||||
func (c *compressWriter) decide(first []byte) {
|
||||
c.decided = true
|
||||
if c.code == 0 {
|
||||
c.code = http.StatusOK
|
||||
}
|
||||
ct := c.Header().Get("Content-Type")
|
||||
sniffed := false
|
||||
if ct == "" && first != nil {
|
||||
// Mirror net/http's sniffing of the first bytes.
|
||||
ct = http.DetectContentType(first)
|
||||
sniffed = true
|
||||
}
|
||||
if !c.mayCompress() || ct == "" || !compressibleContentType(ct) {
|
||||
c.commitPassthrough()
|
||||
return
|
||||
}
|
||||
enc := negotiate(c.ae, c.encs)
|
||||
if enc == nil {
|
||||
c.commitPassthrough()
|
||||
return
|
||||
}
|
||||
cw, err := enc.Compress(c.ResponseWriter)
|
||||
if err != nil {
|
||||
c.commitPassthrough()
|
||||
return
|
||||
}
|
||||
|
||||
h := c.Header()
|
||||
if sniffed {
|
||||
// net/http would otherwise sniff the compressed bytes and label
|
||||
// the response application/octet-stream.
|
||||
h.Set("Content-Type", ct)
|
||||
}
|
||||
// The handler's Content-Length describes the uncompressed
|
||||
// representation; the compressed length is unknown, so the response is
|
||||
// chunked.
|
||||
h.Del("Content-Length")
|
||||
h.Set("Content-Encoding", enc.Token)
|
||||
// Strong validators must be representation-specific.
|
||||
if etag := h.Get("ETag"); etag != "" && !strings.HasPrefix(etag, "W/") {
|
||||
h.Set("ETag", "W/"+etag)
|
||||
}
|
||||
addVary(h, "Accept-Encoding")
|
||||
|
||||
c.cw = cw
|
||||
c.compress = true
|
||||
c.ResponseWriter.WriteHeader(c.code)
|
||||
}
|
||||
|
||||
// commitPassthrough forwards the stashed status to the wrapped writer;
|
||||
// subsequent writes stream through directly.
|
||||
func (c *compressWriter) commitPassthrough() {
|
||||
if c.code == 0 {
|
||||
c.code = http.StatusOK
|
||||
}
|
||||
addVary(c.Header(), "Accept-Encoding")
|
||||
c.ResponseWriter.WriteHeader(c.code)
|
||||
}
|
||||
|
||||
// compressibleContentType reports whether the given content type is worth
|
||||
// live compression. Most types are assumed compressible: only content that
|
||||
// is already compressed by construction is excluded (media codecs, archive
|
||||
// formats), except raw media types that are not, listed below.
|
||||
// +json/+xml suffix types are always compressed.
|
||||
func compressibleContentType(ct string) bool {
|
||||
ct = strings.ToLower(ct)
|
||||
if i := strings.IndexByte(ct, ';'); i >= 0 {
|
||||
ct = strings.TrimSpace(ct[:i])
|
||||
}
|
||||
if strings.HasSuffix(ct, "+json") || strings.HasSuffix(ct, "+xml") {
|
||||
return true
|
||||
}
|
||||
switch {
|
||||
case strings.HasPrefix(ct, "audio/"):
|
||||
return rawAudioContentType(ct)
|
||||
case strings.HasPrefix(ct, "video/"):
|
||||
return rawVideoContentType(ct)
|
||||
}
|
||||
switch ct {
|
||||
case "application/zip",
|
||||
"application/gzip",
|
||||
"application/x-gzip",
|
||||
"application/x-bzip2",
|
||||
"application/x-xz",
|
||||
"application/zstd",
|
||||
"application/x-zstd",
|
||||
"application/x-7z-compressed",
|
||||
"application/x-rar-compressed",
|
||||
"application/vnd.rar",
|
||||
"application/x-compress",
|
||||
"application/x-snappy-framed",
|
||||
"application/x-lz4",
|
||||
// Archive formats that are zip or other compressed containers.
|
||||
"application/java-archive", // jar: zip
|
||||
"application/vnd.android.package-archive", // apk: zip
|
||||
"application/x-apple-diskimage": // dmg: compressed
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// rawAudioContentType reports whether ct is an uncompressed audio format
|
||||
// worth compressing despite audio/ being excluded by default: audio content
|
||||
// is almost always compressed by its codec, but raw PCM is not.
|
||||
func rawAudioContentType(ct string) bool {
|
||||
switch ct {
|
||||
case "audio/wav", "audio/wave", "audio/x-wav", "audio/vnd.wave", // WAV: PCM
|
||||
"audio/pcm",
|
||||
"audio/l16", "audio/l24": // raw linear PCM, RFC 2586
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// rawVideoContentType reports whether ct is an uncompressed video format
|
||||
// worth compressing despite video/ being excluded by default: video content
|
||||
// is almost always compressed by its codec, but raw frame data is not.
|
||||
func rawVideoContentType(ct string) bool {
|
||||
switch ct {
|
||||
case "video/raw", "video/x-raw", "video/x-raw-yuv",
|
||||
"video/x-y4m": // YUV4MPEG2
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// addVary adds token to the response's Vary header, preserving any values
|
||||
// already set by the handler.
|
||||
func addVary(h http.Header, token string) {
|
||||
for _, vary := range h.Values("Vary") {
|
||||
for _, t := range strings.Split(vary, ",") {
|
||||
if strings.EqualFold(strings.TrimSpace(t), token) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
h.Add("Vary", token)
|
||||
}
|
||||
@@ -0,0 +1,547 @@
|
||||
// Copyright (c) Tailscale Inc & contributors
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
package compserve
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
"time"
|
||||
|
||||
"tailscale.com/util/zstdframe"
|
||||
)
|
||||
|
||||
var timeZero time.Time
|
||||
|
||||
// serveCompressed runs inner through CompressWriter with the given
|
||||
// Accept-Encoding header and returns the recorder.
|
||||
func serveCompressed(t *testing.T, ae string, inner http.HandlerFunc) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
r := httptest.NewRequest("GET", "/", nil)
|
||||
if ae != "" {
|
||||
r.Header.Set("Accept-Encoding", ae)
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
cw := CompressWriter(w, r)
|
||||
defer cw.Close()
|
||||
inner.ServeHTTP(cw, r)
|
||||
return w
|
||||
}
|
||||
|
||||
const testBody = `{"hello": "world, this is a moderately sized json response body intended to exceed the minimum compression size threshold"}`
|
||||
|
||||
func TestCompressWriter_Compresses(t *testing.T) {
|
||||
w := serveCompressed(t, "zstd", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write([]byte(strings.Repeat(testBody, 20)))
|
||||
})
|
||||
if got := w.Header().Get("Content-Encoding"); got != "zstd" {
|
||||
t.Errorf("Content-Encoding = %q, want %q", got, "zstd")
|
||||
}
|
||||
if got := w.Code; got != 200 {
|
||||
t.Errorf("code = %d, want 200", got)
|
||||
}
|
||||
decoded, err := zstdframe.AppendDecode(nil, w.Body.Bytes())
|
||||
if err != nil {
|
||||
t.Fatalf("decoding response: %v", err)
|
||||
}
|
||||
if want := strings.Repeat(testBody, 20); string(decoded) != want {
|
||||
t.Errorf("decoded body = %q, want %q", decoded, want)
|
||||
}
|
||||
// The compressed size is not known up front, so no Content-Length is
|
||||
// set; the response is chunked.
|
||||
if got := w.Header().Get("Content-Length"); got != "" {
|
||||
t.Errorf("Content-Length = %q, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompressWriter_VaryAndWeakETag(t *testing.T) {
|
||||
w := serveCompressed(t, "zstd", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("ETag", `"strong-etag"`)
|
||||
w.Header().Set("Vary", "Origin")
|
||||
w.Write([]byte(strings.Repeat(testBody, 20)))
|
||||
})
|
||||
if got := w.Header().Get("ETag"); got != `W/"strong-etag"` {
|
||||
t.Errorf("ETag = %q, want weakened", got)
|
||||
}
|
||||
if vary := strings.Join(w.Header().Values("Vary"), ", "); vary != "Origin, Accept-Encoding" {
|
||||
t.Errorf("Vary = %q, want %q", vary, "Origin, Accept-Encoding")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompressWriter_PreservesWeakETag(t *testing.T) {
|
||||
w := serveCompressed(t, "zstd", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("ETag", `W/"already-weak"`)
|
||||
w.Write([]byte(strings.Repeat(testBody, 20)))
|
||||
})
|
||||
if got := w.Header().Get("ETag"); got != `W/"already-weak"` {
|
||||
t.Errorf("ETag = %q, want unchanged", got)
|
||||
}
|
||||
if got := w.Header().Get("Content-Encoding"); got != "zstd" {
|
||||
t.Errorf("Content-Encoding = %q, want zstd", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompressWriter_NotAccepted(t *testing.T) {
|
||||
want := strings.Repeat(testBody, 20)
|
||||
w := serveCompressed(t, "br", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write([]byte(want))
|
||||
})
|
||||
if got := w.Header().Get("Content-Encoding"); got != "" {
|
||||
t.Errorf("Content-Encoding = %q, want empty", got)
|
||||
}
|
||||
if w.Body.String() != want {
|
||||
t.Error("body was modified")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompressWriter_SkipsWhenAlreadyEncoded(t *testing.T) {
|
||||
want := strings.Repeat(testBody, 20)
|
||||
w := serveCompressed(t, "zstd", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Content-Encoding", "zstd") // e.g. precompressed passthrough
|
||||
w.Write([]byte(want))
|
||||
})
|
||||
if w.Body.String() != want {
|
||||
t.Error("body was modified")
|
||||
}
|
||||
if n := len(w.Header().Values("Content-Encoding")); n != 1 {
|
||||
t.Errorf("Content-Encoding set %d times, want 1", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompressWriter_GzipNotLiveCompressed(t *testing.T) {
|
||||
// Gzip has no Compress hook: live compression is zstd-only, so a
|
||||
// gzip-accepting client gets identity rather than gzip.
|
||||
want := strings.Repeat(testBody, 20)
|
||||
w := serveCompressed(t, "gzip", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write([]byte(want))
|
||||
})
|
||||
if got := w.Header().Get("Content-Encoding"); got != "" {
|
||||
t.Errorf("Content-Encoding = %q, want empty", got)
|
||||
}
|
||||
if w.Body.String() != want {
|
||||
t.Error("body was modified")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompressWriter_SkipsRange(t *testing.T) {
|
||||
want := strings.Repeat(testBody, 20)
|
||||
r := httptest.NewRequest("GET", "/", nil)
|
||||
r.Header.Set("Accept-Encoding", "zstd")
|
||||
r.Header.Set("Range", "bytes=0-99")
|
||||
w := httptest.NewRecorder()
|
||||
cw := CompressWriter(w, r)
|
||||
defer cw.Close()
|
||||
cw.WriteHeader(http.StatusPartialContent)
|
||||
cw.Write([]byte(want))
|
||||
if w.Body.String() != want {
|
||||
t.Error("body was modified")
|
||||
}
|
||||
if got := w.Header().Get("Content-Encoding"); got != "" {
|
||||
t.Errorf("Content-Encoding = %q, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompressWriter_SkipsIncompressibleType(t *testing.T) {
|
||||
// Media codec content types are excluded by category, regardless of
|
||||
// what the bytes look like.
|
||||
want := strings.Repeat(testBody, 20)
|
||||
w := serveCompressed(t, "zstd", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "video/mp4")
|
||||
w.Write([]byte(want))
|
||||
})
|
||||
if got := w.Header().Get("Content-Encoding"); got != "" {
|
||||
t.Errorf("Content-Encoding = %q, want empty", got)
|
||||
}
|
||||
if w.Body.String() != want {
|
||||
t.Error("body was modified")
|
||||
}
|
||||
|
||||
// Raw audio is compressed despite the audio/ exclusion.
|
||||
w = serveCompressed(t, "zstd", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "audio/wav")
|
||||
w.Write([]byte(want))
|
||||
})
|
||||
if got := w.Header().Get("Content-Encoding"); got != "zstd" {
|
||||
t.Errorf("Content-Encoding = %q, want zstd", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompressWriter_SniffsUnsetContentType(t *testing.T) {
|
||||
// No Content-Type: http.DetectContentType sniffs text/plain, which is
|
||||
// compressible. The sniffed type must be persisted on the response:
|
||||
// net/http would otherwise sniff the compressed bytes and label the
|
||||
// response application/octet-stream.
|
||||
w := serveCompressed(t, "zstd", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(strings.Repeat(testBody, 20)))
|
||||
})
|
||||
if got := w.Header().Get("Content-Encoding"); got != "zstd" {
|
||||
t.Errorf("Content-Encoding = %q, want zstd", got)
|
||||
}
|
||||
if got := w.Header().Get("Content-Type"); got != "text/plain; charset=utf-8" {
|
||||
t.Errorf("Content-Type = %q, want text/plain; charset=utf-8", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompressWriter_SkipsNon200(t *testing.T) {
|
||||
want := strings.Repeat(testBody, 20)
|
||||
w := serveCompressed(t, "zstd", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
w.Write([]byte(want))
|
||||
})
|
||||
if w.Code != http.StatusNotFound {
|
||||
t.Errorf("code = %d, want 404", w.Code)
|
||||
}
|
||||
if got := w.Header().Get("Content-Encoding"); got != "" {
|
||||
t.Errorf("Content-Encoding = %q, want empty", got)
|
||||
}
|
||||
if w.Body.String() != want {
|
||||
t.Error("body was modified")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompressWriter_FlushStreamsCompressed(t *testing.T) {
|
||||
// Streaming handlers that flush mid-body remain compressed; each flush
|
||||
// emits the pending compressed bytes to the client.
|
||||
want := strings.Repeat(testBody, 20)
|
||||
w := serveCompressed(t, "zstd", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write([]byte(want[:100]))
|
||||
w.(http.Flusher).Flush()
|
||||
w.Write([]byte(want[100:]))
|
||||
})
|
||||
if got := w.Header().Get("Content-Encoding"); got != "zstd" {
|
||||
t.Errorf("Content-Encoding = %q, want zstd", got)
|
||||
}
|
||||
decoded, err := zstdframe.AppendDecode(nil, w.Body.Bytes())
|
||||
if err != nil {
|
||||
t.Fatalf("decoding response: %v", err)
|
||||
}
|
||||
if string(decoded) != want {
|
||||
t.Error("decoded body mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompressWriter_NoBody(t *testing.T) {
|
||||
// A handler that only sets headers (e.g. a redirect) still produces a
|
||||
// well-formed response.
|
||||
w := serveCompressed(t, "zstd", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Location", "/elsewhere")
|
||||
w.WriteHeader(http.StatusFound)
|
||||
})
|
||||
if w.Code != http.StatusFound {
|
||||
t.Errorf("code = %d, want 302", w.Code)
|
||||
}
|
||||
if got := w.Header().Get("Location"); got != "/elsewhere" {
|
||||
t.Errorf("Location = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompressContentTypes(t *testing.T) {
|
||||
tests := map[string]bool{
|
||||
// Default: compressible.
|
||||
"text/html; charset=utf-8": true,
|
||||
"application/json": true,
|
||||
"application/json; charset=utf-8": true,
|
||||
"application/javascript": true,
|
||||
"application/manifest+json": true,
|
||||
"application/vnd.api+json": true,
|
||||
"application/problem+json": true,
|
||||
"application/xhtml+xml": true,
|
||||
"image/svg+xml": true,
|
||||
"image/svg+xml; charset=utf-8": true,
|
||||
"application/rss+xml": true,
|
||||
"application/wasm": true,
|
||||
"TEXT/PLAIN": true,
|
||||
"application/octet-stream": true, // no-gain guard bails if needed
|
||||
"image/png": true, // no-gain guard bails if needed
|
||||
"image/jpeg; charset=binary": true, // no-gain guard bails if needed
|
||||
"font/woff2": true, // no-gain guard bails if needed
|
||||
"": true,
|
||||
|
||||
// Media codecs are already compressed; excluded by category...
|
||||
"audio/mpeg": false,
|
||||
"audio/ogg": false,
|
||||
"audio/flac": false,
|
||||
"video/mp4": false,
|
||||
"video/webm": false,
|
||||
"video/H264": false,
|
||||
// ...unless raw, uncompressed media.
|
||||
"audio/wav": true,
|
||||
"audio/x-wav": true,
|
||||
"audio/vnd.wave": true,
|
||||
"audio/pcm": true,
|
||||
"audio/L16": true,
|
||||
"audio/l24": true,
|
||||
"video/x-y4m": true,
|
||||
"video/x-raw": true,
|
||||
"video/raw": true,
|
||||
|
||||
// Known compression formats are skipped.
|
||||
"application/zip": false,
|
||||
"application/gzip": false,
|
||||
"application/x-gzip": false,
|
||||
"application/x-bzip2": false,
|
||||
"application/x-xz": false,
|
||||
"application/zstd": false,
|
||||
"application/x-7z-compressed": false,
|
||||
"application/x-rar-compressed": false,
|
||||
"application/vnd.rar": false,
|
||||
"application/x-compress": false,
|
||||
"application/x-snappy-framed": false,
|
||||
"application/x-lz4": false,
|
||||
"application/java-archive": false, // jar: zip
|
||||
"application/vnd.android.package-archive": false, // apk: zip
|
||||
"application/x-apple-diskimage": false, // dmg
|
||||
}
|
||||
for ct, want := range tests {
|
||||
if got := compressibleContentType(ct); got != want {
|
||||
t.Errorf("compressibleContentType(%q) = %v, want %v", ct, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompressWriter_ServeContentIntegration(t *testing.T) {
|
||||
// An inner http.ServeContent handler (raw, non-precompressed file)
|
||||
// should be compressed, with its Content-Length replaced and ETag
|
||||
// weakened.
|
||||
original := []byte(strings.Repeat(testBody, 20))
|
||||
tfs := fstest.MapFS{"data.json": &fstest.MapFile{Data: original}}
|
||||
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
f, err := tfs.Open("data.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
// fs.File does not claim to implement Seeker, but MapFS files do.
|
||||
http.ServeContent(w, r, "data.json", timeZero, f.(io.ReadSeeker))
|
||||
})
|
||||
|
||||
r := httptest.NewRequest("GET", "/", nil)
|
||||
r.Header.Set("Accept-Encoding", "zstd")
|
||||
w := httptest.NewRecorder()
|
||||
cw := CompressWriter(w, r)
|
||||
handler.ServeHTTP(cw, r)
|
||||
// Close before asserting: it is what emits the buffered response.
|
||||
cw.Close()
|
||||
|
||||
if got := w.Header().Get("Content-Encoding"); got != "zstd" {
|
||||
t.Errorf("Content-Encoding = %q, want zstd", got)
|
||||
}
|
||||
// http.ServeContent sets Content-Length for the raw file; streaming
|
||||
// compression must have removed it rather than lie about the size.
|
||||
if got := w.Header().Get("Content-Length"); got != "" {
|
||||
t.Errorf("Content-Length = %q, want empty", got)
|
||||
}
|
||||
decoded, err := zstdframe.AppendDecode(nil, w.Body.Bytes())
|
||||
if err != nil {
|
||||
t.Fatalf("decoding: %v", err)
|
||||
}
|
||||
if !bytes.Equal(decoded, original) {
|
||||
t.Error("decoded body mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompressWriter_DoubleClose(t *testing.T) {
|
||||
// A second Close must not return the pooled encoder to the pool twice;
|
||||
// two concurrent requests would then share one encoder.
|
||||
body := strings.Repeat(testBody, 20)
|
||||
w := serveCompressed(t, "zstd", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write([]byte(body))
|
||||
})
|
||||
// serveCompressed already Closed once; a second Close is a no-op.
|
||||
decoded, err := zstdframe.AppendDecode(nil, w.Body.Bytes())
|
||||
if err != nil {
|
||||
t.Fatalf("decoding response: %v", err)
|
||||
}
|
||||
if string(decoded) != body {
|
||||
t.Error("decoded body mismatch")
|
||||
}
|
||||
|
||||
// The pool is not poisoned: the next request compresses correctly.
|
||||
w2 := serveCompressed(t, "zstd", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write([]byte(body))
|
||||
})
|
||||
decoded2, err := zstdframe.AppendDecode(nil, w2.Body.Bytes())
|
||||
if err != nil {
|
||||
t.Fatalf("decoding second response: %v", err)
|
||||
}
|
||||
if string(decoded2) != body {
|
||||
t.Error("second decoded body mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
// hijackableRecorder wraps a ResponseRecorder, recording whether the
|
||||
// handler used the raw connection.
|
||||
type hijackableRecorder struct {
|
||||
*httptest.ResponseRecorder
|
||||
hijacked bool
|
||||
}
|
||||
|
||||
func (h *hijackableRecorder) Hijack() (net.Conn, *bufio.ReadWriter, error) {
|
||||
h.hijacked = true
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
func TestCompressWriter_HijackBeforeWrite(t *testing.T) {
|
||||
// A handler that hijacks before producing a response gets the raw
|
||||
// connection: no compression is attached, no headers are sent, and the
|
||||
// wrapper adds no io overhead on the hijacked path.
|
||||
hr := &hijackableRecorder{ResponseRecorder: httptest.NewRecorder()}
|
||||
r := httptest.NewRequest("GET", "/", nil)
|
||||
r.Header.Set("Accept-Encoding", "zstd")
|
||||
cw := CompressWriter(hr, r)
|
||||
|
||||
conn, rw, err := cw.Hijack()
|
||||
if err != nil || conn != nil || rw != nil {
|
||||
t.Fatalf("Hijack() = %v, %v, %v", conn, rw, err)
|
||||
}
|
||||
// The wrapper's passthrough path must not forward a header-only
|
||||
// response for a hijacked request.
|
||||
if hr.Code != 200 {
|
||||
t.Errorf("WriteHeader forwarded to underlying writer (code %d)", hr.Code)
|
||||
}
|
||||
if got := hr.Header().Get("Content-Encoding"); got != "" {
|
||||
t.Errorf("Content-Encoding = %q, want empty", got)
|
||||
}
|
||||
if !hr.hijacked {
|
||||
t.Error("underlying writer was not hijacked")
|
||||
}
|
||||
// Post-hijack writes are rejected, not forwarded.
|
||||
if n, err := cw.Write([]byte("x")); err == nil || n != 0 {
|
||||
t.Errorf("post-hijack Write = (%d, %v), want (0, error)", n, err)
|
||||
}
|
||||
cw.Close()
|
||||
}
|
||||
|
||||
func TestCompressWriter_HijackAfterCompression(t *testing.T) {
|
||||
// Hijacking mid-compressed-stream finishes the frame (pooled encoder
|
||||
// returned) and hands over the connection.
|
||||
original := strings.Repeat(testBody, 20)
|
||||
hr := &hijackableRecorder{ResponseRecorder: httptest.NewRecorder()}
|
||||
r := httptest.NewRequest("GET", "/", nil)
|
||||
r.Header.Set("Accept-Encoding", "zstd")
|
||||
cw := CompressWriter(hr, r)
|
||||
if _, err := cw.Write([]byte(original)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := hr.Header().Get("Content-Encoding"); got != "zstd" {
|
||||
t.Fatalf("Content-Encoding = %q, want zstd", got)
|
||||
}
|
||||
if _, _, err := cw.Hijack(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := wBody(hr); !bytes.Equal(got, nil) && len(got) == 0 {
|
||||
t.Error("compressed body was dropped")
|
||||
}
|
||||
decoded, err := zstdframe.AppendDecode(nil, wBody(hr))
|
||||
if err != nil {
|
||||
t.Fatalf("decoding streamed body (frame not finished?): %v", err)
|
||||
}
|
||||
if string(decoded) != original {
|
||||
t.Error("decoded body mismatch")
|
||||
}
|
||||
// Close after hijack must not double-close the pooled encoder.
|
||||
cw.Close()
|
||||
}
|
||||
|
||||
// unwrapOnlyWriter is middleware supporting neither Hijack nor Flush,
|
||||
// only Unwrap, like other middleware that may sit below compressWriter.
|
||||
type unwrapOnlyWriter struct {
|
||||
http.ResponseWriter
|
||||
}
|
||||
|
||||
func (u unwrapOnlyWriter) Unwrap() http.ResponseWriter { return u.ResponseWriter }
|
||||
|
||||
func TestCompressWriter_HijackThroughUnwrap(t *testing.T) {
|
||||
// compressWriter -> unwrapOnlyWriter -> hijackableRecorder: the
|
||||
// hijacker is two layers down. Hijack must walk the Unwrap chain
|
||||
// (via http.ResponseController) and reach it.
|
||||
hr := &hijackableRecorder{ResponseRecorder: httptest.NewRecorder()}
|
||||
r := httptest.NewRequest("GET", "/", nil)
|
||||
r.Header.Set("Accept-Encoding", "zstd")
|
||||
cw := CompressWriter(unwrapOnlyWriter{hr}, r)
|
||||
if _, _, err := cw.Hijack(); err != nil {
|
||||
t.Fatalf("Hijack through middleware: %v", err)
|
||||
}
|
||||
if !hr.hijacked {
|
||||
t.Error("deepest writer was not hijacked")
|
||||
}
|
||||
cw.Close()
|
||||
}
|
||||
|
||||
func TestCompressWriter_HijackNotSupported(t *testing.T) {
|
||||
w := httptest.NewRecorder()
|
||||
r := httptest.NewRequest("GET", "/", nil)
|
||||
cw := CompressWriter(w, r)
|
||||
if _, _, err := cw.Hijack(); !errors.Is(err, http.ErrNotSupported) {
|
||||
t.Errorf("Hijack on non-hijackable writer = %v, want http.ErrNotSupported", err)
|
||||
}
|
||||
cw.Close()
|
||||
}
|
||||
|
||||
func wBody(h *hijackableRecorder) []byte {
|
||||
return h.Body.Bytes()
|
||||
}
|
||||
|
||||
func TestCompressWriter_SkipsUpgradesAndUnusualMethods(t *testing.T) {
|
||||
// Protocol upgrades and non-standard methods get identity responses:
|
||||
// their bodies are not normal response payloads (tunnels, WebDAV
|
||||
// extensions, etc.).
|
||||
want := strings.Repeat(testBody, 20)
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
method string
|
||||
upgrade bool
|
||||
}{
|
||||
{name: "upgrade", method: "GET", upgrade: true},
|
||||
{name: "propfind", method: "PROPFIND"},
|
||||
} {
|
||||
r := httptest.NewRequest(tt.method, "/", nil)
|
||||
r.Header.Set("Accept-Encoding", "zstd")
|
||||
if tt.upgrade {
|
||||
r.Header.Set("Upgrade", "websocket")
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
cw := CompressWriter(w, r)
|
||||
cw.WriteHeader(200)
|
||||
cw.Write([]byte(want))
|
||||
cw.Close()
|
||||
if got := w.Header().Get("Content-Encoding"); got != "" {
|
||||
t.Errorf("%s: Content-Encoding = %q, want empty", tt.name, got)
|
||||
}
|
||||
if w.Body.String() != want {
|
||||
t.Errorf("%s: body was modified", tt.name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompressWriter_HeadRequestPassthrough(t *testing.T) {
|
||||
// HEAD responses have no body; they pass through untouched.
|
||||
r := httptest.NewRequest("HEAD", "/", nil)
|
||||
r.Header.Set("Accept-Encoding", "zstd")
|
||||
w := httptest.NewRecorder()
|
||||
cw := CompressWriter(w, r)
|
||||
defer cw.Close()
|
||||
cw.WriteHeader(200)
|
||||
if got := w.Header().Get("Content-Encoding"); got != "" {
|
||||
t.Errorf("Content-Encoding = %q, want empty", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,306 @@
|
||||
// Copyright (c) Tailscale Inc & contributors
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
// Package compserve serves static files with content negotiation for
|
||||
// precompressed variants, and provides live compression for dynamic
|
||||
// responses.
|
||||
//
|
||||
// An [Encoding] describes a content coding's wire token and where its
|
||||
// precompressed variants live.
|
||||
package compserve
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Encoding describes a single content coding (e.g. zstd) that this package
|
||||
// can negotiate and serve.
|
||||
type Encoding struct {
|
||||
// Token is the Content-Encoding token for this coding, e.g. "zstd".
|
||||
// Comparison against Accept-Encoding is case-insensitive.
|
||||
Token string
|
||||
|
||||
// Ext is the file name extension (including the leading dot) of
|
||||
// precompressed variants as produced by util/precompress, e.g. ".zst".
|
||||
Ext string
|
||||
|
||||
// Decompress, if non-nil, wraps f and yields its decompressed content,
|
||||
// taking ownership of f: the returned ReadCloser's Close closes f, and
|
||||
// if Decompress returns an error it must have closed f already.
|
||||
//
|
||||
// When Decompress is set, a precompressed variant can also be served to
|
||||
// clients that do not accept Token by decompressing it on the fly, so a
|
||||
// file system shipping only precompressed variants can still serve
|
||||
// every client. If nil, the variant is only served passthrough to
|
||||
// clients that accept Token.
|
||||
Decompress func(f fs.File) (io.ReadCloser, error)
|
||||
|
||||
// Compress, if non-nil, wraps w in a streaming compressor used by
|
||||
// [CompressWriter] for live (non-precompressed) response compression:
|
||||
// everything written to the returned WriteCloser is compressed and
|
||||
// forwarded to w, and its Close finishes the stream and releases any
|
||||
// pooled resources. Implementations should use a fast compression
|
||||
// mode: live compression runs per request and must not add meaningful
|
||||
// latency. If nil, the encoding is not offered for live compression.
|
||||
Compress func(w io.Writer) (io.WriteCloser, error)
|
||||
}
|
||||
|
||||
// DefaultEncodings is the set of encodings used when Options does not
|
||||
// specify any: zstd, then gzip for compatibility with file systems built
|
||||
// before zstd-only asset generation. Ties prefer the earlier encoding.
|
||||
var DefaultEncodings = []Encoding{Zstd, Gzip}
|
||||
|
||||
// Options configures ServeFile.
|
||||
type Options struct {
|
||||
// Encodings lists the precompressed variants available in the file
|
||||
// system, in preference order. The zero value selects
|
||||
// [DefaultEncodings].
|
||||
Encodings []Encoding
|
||||
}
|
||||
|
||||
// ServeFile serves the file at path from fsys to w, negotiating a
|
||||
// precompressed variant when the client accepts one and the variant exists.
|
||||
// Clients that do not accept any offered encoding are served the raw file;
|
||||
// if the raw file is absent (e.g. a file system shipping only
|
||||
// precompressed variants), a variant with a Decompress hook is served
|
||||
// decompressed instead.
|
||||
//
|
||||
// ServeFile sets Vary: Accept-Encoding on every response and serves the
|
||||
// content via http.ServeContent with the file's modification time (for
|
||||
// embedded file systems, wrap fsys in [tailscale.com/tsweb/vcstime.FS] to
|
||||
// supply one). Callers may set additional headers (e.g. Cache-Control)
|
||||
// before calling. If ServeFile returns an error it has not written a
|
||||
// response body, so callers may adjust headers and retry with a different
|
||||
// path. If no representation of the file exists, the returned error wraps
|
||||
// fs.ErrNotExist.
|
||||
func ServeFile(w http.ResponseWriter, r *http.Request, fsys fs.FS, path string, opts Options) error {
|
||||
encodings := opts.Encodings
|
||||
if encodings == nil {
|
||||
encodings = DefaultEncodings
|
||||
}
|
||||
// The served representation (an encoded variant or identity) is
|
||||
// selected from Accept-Encoding, so caches must treat the response as
|
||||
// varying even when the negotiated representation is identity.
|
||||
addVary(w.Header(), "Accept-Encoding")
|
||||
|
||||
// Prefer serving an accepted variant passthrough: no server CPU cost.
|
||||
if enc := negotiate(r.Header.Get("Accept-Encoding"), encodings); enc != nil {
|
||||
if f, err := fsys.Open(path + enc.Ext); err == nil {
|
||||
rs, ok := f.(io.ReadSeeker)
|
||||
if !ok {
|
||||
// Fall through to the raw file; it may still be servable.
|
||||
f.Close()
|
||||
} else {
|
||||
w.Header().Set("Content-Encoding", enc.Token)
|
||||
return serveContent(w, r, path, fileModTime(f), rs)
|
||||
}
|
||||
} else if !errors.Is(err, fs.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Serve the raw file if it exists: it is identical to what transcoding
|
||||
// a variant would produce, without the CPU cost.
|
||||
f, err := fsys.Open(path)
|
||||
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
if err == nil {
|
||||
rs, ok := f.(io.ReadSeeker)
|
||||
if !ok {
|
||||
f.Close()
|
||||
return fmt.Errorf("%s is not seekable", path)
|
||||
}
|
||||
return serveContent(w, r, path, fileModTime(f), rs)
|
||||
}
|
||||
|
||||
// The raw file does not exist: transcode a precompressed variant to
|
||||
// identity for clients that do not accept it.
|
||||
for _, e := range encodings {
|
||||
if e.Decompress == nil {
|
||||
continue
|
||||
}
|
||||
f, err := fsys.Open(path + e.Ext)
|
||||
if err != nil {
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
continue
|
||||
}
|
||||
return err
|
||||
}
|
||||
modTime := fileModTime(f)
|
||||
tf := &transcodedFile{fsys: fsys, path: path + e.Ext, dec: e.Decompress}
|
||||
if err := tf.start(f); err != nil {
|
||||
return err
|
||||
}
|
||||
return serveContent(w, r, path, modTime, tf)
|
||||
}
|
||||
return &fs.PathError{Op: "open", Path: path, Err: fs.ErrNotExist}
|
||||
}
|
||||
|
||||
// fileModTime reports f's modification time, or the zero time (which
|
||||
// disables conditional requests in http.ServeContent) if f does not stat.
|
||||
func fileModTime(f fs.File) time.Time {
|
||||
fi, err := f.Stat()
|
||||
if err != nil {
|
||||
return time.Time{}
|
||||
}
|
||||
return fi.ModTime()
|
||||
}
|
||||
|
||||
// serveContent serves rs via http.ServeContent, closing it if it is an
|
||||
// io.Closer.
|
||||
func serveContent(w http.ResponseWriter, r *http.Request, name string, modTime time.Time, rs io.ReadSeeker) error {
|
||||
http.ServeContent(w, r, name, modTime, rs)
|
||||
if c, ok := rs.(io.Closer); ok {
|
||||
// The response body has been written; a close error is not
|
||||
// actionable.
|
||||
c.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// negotiate returns the encoding from offered that the given
|
||||
// Accept-Encoding header value accepts with the highest quality value, or
|
||||
// nil if it accepts none of them. Ties prefer the earlier entry in offered.
|
||||
func negotiate(ae string, offered []Encoding) *Encoding {
|
||||
if ae == "" {
|
||||
return nil
|
||||
}
|
||||
exact, wildcard, hasWildcard := parseAcceptEncoding(ae)
|
||||
var best *Encoding
|
||||
bestQ := 0.0
|
||||
for i := range offered {
|
||||
enc := &offered[i]
|
||||
q, ok := exact[strings.ToLower(enc.Token)]
|
||||
if !ok && hasWildcard {
|
||||
q, ok = wildcard, true
|
||||
}
|
||||
// A quality of zero explicitly rejects the coding, and equal
|
||||
// qualities prefer the earlier entry in offered.
|
||||
if ok && q > bestQ {
|
||||
best, bestQ = enc, q
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
// parseAcceptEncoding parses an Accept-Encoding header value into the
|
||||
// quality values of the explicitly named (lowercased) coding tokens and of
|
||||
// the "*" wildcard, if present. Quality values default to 1 per RFC 9110,
|
||||
// and a malformed quality value rejects the coding.
|
||||
func parseAcceptEncoding(h string) (exact map[string]float64, wildcard float64, hasWildcard bool) {
|
||||
exact = make(map[string]float64)
|
||||
for _, part := range strings.Split(h, ",") {
|
||||
token, params, _ := strings.Cut(part, ";")
|
||||
token = strings.ToLower(strings.TrimSpace(token))
|
||||
if token == "" {
|
||||
continue
|
||||
}
|
||||
q := 1.0
|
||||
for _, p := range strings.FieldsFunc(params, func(r rune) bool { return r == ';' }) {
|
||||
name, value, _ := strings.Cut(strings.TrimSpace(p), "=")
|
||||
if !strings.EqualFold(name, "q") {
|
||||
continue
|
||||
}
|
||||
if v, err := strconv.ParseFloat(strings.TrimSpace(value), 64); err == nil {
|
||||
q = v
|
||||
} else {
|
||||
q = 0
|
||||
}
|
||||
break
|
||||
}
|
||||
if token == "*" {
|
||||
wildcard, hasWildcard = q, true
|
||||
continue
|
||||
}
|
||||
exact[token] = q
|
||||
}
|
||||
return exact, wildcard, hasWildcard
|
||||
}
|
||||
|
||||
// transcodedFile serves the decompressed content of a precompressed file to
|
||||
// clients that do not accept its encoding. It implements io.ReadSeeker so
|
||||
// that it can be used with http.ServeContent; seeking re-opens the
|
||||
// underlying file and decompresses from the start, so it is only suitable
|
||||
// for small files where the double decompression cost is acceptable.
|
||||
type transcodedFile struct {
|
||||
fsys fs.FS
|
||||
path string // path of the compressed file in fsys
|
||||
dec func(f fs.File) (io.ReadCloser, error) // decompressor for that file
|
||||
rc io.ReadCloser // current decompressed stream
|
||||
pos int64 // current position in the decompressed stream
|
||||
}
|
||||
|
||||
// start replaces the decompressed stream with one reading f (whose
|
||||
// ownership is taken) from its start.
|
||||
func (t *transcodedFile) start(f fs.File) error {
|
||||
rc, err := t.dec(f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
t.closeStream()
|
||||
t.rc, t.pos = rc, 0
|
||||
return nil
|
||||
}
|
||||
|
||||
// restart replaces the decompressed stream with a fresh one reading the
|
||||
// underlying file from its start.
|
||||
func (t *transcodedFile) restart() error {
|
||||
f, err := t.fsys.Open(t.path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return t.start(f)
|
||||
}
|
||||
|
||||
func (t *transcodedFile) closeStream() {
|
||||
if t.rc != nil {
|
||||
t.rc.Close()
|
||||
t.rc = nil
|
||||
}
|
||||
}
|
||||
|
||||
func (t *transcodedFile) Read(p []byte) (int, error) {
|
||||
if t.rc == nil {
|
||||
return 0, fs.ErrClosed
|
||||
}
|
||||
n, err := t.rc.Read(p)
|
||||
t.pos += int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (t *transcodedFile) Seek(offset int64, whence int) (int64, error) {
|
||||
switch whence {
|
||||
case io.SeekStart:
|
||||
if err := t.restart(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
_, err := io.CopyN(io.Discard, t, offset)
|
||||
return t.pos, err
|
||||
case io.SeekCurrent:
|
||||
if offset < 0 {
|
||||
return 0, fmt.Errorf("negative offset for SeekCurrent: %w", fs.ErrInvalid)
|
||||
}
|
||||
_, err := io.CopyN(io.Discard, t, offset)
|
||||
return t.pos, err
|
||||
case io.SeekEnd:
|
||||
if offset != 0 {
|
||||
return 0, fmt.Errorf("non-zero offset for SeekEnd: %w", fs.ErrInvalid)
|
||||
}
|
||||
_, err := io.Copy(io.Discard, t)
|
||||
return t.pos, err
|
||||
}
|
||||
return 0, fs.ErrInvalid
|
||||
}
|
||||
|
||||
func (t *transcodedFile) Close() error {
|
||||
t.closeStream()
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,528 @@
|
||||
// Copyright (c) Tailscale Inc & contributors
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
package compserve
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"errors"
|
||||
"io"
|
||||
"io/fs"
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
"time"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
)
|
||||
|
||||
func compressZstd(t *testing.T, data []byte) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
w, err := zstd.NewWriter(&buf, zstd.WithWindowSize(8<<20))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := w.Write(data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func TestNegotiate(t *testing.T) {
|
||||
tests := []struct {
|
||||
ae string // Accept-Encoding header value
|
||||
want string // Token of expected encoding, "" for identity
|
||||
}{
|
||||
{ae: "", want: ""},
|
||||
{ae: "zstd", want: "zstd"},
|
||||
{ae: "zstd, gzip", want: "zstd"},
|
||||
{ae: "gzip, zstd", want: "zstd"}, // preference order wins ties
|
||||
{ae: "gzip", want: "gzip"},
|
||||
{ae: "br", want: ""},
|
||||
{ae: "identity", want: ""}, // identity is never an offered encoding
|
||||
{ae: "identity;q=1, *;q=0", want: ""}, // explicitly accepting only identity
|
||||
{ae: "foo, gzip", want: "gzip"},
|
||||
{ae: "foo, gzip ", want: "gzip"},
|
||||
{ae: "ZSTD", want: "zstd"}, // case-insensitive token match
|
||||
{ae: "zstd;q=0", want: ""},
|
||||
{ae: "zstd;q=0, gzip", want: "gzip"},
|
||||
{ae: "zstd;q=0.5, gzip;q=0.4", want: "zstd"},
|
||||
{ae: "gzip;q=0.9, zstd;q=0.4", want: "gzip"},
|
||||
{ae: "zstd;q=1.2, foo", want: "zstd"}, // lenient: valid float
|
||||
{ae: "zstd;q=abc", want: ""}, // malformed q rejects
|
||||
{ae: "zstd;Q=0", want: ""}, // parameter names are case-insensitive
|
||||
{ae: "*", want: "zstd"},
|
||||
{ae: "*;q=0.1", want: "zstd"},
|
||||
{ae: "*;q=0", want: ""},
|
||||
{ae: "zstd;q=1, *;q=0", want: "zstd"},
|
||||
{ae: "gzip;q=1, *;q=0", want: "gzip"}, // explicit entries beat the wildcard
|
||||
{ae: "br;q=1, *;q=0", want: ""}, // nothing offered is accepted
|
||||
}
|
||||
for _, tt := range tests {
|
||||
got := negotiate(tt.ae, DefaultEncodings)
|
||||
var gotToken string
|
||||
if got != nil {
|
||||
gotToken = got.Token
|
||||
}
|
||||
if gotToken != tt.want {
|
||||
t.Errorf("negotiate(%q) = %q, want %q", tt.ae, gotToken, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func mapFS(files map[string][]byte) fstest.MapFS {
|
||||
tfs := fstest.MapFS{}
|
||||
for name, data := range files {
|
||||
tfs[name] = &fstest.MapFile{Data: data}
|
||||
}
|
||||
return tfs
|
||||
}
|
||||
|
||||
func serveFileRequest(t *testing.T, fsys fs.FS, path, ae string, opts Options) (*httptest.ResponseRecorder, bool) {
|
||||
t.Helper()
|
||||
r := httptest.NewRequest("GET", "/"+path, nil)
|
||||
if ae != "" {
|
||||
r.Header.Set("Accept-Encoding", ae)
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
err := ServeFile(w, r, fsys, path, opts)
|
||||
return w, errors.Is(err, fs.ErrNotExist)
|
||||
}
|
||||
|
||||
func TestServeFile_ZstdPassthrough(t *testing.T) {
|
||||
original := []byte("hello world")
|
||||
compressed := compressZstd(t, original)
|
||||
tfs := mapFS(map[string][]byte{"test.js.zst": compressed})
|
||||
|
||||
w, notExist := serveFileRequest(t, tfs, "test.js", "zstd", Options{})
|
||||
if notExist {
|
||||
t.Fatal("unexpected fs.ErrNotExist")
|
||||
}
|
||||
if got := w.Header().Get("Content-Encoding"); got != "zstd" {
|
||||
t.Errorf("Content-Encoding = %q, want %q", got, "zstd")
|
||||
}
|
||||
// http.ServeContent derives Content-Type from the (original) name via
|
||||
// mime.TypeByExtension; the exact value is platform-dependent (Windows
|
||||
// consults the registry).
|
||||
if got, want := w.Header().Get("Content-Type"), mime.TypeByExtension(".js"); want != "" && got != want {
|
||||
t.Errorf("Content-Type = %q, want %q", got, want)
|
||||
} else if want == "" && got == "" {
|
||||
t.Error("Content-Type not set")
|
||||
}
|
||||
if !bytes.Equal(w.Body.Bytes(), compressed) {
|
||||
t.Errorf("body = decompressed data, want raw compressed passthrough")
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeFile_TranscodeWithoutRawFile(t *testing.T) {
|
||||
// A file system may ship only precompressed variants (as build-webclient
|
||||
// does); clients that do not accept the encoding must still be served.
|
||||
original := []byte("hello world")
|
||||
compressed := compressZstd(t, original)
|
||||
tfs := mapFS(map[string][]byte{"test.js.zst": compressed})
|
||||
|
||||
// Client accepts only gzip: the zstd variant is served decompressed, as
|
||||
// identity.
|
||||
w, notExist := serveFileRequest(t, tfs, "test.js", "gzip", Options{})
|
||||
if notExist {
|
||||
t.Fatal("unexpected fs.ErrNotExist")
|
||||
}
|
||||
if got := w.Header().Get("Content-Encoding"); got != "" {
|
||||
t.Errorf("Content-Encoding = %q, want empty (transparent decompression)", got)
|
||||
}
|
||||
if !bytes.Equal(w.Body.Bytes(), original) {
|
||||
t.Errorf("body = %q, want %q", w.Body.Bytes(), original)
|
||||
}
|
||||
|
||||
// So must clients that accept no encoding at all.
|
||||
w, notExist = serveFileRequest(t, tfs, "test.js", "", Options{})
|
||||
if notExist {
|
||||
t.Fatal("unexpected fs.ErrNotExist")
|
||||
}
|
||||
if got := w.Header().Get("Content-Encoding"); got != "" {
|
||||
t.Errorf("Content-Encoding = %q, want empty", got)
|
||||
}
|
||||
if !bytes.Equal(w.Body.Bytes(), original) {
|
||||
t.Errorf("body = %q, want %q", w.Body.Bytes(), original)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeFile_RawPreferredOverTranscode(t *testing.T) {
|
||||
// When the raw file exists, a client that accepts no encoding is served
|
||||
// it directly; transcoding a variant to identical bytes would only
|
||||
// waste CPU.
|
||||
original := []byte("raw js")
|
||||
compressed := compressZstd(t, []byte("compressed js"))
|
||||
tfs := mapFS(map[string][]byte{
|
||||
"test.js": original,
|
||||
"test.js.zst": compressed,
|
||||
})
|
||||
|
||||
w, notExist := serveFileRequest(t, tfs, "test.js", "", Options{})
|
||||
if notExist {
|
||||
t.Fatal("unexpected fs.ErrNotExist")
|
||||
}
|
||||
if got := w.Header().Get("Content-Encoding"); got != "" {
|
||||
t.Errorf("Content-Encoding = %q, want empty", got)
|
||||
}
|
||||
if !bytes.Equal(w.Body.Bytes(), original) {
|
||||
t.Errorf("body = %q, want %q", w.Body.Bytes(), original)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeFile_PrefersNegotiatedVariant(t *testing.T) {
|
||||
// A client accepting zstd gets the zstd variant even when it also
|
||||
// accepts encodings that are not served (e.g. gzip).
|
||||
tfs := mapFS(map[string][]byte{
|
||||
"test.js.zst": compressZstd(t, []byte("zstd content")),
|
||||
})
|
||||
|
||||
w, notExist := serveFileRequest(t, tfs, "test.js", "zstd, gzip", Options{})
|
||||
if notExist {
|
||||
t.Fatal("unexpected fs.ErrNotExist")
|
||||
}
|
||||
if got := w.Header().Get("Content-Encoding"); got != "zstd" {
|
||||
t.Errorf("Content-Encoding = %q, want %q", got, "zstd")
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeFile_ZstdRejectedServesIdentity(t *testing.T) {
|
||||
// A client that explicitly rejects zstd is served identity content,
|
||||
// transcoded from the only available representation.
|
||||
original := []byte("identity content")
|
||||
tfs := mapFS(map[string][]byte{
|
||||
"test.js.zst": compressZstd(t, original),
|
||||
})
|
||||
|
||||
w, notExist := serveFileRequest(t, tfs, "test.js", "gzip;q=1, zstd;q=0", Options{})
|
||||
if notExist {
|
||||
t.Fatal("unexpected fs.ErrNotExist")
|
||||
}
|
||||
if got := w.Header().Get("Content-Encoding"); got != "" {
|
||||
t.Errorf("Content-Encoding = %q, want empty", got)
|
||||
}
|
||||
if !bytes.Equal(w.Body.Bytes(), original) {
|
||||
t.Errorf("body = %q, want %q", w.Body.Bytes(), original)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeFile_GzipPassthrough(t *testing.T) {
|
||||
// Transitional: pre-zstd file systems ship .gz variants; gzip-accepting
|
||||
// clients get them passthrough.
|
||||
original := []byte("hello world")
|
||||
tfs := mapFS(map[string][]byte{"test.js.gz": original})
|
||||
|
||||
w, notExist := serveFileRequest(t, tfs, "test.js", "gzip", Options{})
|
||||
if notExist {
|
||||
t.Fatal("unexpected fs.ErrNotExist")
|
||||
}
|
||||
if got := w.Header().Get("Content-Encoding"); got != "gzip" {
|
||||
t.Errorf("Content-Encoding = %q, want %q", got, "gzip")
|
||||
}
|
||||
if !bytes.Equal(w.Body.Bytes(), original) {
|
||||
t.Errorf("body = %q, want %q", w.Body.Bytes(), original)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeFile_TranscodeGzip(t *testing.T) {
|
||||
// Transitional: a gz-only file system (the pre-zstd
|
||||
// web-client-prebuilt shape) serves identity content to clients that
|
||||
// do not accept gzip.
|
||||
original := []byte("hello world")
|
||||
var buf bytes.Buffer
|
||||
zw := gzip.NewWriter(&buf)
|
||||
if _, err := zw.Write(original); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tfs := mapFS(map[string][]byte{"test.js.gz": buf.Bytes()})
|
||||
|
||||
w, notExist := serveFileRequest(t, tfs, "test.js", "", Options{})
|
||||
if notExist {
|
||||
t.Fatal("unexpected fs.ErrNotExist")
|
||||
}
|
||||
if got := w.Header().Get("Content-Encoding"); got != "" {
|
||||
t.Errorf("Content-Encoding = %q, want empty", got)
|
||||
}
|
||||
if !bytes.Equal(w.Body.Bytes(), original) {
|
||||
t.Errorf("body = %q, want %q", w.Body.Bytes(), original)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeFile_RawFallback(t *testing.T) {
|
||||
tfs := mapFS(map[string][]byte{"test.js": []byte("raw js")})
|
||||
|
||||
w, notExist := serveFileRequest(t, tfs, "test.js", "zstd, gzip", Options{})
|
||||
if notExist {
|
||||
t.Fatal("unexpected fs.ErrNotExist")
|
||||
}
|
||||
if got := w.Header().Get("Content-Encoding"); got != "" {
|
||||
t.Errorf("Content-Encoding = %q, want empty", got)
|
||||
}
|
||||
if !bytes.Equal(w.Body.Bytes(), []byte("raw js")) {
|
||||
t.Errorf("body = %q, want %q", w.Body.Bytes(), "raw js")
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeFile_Missing(t *testing.T) {
|
||||
tfs := mapFS(map[string][]byte{})
|
||||
|
||||
w, notExist := serveFileRequest(t, tfs, "missing.js", "zstd", Options{})
|
||||
if !notExist {
|
||||
t.Error("expected fs.ErrNotExist for missing file")
|
||||
}
|
||||
if w.Body.Len() != 0 {
|
||||
t.Errorf("body = %q, want no response written", w.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// modTimeFS is an fs.FS whose files report a fixed modification time, like
|
||||
// vcstime-wrapped file systems do for embedded files.
|
||||
type modTimeFS struct {
|
||||
fsys fs.FS
|
||||
mod time.Time
|
||||
}
|
||||
|
||||
func (m modTimeFS) Open(name string) (fs.File, error) {
|
||||
f, err := m.fsys.Open(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return modTimeFile{f, m.mod}, nil
|
||||
}
|
||||
|
||||
type modTimeFile struct {
|
||||
f fs.File
|
||||
mod time.Time
|
||||
}
|
||||
|
||||
func (m modTimeFile) Read(p []byte) (int, error) { return m.f.Read(p) }
|
||||
func (m modTimeFile) Seek(off int64, w int) (int64, error) {
|
||||
return m.f.(io.Seeker).Seek(off, w)
|
||||
}
|
||||
func (m modTimeFile) Close() error { return m.f.Close() }
|
||||
|
||||
func (m modTimeFile) Stat() (fs.FileInfo, error) {
|
||||
fi, err := m.f.Stat()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return modTimeInfo{fi: fi, mod: m.mod}, nil
|
||||
}
|
||||
|
||||
type modTimeInfo struct {
|
||||
fi fs.FileInfo
|
||||
mod time.Time
|
||||
}
|
||||
|
||||
func (m modTimeInfo) Name() string { return m.fi.Name() }
|
||||
func (m modTimeInfo) Size() int64 { return m.fi.Size() }
|
||||
func (m modTimeInfo) Mode() fs.FileMode { return m.fi.Mode() }
|
||||
func (m modTimeInfo) IsDir() bool { return m.fi.IsDir() }
|
||||
func (m modTimeInfo) Sys() any { return m.fi.Sys() }
|
||||
func (m modTimeInfo) ModTime() time.Time { return m.mod }
|
||||
|
||||
func TestServeFile_ModTimeFromFileStat(t *testing.T) {
|
||||
// The served mod time comes from the file's Stat, so wrapping an
|
||||
// embedded FS in vcstime.FS yields working conditional requests.
|
||||
mod := time.Unix(1700000000, 0)
|
||||
tfs := modTimeFS{fsys: mapFS(map[string][]byte{"test.js": []byte("raw js")}), mod: mod}
|
||||
|
||||
w, notExist := serveFileRequest(t, tfs, "test.js", "zstd", Options{})
|
||||
if notExist {
|
||||
t.Fatal("unexpected fs.ErrNotExist")
|
||||
}
|
||||
if got, want := w.Header().Get("Last-Modified"), mod.UTC().Format(http.TimeFormat); got != want {
|
||||
t.Errorf("Last-Modified = %q, want %q", got, want)
|
||||
}
|
||||
|
||||
// And conditional requests get 304s.
|
||||
r := httptest.NewRequest("GET", "/test.js", nil)
|
||||
r.Header.Set("If-Modified-Since", mod.UTC().Format(http.TimeFormat))
|
||||
w2 := httptest.NewRecorder()
|
||||
if err := ServeFile(w2, r, tfs, "test.js", Options{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if w2.Code != http.StatusNotModified {
|
||||
t.Errorf("code = %d, want 304", w2.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeFile_VaryAlwaysSet(t *testing.T) {
|
||||
// Vary must be set even when the negotiated representation is identity,
|
||||
// and on responses to clients sending no Accept-Encoding at all.
|
||||
for _, ae := range []string{"", "zstd", "identity"} {
|
||||
tfs := mapFS(map[string][]byte{"test.js": []byte("raw js")})
|
||||
w, notExist := serveFileRequest(t, tfs, "test.js", ae, Options{})
|
||||
if notExist {
|
||||
t.Fatal("unexpected fs.ErrNotExist")
|
||||
}
|
||||
if got := w.Header().Get("Vary"); got != "Accept-Encoding" {
|
||||
t.Errorf("Accept-Encoding %q: Vary = %q, want Accept-Encoding", ae, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranscodedFile_Seek(t *testing.T) {
|
||||
original := []byte("hello world, this is a test of transcoded seeking")
|
||||
compressed := compressZstd(t, original)
|
||||
tfs := mapFS(map[string][]byte{"test.zst": compressed})
|
||||
|
||||
tf := &transcodedFile{fsys: tfs, path: "test.zst", dec: Zstd.Decompress}
|
||||
if err := tf.restart(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tf.Close()
|
||||
|
||||
// SeekEnd with offset 0 should return the total decompressed size.
|
||||
n, err := tf.Seek(0, io.SeekEnd)
|
||||
if err != nil {
|
||||
t.Fatalf("Seek(0, SeekEnd) error: %v", err)
|
||||
}
|
||||
if n != int64(len(original)) {
|
||||
t.Errorf("Seek(0, SeekEnd) = %d, want %d", n, len(original))
|
||||
}
|
||||
|
||||
// SeekStart with offset 0 should reset to the beginning.
|
||||
n, err = tf.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
t.Fatalf("Seek(0, SeekStart) error: %v", err)
|
||||
}
|
||||
if n != 0 {
|
||||
t.Errorf("Seek(0, SeekStart) = %d, want 0", n)
|
||||
}
|
||||
|
||||
// Read all content after reset.
|
||||
got, err := io.ReadAll(tf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(got, original) {
|
||||
t.Errorf("after Seek(0, SeekStart) + ReadAll: got %q, want %q", got, original)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranscodedFile_SeekCurrentFromNonZeroPosition(t *testing.T) {
|
||||
// Regression test: SeekCurrent must return the absolute stream
|
||||
// position, not the number of bytes skipped. From a non-zero position
|
||||
// these differ.
|
||||
original := []byte("hello world")
|
||||
compressed := compressZstd(t, original)
|
||||
tfs := mapFS(map[string][]byte{"test.zst": compressed})
|
||||
|
||||
tf := &transcodedFile{fsys: tfs, path: "test.zst", dec: Zstd.Decompress}
|
||||
if err := tf.restart(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tf.Close()
|
||||
|
||||
// Read 5 bytes to advance the position, then seek relative.
|
||||
buf := make([]byte, 5)
|
||||
if _, err := io.ReadFull(tf, buf); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
n, err := tf.Seek(3, io.SeekCurrent)
|
||||
if err != nil {
|
||||
t.Fatalf("Seek(3, SeekCurrent) error: %v", err)
|
||||
}
|
||||
if n != 8 {
|
||||
t.Errorf("Seek(3, SeekCurrent) = %d, want 8", n)
|
||||
}
|
||||
|
||||
// Read remaining.
|
||||
got, err := io.ReadAll(tf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(got) != "rld" {
|
||||
t.Errorf("after read+Seek(3, SeekCurrent)+ReadAll: got %q, want %q", got, "rld")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranscodedFile_SeekNegativeCurrentErrors(t *testing.T) {
|
||||
tfs := mapFS(map[string][]byte{"test.zst": compressZstd(t, []byte("hello"))})
|
||||
tf := &transcodedFile{fsys: tfs, path: "test.zst", dec: Zstd.Decompress}
|
||||
if err := tf.restart(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tf.Close()
|
||||
|
||||
if _, err := tf.Seek(-1, io.SeekCurrent); err == nil {
|
||||
t.Error("Seek(-1, SeekCurrent) should return error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranscodedFile_SeekEndNonZeroErrors(t *testing.T) {
|
||||
tfs := mapFS(map[string][]byte{"test.zst": compressZstd(t, []byte("hello"))})
|
||||
tf := &transcodedFile{fsys: tfs, path: "test.zst", dec: Zstd.Decompress}
|
||||
if err := tf.restart(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tf.Close()
|
||||
|
||||
if _, err := tf.Seek(-1, io.SeekEnd); err == nil {
|
||||
t.Error("Seek(-1, SeekEnd) should return error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranscodedFile_ServeContent(t *testing.T) {
|
||||
// Integration test: verify that transcodedFile works correctly with
|
||||
// http.ServeContent, which uses Seek to determine content length and
|
||||
// then serves the content through the same reader.
|
||||
original := []byte("hello world, served via http.ServeContent")
|
||||
compressed := compressZstd(t, original)
|
||||
tfs := mapFS(map[string][]byte{"test.js.zst": compressed})
|
||||
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
tf := &transcodedFile{fsys: tfs, path: "test.js.zst", dec: Zstd.Decompress}
|
||||
if err := tf.restart(); err != nil {
|
||||
http.Error(w, err.Error(), 500)
|
||||
return
|
||||
}
|
||||
defer tf.Close()
|
||||
http.ServeContent(w, r, "test.js", time.Time{}, tf)
|
||||
})
|
||||
|
||||
r := httptest.NewRequest("GET", "/test.js", nil)
|
||||
w := httptest.NewRecorder()
|
||||
handler.ServeHTTP(w, r)
|
||||
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body.String())
|
||||
}
|
||||
if !bytes.Equal(w.Body.Bytes(), original) {
|
||||
t.Errorf("body = %q, want %q", w.Body.String(), original)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranscodedFile_ReusedDecoders(t *testing.T) {
|
||||
// Regression test: pooled zstd decoders must be reusable after a
|
||||
// previous request fully drained (hit EOF on) and returned them.
|
||||
original := []byte("hello world")
|
||||
compressed := compressZstd(t, original)
|
||||
tfs := mapFS(map[string][]byte{"test.zst": compressed})
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
tf := &transcodedFile{fsys: tfs, path: "test.zst", dec: Zstd.Decompress}
|
||||
if err := tf.restart(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := io.ReadAll(tf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tf.Close()
|
||||
if !bytes.Equal(got, original) {
|
||||
t.Errorf("round %d: got %q, want %q", i, got, original)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
// Copyright (c) Tailscale Inc & contributors
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
package compserve
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"io"
|
||||
"io/fs"
|
||||
)
|
||||
|
||||
// Gzip serves gzip-precompressed variants ("*.gz"). It exists for
|
||||
// transitional compatibility with file systems built before zstd-only
|
||||
// asset generation, such as web-client-prebuilt modules published before
|
||||
// cmd/build-webclient stopped writing gzip; file systems built since
|
||||
// contain no .gz variants and never select it.
|
||||
var Gzip = Encoding{
|
||||
Token: "gzip",
|
||||
Ext: ".gz",
|
||||
Decompress: decompressGzip,
|
||||
}
|
||||
|
||||
// decompressGzip wraps f in a streaming gzip decompressor, taking
|
||||
// ownership of f.
|
||||
func decompressGzip(f fs.File) (io.ReadCloser, error) {
|
||||
zr, err := gzip.NewReader(f)
|
||||
if err != nil {
|
||||
f.Close()
|
||||
return nil, err
|
||||
}
|
||||
return &gzipFile{f: f, Reader: zr}, nil
|
||||
}
|
||||
|
||||
// gzipFile is a gzip.Reader reading from f; Close closes both.
|
||||
type gzipFile struct {
|
||||
f fs.File
|
||||
*gzip.Reader
|
||||
}
|
||||
|
||||
func (g *gzipFile) Close() error {
|
||||
g.Reader.Close()
|
||||
return g.f.Close()
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
// Copyright (c) Tailscale Inc & contributors
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
package compserve
|
||||
|
||||
import (
|
||||
"io"
|
||||
"io/fs"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"tailscale.com/util/zstdframe"
|
||||
)
|
||||
|
||||
// Zstd serves zstd-precompressed variants ("*.zst", as generated by
|
||||
// util/precompress). [Encoding.Decompress] lets ServeFile transcode the
|
||||
// variant for clients that do not accept zstd.
|
||||
var Zstd = Encoding{
|
||||
Token: "zstd",
|
||||
Ext: ".zst",
|
||||
Decompress: decompressZstd,
|
||||
Compress: compressZstdLive,
|
||||
}
|
||||
|
||||
// compressZstdLive wraps w in a streaming zstd compressor using the fastest
|
||||
// compression level and a small window suitable for low-latency live
|
||||
// compression. Close on the returned writer finishes the zstd frame and
|
||||
// returns the encoder to the shared pool.
|
||||
func compressZstdLive(w io.Writer) (io.WriteCloser, error) {
|
||||
enc, put := zstdframe.GetStreamingEncoder(
|
||||
zstdframe.FastestCompression,
|
||||
zstdframe.MaxWindowSize(64<<10),
|
||||
)
|
||||
// Encoder.Reset (streaming) cannot fail; only Decoder.Reset can.
|
||||
enc.Reset(w)
|
||||
return &pooledZstdWriter{Encoder: enc, put: put}, nil
|
||||
}
|
||||
|
||||
// pooledZstdWriter is a zstd.Encoder writing to the underlying writer;
|
||||
// Close finishes the frame and returns the encoder to the shared pool.
|
||||
type pooledZstdWriter struct {
|
||||
put func()
|
||||
closed bool
|
||||
*zstd.Encoder
|
||||
}
|
||||
|
||||
func (p *pooledZstdWriter) Close() error {
|
||||
if p.closed {
|
||||
return nil
|
||||
}
|
||||
p.closed = true
|
||||
err := p.Encoder.Close()
|
||||
p.put()
|
||||
return err
|
||||
}
|
||||
|
||||
// decompressZstd wraps f in a streaming zstd decompressor backed by the
|
||||
// shared decoder pool in util/zstdframe (concurrency=1, so streaming decode
|
||||
// neither spawns goroutines nor retains unbounded memory). It takes
|
||||
// ownership of f.
|
||||
func decompressZstd(f fs.File) (io.ReadCloser, error) {
|
||||
dec, put := zstdframe.GetDecoder()
|
||||
if err := dec.Reset(f); err != nil {
|
||||
put()
|
||||
f.Close()
|
||||
return nil, err
|
||||
}
|
||||
return &pooledZstdReader{f: f, put: put, Decoder: dec}, nil
|
||||
}
|
||||
|
||||
// pooledZstdReader is a zstd.Decoder reading from f; Close returns the
|
||||
// decoder to the shared pool and closes f.
|
||||
type pooledZstdReader struct {
|
||||
f fs.File
|
||||
put func()
|
||||
*zstd.Decoder
|
||||
}
|
||||
|
||||
func (p *pooledZstdReader) Close() error {
|
||||
p.put()
|
||||
return p.f.Close()
|
||||
}
|
||||
@@ -28,7 +28,6 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go4.org/mem"
|
||||
"tailscale.com/envknob"
|
||||
"tailscale.com/metrics"
|
||||
"tailscale.com/net/tsaddr"
|
||||
@@ -141,29 +140,6 @@ func allowDebugAccessWithKey(r *http.Request) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// AcceptsEncoding reports whether r accepts the named encoding
|
||||
// ("gzip", "br", etc).
|
||||
func AcceptsEncoding(r *http.Request, enc string) bool {
|
||||
h := r.Header.Get("Accept-Encoding")
|
||||
if h == "" {
|
||||
return false
|
||||
}
|
||||
if !strings.Contains(h, enc) && !mem.ContainsFold(mem.S(h), mem.S(enc)) {
|
||||
return false
|
||||
}
|
||||
remain := h
|
||||
for len(remain) > 0 {
|
||||
var part string
|
||||
part, remain, _ = strings.Cut(remain, ",")
|
||||
part = strings.TrimSpace(part)
|
||||
part, _, _ = strings.Cut(part, ";")
|
||||
if part == enc {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Protected wraps a provided debug handler, h, returning a Handler
|
||||
// that enforces AllowDebugAccess and returns forbidden replies for
|
||||
// unauthorized requests.
|
||||
|
||||
@@ -1141,34 +1141,6 @@ func TestHTTPError_Unwrap(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcceptsEncoding(t *testing.T) {
|
||||
tests := []struct {
|
||||
in, enc string
|
||||
want bool
|
||||
}{
|
||||
{"", "gzip", false},
|
||||
{"gzip", "gzip", true},
|
||||
{"foo,gzip", "gzip", true},
|
||||
{"foo, gzip", "gzip", true},
|
||||
{"foo, gzip ", "gzip", true},
|
||||
{"gzip, foo ", "gzip", true},
|
||||
{"gzip, foo ", "br", false},
|
||||
{"gzip, foo ", "fo", false},
|
||||
{"gzip;q=1.2, foo ", "gzip", true},
|
||||
{" gzip;q=1.2, foo ", "gzip", true},
|
||||
}
|
||||
for i, tt := range tests {
|
||||
h := make(http.Header)
|
||||
if tt.in != "" {
|
||||
h.Set("Accept-Encoding", tt.in)
|
||||
}
|
||||
got := AcceptsEncoding(&http.Request{Header: h}, tt.enc)
|
||||
if got != tt.want {
|
||||
t.Errorf("%d. got %v; want %v", i, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPort80Handler(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
@@ -1,28 +1,26 @@
|
||||
// Copyright (c) Tailscale Inc & contributors
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
// Package precompress provides build- and serving-time support for
|
||||
// precompressed static resources, to avoid the cost of repeatedly compressing
|
||||
// unchanging resources.
|
||||
// Package precompress provides build-time support for precompressing static
|
||||
// resources (using Zstandard), to avoid the cost of repeatedly compressing
|
||||
// unchanging resources. Serving the resulting variants is handled by
|
||||
// tailscale.com/tsweb/compserve.
|
||||
package precompress
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/andybalholm/brotli"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"tailscale.com/tsweb"
|
||||
"tailscale.com/tsweb/compserve"
|
||||
)
|
||||
|
||||
// PrecompressDir compresses static assets in dirPath using Gzip and Brotli, so
|
||||
// that they can be later served with OpenPrecompressedFile.
|
||||
// PrecompressDir compresses static assets in dirPath using Zstandard.
|
||||
func PrecompressDir(dirPath string, options Options) error {
|
||||
var eg errgroup.Group
|
||||
err := fs.WalkDir(os.DirFS(dirPath), ".", func(p string, d fs.DirEntry, err error) error {
|
||||
@@ -60,25 +58,6 @@ type Options struct {
|
||||
ProgressFn func(path string)
|
||||
}
|
||||
|
||||
// OpenPrecompressedFile opens a file from fs, preferring compressed versions
|
||||
// generated by PrecompressDir if possible.
|
||||
func OpenPrecompressedFile(w http.ResponseWriter, r *http.Request, path string, fs fs.FS) (fs.File, error) {
|
||||
if tsweb.AcceptsEncoding(r, "br") {
|
||||
if f, err := fs.Open(path + ".br"); err == nil {
|
||||
w.Header().Set("Content-Encoding", "br")
|
||||
return f, nil
|
||||
}
|
||||
}
|
||||
if tsweb.AcceptsEncoding(r, "gzip") {
|
||||
if f, err := fs.Open(path + ".gz"); err == nil {
|
||||
w.Header().Set("Content-Encoding", "gzip")
|
||||
return f, nil
|
||||
}
|
||||
}
|
||||
|
||||
return fs.Open(path)
|
||||
}
|
||||
|
||||
var compressibleExtensions = map[string]bool{
|
||||
".js": true,
|
||||
".css": true,
|
||||
@@ -94,23 +73,14 @@ func Precompress(path string, options Options) error {
|
||||
return err
|
||||
}
|
||||
|
||||
gzipLevel := gzip.BestCompression
|
||||
zstdLevel := zstd.WithEncoderLevel(zstd.SpeedBestCompression)
|
||||
if options.FastCompression {
|
||||
gzipLevel = gzip.BestSpeed
|
||||
}
|
||||
err = writeCompressed(contents, func(w io.Writer) (io.WriteCloser, error) {
|
||||
return gzip.NewWriterLevel(w, gzipLevel)
|
||||
}, path+".gz", fi.Mode())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
brotliLevel := brotli.BestCompression
|
||||
if options.FastCompression {
|
||||
brotliLevel = brotli.BestSpeed
|
||||
zstdLevel = zstd.WithEncoderLevel(zstd.SpeedFastest)
|
||||
}
|
||||
return writeCompressed(contents, func(w io.Writer) (io.WriteCloser, error) {
|
||||
return brotli.NewWriterLevel(w, brotliLevel), nil
|
||||
}, path+".br", fi.Mode())
|
||||
// Per RFC 8878, encoders should avoid window sizes larger than 8MB, which is the max that Chrome accepts.
|
||||
return zstd.NewWriter(w, zstdLevel, zstd.WithWindowSize(8<<20))
|
||||
}, path+compserve.Zstd.Ext, fi.Mode())
|
||||
}
|
||||
|
||||
func writeCompressed(contents []byte, compressedWriterCreator func(io.Writer) (io.WriteCloser, error), outputPath string, outputMode fs.FileMode) error {
|
||||
|
||||
@@ -125,12 +125,8 @@ type encoderOptions struct {
|
||||
lowMemory bool
|
||||
}
|
||||
|
||||
type encoder struct {
|
||||
pool *sync.Pool
|
||||
*zstd.Encoder
|
||||
}
|
||||
|
||||
func getEncoder(opts ...Option) encoder {
|
||||
// parseEncoderOptions applies opts on top of the default encoder options.
|
||||
func parseEncoderOptions(opts []Option) encoderOptions {
|
||||
eopts := encoderOptions{level: zstd.SpeedDefault, checksum: true}
|
||||
for _, opt := range opts {
|
||||
switch opt := opt.(type) {
|
||||
@@ -144,6 +140,60 @@ func getEncoder(opts ...Option) encoder {
|
||||
eopts.lowMemory = bool(opt)
|
||||
}
|
||||
}
|
||||
return eopts
|
||||
}
|
||||
|
||||
type encoder struct {
|
||||
pool *sync.Pool
|
||||
*zstd.Encoder
|
||||
}
|
||||
|
||||
var streamingEncoderPools sync.Map // map[encoderOptions]*sync.Pool -> *zstd.Encoder
|
||||
|
||||
// GetStreamingEncoder returns an encoder from the shared pool, configured
|
||||
// for streaming (stateful) use: it maintains compression context across
|
||||
// Write calls, and Close finishes the frame. The caller must call
|
||||
// enc.Reset with the destination writer before first use, and enc.Close
|
||||
// when done writing. Call the returned put function exactly once
|
||||
// afterwards to return the encoder to the pool; the encoder must not be
|
||||
// used after that, and must not be used concurrently.
|
||||
func GetStreamingEncoder(opts ...Option) (enc *zstd.Encoder, put func()) {
|
||||
eopts := parseEncoderOptions(opts)
|
||||
|
||||
var pool *sync.Pool
|
||||
if poolCoders() {
|
||||
vpool, ok := streamingEncoderPools.Load(eopts)
|
||||
if !ok {
|
||||
vpool, _ = streamingEncoderPools.LoadOrStore(eopts, new(sync.Pool))
|
||||
}
|
||||
pool = vpool.(*sync.Pool)
|
||||
enc, _ = pool.Get().(*zstd.Encoder)
|
||||
}
|
||||
if enc == nil {
|
||||
// Unlike the stateless encoders above, streaming encoders must not
|
||||
// use SingleSegment framing: the total content size is not known up
|
||||
// front, and the window size must be communicated to decoders.
|
||||
zopts := []zstd.EOption{
|
||||
// Set concurrency=1 to ensure synchronous operation.
|
||||
zstd.WithEncoderConcurrency(1),
|
||||
zstd.WithEncoderLevel(eopts.level),
|
||||
zstd.WithEncoderCRC(eopts.checksum),
|
||||
zstd.WithLowerEncoderMem(eopts.lowMemory),
|
||||
}
|
||||
if eopts.maxWindowLog2 > 0 {
|
||||
zopts = append(zopts, zstd.WithWindowSize(1<<eopts.maxWindowLog2))
|
||||
}
|
||||
enc = must.Get(zstd.NewWriter(nil, zopts...))
|
||||
}
|
||||
return enc, func() {
|
||||
if pool != nil {
|
||||
pool.Put(enc)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func getEncoder(opts ...Option) encoder {
|
||||
eopts := parseEncoderOptions(opts)
|
||||
|
||||
var pool *sync.Pool
|
||||
var enc *zstd.Encoder
|
||||
|
||||
@@ -5,6 +5,7 @@ package zstdframe
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
@@ -60,3 +61,55 @@ func TestSynctestBubbleIsolation(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestSynctestBubbleIsolationStreaming verifies the same isolation property
|
||||
// as TestSynctestBubbleIsolation for the streaming GetDecoder and
|
||||
// GetStreamingEncoder APIs, whose pools are separate from the stateless
|
||||
// coders'.
|
||||
func TestSynctestBubbleIsolationStreaming(t *testing.T) {
|
||||
src := []byte("hello, hello, hello, world, world, world")
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
for range 10 {
|
||||
enc, putEnc := GetStreamingEncoder(FastestCompression)
|
||||
enc.Reset(&bytes.Buffer{})
|
||||
enc.Close()
|
||||
putEnc()
|
||||
|
||||
dec, putDec := GetDecoder()
|
||||
if err := dec.Reset(bytes.NewReader(src)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
putDec()
|
||||
}
|
||||
})
|
||||
|
||||
// A bubble-created streaming coder must never be reused outside the
|
||||
// bubble; prior to the nil-pool guard in GetStreamingEncoder's put
|
||||
// function, put panicked inside the bubble instead.
|
||||
for range 10 {
|
||||
var buf bytes.Buffer
|
||||
enc, putEnc := GetStreamingEncoder(FastestCompression)
|
||||
enc.Reset(&buf)
|
||||
if _, err := enc.Write(src); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := enc.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
putEnc()
|
||||
|
||||
dec, putDec := GetDecoder()
|
||||
if err := dec.Reset(bytes.NewReader(buf.Bytes())); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err := io.ReadAll(dec)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
putDec()
|
||||
if !bytes.Equal(out, src) {
|
||||
t.Fatalf("roundtrip outside bubble = %q, want %q", out, src)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -72,6 +72,17 @@ func AppendDecode(dst, src []byte, opts ...Option) ([]byte, error) {
|
||||
return dec.DecodeAll(src, dst)
|
||||
}
|
||||
|
||||
// GetDecoder returns a decoder from the shared decoder pool for streaming
|
||||
// (io.Reader-style) use, configured with opts. The caller must call
|
||||
// dec.Reset on it before first use and whenever its underlying reader
|
||||
// changes. When done with the decoder, the caller must call the returned
|
||||
// put function exactly once to return it to the pool. The decoder must not
|
||||
// be used after put is called, and must not be used concurrently.
|
||||
func GetDecoder(opts ...Option) (dec *zstd.Decoder, put func()) {
|
||||
d := getDecoder(opts...)
|
||||
return d.Decoder, func() { putDecoder(d) }
|
||||
}
|
||||
|
||||
// NextSize parses the next frame (regardless of whether it is a
|
||||
// data frame or a metadata frame) and returns the total size of the frame.
|
||||
// The frame can be skipped by slicing n bytes from b (e.g., b[n:]).
|
||||
|
||||
Reference in new issue
Block a user