admin/cozystack
1
0 Fork 0
mirror of https://github.com/cozystack/cozystack.git synced 2026-10-23 03:27:50 +09:00
Code Issues Packages Projects Releases Wiki Activity

build(seaweedfs): build SeaweedFS in-tree with a bounded read-ahead

Browse Source 
The S3 gateway is OOM-killed while serving a large object. SeaweedFS
starts read-ahead downloads after every read of a sequential stream and
counts only the downloads it newly starts toward its prefetch window,
so a GET reading 256 KiB at a time out of 4 MiB chunks pulls up to
about a gigabyte of the object into memory, past the 512Mi limit of
the S3 deployment. No SeaweedFS release carries a fix yet, so there is
no release to move to instead.

Build weed from the 4.45 sources with a patch that keeps the window at
the next prefetchCount chunks, and lay it over the upstream image of
the same tag so the rest of the image stays upstream's own. The image
build runs the tests of the packages the patch touches, including the
regression test the patch adds, because go build never compiles them.

Assisted-by: LLM
Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
This commit is contained in:
Alexey Artamonov authored and Aleksei Sviridkin committed 2026-10-13 16:22:43 +03:00
1 parent c0449eeabb
commit 1913104bd3
5 files changed
+179 -2
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Makefile
+1
View File
@@ -37,6 +37,7 @@ build: build-deps
make -C packages/system/capi-providers-infraprovider image
make -C packages/system/multus image
make -C packages/system/bucket image
make -C packages/system/seaweedfs image
make -C packages/system/objectstorage-controller image
make -C packages/system/securitygroup-controller image
make -C packages/system/grafana-operator image
hack/build-matrix_test.bats
+2 -2
View File
@@ -110,7 +110,7 @@ load test_helper
tmp=$(mktemp)
echo "packages/system/seaweedfs/values.yaml" > "$tmp"
out=$(hack/build-matrix.sh "$tmp")
[ "$out" = '["packages/system/objectstorage-controller"]' ]
[ "$out" = '["packages/system/seaweedfs","packages/system/objectstorage-controller"]' ]
rm -f "$tmp"
}
@@ -118,7 +118,7 @@ load test_helper
tmp=$(mktemp)
printf 'packages/system/seaweedfs/values.yaml\npackages/system/objectstorage-controller/values.yaml\n' > "$tmp"
out=$(hack/build-matrix.sh "$tmp")
[ "$out" = '["packages/system/objectstorage-controller"]' ]
[ "$out" = '["packages/system/seaweedfs","packages/system/objectstorage-controller"]' ]
rm -f "$tmp"
}
packages/system/seaweedfs/Makefile
+24
View File
@@ -1,11 +1,35 @@
export NAME=seaweedfs-system
SEAWEEDFS_TAG=$(shell grep -F 'ARG VERSION=' images/seaweedfs/Dockerfile | cut -f2 -d=)
include ../../../hack/common-envs.mk
include ../../../hack/package.mk
image:
docker buildx build images/seaweedfs \
$(call image-tags,seaweedfs,$(SEAWEEDFS_TAG)) \
$(call cache-args,seaweedfs) \
--metadata-file images/seaweedfs.json \
$(BUILDX_ARGS)
REPOSITORY="$(REGISTRY)/seaweedfs" \
yq -i '.seaweedfs.image.repository = strenv(REPOSITORY)' values.yaml
TAG=$(IMAGE_TAG)@$$(yq e '."containerimage.digest"' images/seaweedfs.json -o json -r) \
yq -i '.seaweedfs.image.tag = strenv(TAG)' values.yaml
rm -f images/seaweedfs.json
test:
helm unittest .
$(MAKE) test-guard-fail-closed
$(MAKE) test-image-version
# The image replaces weed in the upstream image of ARG VERSION, so a chart moved
# by `make update` without the Dockerfile would run the previous release's
# binary under the new chart.
.PHONY: test-image-version
test-image-version:
@chart=$$(yq e '.appVersion' charts/seaweedfs/Chart.yaml); \
[ "$(SEAWEEDFS_TAG)" = "$$chart" ] \
|| { echo "FAIL: images/seaweedfs/Dockerfile builds $(SEAWEEDFS_TAG), the vendored chart is $$chart"; exit 1; }
@echo "ok: images/seaweedfs builds the chart's appVersion $(SEAWEEDFS_TAG)"
# The naming guard refuses to render an UPGRADE that cannot see the cluster
# (templates/naming-guard.yaml): a client-side lookup silently returns nothing,
packages/system/seaweedfs/images/seaweedfs/Dockerfile
+50
View File
@@ -0,0 +1,50 @@
# Rebuild the weed binary from the upstream sources with the Cozystack patches
# applied, and lay it over the upstream image of the same tag. Only the Go
# binary changes: the Rust volume server and worker, the entrypoint and the
# runtime packages are the upstream image's own, byte for byte.
#
# patches/reader-cache-prefetch-window.diff bounds the read-ahead of a
# sequential chunk reader to the prefetch count it was given. Upstream counts
# only the downloads it newly starts toward that window and runs the prefetch
# after every ReadAt, so the S3 GET path, reading 256 KiB at a time out of
# 4 MiB chunks, pushes the window four chunks further on each read until the
# ReaderCache limit of 256 downloaders is reached. The gateway then holds up to
# about a gigabyte of a single object in memory, is OOM-killed under the 512Mi
# limit the S3 deployment gets from its resources preset, and the client sees
# the response cut off tens of megabytes in:
# https://github.com/cozystack/cozystack/issues/3782. Upstream carries the same
# defect on master, tracked as seaweedfs/seaweedfs#11672; drop the patch on the
# first release that carries a fix.
# Keep in step with the chart appVersion pulled by `make update`, and with the
# base image below: the patched binary replaces this version's, so building it
# from another tag would run one version's weed against the rest of another's
# image.
ARG VERSION=4.45
FROM --platform=$BUILDPLATFORM golang:1.26@sha256:eb36c1664dd974cde625f736e02c204383deebe03977365caaec5bf49f794348 AS builder
ARG GOPROXY
ARG VERSION
ARG TARGETOS
ARG TARGETARCH
WORKDIR /workspace
RUN curl -sSL https://github.com/seaweedfs/seaweedfs/archive/refs/tags/${VERSION}.tar.gz | tar -xzf- --strip=1
COPY patches /patches
RUN git apply /patches/*.diff
# go build never compiles a _test.go, so nothing else runs the tests the
# patches carry. The list is read into a variable first: a for loop drops the
# exit status of a command substitution, so a patch git cannot read would leave
# the list empty and the step green.
RUN files=$(git apply --numstat /patches/*.diff) || exit 1; \
for dir in $(printf '%s\n' "$files" | cut -f3 | grep '\.go$' | xargs -r -n1 dirname | sort -u); do \
go test -race "./$dir" || exit 1; \
done
RUN cd weed && GOOS=$TARGETOS GOARCH=$TARGETARCH CGO_ENABLED=0 go build -o /workspace/weed-bin .
FROM chrislusf/seaweedfs:${VERSION}@sha256:fc9f76fa993ad69966ffeb2f65d0318fcae39c6f8e20cf68ef7b3a5cb97769e5
COPY --from=builder /workspace/weed-bin /usr/bin/weed
packages/system/seaweedfs/images/seaweedfs/patches/reader-cache-prefetch-window.diff
+102
View File
@@ -0,0 +1,102 @@
diff --git a/weed/filer/reader_cache.go b/weed/filer/reader_cache.go
--- a/weed/filer/reader_cache.go
+++ b/weed/filer/reader_cache.go
@@ -75,8 +75,10 @@
return
}
- cached := 0
- for x := chunkViews; x != nil && cached < count; x = x.Next {
+ // The window is the next count chunks, whether or not a download for them
+ // is already running: counting only newly started downloads would push the
+ // window count chunks further on every call.
+ for x, i := chunkViews, 0; x != nil && i < count; x, i = x.Next, i+1 {
chunkView := x.Value
if _, found := rc.downloaders[chunkView.FileId]; found {
continue
@@ -98,7 +100,6 @@
go cacher.startCaching()
<-cacher.cacheStartedCh
rc.downloaders[chunkView.FileId] = cacher
- cached++
}
return
diff --git a/weed/filer/reader_at_prefetch_window_test.go b/weed/filer/reader_at_prefetch_window_test.go
new file mode 100644
--- /dev/null
+++ b/weed/filer/reader_at_prefetch_window_test.go
@@ -0,0 +1,73 @@
+package filer
+
+import (
+ "context"
+ "fmt"
+ "strconv"
+ "strings"
+ "sync/atomic"
+ "testing"
+
+ "github.com/seaweedfs/seaweedfs/weed/util/chunk_cache"
+ util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+)
+
+// A sequential reader that consumes a chunk in several small ReadAt calls, as
+// the S3 GET path does with its 256 KiB copy buffer over 4 MiB chunks, must
+// keep its read-ahead at prefetchCount chunks. Each ReadAt triggers a
+// prefetch; if chunks already being fetched are not counted toward the
+// window, every call starts prefetchCount more downloads and the read-ahead
+// runs to the ReaderCache downloader limit, buffering most of the object.
+func TestChunkReadAtSequentialPrefetchStaysWithinWindow(t *testing.T) {
+ const chunkSize = 64 << 10
+ const chunkCount = 64
+ const sliceSize = 16 << 10
+
+ var readerChunk atomic.Int64
+ var maxLead atomic.Int64
+ rc := NewReaderCache(256, (*chunk_cache.TieredChunkCache)(nil), func(context.Context, string) ([]string, error) {
+ return []string{"unused"}, nil
+ }, nil)
+ defer rc.destroy()
+ rc.fetchChunkDataFn = func(_ context.Context, buffer []byte, _ []string, _ []byte, _ bool, _ bool, _ int64, fileId string, _ util_http.RefreshUrlsFunc) (int, error) {
+ idx, err := strconv.Atoi(strings.TrimPrefix(fileId, "chunk"))
+ if err != nil {
+ return 0, err
+ }
+ lead := int64(idx) - readerChunk.Load()
+ for {
+ cur := maxLead.Load()
+ if lead <= cur || maxLead.CompareAndSwap(cur, lead) {
+ break
+ }
+ }
+ return len(buffer), nil
+ }
+
+ views := NewIntervalList[*ChunkView]()
+ for i := 0; i < chunkCount; i++ {
+ views.AppendInterval(&Interval[*ChunkView]{
+ StartOffset: int64(i * chunkSize),
+ StopOffset: int64((i + 1) * chunkSize),
+ Value: &ChunkView{
+ FileId: fmt.Sprintf("chunk%d", i),
+ ViewSize: chunkSize,
+ ViewOffset: int64(i * chunkSize),
+ ChunkSize: chunkSize,
+ },
+ })
+ }
+ reader := NewChunkReaderAtFromClient(context.Background(), rc, views, chunkSize*chunkCount, DefaultPrefetchCount)
+
+ buf := make([]byte, sliceSize)
+ for off := int64(0); off < chunkSize*chunkCount; off += sliceSize {
+ readerChunk.Store(off / chunkSize)
+ if _, err := reader.ReadAt(buf, off); err != nil && off+sliceSize < chunkSize*chunkCount {
+ t.Fatalf("ReadAt(%d): %v", off, err)
+ }
+ }
+
+ if got := maxLead.Load(); got > DefaultPrefetchCount {
+ t.Fatalf("read-ahead reached %d chunks past the reader, want at most %d", got, DefaultPrefetchCount)
+ }
+}