mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
fix(seaweedfs): stop the master crashing on the crowded volume map
SeaweedFS 4.45 updates the crowded-volume map of a volume layout under the read lock, while Assign reads that map under the same read lock to count writable volumes. Under a steady write load the Go runtime aborts the master with "concurrent map read and map write", a fatal error that recover cannot catch, and the S3 gateway answers 500 until a master is elected again. Taking the write lock for the update, as every other mutation of the map already does, removes the race. Upstream fixed it in seaweedfs/seaweedfs#11216, released in 4.46. The patch carries that change and its regression test unchanged, so the build runs the test and the patch stops applying, and is dropped, once the vendored release carries the fix. Signed-off-by: Mattia Eleuteri <mattia@hidora.io> Assisted-by: LLM
This commit is contained in:
2 files changed
+96
No files matched your search
@@ -16,6 +16,15 @@
|
||||
# seaweedfs/seaweedfs#11674 after 4.48; drop the patch on the first vendored
|
||||
# release that carries it.
|
||||
#
|
||||
# patches/volume-layout-crowded-lock.diff makes SetVolumeCrowded take the write
|
||||
# lock of the volume layout. Upstream 4.45 mutates the crowded map under the
|
||||
# read lock, while Assign reads it under the same read lock through
|
||||
# GetWritableVolumeCount, so under a steady stream of writes the master dies on
|
||||
# a fatal "concurrent map read and map write", which recover cannot catch, and
|
||||
# S3 answers 500 until it is back. Backport of seaweedfs/seaweedfs#11216,
|
||||
# released in 4.46; drop the patch on the first vendored release that carries
|
||||
# it.
|
||||
#
|
||||
# `make update` re-pins VERSION from the vendored chart, SOURCE_SHA256 from the
|
||||
# source archive of that tag, UPSTREAM_DIGEST from the upstream image of that
|
||||
# tag and the builder from upstream's docker/Dockerfile.go_build, and checks
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
diff --git a/weed/topology/crowded_writable_test.go b/weed/topology/crowded_writable_test.go
|
||||
--- a/weed/topology/crowded_writable_test.go
|
||||
+++ b/weed/topology/crowded_writable_test.go
|
||||
@@ -1,6 +1,7 @@
|
||||
package topology
|
||||
|
||||
import (
|
||||
+ "sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -73,3 +74,52 @@ func TestCrowdedCountStillMatchesWritables(t *testing.T) {
|
||||
t.Errorf("writable=%d crowded=%d, want 1 and 1", writable, crowded)
|
||||
}
|
||||
}
|
||||
+
|
||||
+// SetVolumeCrowded mutates the crowded map while GetWritableVolumeCount reads
|
||||
+// it on the Assign hot path. Under RLock both could run at once and the Go
|
||||
+// runtime aborts with "concurrent map read and map write". Run with -race to
|
||||
+// reproduce the original bug; the write lock makes this pass.
|
||||
+func TestSetVolumeCrowdedNoRaceWithGetWritableVolumeCount(t *testing.T) {
|
||||
+ rp, _ := super_block.NewReplicaPlacementFromString("000")
|
||||
+ vl := NewVolumeLayout(rp, needle.EMPTY_TTL, types.HardDriveType, 10000, false)
|
||||
+ // GetWritableVolumeCount iterates vl.writables, so give it something to
|
||||
+ // read while the crowded map is being mutated.
|
||||
+ vl.writables = []needle.VolumeId{1, 2, 3}
|
||||
+
|
||||
+ var wg sync.WaitGroup
|
||||
+ stop := make(chan struct{})
|
||||
+ for range 8 {
|
||||
+ wg.Add(1)
|
||||
+ go func() {
|
||||
+ defer wg.Done()
|
||||
+ for {
|
||||
+ select {
|
||||
+ case <-stop:
|
||||
+ return
|
||||
+ default:
|
||||
+ vl.SetVolumeCrowded(needle.VolumeId(2))
|
||||
+ }
|
||||
+ }
|
||||
+ }()
|
||||
+ }
|
||||
+ for range 8 {
|
||||
+ wg.Add(1)
|
||||
+ go func() {
|
||||
+ defer wg.Done()
|
||||
+ for {
|
||||
+ select {
|
||||
+ case <-stop:
|
||||
+ return
|
||||
+ default:
|
||||
+ vl.GetWritableVolumeCount()
|
||||
+ }
|
||||
+ }
|
||||
+ }()
|
||||
+ }
|
||||
+
|
||||
+ // The race detector needs only a brief window to flag a concurrent
|
||||
+ // map read/write; let the goroutines hammer the lock briefly.
|
||||
+ time.Sleep(50 * time.Millisecond)
|
||||
+ close(stop)
|
||||
+ wg.Wait()
|
||||
+}
|
||||
diff --git a/weed/topology/volume_layout.go b/weed/topology/volume_layout.go
|
||||
--- a/weed/topology/volume_layout.go
|
||||
+++ b/weed/topology/volume_layout.go
|
||||
@@ -1019,11 +1019,14 @@ func (vl *VolumeLayout) setVolumeCrowded(vid needle.VolumeId) {
|
||||
}
|
||||
|
||||
func (vl *VolumeLayout) SetVolumeCrowded(vid needle.VolumeId) {
|
||||
- // since delete is guarded by accessLock.Lock(),
|
||||
- // and is always called in sequential order,
|
||||
- // RLock() should be safe enough
|
||||
- vl.accessLock.RLock()
|
||||
- defer vl.accessLock.RUnlock()
|
||||
+ // setVolumeCrowded mutates the crowded map, and GetWritableVolumeCount
|
||||
+ // reads it under RLock on the Assign hot path. Two concurrent RLock
|
||||
+ // holders with one writing the map triggers a fatal
|
||||
+ // "concurrent map read and map write". Take the write lock: this path
|
||||
+ // is a low-frequency single consumer driven by the crowded-volume
|
||||
+ // event loop, and every other mutation of crowded already holds Lock().
|
||||
+ vl.accessLock.Lock()
|
||||
+ defer vl.accessLock.Unlock()
|
||||
|
||||
vl.setVolumeCrowded(vid)
|
||||
}
|
||||
Reference in new issue
Block a user