mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 20:23:39 +09:00
Shutdown stopped the clients registered at that moment and then shut the HTTP server down. A stream accepted in between registered after the stop loop, so nothing ended it, and http.Server.Shutdown waited for its handler. The kube-ovn plunger calls Shutdown without a deadline, so such a stream delayed stopping until the manager's graceful-shutdown timeout and then failed it. Shutdown now marks the server as closing under the same lock, and a stream that opens after that is answered 503 without being registered. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <f@lex.la>
276 lines
7.7 KiB
Go
276 lines
7.7 KiB
Go
/*
|
|
Copyright 2026 The Cozystack Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package sse
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"log"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// streamWriter accepts the first okWrites writes and fails every one after.
|
|
type streamWriter struct {
|
|
mu sync.Mutex
|
|
header http.Header
|
|
okWrites int
|
|
body strings.Builder
|
|
}
|
|
|
|
func (w *streamWriter) Header() http.Header { return w.header }
|
|
func (w *streamWriter) WriteHeader(int) {}
|
|
func (w *streamWriter) Flush() {}
|
|
|
|
func (w *streamWriter) Write(p []byte) (int, error) {
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
if w.okWrites == 0 {
|
|
return 0, errors.New("connection reset")
|
|
}
|
|
w.okWrites--
|
|
return w.body.Write(p)
|
|
}
|
|
|
|
func (w *streamWriter) String() string {
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
return w.body.String()
|
|
}
|
|
|
|
// serve runs the stream handler for a client that never disconnects and
|
|
// reports whether the handler gave the client up within a second.
|
|
func serve(t *testing.T, s *Server, w *streamWriter, publish string) bool {
|
|
t.Helper()
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
t.Cleanup(cancel)
|
|
r := httptest.NewRequest(http.MethodGet, "/stream", nil).WithContext(ctx)
|
|
done := make(chan struct{})
|
|
go func() {
|
|
s.handleStream(w, r)
|
|
close(done)
|
|
}()
|
|
if publish != "" {
|
|
deadline := time.Now().Add(time.Second)
|
|
for len(s.snapshotClients()) == 0 && time.Now().Before(deadline) {
|
|
time.Sleep(time.Millisecond)
|
|
}
|
|
// Onto the client's queue, which only the pump reads, so the failing
|
|
// write is the pump's and not the handler's write of latest.
|
|
for _, c := range s.snapshotClients() {
|
|
c.ch <- publish
|
|
}
|
|
}
|
|
select {
|
|
case <-done:
|
|
return len(s.snapshotClients()) == 0
|
|
case <-time.After(time.Second):
|
|
return false
|
|
}
|
|
}
|
|
|
|
func (s *Server) snapshotClients() []*client {
|
|
s.clientsMu.RLock()
|
|
defer s.clientsMu.RUnlock()
|
|
var out []*client
|
|
for c := range s.clients {
|
|
out = append(out, c)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func newTestServer(heartbeat time.Duration) *Server {
|
|
return New(Options{Heartbeat: heartbeat, Logger: log.New(io.Discard, "", 0)})
|
|
}
|
|
|
|
// A client whose connection stops taking writes is dropped at once, at
|
|
// whichever write first fails, instead of holding its handler until the
|
|
// request context ends.
|
|
func TestStreamDropsClientOnWriteFailure(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
okWrites int
|
|
latest string
|
|
publish string
|
|
heartbeat time.Duration
|
|
}{
|
|
{name: "connect comment", okWrites: 0, heartbeat: time.Hour},
|
|
{name: "latest payload", okWrites: 1, latest: "state", heartbeat: time.Hour},
|
|
{name: "published event", okWrites: 1, publish: "event", heartbeat: time.Hour},
|
|
{name: "heartbeat", okWrites: 1, heartbeat: 10 * time.Millisecond},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
s := newTestServer(tc.heartbeat)
|
|
if tc.latest != "" {
|
|
s.Publish(tc.latest)
|
|
}
|
|
if !serve(t, s, &streamWriter{header: http.Header{}, okWrites: tc.okWrites}, tc.publish) {
|
|
t.Fatal("the handler kept a client whose writes fail")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// Each line of a payload is its own data field, and the event ends with an
|
|
// empty line.
|
|
func TestStreamFramesMultilinePayload(t *testing.T) {
|
|
s := newTestServer(time.Hour)
|
|
s.Publish("one\ntwo\n")
|
|
w := &streamWriter{header: http.Header{}, okWrites: 100}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
r := httptest.NewRequest(http.MethodGet, "/stream", nil).WithContext(ctx)
|
|
done := make(chan struct{})
|
|
go func() {
|
|
s.handleStream(w, r)
|
|
close(done)
|
|
}()
|
|
deadline := time.Now().Add(time.Second)
|
|
for !strings.Contains(w.String(), "data: two\n\n") && time.Now().Before(deadline) {
|
|
time.Sleep(time.Millisecond)
|
|
}
|
|
cancel()
|
|
<-done
|
|
if got := w.String(); !strings.Contains(got, "data: one\ndata: two\n\n") {
|
|
t.Fatalf("stream = %q, want both lines framed as one event", got)
|
|
}
|
|
if len(s.snapshotClients()) != 0 {
|
|
t.Fatal("a disconnected client stayed registered")
|
|
}
|
|
}
|
|
|
|
// Shutdown ends every open stream, and the handler that then returns does
|
|
// not close the client a second time.
|
|
func TestShutdownEndsOpenStreams(t *testing.T) {
|
|
s := newTestServer(time.Hour)
|
|
w := &streamWriter{header: http.Header{}, okWrites: 100}
|
|
r := httptest.NewRequest(http.MethodGet, "/stream", nil)
|
|
done := make(chan struct{})
|
|
go func() {
|
|
s.handleStream(w, r)
|
|
close(done)
|
|
}()
|
|
deadline := time.Now().Add(time.Second)
|
|
for len(s.snapshotClients()) == 0 && time.Now().Before(deadline) {
|
|
time.Sleep(time.Millisecond)
|
|
}
|
|
if err := s.Shutdown(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
select {
|
|
case <-done:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("Shutdown left the stream open")
|
|
}
|
|
}
|
|
|
|
// blockingWriter holds every write after the first until release is closed.
|
|
type blockingWriter struct {
|
|
streamWriter
|
|
writing chan struct{}
|
|
release chan struct{}
|
|
returned chan struct{}
|
|
}
|
|
|
|
func (w *blockingWriter) Write(p []byte) (int, error) {
|
|
if w.String() == "" {
|
|
return w.streamWriter.Write(p)
|
|
}
|
|
select {
|
|
case w.writing <- struct{}{}:
|
|
default:
|
|
}
|
|
<-w.release
|
|
return w.streamWriter.Write(p)
|
|
}
|
|
|
|
// The handler does not return while its pump is still writing: net/http does
|
|
// not allow a write to a response after ServeHTTP has returned.
|
|
func TestStreamWaitsForPumpBeforeReturning(t *testing.T) {
|
|
s := newTestServer(time.Hour)
|
|
w := &blockingWriter{
|
|
streamWriter: streamWriter{header: http.Header{}, okWrites: 100},
|
|
writing: make(chan struct{}, 1),
|
|
release: make(chan struct{}),
|
|
returned: make(chan struct{}),
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
r := httptest.NewRequest(http.MethodGet, "/stream", nil).WithContext(ctx)
|
|
go func() {
|
|
s.handleStream(w, r)
|
|
close(w.returned)
|
|
}()
|
|
deadline := time.Now().Add(time.Second)
|
|
for len(s.snapshotClients()) == 0 && time.Now().Before(deadline) {
|
|
time.Sleep(time.Millisecond)
|
|
}
|
|
// Straight onto the client's queue, which only the pump reads: Publish
|
|
// would also set latest, which the handler itself may still write.
|
|
s.snapshotClients()[0].ch <- "event"
|
|
select {
|
|
case <-w.writing:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("the pump never wrote the event")
|
|
}
|
|
cancel()
|
|
select {
|
|
case <-w.returned:
|
|
close(w.release)
|
|
t.Fatal("the handler returned while its pump was writing")
|
|
case <-time.After(100 * time.Millisecond):
|
|
}
|
|
close(w.release)
|
|
select {
|
|
case <-w.returned:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("the handler did not return once its pump finished")
|
|
}
|
|
}
|
|
|
|
// A stream that opens once Shutdown has run is refused instead of being
|
|
// registered after the clients were stopped, where nothing would end it.
|
|
func TestStreamOpenedDuringShutdownIsRefused(t *testing.T) {
|
|
s := newTestServer(time.Hour)
|
|
if err := s.Shutdown(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
t.Cleanup(cancel)
|
|
rec := httptest.NewRecorder()
|
|
r := httptest.NewRequest(http.MethodGet, "/stream", nil).WithContext(ctx)
|
|
done := make(chan struct{})
|
|
go func() {
|
|
s.handleStream(rec, r)
|
|
close(done)
|
|
}()
|
|
select {
|
|
case <-done:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("a stream opened after Shutdown stayed open")
|
|
}
|
|
if rec.Code != http.StatusServiceUnavailable {
|
|
t.Errorf("status = %d, want %d", rec.Code, http.StatusServiceUnavailable)
|
|
}
|
|
if len(s.snapshotClients()) != 0 {
|
|
t.Error("a stream opened after Shutdown was registered")
|
|
}
|
|
}
|