diff --git a/cmd/desktop/app.go b/cmd/desktop/app.go index 88c863e0..8fbfbfa9 100644 --- a/cmd/desktop/app.go +++ b/cmd/desktop/app.go @@ -17,13 +17,15 @@ type App struct { apiLanBaseURL string listenPublic bool shutdownCh chan struct{} + shutdownDone chan struct{} pickingDir atomic.Bool } // NewApp creates a new App application struct. func NewApp() *App { return &App{ - shutdownCh: make(chan struct{}, 1), + shutdownCh: make(chan struct{}, 1), + shutdownDone: make(chan struct{}), } } @@ -33,7 +35,11 @@ func (a *App) startup(ctx context.Context) { } func (a *App) shutdown(ctx context.Context) { - a.shutdownCh <- struct{}{} + select { + case a.shutdownCh <- struct{}{}: + default: + } + <-a.shutdownDone } // GetAPIBaseURL returns the local HTTP base URL for REST API calls (e.g. http://127.0.0.1:PORT/api/v1). diff --git a/cmd/desktop/main.go b/cmd/desktop/main.go index 7d9c9b57..ef6bbfaa 100644 --- a/cmd/desktop/main.go +++ b/cmd/desktop/main.go @@ -268,6 +268,7 @@ func main() { // Run backend in a separate goroutine go func() { + defer close(app.shutdownDone) err := c.Invoke(func( cfg *config.Config, router *gin.Engine, @@ -297,23 +298,28 @@ func main() { } } - // Handle graceful shutdown from Wails OnShutdown hook + // Wails OnShutdown returns only after this goroutine closes + // serverStopped. Closing the listener by hand makes Serve return + // "use of closed network connection" instead of ErrServerClosed, + // and the error path calls logger.Fatalf → os.Exit(1) before cleanup. + serverStopped := make(chan struct{}) go func() { + defer close(serverStopped) <-app.shutdownCh logger.Infof(context.Background(), "Wails shutting down, stopping Go backend...") - listener.Close() - shutdownTimeout := cfg.Server.ShutdownTimeout - if shutdownTimeout == 0 { - shutdownTimeout = 30 * time.Second - } - shutdownCtx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) + drainBudget, cleanupBudget := runtime.ShutdownBudgets(cfg.Server.ShutdownTimeout) + shutdownCtx, cancel := context.WithTimeout(context.Background(), drainBudget) defer cancel() if err := server.Shutdown(shutdownCtx); err != nil { server.Close() } - resourceCleaner.Cleanup(shutdownCtx) + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), cleanupBudget) + defer cleanupCancel() + if errs := resourceCleaner.Cleanup(cleanupCtx); len(errs) > 0 { + logger.Errorf(context.Background(), "Errors occurred during resource cleanup: %v", errs) + } }() // Also listen for OS signals just in case @@ -321,13 +327,17 @@ func main() { signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) go func() { <-signals - app.shutdownCh <- struct{}{} // trigger shutdown + select { + case app.shutdownCh <- struct{}{}: + default: + } }() logger.Infof(context.Background(), "Server is running at %s (proxy -> %s)", tcpAddr.String(), app.backendURL) if err := server.Serve(listener); err != nil && err != http.ErrServerClosed { return fmt.Errorf("server error: %v", err) } + <-serverStopped return nil }) if err != nil { diff --git a/cmd/server/main.go b/cmd/server/main.go index 021a8bd2..ff509a4f 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -99,31 +99,34 @@ func main() { sig := <-signals logger.Infof(context.Background(), "Received signal: %v, starting server shutdown...", sig) - // Close listener first to release port immediately, - // so the next process can bind during our graceful drain. - listener.Close() - - shutdownTimeout := cfg.Server.ShutdownTimeout - if shutdownTimeout == 0 { - shutdownTimeout = 30 * time.Second - } - shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), shutdownTimeout) + drainBudget, cleanupBudget := runtime.ShutdownBudgets(cfg.Server.ShutdownTimeout) + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), drainBudget) defer shutdownCancel() - // Second signal → force close all connections immediately + // Second signal → force close all connections immediately. go func() { sig := <-signals logger.Warnf(context.Background(), "Received second signal: %v, forcing shutdown...", sig) server.Close() }() + // Do NOT close the listener manually before Shutdown. Serve would + // return the raw accept error ("use of closed network connection") + // instead of ErrServerClosed, and logger.Fatalf → os.Exit(1) would + // kill the process before ResourceCleaner runs. Shutdown closes + // listeners first, so the port is released before the drain. if err := server.Shutdown(shutdownCtx); err != nil { logger.Errorf(context.Background(), "Server forced to shutdown: %v", err) server.Close() } logger.Info(context.Background(), "Cleaning up resources...") - errs := resourceCleaner.Cleanup(shutdownCtx) + // Fresh context sized from the reserved slice of ShutdownTimeout. + // Drain must not hand cleanup an already-expired context, and the + // two slices must still add up to one supervisor grace period. + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), cleanupBudget) + defer cleanupCancel() + errs := resourceCleaner.Cleanup(cleanupCtx) if len(errs) > 0 { logger.Errorf(context.Background(), "Errors occurred during resource cleanup: %v", errs) } diff --git a/cmd/server/shutdown_test.go b/cmd/server/shutdown_test.go new file mode 100644 index 00000000..443d8191 --- /dev/null +++ b/cmd/server/shutdown_test.go @@ -0,0 +1,71 @@ +package main + +import ( + "context" + "errors" + "net" + "net/http" + "testing" + "time" +) + +func startTestServer(t *testing.T) (*http.Server, net.Listener, <-chan error) { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + srv := &http.Server{Handler: http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})} + errCh := make(chan error, 1) + go func() { + errCh <- srv.Serve(ln) + }() + waitUntilAccepting(t, ln.Addr().String()) + return srv, ln, errCh +} + +func waitUntilAccepting(t *testing.T, addr string) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + conn, err := net.DialTimeout("tcp", addr, 50*time.Millisecond) + if err == nil { + _ = conn.Close() + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("server at %s did not accept", addr) +} + +func TestShutdownReturnsErrServerClosed(t *testing.T) { + srv, _, errCh := startTestServer(t) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + if err := srv.Shutdown(ctx); err != nil { + t.Fatalf("Shutdown: %v", err) + } + select { + case err := <-errCh: + if !errors.Is(err, http.ErrServerClosed) { + t.Fatalf("Serve err = %v, want ErrServerClosed", err) + } + case <-time.After(2 * time.Second): + t.Fatal("Serve did not return") + } +} + +func TestManualListenerCloseIsNotErrServerClosed(t *testing.T) { + _, ln, errCh := startTestServer(t) + if err := ln.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + select { + case err := <-errCh: + if err == nil || errors.Is(err, http.ErrServerClosed) { + t.Fatalf("Serve err = %v, want the raw accept error", err) + } + case <-time.After(2 * time.Second): + t.Fatal("Serve did not return") + } +} diff --git a/internal/application/service/audit_log_retention.go b/internal/application/service/audit_log_retention.go index 50e213ff..6ab9f7e9 100644 --- a/internal/application/service/audit_log_retention.go +++ b/internal/application/service/audit_log_retention.go @@ -8,6 +8,7 @@ import ( "github.com/Tencent/WeKnora/internal/config" "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/runtime" "github.com/Tencent/WeKnora/internal/types/interfaces" ) @@ -102,6 +103,13 @@ func (r *AuditLogRetentionRunner) Start(ctx context.Context) { // If Start was never called, Stop returns immediately (no doneCh to // wait on — see the `started` flag in the struct comment). func (r *AuditLogRetentionRunner) Stop() { + r.StopWithin(0) +} + +// StopWithin is Stop with a bound. timeout <= 0 waits until the loop +// returns. A positive timeout continues shutdown while a purge is still +// inside its own database deadline, instead of holding every later hook. +func (r *AuditLogRetentionRunner) StopWithin(timeout time.Duration) { if r == nil { return } @@ -111,7 +119,10 @@ func (r *AuditLogRetentionRunner) Stop() { r.stopOnce.Do(func() { close(r.stopCh) }) - <-r.doneCh + if runtime.WaitFor(r.doneCh, timeout) { + return + } + logger.Warnf(context.Background(), "[audit-retention] sweep still running after %s; continuing shutdown", timeout) } // loop runs the actual sweep cadence. Uses a fresh context.Background diff --git a/internal/application/service/knowledge_housekeeping.go b/internal/application/service/knowledge_housekeeping.go index a9b4c1e3..4cb126b5 100644 --- a/internal/application/service/knowledge_housekeeping.go +++ b/internal/application/service/knowledge_housekeeping.go @@ -30,6 +30,7 @@ import ( "github.com/Tencent/WeKnora/internal/config" werrors "github.com/Tencent/WeKnora/internal/errors" "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/runtime" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" "github.com/robfig/cron/v3" @@ -119,13 +120,20 @@ func (h *HousekeepingService) Start(ctx context.Context) error { // Stop halts the cron and waits for in-flight sweeps to finish. func (h *HousekeepingService) Stop() { + h.StopWithin(0) +} + +// StopWithin is Stop with a bound. timeout <= 0 waits for the in-flight sweep. +func (h *HousekeepingService) StopWithin(timeout time.Duration) { h.mu.Lock() defer h.mu.Unlock() if !h.started { return } - c := h.cron.Stop() - <-c.Done() + if !runtime.WaitFor(h.cron.Stop().Done(), timeout) { + logger.Warnf(context.Background(), + "[Housekeeping] in-flight sweep still running after %s; continuing shutdown", timeout) + } h.started = false } diff --git a/internal/application/service/tenant_skill_reaper.go b/internal/application/service/tenant_skill_reaper.go index ae1913e9..a98c37bb 100644 --- a/internal/application/service/tenant_skill_reaper.go +++ b/internal/application/service/tenant_skill_reaper.go @@ -8,6 +8,7 @@ import ( "github.com/Tencent/WeKnora/internal/application/repository" "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/runtime" "github.com/Tencent/WeKnora/internal/sandbox" "github.com/Tencent/WeKnora/internal/tracing/langfuse" "github.com/Tencent/WeKnora/internal/types" @@ -874,12 +875,19 @@ func (s *TenantSkillService) Start(ctx context.Context) error { // Stop halts the cron and waits for in-flight sweeps to finish. func (s *TenantSkillService) Stop() { + s.StopWithin(0) +} + +// StopWithin is Stop with a bound. timeout <= 0 waits for the in-flight sweep. +func (s *TenantSkillService) StopWithin(timeout time.Duration) { s.cronMu.Lock() defer s.cronMu.Unlock() if !s.started { return } - c := s.cron.Stop() - <-c.Done() + if !runtime.WaitFor(s.cron.Stop().Done(), timeout) { + logger.Warnf(context.Background(), + "[skill] in-flight reaper still running after %s; continuing shutdown", timeout) + } s.started = false } diff --git a/internal/container/cleanup.go b/internal/container/cleanup.go index fd14357c..358172f1 100644 --- a/internal/container/cleanup.go +++ b/internal/container/cleanup.go @@ -4,38 +4,43 @@ import ( "context" "log" "sync" + "time" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" ) +// cleanupStepTimeout bounds one shutdown hook that waits on in-flight work +// (cron jobs, retention sweeps). Several of those hooks run before child +// processes are reaped; an unbounded wait used to hold SIGTERM until the +// supervisor killed the process and skipped the rest. +const cleanupStepTimeout = 5 * time.Second + +type cleanupEntry struct { + name string + fn types.CleanupFunc +} + // ResourceCleaner is a resource cleaner that can be used to clean up resources type ResourceCleaner struct { mu sync.Mutex - cleanups []types.CleanupFunc + cleanups []cleanupEntry } // NewResourceCleaner creates a new resource cleaner func NewResourceCleaner() interfaces.ResourceCleaner { return &ResourceCleaner{ - cleanups: make([]types.CleanupFunc, 0), + cleanups: make([]cleanupEntry, 0), } } -// Register registers a cleanup function -// Note: the cleanup function will be executed in reverse order (the last registered will be executed first) +// Register registers a cleanup function. +// The cleanup function runs in reverse order (the last registered runs first). func (c *ResourceCleaner) Register(cleanup types.CleanupFunc) { - if cleanup == nil { - return - } - - c.mu.Lock() - defer c.mu.Unlock() - - c.cleanups = append(c.cleanups, cleanup) + c.register("", cleanup) } -// RegisterWithName registers a cleanup function with a name, for logging tracking +// RegisterWithName registers a cleanup function with a name, for logging tracking. func (c *ResourceCleaner) RegisterWithName(name string, cleanup types.CleanupFunc) { if cleanup == nil { return @@ -52,28 +57,64 @@ func (c *ResourceCleaner) RegisterWithName(name string, cleanup types.CleanupFun return err } - c.Register(wrappedCleanup) + c.register(name, wrappedCleanup) } -// Cleanup executes all cleanup functions -// Even if a cleanup function fails, other cleanup functions will still be executed +func (c *ResourceCleaner) register(name string, cleanup types.CleanupFunc) { + if cleanup == nil { + return + } + + c.mu.Lock() + defer c.mu.Unlock() + c.cleanups = append(c.cleanups, cleanupEntry{name: name, fn: cleanup}) +} + +// Promote moves every callback registered under name to the end of the list. +// Cleanup runs in reverse, so the promoted callbacks run first. BrowserSkill +// is registered during early container wiring and would otherwise run last, +// after hooks that wait on in-flight jobs. +func (c *ResourceCleaner) Promote(name string) { + if name == "" { + return + } + + c.mu.Lock() + defer c.mu.Unlock() + + matched := make([]cleanupEntry, 0, 1) + rest := make([]cleanupEntry, 0, len(c.cleanups)) + for _, entry := range c.cleanups { + if entry.name == name { + matched = append(matched, entry) + continue + } + rest = append(rest, entry) + } + c.cleanups = append(rest, matched...) +} + +// Cleanup executes all cleanup functions. +// A cancelled or expired context does not skip remaining functions: the +// deadline used to return on the first select and drop every later hook, +// including ones registered earliest (BrowserSkill). Context errors are not +// reported as hook failures when the hooks themselves still ran. func (c *ResourceCleaner) Cleanup(ctx context.Context) (errs []error) { c.mu.Lock() defer c.mu.Unlock() - // Execute cleanup functions in reverse order (the last registered will be executed first) - for i := len(c.cleanups) - 1; i >= 0; i-- { - select { - case <-ctx.Done(): - errs = append(errs, ctx.Err()) - return errs - default: - if err := c.cleanups[i](); err != nil { - errs = append(errs, err) - } - } + if ctx == nil { + ctx = context.Background() } + for i := len(c.cleanups) - 1; i >= 0; i-- { + if err := c.cleanups[i].fn(); err != nil { + errs = append(errs, err) + } + } + if err := ctx.Err(); err != nil { + log.Printf("Resource cleanup continued past its deadline: %v", err) + } return errs } @@ -82,5 +123,5 @@ func (c *ResourceCleaner) Reset() { c.mu.Lock() defer c.mu.Unlock() - c.cleanups = make([]types.CleanupFunc, 0) + c.cleanups = make([]cleanupEntry, 0) } diff --git a/internal/container/cleanup_test.go b/internal/container/cleanup_test.go new file mode 100644 index 00000000..37a3deba --- /dev/null +++ b/internal/container/cleanup_test.go @@ -0,0 +1,76 @@ +package container + +import ( + "context" + "errors" + "testing" + "time" +) + +func TestCleanupRunsRemainingHooksAfterDeadline(t *testing.T) { + cleaner := NewResourceCleaner() + var ran []string + // Registered first, so it runs last unless promoted. This is BrowserSkill's + // position: an earlier slow hook must not cause it to be skipped. + cleaner.RegisterWithName("tail", func() error { + ran = append(ran, "tail") + return nil + }) + cleaner.RegisterWithName("slow", func() error { + time.Sleep(40 * time.Millisecond) + ran = append(ran, "slow") + return errors.New("slow failed") + }) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + errs := cleaner.Cleanup(ctx) + if len(ran) != 2 || ran[0] != "slow" || ran[1] != "tail" { + t.Fatalf("ran = %v, want slow then tail", ran) + } + if len(errs) != 1 || errs[0].Error() != "slow failed" { + t.Fatalf("errs = %v, want only the hook error", errs) + } +} + +func TestPromoteRunsNamedHookFirst(t *testing.T) { + cleaner := NewResourceCleaner() + var ran []string + cleaner.RegisterWithName("BrowserSkill", func() error { + ran = append(ran, "browser") + return nil + }) + cleaner.RegisterWithName("later", func() error { + ran = append(ran, "later") + return nil + }) + cleaner.Promote("BrowserSkill") + + if errs := cleaner.Cleanup(context.Background()); len(errs) != 0 { + t.Fatalf("errs = %v", errs) + } + if len(ran) != 2 || ran[0] != "browser" || ran[1] != "later" { + t.Fatalf("ran = %v, want browser then later", ran) + } +} + +func TestCleanupAlreadyCanceledContextStillRunsHooks(t *testing.T) { + cleaner := NewResourceCleaner() + var n int + cleaner.Register(func() error { + n++ + return nil + }) + cleaner.Register(func() error { + n++ + return nil + }) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if errs := cleaner.Cleanup(ctx); len(errs) != 0 { + t.Fatalf("errs = %v", errs) + } + if n != 2 { + t.Fatalf("ran %d hooks, want 2", n) + } +} diff --git a/internal/container/container.go b/internal/container/container.go index f2d81107..3a5a52ad 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -615,6 +615,14 @@ func BuildContainer(container *dig.Container) *dig.Container { // them only after the matching handlers are ready. must(container.Invoke(recoverPendingWikiTasks)) + // BrowserSkill is registered when its manager is constructed, which is + // early, so reverse-order cleanup would run it last. Force the manager + // to exist, then run that hook first so a slow cron stop cannot leave + // the daemon alive until the process is killed. + must(container.Invoke(func(cleaner interfaces.ResourceCleaner, _ *browserskill.Manager) { + cleaner.Promote("BrowserSkill") + })) + logger.Infof(ctx, "[Container] Container initialization completed successfully") return container } @@ -1911,7 +1919,7 @@ func startDataSourceScheduler(scheduler *datasource.Scheduler, cleaner interface } cleaner.RegisterWithName("DataSourceScheduler", func() error { - scheduler.Stop() + scheduler.StopWithin(cleanupStepTimeout) return nil }) } @@ -1929,7 +1937,7 @@ func startHousekeepingService(svc *service.HousekeepingService, cleaner interfac logger.Warnf(context.Background(), "[Container] housekeeping start failed: %v", err) } cleaner.RegisterWithName("KnowledgeHousekeeping", func() error { - svc.Stop() + svc.StopWithin(cleanupStepTimeout) return nil }) } @@ -1945,7 +1953,7 @@ func startTenantSkillReaper(svc *service.TenantSkillService, cleaner interfaces. logger.Warnf(context.Background(), "[Container] tenant skill reaper start failed: %v", err) } cleaner.RegisterWithName("TenantSkillReaper", func() error { - svc.Stop() + svc.StopWithin(cleanupStepTimeout) return nil }) } @@ -2021,7 +2029,7 @@ func startAuditLogRetention( ) { runner.Start(context.Background()) cleaner.RegisterWithName("AuditLogRetentionRunner", func() error { - runner.Stop() + runner.StopWithin(cleanupStepTimeout) return nil }) } diff --git a/internal/datasource/scheduler.go b/internal/datasource/scheduler.go index 0419d1ca..fd595445 100644 --- a/internal/datasource/scheduler.go +++ b/internal/datasource/scheduler.go @@ -8,6 +8,7 @@ import ( "time" "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/runtime" "github.com/Tencent/WeKnora/internal/tracing/langfuse" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" @@ -92,8 +93,21 @@ func (s *Scheduler) Start(ctx context.Context) error { // Stop gracefully stops the cron runner and waits for running jobs to finish. func (s *Scheduler) Stop() { - ctx := s.cron.Stop() - <-ctx.Done() + s.StopWithin(0) +} + +// StopWithin is Stop with a bound. timeout <= 0 waits until in-flight jobs +// finish. A positive timeout lets shutdown continue so one long sync cannot +// hold every later cleanup hook, including child-process reaping. +func (s *Scheduler) StopWithin(timeout time.Duration) { + if s == nil || s.cron == nil { + return + } + if runtime.WaitFor(s.cron.Stop().Done(), timeout) { + return + } + logger.Warnf(context.Background(), + "[Scheduler] in-flight sync still running after %s; continuing shutdown", timeout) } // AddOrUpdate registers (or re-registers) a cron entry for the given data source. diff --git a/internal/runtime/shutdown.go b/internal/runtime/shutdown.go new file mode 100644 index 00000000..90ae1fa0 --- /dev/null +++ b/internal/runtime/shutdown.go @@ -0,0 +1,50 @@ +package runtime + +import "time" + +// DefaultShutdownTimeout is used when server.shutdown_timeout is unset. +const DefaultShutdownTimeout = 30 * time.Second + +// CleanupBudget is reserved from the configured shutdown timeout so connection +// drain cannot consume the whole grace period before process cleanup runs. +// Supervisors often use the same number for termination grace (Kubernetes +// defaults to 30s), so drain and cleanup share one budget instead of stacking. +const CleanupBudget = 5 * time.Second + +// ShutdownBudgets splits total into an HTTP drain budget and a cleanup budget. +// Their sum equals total. total <= 0 uses DefaultShutdownTimeout. When total +// is too small to reserve CleanupBudget, the two halves are equal. +func ShutdownBudgets(total time.Duration) (drain, cleanup time.Duration) { + if total <= 0 { + total = DefaultShutdownTimeout + } + cleanup = CleanupBudget + if cleanup >= total { + cleanup = total / 2 + if cleanup <= 0 { + cleanup = total + } + } + drain = total - cleanup + if drain <= 0 { + drain = total + } + return drain, cleanup +} + +// WaitFor reports whether ch was received before timeout. timeout <= 0 waits +// until ch is received. The boolean is false only when the timeout wins. +func WaitFor(ch <-chan struct{}, timeout time.Duration) bool { + if timeout <= 0 { + <-ch + return true + } + timer := time.NewTimer(timeout) + defer timer.Stop() + select { + case <-ch: + return true + case <-timer.C: + return false + } +} diff --git a/internal/runtime/shutdown_test.go b/internal/runtime/shutdown_test.go new file mode 100644 index 00000000..84f8f4a2 --- /dev/null +++ b/internal/runtime/shutdown_test.go @@ -0,0 +1,62 @@ +package runtime + +import ( + "testing" + "time" +) + +func TestShutdownBudgetsShareOneDeadline(t *testing.T) { + tests := []struct { + name string + total time.Duration + wantDrain time.Duration + wantCleanup time.Duration + }{ + {name: "unset", total: 0, wantDrain: 25 * time.Second, wantCleanup: 5 * time.Second}, + {name: "default", total: 30 * time.Second, wantDrain: 25 * time.Second, wantCleanup: 5 * time.Second}, + {name: "fits reserve", total: 10 * time.Second, wantDrain: 5 * time.Second, wantCleanup: 5 * time.Second}, + {name: "split small", total: 4 * time.Second, wantDrain: 2 * time.Second, wantCleanup: 2 * time.Second}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + drain, cleanup := ShutdownBudgets(tt.total) + if drain != tt.wantDrain || cleanup != tt.wantCleanup { + t.Fatalf("ShutdownBudgets(%s) = %s drain + %s cleanup, want %s + %s", + tt.total, drain, cleanup, tt.wantDrain, tt.wantCleanup) + } + total := tt.total + if total <= 0 { + total = DefaultShutdownTimeout + } + if drain+cleanup != total { + t.Fatalf("budgets sum to %s, want %s", drain+cleanup, total) + } + }) + } +} + +func TestWaitFor(t *testing.T) { + closed := make(chan struct{}) + close(closed) + if !WaitFor(closed, time.Millisecond) { + t.Fatal("closed channel should be received") + } + + never := make(chan struct{}) + start := time.Now() + if WaitFor(never, 20*time.Millisecond) { + t.Fatal("timeout should lose") + } + if elapsed := time.Since(start); elapsed > 500*time.Millisecond { + t.Fatalf("timeout waited %s", elapsed) + } + + later := make(chan struct{}) + go func() { + time.Sleep(15 * time.Millisecond) + close(later) + }() + if !WaitFor(later, 0) { + t.Fatal("non-positive timeout should wait for the channel") + } +} diff --git a/internal/types/interfaces/resource.go b/internal/types/interfaces/resource.go index c7ceb24f..d85cbcc3 100644 --- a/internal/types/interfaces/resource.go +++ b/internal/types/interfaces/resource.go @@ -12,6 +12,9 @@ import ( type ResourceCleaner interface { Register(cleanup types.CleanupFunc) RegisterWithName(name string, cleanup types.CleanupFunc) + // Promote moves callbacks registered under name so Cleanup runs them + // first. Cleanup's order is reverse registration order. + Promote(name string) Cleanup(ctx context.Context) []error } diff --git a/website-docs/01-getting-started/04-configuration.md b/website-docs/01-getting-started/04-configuration.md index 79a910f6..80bcd3ea 100644 --- a/website-docs/01-getting-started/04-configuration.md +++ b/website-docs/01-getting-started/04-configuration.md @@ -47,7 +47,7 @@ flowchart LR | `server.port` | int | 8080 | HTTP 监听端口,校验范围 1–65535 | | `server.host` | string | "0.0.0.0" | 监听地址 | | `server.log_path` | string | 空 | 日志文件路径(也可用环境变量 `LOG_PATH`) | -| `server.shutdown_timeout` | duration | 30s | 优雅停机超时 | +| `server.shutdown_timeout` | duration | 30s | 优雅停机总预算。连接排空与资源清理共用,默认留 5s 给清理 | ### conversation(`ConversationConfig`)——检索问答管线 diff --git a/website-docs/02-architecture/02-backend-design.md b/website-docs/02-architecture/02-backend-design.md index 448e9070..ef91e181 100644 --- a/website-docs/02-architecture/02-backend-design.md +++ b/website-docs/02-architecture/02-backend-design.md @@ -119,10 +119,10 @@ flowchart TD H --> I["systemSettingSvc.SubscribeRedis(ctx)
订阅 system_settings 变更 (Lite 模式 no-op)"] I --> J["signal.Notify(shutdownSignals) + server.Serve(listener)"] J --> K{"收到第一个信号?"} - K -->|是| L["listener.Close() 立即释放端口
server.Shutdown(ctx, ShutdownTimeout 默认 30s) 优雅排空"] + K -->|是| L["server.Shutdown 先关 listener 再排空
排空预算 = ShutdownTimeout − 5s"] L --> M{"排空期间收到第二个信号?"} M -->|是| N["server.Close() 强制断开所有连接"] - M -->|否| O["resourceCleaner.Cleanup(ctx)
ants 池 / Langfuse / 调度器逐个析构"] + M -->|否| O["resourceCleaner.Cleanup
BrowserSkill 最先执行;到期不跳过后续钩子"] N --> O O --> P["进程退出"] ``` @@ -130,7 +130,7 @@ flowchart TD 要点: - **引导失败不阻断启动**:`bootstrap.go` 对失败记录 `logger.Warnf`。系统管理员引导仅在部署尚无系统管理员时生效,重启不会恢复已撤销的权限; -- **两段式优雅退出**:第一个 SIGTERM/SIGINT 先关 listener(新进程可立即绑定端口)再 `Shutdown` 排空存量连接;第二个信号强制 `Close`; +- **两段式优雅退出**:第一个 SIGTERM/SIGINT 调用 `server.Shutdown`(它会先关闭 listener,端口随即可被新进程绑定),在 `ShutdownTimeout` 里为资源清理预留 5s,其余时间排空存量连接。不要在 `Shutdown` 之前手动 `listener.Close()`:`Serve` 会返回 `use of closed network connection` 而不是 `ErrServerClosed`,`logger.Fatalf` 会在清理前把进程杀掉。第二个信号强制 `Close`; - **端口占用重试**:`listenWithRetry` 以 300ms 起步指数退避重试 10 次(滚动重启场景旧进程尚未释放端口时避免直接失败)。 ## 路由组织与 RBAC 装配(internal/router) {#_4-路由组织与-rbac-装配-internal-router}