diff --git a/cmd/desktop/app.go b/cmd/desktop/app.go
index 88c863e0..8fbfbfa9 100644
--- a/cmd/desktop/app.go
+++ b/cmd/desktop/app.go
@@ -17,13 +17,15 @@ type App struct {
apiLanBaseURL string
listenPublic bool
shutdownCh chan struct{}
+ shutdownDone chan struct{}
pickingDir atomic.Bool
}
// NewApp creates a new App application struct.
func NewApp() *App {
return &App{
- shutdownCh: make(chan struct{}, 1),
+ shutdownCh: make(chan struct{}, 1),
+ shutdownDone: make(chan struct{}),
}
}
@@ -33,7 +35,11 @@ func (a *App) startup(ctx context.Context) {
}
func (a *App) shutdown(ctx context.Context) {
- a.shutdownCh <- struct{}{}
+ select {
+ case a.shutdownCh <- struct{}{}:
+ default:
+ }
+ <-a.shutdownDone
}
// GetAPIBaseURL returns the local HTTP base URL for REST API calls (e.g. http://127.0.0.1:PORT/api/v1).
diff --git a/cmd/desktop/main.go b/cmd/desktop/main.go
index 7d9c9b57..ef6bbfaa 100644
--- a/cmd/desktop/main.go
+++ b/cmd/desktop/main.go
@@ -268,6 +268,7 @@ func main() {
// Run backend in a separate goroutine
go func() {
+ defer close(app.shutdownDone)
err := c.Invoke(func(
cfg *config.Config,
router *gin.Engine,
@@ -297,23 +298,28 @@ func main() {
}
}
- // Handle graceful shutdown from Wails OnShutdown hook
+ // Wails OnShutdown returns only after this goroutine closes
+ // serverStopped. Closing the listener by hand makes Serve return
+ // "use of closed network connection" instead of ErrServerClosed,
+ // and the error path calls logger.Fatalf → os.Exit(1) before cleanup.
+ serverStopped := make(chan struct{})
go func() {
+ defer close(serverStopped)
<-app.shutdownCh
logger.Infof(context.Background(), "Wails shutting down, stopping Go backend...")
- listener.Close()
- shutdownTimeout := cfg.Server.ShutdownTimeout
- if shutdownTimeout == 0 {
- shutdownTimeout = 30 * time.Second
- }
- shutdownCtx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
+ drainBudget, cleanupBudget := runtime.ShutdownBudgets(cfg.Server.ShutdownTimeout)
+ shutdownCtx, cancel := context.WithTimeout(context.Background(), drainBudget)
defer cancel()
if err := server.Shutdown(shutdownCtx); err != nil {
server.Close()
}
- resourceCleaner.Cleanup(shutdownCtx)
+ cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), cleanupBudget)
+ defer cleanupCancel()
+ if errs := resourceCleaner.Cleanup(cleanupCtx); len(errs) > 0 {
+ logger.Errorf(context.Background(), "Errors occurred during resource cleanup: %v", errs)
+ }
}()
// Also listen for OS signals just in case
@@ -321,13 +327,17 @@ func main() {
signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
go func() {
<-signals
- app.shutdownCh <- struct{}{} // trigger shutdown
+ select {
+ case app.shutdownCh <- struct{}{}:
+ default:
+ }
}()
logger.Infof(context.Background(), "Server is running at %s (proxy -> %s)", tcpAddr.String(), app.backendURL)
if err := server.Serve(listener); err != nil && err != http.ErrServerClosed {
return fmt.Errorf("server error: %v", err)
}
+ <-serverStopped
return nil
})
if err != nil {
diff --git a/cmd/server/main.go b/cmd/server/main.go
index 021a8bd2..ff509a4f 100644
--- a/cmd/server/main.go
+++ b/cmd/server/main.go
@@ -99,31 +99,34 @@ func main() {
sig := <-signals
logger.Infof(context.Background(), "Received signal: %v, starting server shutdown...", sig)
- // Close listener first to release port immediately,
- // so the next process can bind during our graceful drain.
- listener.Close()
-
- shutdownTimeout := cfg.Server.ShutdownTimeout
- if shutdownTimeout == 0 {
- shutdownTimeout = 30 * time.Second
- }
- shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), shutdownTimeout)
+ drainBudget, cleanupBudget := runtime.ShutdownBudgets(cfg.Server.ShutdownTimeout)
+ shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), drainBudget)
defer shutdownCancel()
- // Second signal → force close all connections immediately
+ // Second signal → force close all connections immediately.
go func() {
sig := <-signals
logger.Warnf(context.Background(), "Received second signal: %v, forcing shutdown...", sig)
server.Close()
}()
+ // Do NOT close the listener manually before Shutdown. Serve would
+ // return the raw accept error ("use of closed network connection")
+ // instead of ErrServerClosed, and logger.Fatalf → os.Exit(1) would
+ // kill the process before ResourceCleaner runs. Shutdown closes
+ // listeners first, so the port is released before the drain.
if err := server.Shutdown(shutdownCtx); err != nil {
logger.Errorf(context.Background(), "Server forced to shutdown: %v", err)
server.Close()
}
logger.Info(context.Background(), "Cleaning up resources...")
- errs := resourceCleaner.Cleanup(shutdownCtx)
+ // Fresh context sized from the reserved slice of ShutdownTimeout.
+ // Drain must not hand cleanup an already-expired context, and the
+ // two slices must still add up to one supervisor grace period.
+ cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), cleanupBudget)
+ defer cleanupCancel()
+ errs := resourceCleaner.Cleanup(cleanupCtx)
if len(errs) > 0 {
logger.Errorf(context.Background(), "Errors occurred during resource cleanup: %v", errs)
}
diff --git a/cmd/server/shutdown_test.go b/cmd/server/shutdown_test.go
new file mode 100644
index 00000000..443d8191
--- /dev/null
+++ b/cmd/server/shutdown_test.go
@@ -0,0 +1,71 @@
+package main
+
+import (
+ "context"
+ "errors"
+ "net"
+ "net/http"
+ "testing"
+ "time"
+)
+
+func startTestServer(t *testing.T) (*http.Server, net.Listener, <-chan error) {
+ t.Helper()
+ ln, err := net.Listen("tcp", "127.0.0.1:0")
+ if err != nil {
+ t.Fatalf("listen: %v", err)
+ }
+ srv := &http.Server{Handler: http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})}
+ errCh := make(chan error, 1)
+ go func() {
+ errCh <- srv.Serve(ln)
+ }()
+ waitUntilAccepting(t, ln.Addr().String())
+ return srv, ln, errCh
+}
+
+func waitUntilAccepting(t *testing.T, addr string) {
+ t.Helper()
+ deadline := time.Now().Add(2 * time.Second)
+ for time.Now().Before(deadline) {
+ conn, err := net.DialTimeout("tcp", addr, 50*time.Millisecond)
+ if err == nil {
+ _ = conn.Close()
+ return
+ }
+ time.Sleep(10 * time.Millisecond)
+ }
+ t.Fatalf("server at %s did not accept", addr)
+}
+
+func TestShutdownReturnsErrServerClosed(t *testing.T) {
+ srv, _, errCh := startTestServer(t)
+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+ defer cancel()
+ if err := srv.Shutdown(ctx); err != nil {
+ t.Fatalf("Shutdown: %v", err)
+ }
+ select {
+ case err := <-errCh:
+ if !errors.Is(err, http.ErrServerClosed) {
+ t.Fatalf("Serve err = %v, want ErrServerClosed", err)
+ }
+ case <-time.After(2 * time.Second):
+ t.Fatal("Serve did not return")
+ }
+}
+
+func TestManualListenerCloseIsNotErrServerClosed(t *testing.T) {
+ _, ln, errCh := startTestServer(t)
+ if err := ln.Close(); err != nil {
+ t.Fatalf("Close: %v", err)
+ }
+ select {
+ case err := <-errCh:
+ if err == nil || errors.Is(err, http.ErrServerClosed) {
+ t.Fatalf("Serve err = %v, want the raw accept error", err)
+ }
+ case <-time.After(2 * time.Second):
+ t.Fatal("Serve did not return")
+ }
+}
diff --git a/internal/application/service/audit_log_retention.go b/internal/application/service/audit_log_retention.go
index 50e213ff..6ab9f7e9 100644
--- a/internal/application/service/audit_log_retention.go
+++ b/internal/application/service/audit_log_retention.go
@@ -8,6 +8,7 @@ import (
"github.com/Tencent/WeKnora/internal/config"
"github.com/Tencent/WeKnora/internal/logger"
+ "github.com/Tencent/WeKnora/internal/runtime"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
@@ -102,6 +103,13 @@ func (r *AuditLogRetentionRunner) Start(ctx context.Context) {
// If Start was never called, Stop returns immediately (no doneCh to
// wait on — see the `started` flag in the struct comment).
func (r *AuditLogRetentionRunner) Stop() {
+ r.StopWithin(0)
+}
+
+// StopWithin is Stop with a bound. timeout <= 0 waits until the loop
+// returns. A positive timeout continues shutdown while a purge is still
+// inside its own database deadline, instead of holding every later hook.
+func (r *AuditLogRetentionRunner) StopWithin(timeout time.Duration) {
if r == nil {
return
}
@@ -111,7 +119,10 @@ func (r *AuditLogRetentionRunner) Stop() {
r.stopOnce.Do(func() {
close(r.stopCh)
})
- <-r.doneCh
+ if runtime.WaitFor(r.doneCh, timeout) {
+ return
+ }
+ logger.Warnf(context.Background(), "[audit-retention] sweep still running after %s; continuing shutdown", timeout)
}
// loop runs the actual sweep cadence. Uses a fresh context.Background
diff --git a/internal/application/service/knowledge_housekeeping.go b/internal/application/service/knowledge_housekeeping.go
index a9b4c1e3..4cb126b5 100644
--- a/internal/application/service/knowledge_housekeeping.go
+++ b/internal/application/service/knowledge_housekeeping.go
@@ -30,6 +30,7 @@ import (
"github.com/Tencent/WeKnora/internal/config"
werrors "github.com/Tencent/WeKnora/internal/errors"
"github.com/Tencent/WeKnora/internal/logger"
+ "github.com/Tencent/WeKnora/internal/runtime"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/robfig/cron/v3"
@@ -119,13 +120,20 @@ func (h *HousekeepingService) Start(ctx context.Context) error {
// Stop halts the cron and waits for in-flight sweeps to finish.
func (h *HousekeepingService) Stop() {
+ h.StopWithin(0)
+}
+
+// StopWithin is Stop with a bound. timeout <= 0 waits for the in-flight sweep.
+func (h *HousekeepingService) StopWithin(timeout time.Duration) {
h.mu.Lock()
defer h.mu.Unlock()
if !h.started {
return
}
- c := h.cron.Stop()
- <-c.Done()
+ if !runtime.WaitFor(h.cron.Stop().Done(), timeout) {
+ logger.Warnf(context.Background(),
+ "[Housekeeping] in-flight sweep still running after %s; continuing shutdown", timeout)
+ }
h.started = false
}
diff --git a/internal/application/service/tenant_skill_reaper.go b/internal/application/service/tenant_skill_reaper.go
index ae1913e9..a98c37bb 100644
--- a/internal/application/service/tenant_skill_reaper.go
+++ b/internal/application/service/tenant_skill_reaper.go
@@ -8,6 +8,7 @@ import (
"github.com/Tencent/WeKnora/internal/application/repository"
"github.com/Tencent/WeKnora/internal/logger"
+ "github.com/Tencent/WeKnora/internal/runtime"
"github.com/Tencent/WeKnora/internal/sandbox"
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
"github.com/Tencent/WeKnora/internal/types"
@@ -874,12 +875,19 @@ func (s *TenantSkillService) Start(ctx context.Context) error {
// Stop halts the cron and waits for in-flight sweeps to finish.
func (s *TenantSkillService) Stop() {
+ s.StopWithin(0)
+}
+
+// StopWithin is Stop with a bound. timeout <= 0 waits for the in-flight sweep.
+func (s *TenantSkillService) StopWithin(timeout time.Duration) {
s.cronMu.Lock()
defer s.cronMu.Unlock()
if !s.started {
return
}
- c := s.cron.Stop()
- <-c.Done()
+ if !runtime.WaitFor(s.cron.Stop().Done(), timeout) {
+ logger.Warnf(context.Background(),
+ "[skill] in-flight reaper still running after %s; continuing shutdown", timeout)
+ }
s.started = false
}
diff --git a/internal/container/cleanup.go b/internal/container/cleanup.go
index fd14357c..358172f1 100644
--- a/internal/container/cleanup.go
+++ b/internal/container/cleanup.go
@@ -4,38 +4,43 @@ import (
"context"
"log"
"sync"
+ "time"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
+// cleanupStepTimeout bounds one shutdown hook that waits on in-flight work
+// (cron jobs, retention sweeps). Several of those hooks run before child
+// processes are reaped; an unbounded wait used to hold SIGTERM until the
+// supervisor killed the process and skipped the rest.
+const cleanupStepTimeout = 5 * time.Second
+
+type cleanupEntry struct {
+ name string
+ fn types.CleanupFunc
+}
+
// ResourceCleaner is a resource cleaner that can be used to clean up resources
type ResourceCleaner struct {
mu sync.Mutex
- cleanups []types.CleanupFunc
+ cleanups []cleanupEntry
}
// NewResourceCleaner creates a new resource cleaner
func NewResourceCleaner() interfaces.ResourceCleaner {
return &ResourceCleaner{
- cleanups: make([]types.CleanupFunc, 0),
+ cleanups: make([]cleanupEntry, 0),
}
}
-// Register registers a cleanup function
-// Note: the cleanup function will be executed in reverse order (the last registered will be executed first)
+// Register registers a cleanup function.
+// The cleanup function runs in reverse order (the last registered runs first).
func (c *ResourceCleaner) Register(cleanup types.CleanupFunc) {
- if cleanup == nil {
- return
- }
-
- c.mu.Lock()
- defer c.mu.Unlock()
-
- c.cleanups = append(c.cleanups, cleanup)
+ c.register("", cleanup)
}
-// RegisterWithName registers a cleanup function with a name, for logging tracking
+// RegisterWithName registers a cleanup function with a name, for logging tracking.
func (c *ResourceCleaner) RegisterWithName(name string, cleanup types.CleanupFunc) {
if cleanup == nil {
return
@@ -52,28 +57,64 @@ func (c *ResourceCleaner) RegisterWithName(name string, cleanup types.CleanupFun
return err
}
- c.Register(wrappedCleanup)
+ c.register(name, wrappedCleanup)
}
-// Cleanup executes all cleanup functions
-// Even if a cleanup function fails, other cleanup functions will still be executed
+func (c *ResourceCleaner) register(name string, cleanup types.CleanupFunc) {
+ if cleanup == nil {
+ return
+ }
+
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ c.cleanups = append(c.cleanups, cleanupEntry{name: name, fn: cleanup})
+}
+
+// Promote moves every callback registered under name to the end of the list.
+// Cleanup runs in reverse, so the promoted callbacks run first. BrowserSkill
+// is registered during early container wiring and would otherwise run last,
+// after hooks that wait on in-flight jobs.
+func (c *ResourceCleaner) Promote(name string) {
+ if name == "" {
+ return
+ }
+
+ c.mu.Lock()
+ defer c.mu.Unlock()
+
+ matched := make([]cleanupEntry, 0, 1)
+ rest := make([]cleanupEntry, 0, len(c.cleanups))
+ for _, entry := range c.cleanups {
+ if entry.name == name {
+ matched = append(matched, entry)
+ continue
+ }
+ rest = append(rest, entry)
+ }
+ c.cleanups = append(rest, matched...)
+}
+
+// Cleanup executes all cleanup functions.
+// A cancelled or expired context does not skip remaining functions: the
+// deadline used to return on the first select and drop every later hook,
+// including ones registered earliest (BrowserSkill). Context errors are not
+// reported as hook failures when the hooks themselves still ran.
func (c *ResourceCleaner) Cleanup(ctx context.Context) (errs []error) {
c.mu.Lock()
defer c.mu.Unlock()
- // Execute cleanup functions in reverse order (the last registered will be executed first)
- for i := len(c.cleanups) - 1; i >= 0; i-- {
- select {
- case <-ctx.Done():
- errs = append(errs, ctx.Err())
- return errs
- default:
- if err := c.cleanups[i](); err != nil {
- errs = append(errs, err)
- }
- }
+ if ctx == nil {
+ ctx = context.Background()
}
+ for i := len(c.cleanups) - 1; i >= 0; i-- {
+ if err := c.cleanups[i].fn(); err != nil {
+ errs = append(errs, err)
+ }
+ }
+ if err := ctx.Err(); err != nil {
+ log.Printf("Resource cleanup continued past its deadline: %v", err)
+ }
return errs
}
@@ -82,5 +123,5 @@ func (c *ResourceCleaner) Reset() {
c.mu.Lock()
defer c.mu.Unlock()
- c.cleanups = make([]types.CleanupFunc, 0)
+ c.cleanups = make([]cleanupEntry, 0)
}
diff --git a/internal/container/cleanup_test.go b/internal/container/cleanup_test.go
new file mode 100644
index 00000000..37a3deba
--- /dev/null
+++ b/internal/container/cleanup_test.go
@@ -0,0 +1,76 @@
+package container
+
+import (
+ "context"
+ "errors"
+ "testing"
+ "time"
+)
+
+func TestCleanupRunsRemainingHooksAfterDeadline(t *testing.T) {
+ cleaner := NewResourceCleaner()
+ var ran []string
+ // Registered first, so it runs last unless promoted. This is BrowserSkill's
+ // position: an earlier slow hook must not cause it to be skipped.
+ cleaner.RegisterWithName("tail", func() error {
+ ran = append(ran, "tail")
+ return nil
+ })
+ cleaner.RegisterWithName("slow", func() error {
+ time.Sleep(40 * time.Millisecond)
+ ran = append(ran, "slow")
+ return errors.New("slow failed")
+ })
+
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond)
+ defer cancel()
+ errs := cleaner.Cleanup(ctx)
+ if len(ran) != 2 || ran[0] != "slow" || ran[1] != "tail" {
+ t.Fatalf("ran = %v, want slow then tail", ran)
+ }
+ if len(errs) != 1 || errs[0].Error() != "slow failed" {
+ t.Fatalf("errs = %v, want only the hook error", errs)
+ }
+}
+
+func TestPromoteRunsNamedHookFirst(t *testing.T) {
+ cleaner := NewResourceCleaner()
+ var ran []string
+ cleaner.RegisterWithName("BrowserSkill", func() error {
+ ran = append(ran, "browser")
+ return nil
+ })
+ cleaner.RegisterWithName("later", func() error {
+ ran = append(ran, "later")
+ return nil
+ })
+ cleaner.Promote("BrowserSkill")
+
+ if errs := cleaner.Cleanup(context.Background()); len(errs) != 0 {
+ t.Fatalf("errs = %v", errs)
+ }
+ if len(ran) != 2 || ran[0] != "browser" || ran[1] != "later" {
+ t.Fatalf("ran = %v, want browser then later", ran)
+ }
+}
+
+func TestCleanupAlreadyCanceledContextStillRunsHooks(t *testing.T) {
+ cleaner := NewResourceCleaner()
+ var n int
+ cleaner.Register(func() error {
+ n++
+ return nil
+ })
+ cleaner.Register(func() error {
+ n++
+ return nil
+ })
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ if errs := cleaner.Cleanup(ctx); len(errs) != 0 {
+ t.Fatalf("errs = %v", errs)
+ }
+ if n != 2 {
+ t.Fatalf("ran %d hooks, want 2", n)
+ }
+}
diff --git a/internal/container/container.go b/internal/container/container.go
index f2d81107..3a5a52ad 100644
--- a/internal/container/container.go
+++ b/internal/container/container.go
@@ -615,6 +615,14 @@ func BuildContainer(container *dig.Container) *dig.Container {
// them only after the matching handlers are ready.
must(container.Invoke(recoverPendingWikiTasks))
+ // BrowserSkill is registered when its manager is constructed, which is
+ // early, so reverse-order cleanup would run it last. Force the manager
+ // to exist, then run that hook first so a slow cron stop cannot leave
+ // the daemon alive until the process is killed.
+ must(container.Invoke(func(cleaner interfaces.ResourceCleaner, _ *browserskill.Manager) {
+ cleaner.Promote("BrowserSkill")
+ }))
+
logger.Infof(ctx, "[Container] Container initialization completed successfully")
return container
}
@@ -1911,7 +1919,7 @@ func startDataSourceScheduler(scheduler *datasource.Scheduler, cleaner interface
}
cleaner.RegisterWithName("DataSourceScheduler", func() error {
- scheduler.Stop()
+ scheduler.StopWithin(cleanupStepTimeout)
return nil
})
}
@@ -1929,7 +1937,7 @@ func startHousekeepingService(svc *service.HousekeepingService, cleaner interfac
logger.Warnf(context.Background(), "[Container] housekeeping start failed: %v", err)
}
cleaner.RegisterWithName("KnowledgeHousekeeping", func() error {
- svc.Stop()
+ svc.StopWithin(cleanupStepTimeout)
return nil
})
}
@@ -1945,7 +1953,7 @@ func startTenantSkillReaper(svc *service.TenantSkillService, cleaner interfaces.
logger.Warnf(context.Background(), "[Container] tenant skill reaper start failed: %v", err)
}
cleaner.RegisterWithName("TenantSkillReaper", func() error {
- svc.Stop()
+ svc.StopWithin(cleanupStepTimeout)
return nil
})
}
@@ -2021,7 +2029,7 @@ func startAuditLogRetention(
) {
runner.Start(context.Background())
cleaner.RegisterWithName("AuditLogRetentionRunner", func() error {
- runner.Stop()
+ runner.StopWithin(cleanupStepTimeout)
return nil
})
}
diff --git a/internal/datasource/scheduler.go b/internal/datasource/scheduler.go
index 0419d1ca..fd595445 100644
--- a/internal/datasource/scheduler.go
+++ b/internal/datasource/scheduler.go
@@ -8,6 +8,7 @@ import (
"time"
"github.com/Tencent/WeKnora/internal/logger"
+ "github.com/Tencent/WeKnora/internal/runtime"
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
@@ -92,8 +93,21 @@ func (s *Scheduler) Start(ctx context.Context) error {
// Stop gracefully stops the cron runner and waits for running jobs to finish.
func (s *Scheduler) Stop() {
- ctx := s.cron.Stop()
- <-ctx.Done()
+ s.StopWithin(0)
+}
+
+// StopWithin is Stop with a bound. timeout <= 0 waits until in-flight jobs
+// finish. A positive timeout lets shutdown continue so one long sync cannot
+// hold every later cleanup hook, including child-process reaping.
+func (s *Scheduler) StopWithin(timeout time.Duration) {
+ if s == nil || s.cron == nil {
+ return
+ }
+ if runtime.WaitFor(s.cron.Stop().Done(), timeout) {
+ return
+ }
+ logger.Warnf(context.Background(),
+ "[Scheduler] in-flight sync still running after %s; continuing shutdown", timeout)
}
// AddOrUpdate registers (or re-registers) a cron entry for the given data source.
diff --git a/internal/runtime/shutdown.go b/internal/runtime/shutdown.go
new file mode 100644
index 00000000..90ae1fa0
--- /dev/null
+++ b/internal/runtime/shutdown.go
@@ -0,0 +1,50 @@
+package runtime
+
+import "time"
+
+// DefaultShutdownTimeout is used when server.shutdown_timeout is unset.
+const DefaultShutdownTimeout = 30 * time.Second
+
+// CleanupBudget is reserved from the configured shutdown timeout so connection
+// drain cannot consume the whole grace period before process cleanup runs.
+// Supervisors often use the same number for termination grace (Kubernetes
+// defaults to 30s), so drain and cleanup share one budget instead of stacking.
+const CleanupBudget = 5 * time.Second
+
+// ShutdownBudgets splits total into an HTTP drain budget and a cleanup budget.
+// Their sum equals total. total <= 0 uses DefaultShutdownTimeout. When total
+// is too small to reserve CleanupBudget, the two halves are equal.
+func ShutdownBudgets(total time.Duration) (drain, cleanup time.Duration) {
+ if total <= 0 {
+ total = DefaultShutdownTimeout
+ }
+ cleanup = CleanupBudget
+ if cleanup >= total {
+ cleanup = total / 2
+ if cleanup <= 0 {
+ cleanup = total
+ }
+ }
+ drain = total - cleanup
+ if drain <= 0 {
+ drain = total
+ }
+ return drain, cleanup
+}
+
+// WaitFor reports whether ch was received before timeout. timeout <= 0 waits
+// until ch is received. The boolean is false only when the timeout wins.
+func WaitFor(ch <-chan struct{}, timeout time.Duration) bool {
+ if timeout <= 0 {
+ <-ch
+ return true
+ }
+ timer := time.NewTimer(timeout)
+ defer timer.Stop()
+ select {
+ case <-ch:
+ return true
+ case <-timer.C:
+ return false
+ }
+}
diff --git a/internal/runtime/shutdown_test.go b/internal/runtime/shutdown_test.go
new file mode 100644
index 00000000..84f8f4a2
--- /dev/null
+++ b/internal/runtime/shutdown_test.go
@@ -0,0 +1,62 @@
+package runtime
+
+import (
+ "testing"
+ "time"
+)
+
+func TestShutdownBudgetsShareOneDeadline(t *testing.T) {
+ tests := []struct {
+ name string
+ total time.Duration
+ wantDrain time.Duration
+ wantCleanup time.Duration
+ }{
+ {name: "unset", total: 0, wantDrain: 25 * time.Second, wantCleanup: 5 * time.Second},
+ {name: "default", total: 30 * time.Second, wantDrain: 25 * time.Second, wantCleanup: 5 * time.Second},
+ {name: "fits reserve", total: 10 * time.Second, wantDrain: 5 * time.Second, wantCleanup: 5 * time.Second},
+ {name: "split small", total: 4 * time.Second, wantDrain: 2 * time.Second, wantCleanup: 2 * time.Second},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ drain, cleanup := ShutdownBudgets(tt.total)
+ if drain != tt.wantDrain || cleanup != tt.wantCleanup {
+ t.Fatalf("ShutdownBudgets(%s) = %s drain + %s cleanup, want %s + %s",
+ tt.total, drain, cleanup, tt.wantDrain, tt.wantCleanup)
+ }
+ total := tt.total
+ if total <= 0 {
+ total = DefaultShutdownTimeout
+ }
+ if drain+cleanup != total {
+ t.Fatalf("budgets sum to %s, want %s", drain+cleanup, total)
+ }
+ })
+ }
+}
+
+func TestWaitFor(t *testing.T) {
+ closed := make(chan struct{})
+ close(closed)
+ if !WaitFor(closed, time.Millisecond) {
+ t.Fatal("closed channel should be received")
+ }
+
+ never := make(chan struct{})
+ start := time.Now()
+ if WaitFor(never, 20*time.Millisecond) {
+ t.Fatal("timeout should lose")
+ }
+ if elapsed := time.Since(start); elapsed > 500*time.Millisecond {
+ t.Fatalf("timeout waited %s", elapsed)
+ }
+
+ later := make(chan struct{})
+ go func() {
+ time.Sleep(15 * time.Millisecond)
+ close(later)
+ }()
+ if !WaitFor(later, 0) {
+ t.Fatal("non-positive timeout should wait for the channel")
+ }
+}
diff --git a/internal/types/interfaces/resource.go b/internal/types/interfaces/resource.go
index c7ceb24f..d85cbcc3 100644
--- a/internal/types/interfaces/resource.go
+++ b/internal/types/interfaces/resource.go
@@ -12,6 +12,9 @@ import (
type ResourceCleaner interface {
Register(cleanup types.CleanupFunc)
RegisterWithName(name string, cleanup types.CleanupFunc)
+ // Promote moves callbacks registered under name so Cleanup runs them
+ // first. Cleanup's order is reverse registration order.
+ Promote(name string)
Cleanup(ctx context.Context) []error
}
diff --git a/website-docs/01-getting-started/04-configuration.md b/website-docs/01-getting-started/04-configuration.md
index 79a910f6..80bcd3ea 100644
--- a/website-docs/01-getting-started/04-configuration.md
+++ b/website-docs/01-getting-started/04-configuration.md
@@ -47,7 +47,7 @@ flowchart LR
| `server.port` | int | 8080 | HTTP 监听端口,校验范围 1–65535 |
| `server.host` | string | "0.0.0.0" | 监听地址 |
| `server.log_path` | string | 空 | 日志文件路径(也可用环境变量 `LOG_PATH`) |
-| `server.shutdown_timeout` | duration | 30s | 优雅停机超时 |
+| `server.shutdown_timeout` | duration | 30s | 优雅停机总预算。连接排空与资源清理共用,默认留 5s 给清理 |
### conversation(`ConversationConfig`)——检索问答管线
diff --git a/website-docs/02-architecture/02-backend-design.md b/website-docs/02-architecture/02-backend-design.md
index 448e9070..ef91e181 100644
--- a/website-docs/02-architecture/02-backend-design.md
+++ b/website-docs/02-architecture/02-backend-design.md
@@ -119,10 +119,10 @@ flowchart TD
H --> I["systemSettingSvc.SubscribeRedis(ctx)
订阅 system_settings 变更 (Lite 模式 no-op)"]
I --> J["signal.Notify(shutdownSignals) + server.Serve(listener)"]
J --> K{"收到第一个信号?"}
- K -->|是| L["listener.Close() 立即释放端口
server.Shutdown(ctx, ShutdownTimeout 默认 30s) 优雅排空"]
+ K -->|是| L["server.Shutdown 先关 listener 再排空
排空预算 = ShutdownTimeout − 5s"]
L --> M{"排空期间收到第二个信号?"}
M -->|是| N["server.Close() 强制断开所有连接"]
- M -->|否| O["resourceCleaner.Cleanup(ctx)
ants 池 / Langfuse / 调度器逐个析构"]
+ M -->|否| O["resourceCleaner.Cleanup
BrowserSkill 最先执行;到期不跳过后续钩子"]
N --> O
O --> P["进程退出"]
```
@@ -130,7 +130,7 @@ flowchart TD
要点:
- **引导失败不阻断启动**:`bootstrap.go` 对失败记录 `logger.Warnf`。系统管理员引导仅在部署尚无系统管理员时生效,重启不会恢复已撤销的权限;
-- **两段式优雅退出**:第一个 SIGTERM/SIGINT 先关 listener(新进程可立即绑定端口)再 `Shutdown` 排空存量连接;第二个信号强制 `Close`;
+- **两段式优雅退出**:第一个 SIGTERM/SIGINT 调用 `server.Shutdown`(它会先关闭 listener,端口随即可被新进程绑定),在 `ShutdownTimeout` 里为资源清理预留 5s,其余时间排空存量连接。不要在 `Shutdown` 之前手动 `listener.Close()`:`Serve` 会返回 `use of closed network connection` 而不是 `ErrServerClosed`,`logger.Fatalf` 会在清理前把进程杀掉。第二个信号强制 `Close`;
- **端口占用重试**:`listenWithRetry` 以 300ms 起步指数退避重试 10 次(滚动重启场景旧进程尚未释放端口时避免直接失败)。
## 路由组织与 RBAC 装配(internal/router) {#_4-路由组织与-rbac-装配-internal-router}