From a8eb7e28fa4045d2d99966f5ff99b68b9a555a93 Mon Sep 17 00:00:00 2001 From: lyingbug Date: Tue, 20 Oct 2026 13:00:12 +0800 Subject: [PATCH] fix(automation): enforce execution identity, cancellation, and scheduling limits (#4506) * fix(automation): preserve execution identity and scheduler guarantees * fix(ci): satisfy lint checks for automation regressions --- .../service/agent_collaboration.go | 41 +- .../service/agent_collaboration_inherited.go | 6 +- .../service/agent_collaboration_recovery.go | 5 + internal/application/service/automation.go | 23 + .../service/automation_regression_test.go | 462 ++++++++++++++++++ .../application/service/automation_runner.go | 28 +- .../service/automation_scheduler.go | 85 +++- .../application/service/session_agent_qa.go | 1 + internal/container/container.go | 1 + internal/database/migration_source_test.go | 22 + ...gration_sqlite_automation_duration_test.go | 36 ++ .../migration_sqlite_versioned_schema_test.go | 14 +- internal/types/agent_prompt.go | 6 +- internal/types/automation.go | 3 + ...048_automation_execution_duration.down.sql | 1 + ...00048_automation_execution_duration.up.sql | 1 + ...131_automation_execution_duration.down.sql | 1 + ...00131_automation_execution_duration.up.sql | 1 + 18 files changed, 695 insertions(+), 42 deletions(-) create mode 100644 internal/application/service/automation_regression_test.go create mode 100644 internal/database/migration_source_test.go create mode 100644 internal/database/migration_sqlite_automation_duration_test.go create mode 100644 migrations/sqlite/000048_automation_execution_duration.down.sql create mode 100644 migrations/sqlite/000048_automation_execution_duration.up.sql create mode 100644 migrations/versioned/000131_automation_execution_duration.down.sql create mode 100644 migrations/versioned/000131_automation_execution_duration.up.sql diff --git a/internal/application/service/agent_collaboration.go b/internal/application/service/agent_collaboration.go index 19781462..6e9ca512 100644 --- a/internal/application/service/agent_collaboration.go +++ b/internal/application/service/agent_collaboration.go @@ -184,12 +184,37 @@ func (s *AgentCollaboration) prune(ctx context.Context) error { type delegatedConfig struct { Agent types.CustomAgent `json:"agent"` Access types.AssistantAccessLevel `json:"access"` + // Durable execution identity follows children across scheduler instances + // and approval resumes; it must not turn an automation into a human turn. + ActingVia string `json:"acting_via,omitempty"` + Mode types.RunMode `json:"mode,omitempty"` // AssistantParent records that the delegating run was the WeKnora // assistant, whose level stays under the workspace's assistant ceiling. AssistantParent bool `json:"assistant_parent,omitempty"` Inherited *inheritedAgentConfig `json:"inherited,omitempty"` } +func (d *delegatedConfig) captureExecution(ctx context.Context) { + d.ActingVia = types.ActingViaFromContext(ctx) + d.Mode = types.RunModeAttended + if mode, ok := ctx.Value(types.RunModeContextKey).(types.RunMode); ok { + d.Mode = mode.Normalize() + } + if d.ActingVia != "" { + d.Mode = types.RunModeDeferred + } +} + +func (d *delegatedConfig) runMode() types.RunMode { + if d.ActingVia != "" { + return types.RunModeDeferred + } + if d.Mode == "" { + return types.RunModeAttended // snapshots created before mode was stored + } + return d.Mode.Normalize() +} + // level is the child's access level: the delegating run's, capped by the // child agent's own ceiling and, under an assistant parent, by the // workspace's current assistant ceiling. @@ -222,7 +247,9 @@ func (s *AgentCollaboration) resolveDelegated( id == types.BuiltinSkillInstallerID { return "", errors.New("only same-workspace reasoning agents without management privileges may be delegated") } - data, err := json.Marshal(delegatedConfig{Agent: *a, Access: level, AssistantParent: assistantParent}) + saved := delegatedConfig{Agent: *a, Access: level, AssistantParent: assistantParent} + saved.captureExecution(ctx) + data, err := json.Marshal(saved) return string(data), err } @@ -377,7 +404,11 @@ func (s *AgentCollaboration) workerContext(ctx context.Context, scope collaborat ctx = context.WithValue(ctx, types.TenantRoleContextKey, membership.Role) ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenant) ctx = types.WithCaller(ctx, types.Caller{TenantID: scope.TenantID, UserID: scope.UserID, Role: membership.Role}) - ctx = types.WithPrincipal(ctx, types.Principal{Type: types.PrincipalWebUser, ID: scope.UserID}) + principal := types.Principal{Type: types.PrincipalWebUser, ID: scope.UserID} + if previous, ok := types.PrincipalFromContext(ctx); ok && previous.ID == scope.UserID { + principal.ActingVia = previous.ActingVia + } + ctx = types.WithPrincipal(ctx, principal) return ctx, nil } @@ -397,6 +428,10 @@ func (s *AgentCollaboration) run(ctx context.Context, w collaboration.Work) (col if err = json.Unmarshal([]byte(w.Member.ConfigJSON), &saved); err != nil { return result, err } + ctx = types.WithPrincipal(ctx, types.Principal{ + Type: types.PrincipalWebUser, ID: w.Scope.UserID, ActingVia: saved.ActingVia, + }) + ctx = types.WithRunMode(ctx, saved.runMode()) // Both modes pin configuration, but reauthorize the source before each turn. if saved.Inherited == nil { if _, err = s.resolve(ctx, w.Member.AgentID, types.AssistantAccessReadOnly); err != nil { @@ -610,7 +645,7 @@ func (s *AgentCollaboration) run(ctx context.Context, w collaboration.Work) (col ctx, &types.QARequest{ Resume: resume, - Mode: types.RunModeAttended, + Mode: saved.runMode(), GrantSessionID: w.Scope.SessionID, Suspendable: true, Session: session, diff --git a/internal/application/service/agent_collaboration_inherited.go b/internal/application/service/agent_collaboration_inherited.go index e2d8e806..b2b5211b 100644 --- a/internal/application/service/agent_collaboration_inherited.go +++ b/internal/application/service/agent_collaboration_inherited.go @@ -75,10 +75,12 @@ func inheritedChildSnapshot(ctx context.Context, cfg *types.AgentConfig, toolNam } } // Marshal immediately: later changes to parent config cannot change queued work. - data, err := json.Marshal(delegatedConfig{ + saved := delegatedConfig{ Agent: *source.Agent, Access: cfg.AssistantAccess, AssistantParent: cfg.OperationsAgentID != "", Inherited: inherited, - }) + } + saved.captureExecution(ctx) + data, err := json.Marshal(saved) return string(data), err } diff --git a/internal/application/service/agent_collaboration_recovery.go b/internal/application/service/agent_collaboration_recovery.go index 223f2c3c..9ba5f3c7 100644 --- a/internal/application/service/agent_collaboration_recovery.go +++ b/internal/application/service/agent_collaboration_recovery.go @@ -223,6 +223,11 @@ func (s *AgentCollaboration) resumeParent(ctx context.Context, b *collaboration. if err = json.Unmarshal(snap.Request, &saved); err != nil { return err } + if saved.ActingVia != "" { + // Automation leases, cancellation and duration belong to their own + // scheduler. Never revive one through ordinary parent recovery. + return errors.New("scheduler-owned parent cannot resume through collaboration recovery") + } if saved.AgentID == "" || saved.SharedAgentReadOnly { return errors.New("invalid parent recovery identity") } diff --git a/internal/application/service/automation.go b/internal/application/service/automation.go index f5f07d7e..4f24789e 100644 --- a/internal/application/service/automation.go +++ b/internal/application/service/automation.go @@ -80,6 +80,8 @@ type AutomationService struct { // Events publishes a run to its message's event stream; see // AgentCollaboration.ChildEvents. Events func(context.Context, *types.Session, *types.Message, *event.EventBus) func(context.Context) error + // StopDelegation fences children and discards recoverable parent waits. + StopDelegation func(context.Context, uint64, string, string, string) error cfg AutomationConfig instance string @@ -621,16 +623,37 @@ func (s *AutomationService) cancelRun(ctx context.Context, run *types.Automation Update("cancel_requested", true).Error; err != nil { return err } + stopErr := s.stopRunDelegation(ctx, run) s.mu.Lock() cancel := s.running[run.ID] s.mu.Unlock() if cancel != nil { cancel() } + return stopErr } return nil } +func (s *AutomationService) stopRunDelegation(ctx context.Context, run *types.AutomationRun) error { + if s.StopDelegation == nil { + return nil + } + // The caller may hold the row from before the executor created its turn. + var current types.AutomationRun + if err := s.db.WithContext(ctx).Where("id = ?", run.ID).Take(¤t).Error; err != nil { + return err + } + if current.SessionID == "" || current.AssistantMessageID == "" { + return nil + } + a, err := s.automationOf(ctx, ¤t) + if err != nil { + return err + } + return s.StopDelegation(ctx, a.TenantID, a.OwnerUserID, current.SessionID, current.AssistantMessageID) +} + // closePrompt ends the open prompt id a run waits on and completes the // run's message, as the prompt reaper would. func (s *AutomationService) closePrompt(ctx context.Context, id, status, reason string) { diff --git a/internal/application/service/automation_regression_test.go b/internal/application/service/automation_regression_test.go new file mode 100644 index 00000000..aa22ad3c --- /dev/null +++ b/internal/application/service/automation_regression_test.go @@ -0,0 +1,462 @@ +package service + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "sync" + "testing" + "time" + + "github.com/Tencent/WeKnora/internal/agent" + "github.com/Tencent/WeKnora/internal/agent/collaboration" + "github.com/Tencent/WeKnora/internal/event" + "github.com/Tencent/WeKnora/internal/types" + "github.com/stretchr/testify/require" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +func TestAutomationDelegatedExecutionKeepsIdentity(t *testing.T) { + for _, inherited := range []bool{false, true} { + t.Run(fmt.Sprintf("inherited=%t", inherited), func(t *testing.T) { + svc, agents, _, ctx := collaborationService(t) + via := types.AutomationActingVia("scheduled-task") + parent := types.WithPrincipal(ctx, types.Principal{Type: types.PrincipalWebUser, ID: "u", ActingVia: via}) + parent = types.WithRunMode(parent, types.RunModeDeferred) + var snapshot string + var err error + if inherited { + parent = context.WithValue(parent, childSourceKey{}, childSource{Agent: agents.agent, ModelID: "model"}) + snapshot, err = inheritedChildSnapshot( + parent, + &types.AgentConfig{AssistantAccess: types.AssistantAccessAutoWrite}, + nil, + ) + } else { + snapshot, err = svc.resolve(parent, "a", types.AssistantAccessAutoWrite) + } + require.NoError(t, err) + scope := collaboration.Scope{TenantID: automationTenant, UserID: "u", SessionID: "parent", ID: "answer"} + require.NoError(t, svc.Runtime.Store.Ensure(ctx, scope)) + _, err = svc.Runtime.Spawn(ctx, scope, "spawn", "child", "a", snapshot, "work") + require.NoError(t, err) + require.NoError(t, svc.Runtime.Store.Change(ctx, scope, func(g *collaboration.Group) error { + r := g.Members[0].Runs[0] + r.State = "running" + r.Owner = "worker" + r.Lease = time.Now().Add(time.Minute) + return nil + })) + called := false + svc.Bind( + &collaborationSessions{ + run: func(child context.Context, req *types.QARequest, bus *event.EventBus) error { + called = true + require.Equal(t, via, types.ActingViaFromContext(child)) + require.Equal(t, types.RunModeDeferred, req.Mode) + require.False(t, req.Mode.Attended(), "ask_user must stay unavailable") + _, _, err := automationOwner(child) + require.Error(t, err, "delegation must not bypass the automation management guard") + return complete(child, bus, event.AgentCompleteData{FinalAnswer: "done"}) + }, + }, + ) + g, err := svc.Runtime.Store.Read(ctx, scope) + require.NoError(t, err) + // A scheduler starts from its own context, not the parent's request context. + _, err = svc.run( + context.Background(), + collaboration.Work{Scope: scope, Member: *g.Members[0], Run: *g.Members[0].Runs[0]}, + ) + require.NoError(t, err) + require.True(t, called) + }) + } +} + +func TestAutomationCancelStopsDelegationAndRecovery(t *testing.T) { + f := newAutomationFixture(t) + require.NoError(t, f.db.AutoMigrate(&collaboration.Record{})) + delegated := NewAgentCollaboration(f.db, f.agents, f.messages, f.users, f.members, f.svc.tenants) + f.svc.StopDelegation = delegated.StopOwned + a := f.create(t, types.AutomationPatch{}) + queued, err := f.svc.RunNow(f.ctx, a.ID) + require.NoError(t, err) + entered := make(chan collaboration.Scope, 1) + f.sessions.run = func(ctx context.Context, req *types.QARequest, _ *event.EventBus) error { + scope := collaboration.Scope{ + TenantID: automationTenant, + UserID: "u", + SessionID: req.Session.ID, + ID: req.AssistantMessageID, + } + b, err := delegated.Runtime.AcquireParent(ctx, scope) + if err != nil { + return err + } + _, err = delegated.Runtime.Spawn(b.Context(ctx), scope, "spawn", "child", "a", "{}", "work") + if err != nil { + return err + } + _, err = delegated.Runtime.Spawn(b.Context(ctx), scope, "spawn-running", "running_child", "a", "{}", "work") + if err != nil { + return err + } + if err = delegated.Runtime.Store.Change(b.Context(ctx), scope, func(g *collaboration.Group) error { + r := g.Members[1].Runs[0] + r.State, r.Owner, r.Lease = "running", "remote-worker", time.Now().Add(time.Minute) + return nil + }); err != nil { + return err + } + saved := resumableAgentRequest(ctx, req) + data, err := json.Marshal( + agent.RunSnapshot{Version: agent.RunSnapshotVersion, Request: mustJSON(t, saved), CollaborationWait: true}, + ) + if err != nil { + return err + } + if err = b.SaveWait(ctx, data, time.Now().Add(time.Minute)); err != nil { + return err + } + entered <- scope + <-ctx.Done() + // Exercise the same cleanup as an engine cancelled while in wait_agent. + b.End(true) + return ctx.Err() + } + require.NoError(t, f.svc.startQueued(f.ctx, f.now)) + var scope collaboration.Scope + select { + case scope = <-entered: + case <-time.After(5 * time.Second): + t.Fatal("run did not start") + } + _, err = f.svc.CancelRun(f.ctx, queued.ID) + require.NoError(t, err) + done := f.settleRun(t, queued.ID) + require.Equal(t, types.AutomationRunCancelled, done.Status) + g, err := delegated.Runtime.Store.Read(f.ctx, scope) + require.NoError(t, err) + require.NotEqual(t, "active", g.Status) + require.Nil(t, g.ParentWait, "cancelled checkpoints must not be recovered") + require.True(t, g.Members[0].Runs[0].CancelRequested) + require.Equal(t, "cancelled", g.Members[0].Runs[0].State) + require.True(t, g.Members[1].Runs[0].CancelRequested) + require.ErrorIs(t, delegated.Runtime.CheckOwner(f.ctx, collaboration.Work{ + Scope: scope, Member: *g.Members[1], Run: *g.Members[1].Runs[0], + }), collaboration.ErrConflict, "a remote child must lose permission to execute tools immediately") +} + +func mustJSON(t *testing.T, v any) json.RawMessage { + t.Helper() + data, err := json.Marshal(v) + require.NoError(t, err) + return data +} + +func TestAutomationParentRecoveryDoesNotEscapeScheduler(t *testing.T) { + svc, _, messages, ctx := collaborationService(t) + scope := collaboration.Scope{TenantID: automationTenant, UserID: "u", SessionID: "parent", ID: "answer"} + b, err := svc.Runtime.AcquireParent(ctx, scope) + require.NoError(t, err) + t.Cleanup(func() { b.End(false) }) + require.NoError(t, messages.UpdateMessage(ctx, &types.Message{ID: scope.ID, SessionID: scope.SessionID})) + svc.streams = &collaborationStreams{} + svc.ParentEvents = func( + context.Context, *types.Session, *types.Message, *event.EventBus, + ) func(context.Context) error { + return func(context.Context) error { return nil } + } + called := false + svc.Bind( + &collaborationSessions{ + run: func(context.Context, *types.QARequest, *event.EventBus) error { called = true; return nil }, + }, + ) + ctx = types.WithPrincipal( + ctx, + types.Principal{Type: types.PrincipalWebUser, ID: "u", ActingVia: types.AutomationActingVia("a")}, + ) + req := resumableAgentRequest( + ctx, + &types.QARequest{CustomAgent: &types.CustomAgent{ID: "a"}, Mode: types.RunModeDeferred}, + ) + snapshot := mustJSON( + t, + agent.RunSnapshot{Version: agent.RunSnapshotVersion, Request: mustJSON(t, req), CollaborationWait: true}, + ) + err = svc.resumeParent(context.Background(), b, snapshot) + require.ErrorContains(t, err, "scheduler-owned parent") + require.False(t, called) +} + +func TestAutomationStartSerializesDifferentRunsAcrossInstances(t *testing.T) { + f := newAutomationFixture(t) + a := f.create(t, types.AutomationPatch{OverlapPolicy: ptr(types.AutomationPolicyQueue)}) + rows := []*types.AutomationRun{ + { + AutomationID: a.ID, + TenantID: automationTenant, + ScheduledAt: f.now.Add(-time.Hour), + TriggerKind: types.AutomationRunSchedule, + Status: types.AutomationRunQueued, + Resume: true, + }, + { + AutomationID: a.ID, + TenantID: automationTenant, + ScheduledAt: f.now, + TriggerKind: types.AutomationRunSchedule, + Status: types.AutomationRunQueued, + }, + } + for _, r := range rows { + require.NoError(t, f.db.Create(r).Error) + } + // A separate connection pool represents another scheduler process. + db, err := gorm.Open( + sqlite.Open(f.db.Dialector.(*sqlite.Dialector).DSN), + &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}, + ) + require.NoError(t, err) + raw, err := db.DB() + require.NoError(t, err) + t.Cleanup(func() { _ = raw.Close() }) + other := &AutomationService{db: db, instance: "second-instance"} + arrived := make(chan struct{}, 2) + release := make(chan struct{}) + for _, db := range []*gorm.DB{f.db, db} { + require.NoError(t, db.Callback().Query().After("gorm:query").Register("admission_barrier", func(db *gorm.DB) { + if n, ok := db.Statement.Dest.(*int64); ok && *n == 0 && db.Statement.Table == "automation_runs" && + strings.Contains(db.Statement.SQL.String(), "count(*)") { + arrived <- struct{}{} + <-release + } + })) + } + var wg sync.WaitGroup + wg.Add(2) + started := make([]bool, 2) + errs := make([]error, 2) + go func() { defer wg.Done(); started[0], errs[0] = f.svc.startRun(f.ctx, rows[0], f.now) }() + select { + case <-arrived: + case <-time.After(5 * time.Second): + close(release) + t.Fatal("first admission did not reach check") + } + go func() { defer wg.Done(); started[1], errs[1] = other.startRun(f.ctx, rows[1], f.now) }() + // The old implementation lets both count queries observe zero. With the + // automation lock the second query cannot run until the first commits. + select { + case <-arrived: + case <-time.After(100 * time.Millisecond): + } + close(release) + wg.Wait() + require.NoError(t, errs[0]) + require.NoError(t, errs[1]) + require.True(t, started[0]) + require.False(t, started[1]) + var active int64 + require.NoError( + t, + f.db.Model(&types.AutomationRun{}). + Where("automation_id = ? AND status = ?", a.ID, types.AutomationRunRunning). + Count(&active). + Error, + ) + require.EqualValues(t, 1, active) +} + +func TestAutomationQueuedRunsSkipBlockedHead(t *testing.T) { + f := newAutomationFixture(t) + for i := 0; i < 16; i++ { + a := f.create( + t, + types.AutomationPatch{ + Name: ptr(fmt.Sprintf("blocked%d", i)), + OverlapPolicy: ptr(types.AutomationPolicyQueue), + }, + ) + for j, status := range []string{types.AutomationRunAwaitingApproval, types.AutomationRunQueued} { + require.NoError( + t, + f.db.Create( + &types.AutomationRun{ + AutomationID: a.ID, + TenantID: automationTenant, + ScheduledAt: f.now.Add(time.Duration(j-2) * time.Hour), + TriggerKind: types.AutomationRunSchedule, + Status: status, + }, + ).Error, + ) + } + } + a := f.create(t, types.AutomationPatch{Name: ptr("runnable")}) + r, err := f.svc.RunNow(f.ctx, a.ID) + require.NoError(t, err) + require.NoError(t, f.svc.startQueued(context.Background(), f.now)) + require.Eventually(t, func() bool { + var row types.AutomationRun + return f.db.First(&row, "id = ?", r.ID).Error == nil && row.Status == types.AutomationRunSucceeded && + len(f.svc.slots) == 0 + }, 5*time.Second, 10*time.Millisecond) +} + +func TestAutomationApprovalReconciliationVisitsEveryBatch(t *testing.T) { + f := newAutomationFixture(t) + f.svc.cfg.MaxPerTenant = 200 + var expired, answered *types.AutomationRun + for i := 0; i < 102; i++ { + timeout := 10080 + updated := f.now.Add(-time.Hour) + if i == 100 { + timeout = 5 + updated = f.now.Add(-10 * time.Minute) + } + a := f.create( + t, + types.AutomationPatch{Name: ptr(fmt.Sprintf("approval%d", i)), ApprovalTimeoutMinutes: ptr(timeout)}, + ) + status := types.AgentPromptSuspended + if i == 0 || i == 101 { + status = types.AgentPromptAnswered + } + p := &types.AgentPrompt{ID: fmt.Sprintf("p%d", i), Status: status} + require.NoError(t, f.broker.Store().Create(f.ctx, p)) + r := &types.AutomationRun{ + ID: fmt.Sprintf("run-%03d", i), + AutomationID: a.ID, + TenantID: automationTenant, + ScheduledAt: updated, + TriggerKind: types.AutomationRunSchedule, + Status: types.AutomationRunAwaitingApproval, + PendingPromptID: p.ID, + UpdatedAt: updated, + } + require.NoError(t, f.db.Create(r).Error) + if i == 100 { + expired = r + } + if i == 101 { + answered = r + } + } + require.NoError(t, f.svc.reconcileAwaiting(f.ctx, f.now)) + require.NoError(t, f.db.First(expired, "id = ?", expired.ID).Error) + require.Equal(t, types.AutomationRunFailed, expired.Status) + require.NoError(t, f.db.First(answered, "id = ?", answered.ID).Error) + require.Equal(t, types.AutomationRunQueued, answered.Status) + require.True(t, answered.Resume) +} + +func TestAutomationApprovalResumeUsesRemainingExecutionBudget(t *testing.T) { + f := newAutomationFixture(t) + f.svc.cfg.DefaultMaxDuration = time.Second + a := f.create(t, types.AutomationPatch{}) + f.suspendOnApproval(t, "op:x") + original := f.sessions.run + f.sessions.run = func(ctx context.Context, req *types.QARequest, bus *event.EventBus) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(600 * time.Millisecond): + } + return original(ctx, req, bus) + } + queued, err := f.svc.RunNow(f.ctx, a.ID) + require.NoError(t, err) + suspended := f.settleRun(t, queued.ID) + require.Equal(t, types.AutomationRunAwaitingApproval, suspended.Status) + // Approval waiting is free, even across service restarts. + f.now = f.now.Add(12 * time.Hour) + restarted := NewAutomationService( + f.db, + f.agents, + f.users, + f.members, + f.svc.tenants, + f.messages, + f.svc.grants, + f.svc.notices, + ) + restarted.Configure(AutomationConfig{DefaultMaxDuration: time.Second}) + restarted.now = func() time.Time { return f.now } + restarted.Bind(f.sessions, f.broker, nil, nil) + f.svc = restarted + ok, err := f.broker.Store().Transition(f.ctx, suspended.PendingPromptID, types.AgentPromptTransition{ + From: []string{types.AgentPromptSuspended}, + To: types.AgentPromptAnswered, + Response: types.JSON(`{"approved":true}`), + }) + require.NoError(t, err) + require.True(t, ok) + resumed := f.settleRun(t, queued.ID) + require.Equal(t, types.AutomationRunFailed, resumed.Status) + require.Contains(t, resumed.Error, "longer than 1s") + require.NotNil(t, f.sessions.last().req.Resume, "waiting time must not exhaust the execution budget") +} + +func TestAutomationStaleFinalizerDoesNotCancelResumedDelegation(t *testing.T) { + f := newAutomationFixture(t) + a := f.create(t, types.AutomationPatch{}) + r, err := f.svc.RunNow(f.ctx, a.ID) + require.NoError(t, err) + require.NoError(t, f.db.Model(r).Updates(map[string]any{ + "status": types.AutomationRunRunning, "session_id": "session", "assistant_message_id": "answer", + }).Error) + stopped := false + f.svc.StopDelegation = func(context.Context, uint64, string, string, string) error { stopped = true; return nil } + ok, err := f.svc.finishRun( + f.ctx, + r, + []string{types.AutomationRunAwaitingApproval}, + types.AutomationRunFailed, + "stale timeout", + nil, + ) + require.NoError(t, err) + require.False(t, ok) + require.False(t, stopped) + ok, err = f.svc.finishRun( + f.ctx, + r, + []string{types.AutomationRunRunning}, + types.AutomationRunNeedsAttention, + "executor lost", + nil, + ) + require.NoError(t, err) + require.True(t, ok) + require.True(t, stopped) +} + +func TestAutomationCancellationWinsApprovalRace(t *testing.T) { + f := newAutomationFixture(t) + a := f.create(t, types.AutomationPatch{}) + f.suspendOnApproval(t, "op:x") + queued, err := f.svc.RunNow(f.ctx, a.ID) + require.NoError(t, err) + r := f.settleRun(t, queued.ID) + require.Equal(t, types.AutomationRunAwaitingApproval, r.Status) + // A remote cancellation can land after execute checked its context but + // before it records the approval suspension. Do not resume that run. + require.NoError(t, f.db.Model(r).Update("cancel_requested", true).Error) + ok, err := f.broker.Store().Transition(f.ctx, r.PendingPromptID, types.AgentPromptTransition{ + From: []string{types.AgentPromptSuspended}, + To: types.AgentPromptAnswered, + Response: types.JSON(`{"approved":true}`), + }) + require.NoError(t, err) + require.True(t, ok) + require.NoError(t, f.svc.queueResume(f.ctx, r.ID, r.PendingPromptID)) + require.NoError(t, f.svc.reconcileAwaiting(f.ctx, f.now)) + require.NoError(t, f.db.First(&r, "id = ?", r.ID).Error) + require.Equal(t, types.AutomationRunCancelled, r.Status) +} diff --git a/internal/application/service/automation_runner.go b/internal/application/service/automation_runner.go index e10c4e39..34724bac 100644 --- a/internal/application/service/automation_runner.go +++ b/internal/application/service/automation_runner.go @@ -36,6 +36,7 @@ type runOutcome struct { answer string usage *types.TokenUsage promptID string + elapsed time.Duration // pause, when set, pauses the automation with this reason. pause string } @@ -56,6 +57,7 @@ func (s *AutomationService) execute(parent context.Context, run *types.Automatio }() go s.heartbeat(ctx, run.ID, cancel) + started := time.Now() out := func() (out runOutcome) { defer func() { if r := recover(); r != nil { @@ -64,11 +66,15 @@ func (s *AutomationService) execute(parent context.Context, run *types.Automatio }() return s.runOnce(ctx, run) }() - if ctx.Err() != nil && out.status != types.AutomationRunAwaitingApproval { + out.elapsed = time.Since(started) + if ctx.Err() != nil { var row types.AutomationRun if err := s.db.WithContext(context.WithoutCancel(ctx)).Select("cancel_requested"). Where("id = ?", run.ID).Take(&row).Error; err == nil && row.CancelRequested { out.status, out.reason = types.AutomationRunCancelled, "cancelled by the owner" + if out.promptID != "" { + s.closePrompt(context.WithoutCancel(ctx), out.promptID, types.AgentPromptCanceled, out.reason) + } } } s.record(ctx, run, out) @@ -109,7 +115,10 @@ func (s *AutomationService) heartbeat(ctx context.Context, runID string, cancel func (s *AutomationService) record(ctx context.Context, run *types.AutomationRun, out runOutcome) { ctx = context.WithoutCancel(ctx) run.Answer = clipRunes(out.answer, maxStoredAnswerRunes) - updates := map[string]any{"answer": run.Answer, "updated_at": s.now()} + updates := map[string]any{ + "answer": run.Answer, "updated_at": s.now(), + "execution_duration_ms": gorm.Expr("execution_duration_ms + ?", out.elapsed.Milliseconds()), + } if out.usage != nil { if data, err := json.Marshal(out.usage); err == nil { updates["usage"] = types.JSON(data) @@ -204,6 +213,7 @@ func (s *AutomationService) ownerContext(ctx context.Context, a *types.Automatio // runOnce executes run, or continues it from its answered approval. func (s *AutomationService) runOnce(ctx context.Context, run *types.AutomationRun) runOutcome { + started := time.Now() if s.sessions == nil { return failed("the agent executor is unavailable") } @@ -234,7 +244,14 @@ func (s *AutomationService) runOnce(ctx context.Context, run *types.AutomationRu if m := a.Limits.MaxDurationMinutes; m > 0 { limit = time.Duration(m) * time.Minute } - ctx, cancel := context.WithTimeout(ctx, limit) + remaining := limit - time.Duration(run.ExecutionDurationMS)*time.Millisecond - time.Since(started) + if remaining <= 0 { + if run.Resume && run.PendingPromptID != "" { + s.closePrompt(ctx, run.PendingPromptID, types.AgentPromptFailed, "execution time limit exceeded") + } + return failed(fmt.Sprintf("the run took longer than %s", limit)) + } + ctx, cancel := context.WithTimeout(ctx, remaining) defer cancel() var resume *types.AgentResume @@ -524,7 +541,10 @@ func (s *AutomationService) agentQA( } mu.Lock() defer mu.Unlock() - answer.IsCompleted = suspended == nil + answer.IsCompleted = suspended == nil || err != nil + if answer.IsCompleted { + answer.RunState = "" + } answer.UpdatedAt = s.now() if err != nil && answer.Content == "" { answer.Content = err.Error() diff --git a/internal/application/service/automation_scheduler.go b/internal/application/service/automation_scheduler.go index 6c2196c7..c248c2ff 100644 --- a/internal/application/service/automation_scheduler.go +++ b/internal/application/service/automation_scheduler.go @@ -163,6 +163,9 @@ func (s *AutomationService) startQueued(ctx context.Context, now time.Time) erro } var queued []*types.AutomationRun if err := s.db.WithContext(ctx).Where("status = ?", types.AutomationRunQueued). + Where(`NOT EXISTS (SELECT 1 FROM automation_runs AS holding + WHERE holding.automation_id = automation_runs.automation_id AND holding.status IN ? )`, + []string{types.AutomationRunRunning, types.AutomationRunAwaitingApproval}). Order("resume DESC, scheduled_at").Limit(free * 4).Find(&queued).Error; err != nil { return err } @@ -186,16 +189,6 @@ func (s *AutomationService) startQueued(ctx context.Context, now time.Time) erro } func (s *AutomationService) startRun(ctx context.Context, run *types.AutomationRun, now time.Time) (bool, error) { - var holding int64 - if err := s.db.WithContext(ctx).Model(&types.AutomationRun{}). - Where("automation_id = ? AND id <> ? AND status IN ?", run.AutomationID, run.ID, []string{ - types.AutomationRunRunning, types.AutomationRunAwaitingApproval, - }).Count(&holding).Error; err != nil { - return false, err - } - if holding > 0 { - return false, nil - } lease := now.Add(s.cfg.Lease) updates := map[string]any{ "status": types.AutomationRunRunning, "owner": s.instance, "lease_until": lease, "updated_at": now, @@ -203,10 +196,33 @@ func (s *AutomationService) startRun(ctx context.Context, run *types.AutomationR if run.StartedAt == nil { updates["started_at"] = now } - res := s.db.WithContext(ctx).Model(&types.AutomationRun{}). - Where("id = ? AND status = ?", run.ID, types.AutomationRunQueued).Updates(updates) - if res.Error != nil || res.RowsAffected == 0 { - return false, res.Error + started := false + err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + // Take the automation's write lock before reading its runs. A run-row + // CAS alone cannot serialize two different queued runs. Writing first + // also avoids a deferred read-to-write transaction upgrade on SQLite. + locked := tx.Model(&types.Automation{}).Where("id = ?", run.AutomationID). + UpdateColumn("version", gorm.Expr("version + 1")) + if locked.Error != nil || locked.RowsAffected == 0 { + return locked.Error + } + var holding int64 + if err := tx.Model(&types.AutomationRun{}). + Where("automation_id = ? AND id <> ? AND status IN ?", run.AutomationID, run.ID, []string{ + types.AutomationRunRunning, types.AutomationRunAwaitingApproval, + }).Count(&holding).Error; err != nil { + return err + } + if holding > 0 { + return nil + } + res := tx.Model(&types.AutomationRun{}). + Where("id = ? AND status = ?", run.ID, types.AutomationRunQueued).Updates(updates) + started = res.Error == nil && res.RowsAffected > 0 + return res.Error + }) + if err != nil || !started { + return false, err } run.Status, run.Owner, run.LeaseUntil = types.AutomationRunRunning, s.instance, &lease if run.StartedAt == nil { @@ -266,20 +282,31 @@ func (s *AutomationService) reconcileAwaiting(ctx context.Context, now time.Time if s.broker == nil { return nil } - var waiting []*types.AutomationRun - if err := s.db.WithContext(ctx).Where("status = ?", types.AutomationRunAwaitingApproval). - Order("updated_at").Limit(automationClaimBatch * 2).Find(&waiting).Error; err != nil { - return err - } - for _, run := range waiting { - if err := s.reconcileRun(ctx, run, now); err != nil { - logger.Warnf(ctx, "Reconcile automation run %s: %v", run.ID, err) + // Keyset pagination visits later approvals even when earlier ones stay + // open. Offsets would skip rows as reconciled runs leave the waiting set. + for cursor := ""; ; { + var waiting []*types.AutomationRun + if err := s.db.WithContext(ctx). + Where("status = ? AND id > ?", types.AutomationRunAwaitingApproval, cursor). + Order("id").Limit(automationClaimBatch * 2).Find(&waiting).Error; err != nil { + return err } + for _, run := range waiting { + if err := s.reconcileRun(ctx, run, now); err != nil { + logger.Warnf(ctx, "Reconcile automation run %s: %v", run.ID, err) + } + } + if len(waiting) < automationClaimBatch*2 { + return nil + } + cursor = waiting[len(waiting)-1].ID } - return nil } func (s *AutomationService) reconcileRun(ctx context.Context, run *types.AutomationRun, now time.Time) error { + if run.CancelRequested { + return s.cancelRun(ctx, run, "cancelled by the owner") + } waiting := []string{types.AutomationRunAwaitingApproval} prompt, err := s.broker.Store().Get(ctx, run.PendingPromptID) if err != nil { @@ -324,7 +351,8 @@ func (s *AutomationService) reconcileRun(ctx context.Context, run *types.Automat // queue, to continue from the answer. func (s *AutomationService) queueResume(ctx context.Context, runID, promptID string) error { res := s.db.WithContext(ctx).Model(&types.AutomationRun{}). - Where("id = ? AND status = ? AND pending_prompt_id = ?", runID, types.AutomationRunAwaitingApproval, promptID). + Where("id = ? AND status = ? AND pending_prompt_id = ? AND cancel_requested = ?", + runID, types.AutomationRunAwaitingApproval, promptID, false). Updates(map[string]any{"status": types.AutomationRunQueued, "resume": true, "updated_at": s.now()}) if res.Error == nil && res.RowsAffected > 0 { s.Wake() @@ -407,8 +435,15 @@ func (s *AutomationService) finishRun( return false, res.Error } run.Status, run.Error, run.FinishedAt = status, reason, &now + // Only the winning finalizer may stop work. A stale approval reaper must + // not cancel a run that another scheduler has already resumed. + var stopErr error + if status == types.AutomationRunFailed || status == types.AutomationRunCancelled || + status == types.AutomationRunNeedsAttention { + stopErr = s.stopRunDelegation(context.WithoutCancel(ctx), run) + } s.settle(ctx, run) - return true, nil + return true, stopErr } // settle records a finished run on its automation, pauses it after too many diff --git a/internal/application/service/session_agent_qa.go b/internal/application/service/session_agent_qa.go index 22db2983..92448798 100644 --- a/internal/application/service/session_agent_qa.go +++ b/internal/application/service/session_agent_qa.go @@ -431,6 +431,7 @@ func (s *sessionService) AgentQA( // run elsewhere. Session, attachments and images are reloaded from storage. func resumableAgentRequest(ctx context.Context, req *types.QARequest) types.ResumableAgentRequest { r := types.ResumableAgentRequest{ + ActingVia: types.ActingViaFromContext(ctx), Query: req.Query, SummaryModelID: req.SummaryModelID, ReasoningEffort: req.ReasoningEffort, diff --git a/internal/container/container.go b/internal/container/container.go index a8fd408d..8121e8d9 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -655,6 +655,7 @@ func BuildContainer(container *dig.Container) *dig.Container { } automations.Bind(sessionService, broker, runs, streams) automations.Events = sessions.CollaborationEvents + automations.StopDelegation = collaboration.StopOwned broker.SetRunHooks(runs.Hooks()) runs.StopDelegation = func(ctx context.Context, p *types.AgentPrompt) error { return collaboration.StopPrompt(ctx, p) diff --git a/internal/database/migration_source_test.go b/internal/database/migration_source_test.go new file mode 100644 index 00000000..e2be2cf5 --- /dev/null +++ b/internal/database/migration_source_test.go @@ -0,0 +1,22 @@ +package database + +import ( + "path/filepath" + "testing" + + "github.com/golang-migrate/migrate/v4/source" + "github.com/stretchr/testify/require" +) + +// Check both migration registries without requiring a live PostgreSQL server. +// Independently merged migrations must not reuse a version in either dialect. +func TestMigrationSourcesHaveUniqueVersions(t *testing.T) { + root := sqliteRepoRoot(t) + for _, dialect := range []string{"sqlite", "versioned"} { + t.Run(dialect, func(t *testing.T) { + migrations, err := source.Open("file://" + filepath.Join(root, "migrations", dialect)) + require.NoError(t, err) + require.NoError(t, migrations.Close()) + }) + } +} diff --git a/internal/database/migration_sqlite_automation_duration_test.go b/internal/database/migration_sqlite_automation_duration_test.go new file mode 100644 index 00000000..bd357aa9 --- /dev/null +++ b/internal/database/migration_sqlite_automation_duration_test.go @@ -0,0 +1,36 @@ +package database + +import ( + "fmt" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSQLiteAutomationDurationUpgradePreservesExistingRuns(t *testing.T) { + repoRoot := sqliteRepoRoot(t) + for _, version := range []int{45, 47} { + t.Run(fmt.Sprintf("from-v%d", version), func(t *testing.T) { + chdirAndRestore(t, copySQLiteMigrationsThrough(t, repoRoot, version)) + dbPath := filepath.Join(t.TempDir(), "automations.db") + require.NoError(t, RunMigrationsWithOptions("sqlite3://unused", MigrationOptions{SQLiteDBPath: dbPath})) + db := openSQLiteDB(t, dbPath) + _, err := db.Exec(`INSERT INTO automation_runs + (id, automation_id, tenant_id, scheduled_at, trigger_kind, status, answer, created_at, updated_at) + VALUES ('existing-run', 'automation', 1, CURRENT_TIMESTAMP, 'schedule', + 'awaiting_approval', 'saved answer', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + require.NoError(t, err) + + chdirAndRestore(t, repoRoot) + require.NoError(t, RunMigrationsWithOptions("sqlite3://unused", MigrationOptions{SQLiteDBPath: dbPath})) + var duration int64 + var answer, status string + require.NoError(t, db.QueryRow(`SELECT execution_duration_ms, answer, status + FROM automation_runs WHERE id = 'existing-run'`).Scan(&duration, &answer, &status)) + require.Zero(t, duration) + require.Equal(t, "saved answer", answer) + require.Equal(t, "awaiting_approval", status) + }) + } +} diff --git a/internal/database/migration_sqlite_versioned_schema_test.go b/internal/database/migration_sqlite_versioned_schema_test.go index 043b976d..5e72bf99 100644 --- a/internal/database/migration_sqlite_versioned_schema_test.go +++ b/internal/database/migration_sqlite_versioned_schema_test.go @@ -52,6 +52,7 @@ var versionedSQLiteTables = []string{ // versionedSQLiteColumns maps each existing table to the columns that the // versioned migrations add and the SQLite baseline was missing. var versionedSQLiteColumns = map[string][]string{ + "automation_runs": {"execution_duration_ms"}, // 000048 "model_catalog_configs": {"version", "overlay", "history", "updated_by", "updated_at"}, // 000031 "memory_subjects": {"extraction_state"}, // 000094 "memory_items": {"replaces_id"}, // 000094 @@ -86,7 +87,7 @@ var versionedSQLiteColumns = map[string][]string{ }, // 000028 } -const expectedSQLiteMigrationVersion = 47 +const expectedSQLiteMigrationVersion = 48 func TestSQLiteMigrationsCreateVersionedSchema(t *testing.T) { repoRoot := sqliteRepoRoot(t) @@ -262,19 +263,20 @@ func TestSQLiteSandboxAllocationMigrationRollsBack(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { _, _ = m.Close() }) - require.NoError(t, m.Steps(-1)) + // Target the allocation migration even when newer migrations exist. + require.NoError(t, m.Migrate(46)) version, dirty = sqliteMigrationState(t, sqlDB) - require.Equal(t, expectedSQLiteMigrationVersion-1, version) + require.Equal(t, 46, version) require.False(t, dirty) for _, table := range []string{"session_sandbox_assignments", "sandbox_user_machines"} { require.Falsef(t, sqliteTableExists(t, sqlDB, table), "down must drop %s", table) } require.True(t, sqliteColumnExists(t, sqlDB, "sessions", "owner_auth_source"), - "rolling back the allocation tables keeps the owner source of 000034") + "rolling back the allocation tables keeps the owner source of 000046") - require.NoError(t, m.Steps(1)) + require.NoError(t, m.Migrate(47)) version, dirty = sqliteMigrationState(t, sqlDB) - require.Equal(t, expectedSQLiteMigrationVersion, version) + require.Equal(t, 47, version) require.False(t, dirty) require.True(t, sqliteTableExists(t, sqlDB, "session_sandbox_assignments")) require.True(t, sqliteTableExists(t, sqlDB, "sandbox_user_machines")) diff --git a/internal/types/agent_prompt.go b/internal/types/agent_prompt.go index 1365f3b4..8a47fbed 100644 --- a/internal/types/agent_prompt.go +++ b/internal/types/agent_prompt.go @@ -168,8 +168,10 @@ func (p *AgentPrompt) TimelineView() *PendingPromptView { // ResumableAgentRequest is the part of a QARequest needed to rebuild the run // on another replica. It holds the original query and durable request settings. type ResumableAgentRequest struct { - AgentID string `json:"agent_id"` - Query string `json:"query,omitempty"` + // ActingVia keeps scheduler-owned runs out of ordinary chat recovery. + ActingVia string `json:"acting_via,omitempty"` + AgentID string `json:"agent_id"` + Query string `json:"query,omitempty"` // AgentTenantID is the agent owner's workspace; it differs from the // session's when the agent is shared. AgentTenantID uint64 `json:"agent_tenant_id,omitempty"` diff --git a/internal/types/automation.go b/internal/types/automation.go index a88ba78a..316762fe 100644 --- a/internal/types/automation.go +++ b/internal/types/automation.go @@ -170,6 +170,9 @@ type AutomationRun struct { Resume bool `json:"-" gorm:"not null;default:false"` // CancelRequested stops the run at its next check. CancelRequested bool `json:"cancel_requested" gorm:"not null;default:false"` + // ExecutionDurationMS accumulates execution segments across approvals; + // queued time and time spent waiting for an answer do not consume it. + ExecutionDurationMS int64 `json:"-" gorm:"not null;default:0"` // Answer is the start of the final answer, for listings. Answer string `json:"answer" gorm:"type:text;not null;default:''" extensions:"x-agent,x-agent-response"` Error string `json:"error" gorm:"type:text;not null;default:''" extensions:"x-agent,x-agent-response"` diff --git a/migrations/sqlite/000048_automation_execution_duration.down.sql b/migrations/sqlite/000048_automation_execution_duration.down.sql new file mode 100644 index 00000000..1193a795 --- /dev/null +++ b/migrations/sqlite/000048_automation_execution_duration.down.sql @@ -0,0 +1 @@ +ALTER TABLE automation_runs DROP COLUMN execution_duration_ms; diff --git a/migrations/sqlite/000048_automation_execution_duration.up.sql b/migrations/sqlite/000048_automation_execution_duration.up.sql new file mode 100644 index 00000000..1d44f81c --- /dev/null +++ b/migrations/sqlite/000048_automation_execution_duration.up.sql @@ -0,0 +1 @@ +ALTER TABLE automation_runs ADD COLUMN execution_duration_ms BIGINT NOT NULL DEFAULT 0; diff --git a/migrations/versioned/000131_automation_execution_duration.down.sql b/migrations/versioned/000131_automation_execution_duration.down.sql new file mode 100644 index 00000000..1193a795 --- /dev/null +++ b/migrations/versioned/000131_automation_execution_duration.down.sql @@ -0,0 +1 @@ +ALTER TABLE automation_runs DROP COLUMN execution_duration_ms; diff --git a/migrations/versioned/000131_automation_execution_duration.up.sql b/migrations/versioned/000131_automation_execution_duration.up.sql new file mode 100644 index 00000000..1d44f81c --- /dev/null +++ b/migrations/versioned/000131_automation_execution_duration.up.sql @@ -0,0 +1 @@ +ALTER TABLE automation_runs ADD COLUMN execution_duration_ms BIGINT NOT NULL DEFAULT 0;