fix(automation): enforce execution identity, cancellation, and scheduling limits (#4506)
* fix(automation): preserve execution identity and scheduler guarantees * fix(ci): satisfy lint checks for automation regressionsmain
parent
8dab7550f5
commit
a8eb7e28fa
|
|
@ -184,12 +184,37 @@ func (s *AgentCollaboration) prune(ctx context.Context) error {
|
|||
type delegatedConfig struct {
|
||||
Agent types.CustomAgent `json:"agent"`
|
||||
Access types.AssistantAccessLevel `json:"access"`
|
||||
// Durable execution identity follows children across scheduler instances
|
||||
// and approval resumes; it must not turn an automation into a human turn.
|
||||
ActingVia string `json:"acting_via,omitempty"`
|
||||
Mode types.RunMode `json:"mode,omitempty"`
|
||||
// AssistantParent records that the delegating run was the WeKnora
|
||||
// assistant, whose level stays under the workspace's assistant ceiling.
|
||||
AssistantParent bool `json:"assistant_parent,omitempty"`
|
||||
Inherited *inheritedAgentConfig `json:"inherited,omitempty"`
|
||||
}
|
||||
|
||||
func (d *delegatedConfig) captureExecution(ctx context.Context) {
|
||||
d.ActingVia = types.ActingViaFromContext(ctx)
|
||||
d.Mode = types.RunModeAttended
|
||||
if mode, ok := ctx.Value(types.RunModeContextKey).(types.RunMode); ok {
|
||||
d.Mode = mode.Normalize()
|
||||
}
|
||||
if d.ActingVia != "" {
|
||||
d.Mode = types.RunModeDeferred
|
||||
}
|
||||
}
|
||||
|
||||
func (d *delegatedConfig) runMode() types.RunMode {
|
||||
if d.ActingVia != "" {
|
||||
return types.RunModeDeferred
|
||||
}
|
||||
if d.Mode == "" {
|
||||
return types.RunModeAttended // snapshots created before mode was stored
|
||||
}
|
||||
return d.Mode.Normalize()
|
||||
}
|
||||
|
||||
// level is the child's access level: the delegating run's, capped by the
|
||||
// child agent's own ceiling and, under an assistant parent, by the
|
||||
// workspace's current assistant ceiling.
|
||||
|
|
@ -222,7 +247,9 @@ func (s *AgentCollaboration) resolveDelegated(
|
|||
id == types.BuiltinSkillInstallerID {
|
||||
return "", errors.New("only same-workspace reasoning agents without management privileges may be delegated")
|
||||
}
|
||||
data, err := json.Marshal(delegatedConfig{Agent: *a, Access: level, AssistantParent: assistantParent})
|
||||
saved := delegatedConfig{Agent: *a, Access: level, AssistantParent: assistantParent}
|
||||
saved.captureExecution(ctx)
|
||||
data, err := json.Marshal(saved)
|
||||
return string(data), err
|
||||
}
|
||||
|
||||
|
|
@ -377,7 +404,11 @@ func (s *AgentCollaboration) workerContext(ctx context.Context, scope collaborat
|
|||
ctx = context.WithValue(ctx, types.TenantRoleContextKey, membership.Role)
|
||||
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenant)
|
||||
ctx = types.WithCaller(ctx, types.Caller{TenantID: scope.TenantID, UserID: scope.UserID, Role: membership.Role})
|
||||
ctx = types.WithPrincipal(ctx, types.Principal{Type: types.PrincipalWebUser, ID: scope.UserID})
|
||||
principal := types.Principal{Type: types.PrincipalWebUser, ID: scope.UserID}
|
||||
if previous, ok := types.PrincipalFromContext(ctx); ok && previous.ID == scope.UserID {
|
||||
principal.ActingVia = previous.ActingVia
|
||||
}
|
||||
ctx = types.WithPrincipal(ctx, principal)
|
||||
return ctx, nil
|
||||
}
|
||||
|
||||
|
|
@ -397,6 +428,10 @@ func (s *AgentCollaboration) run(ctx context.Context, w collaboration.Work) (col
|
|||
if err = json.Unmarshal([]byte(w.Member.ConfigJSON), &saved); err != nil {
|
||||
return result, err
|
||||
}
|
||||
ctx = types.WithPrincipal(ctx, types.Principal{
|
||||
Type: types.PrincipalWebUser, ID: w.Scope.UserID, ActingVia: saved.ActingVia,
|
||||
})
|
||||
ctx = types.WithRunMode(ctx, saved.runMode())
|
||||
// Both modes pin configuration, but reauthorize the source before each turn.
|
||||
if saved.Inherited == nil {
|
||||
if _, err = s.resolve(ctx, w.Member.AgentID, types.AssistantAccessReadOnly); err != nil {
|
||||
|
|
@ -610,7 +645,7 @@ func (s *AgentCollaboration) run(ctx context.Context, w collaboration.Work) (col
|
|||
ctx,
|
||||
&types.QARequest{
|
||||
Resume: resume,
|
||||
Mode: types.RunModeAttended,
|
||||
Mode: saved.runMode(),
|
||||
GrantSessionID: w.Scope.SessionID,
|
||||
Suspendable: true,
|
||||
Session: session,
|
||||
|
|
|
|||
|
|
@ -75,10 +75,12 @@ func inheritedChildSnapshot(ctx context.Context, cfg *types.AgentConfig, toolNam
|
|||
}
|
||||
}
|
||||
// Marshal immediately: later changes to parent config cannot change queued work.
|
||||
data, err := json.Marshal(delegatedConfig{
|
||||
saved := delegatedConfig{
|
||||
Agent: *source.Agent, Access: cfg.AssistantAccess, AssistantParent: cfg.OperationsAgentID != "",
|
||||
Inherited: inherited,
|
||||
})
|
||||
}
|
||||
saved.captureExecution(ctx)
|
||||
data, err := json.Marshal(saved)
|
||||
return string(data), err
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -223,6 +223,11 @@ func (s *AgentCollaboration) resumeParent(ctx context.Context, b *collaboration.
|
|||
if err = json.Unmarshal(snap.Request, &saved); err != nil {
|
||||
return err
|
||||
}
|
||||
if saved.ActingVia != "" {
|
||||
// Automation leases, cancellation and duration belong to their own
|
||||
// scheduler. Never revive one through ordinary parent recovery.
|
||||
return errors.New("scheduler-owned parent cannot resume through collaboration recovery")
|
||||
}
|
||||
if saved.AgentID == "" || saved.SharedAgentReadOnly {
|
||||
return errors.New("invalid parent recovery identity")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -80,6 +80,8 @@ type AutomationService struct {
|
|||
// Events publishes a run to its message's event stream; see
|
||||
// AgentCollaboration.ChildEvents.
|
||||
Events func(context.Context, *types.Session, *types.Message, *event.EventBus) func(context.Context) error
|
||||
// StopDelegation fences children and discards recoverable parent waits.
|
||||
StopDelegation func(context.Context, uint64, string, string, string) error
|
||||
|
||||
cfg AutomationConfig
|
||||
instance string
|
||||
|
|
@ -621,16 +623,37 @@ func (s *AutomationService) cancelRun(ctx context.Context, run *types.Automation
|
|||
Update("cancel_requested", true).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
stopErr := s.stopRunDelegation(ctx, run)
|
||||
s.mu.Lock()
|
||||
cancel := s.running[run.ID]
|
||||
s.mu.Unlock()
|
||||
if cancel != nil {
|
||||
cancel()
|
||||
}
|
||||
return stopErr
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *AutomationService) stopRunDelegation(ctx context.Context, run *types.AutomationRun) error {
|
||||
if s.StopDelegation == nil {
|
||||
return nil
|
||||
}
|
||||
// The caller may hold the row from before the executor created its turn.
|
||||
var current types.AutomationRun
|
||||
if err := s.db.WithContext(ctx).Where("id = ?", run.ID).Take(¤t).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if current.SessionID == "" || current.AssistantMessageID == "" {
|
||||
return nil
|
||||
}
|
||||
a, err := s.automationOf(ctx, ¤t)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.StopDelegation(ctx, a.TenantID, a.OwnerUserID, current.SessionID, current.AssistantMessageID)
|
||||
}
|
||||
|
||||
// closePrompt ends the open prompt id a run waits on and completes the
|
||||
// run's message, as the prompt reaper would.
|
||||
func (s *AutomationService) closePrompt(ctx context.Context, id, status, reason string) {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,462 @@
|
|||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/agent"
|
||||
"github.com/Tencent/WeKnora/internal/agent/collaboration"
|
||||
"github.com/Tencent/WeKnora/internal/event"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/logger"
|
||||
)
|
||||
|
||||
func TestAutomationDelegatedExecutionKeepsIdentity(t *testing.T) {
|
||||
for _, inherited := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("inherited=%t", inherited), func(t *testing.T) {
|
||||
svc, agents, _, ctx := collaborationService(t)
|
||||
via := types.AutomationActingVia("scheduled-task")
|
||||
parent := types.WithPrincipal(ctx, types.Principal{Type: types.PrincipalWebUser, ID: "u", ActingVia: via})
|
||||
parent = types.WithRunMode(parent, types.RunModeDeferred)
|
||||
var snapshot string
|
||||
var err error
|
||||
if inherited {
|
||||
parent = context.WithValue(parent, childSourceKey{}, childSource{Agent: agents.agent, ModelID: "model"})
|
||||
snapshot, err = inheritedChildSnapshot(
|
||||
parent,
|
||||
&types.AgentConfig{AssistantAccess: types.AssistantAccessAutoWrite},
|
||||
nil,
|
||||
)
|
||||
} else {
|
||||
snapshot, err = svc.resolve(parent, "a", types.AssistantAccessAutoWrite)
|
||||
}
|
||||
require.NoError(t, err)
|
||||
scope := collaboration.Scope{TenantID: automationTenant, UserID: "u", SessionID: "parent", ID: "answer"}
|
||||
require.NoError(t, svc.Runtime.Store.Ensure(ctx, scope))
|
||||
_, err = svc.Runtime.Spawn(ctx, scope, "spawn", "child", "a", snapshot, "work")
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, svc.Runtime.Store.Change(ctx, scope, func(g *collaboration.Group) error {
|
||||
r := g.Members[0].Runs[0]
|
||||
r.State = "running"
|
||||
r.Owner = "worker"
|
||||
r.Lease = time.Now().Add(time.Minute)
|
||||
return nil
|
||||
}))
|
||||
called := false
|
||||
svc.Bind(
|
||||
&collaborationSessions{
|
||||
run: func(child context.Context, req *types.QARequest, bus *event.EventBus) error {
|
||||
called = true
|
||||
require.Equal(t, via, types.ActingViaFromContext(child))
|
||||
require.Equal(t, types.RunModeDeferred, req.Mode)
|
||||
require.False(t, req.Mode.Attended(), "ask_user must stay unavailable")
|
||||
_, _, err := automationOwner(child)
|
||||
require.Error(t, err, "delegation must not bypass the automation management guard")
|
||||
return complete(child, bus, event.AgentCompleteData{FinalAnswer: "done"})
|
||||
},
|
||||
},
|
||||
)
|
||||
g, err := svc.Runtime.Store.Read(ctx, scope)
|
||||
require.NoError(t, err)
|
||||
// A scheduler starts from its own context, not the parent's request context.
|
||||
_, err = svc.run(
|
||||
context.Background(),
|
||||
collaboration.Work{Scope: scope, Member: *g.Members[0], Run: *g.Members[0].Runs[0]},
|
||||
)
|
||||
require.NoError(t, err)
|
||||
require.True(t, called)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAutomationCancelStopsDelegationAndRecovery(t *testing.T) {
|
||||
f := newAutomationFixture(t)
|
||||
require.NoError(t, f.db.AutoMigrate(&collaboration.Record{}))
|
||||
delegated := NewAgentCollaboration(f.db, f.agents, f.messages, f.users, f.members, f.svc.tenants)
|
||||
f.svc.StopDelegation = delegated.StopOwned
|
||||
a := f.create(t, types.AutomationPatch{})
|
||||
queued, err := f.svc.RunNow(f.ctx, a.ID)
|
||||
require.NoError(t, err)
|
||||
entered := make(chan collaboration.Scope, 1)
|
||||
f.sessions.run = func(ctx context.Context, req *types.QARequest, _ *event.EventBus) error {
|
||||
scope := collaboration.Scope{
|
||||
TenantID: automationTenant,
|
||||
UserID: "u",
|
||||
SessionID: req.Session.ID,
|
||||
ID: req.AssistantMessageID,
|
||||
}
|
||||
b, err := delegated.Runtime.AcquireParent(ctx, scope)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = delegated.Runtime.Spawn(b.Context(ctx), scope, "spawn", "child", "a", "{}", "work")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = delegated.Runtime.Spawn(b.Context(ctx), scope, "spawn-running", "running_child", "a", "{}", "work")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = delegated.Runtime.Store.Change(b.Context(ctx), scope, func(g *collaboration.Group) error {
|
||||
r := g.Members[1].Runs[0]
|
||||
r.State, r.Owner, r.Lease = "running", "remote-worker", time.Now().Add(time.Minute)
|
||||
return nil
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
saved := resumableAgentRequest(ctx, req)
|
||||
data, err := json.Marshal(
|
||||
agent.RunSnapshot{Version: agent.RunSnapshotVersion, Request: mustJSON(t, saved), CollaborationWait: true},
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = b.SaveWait(ctx, data, time.Now().Add(time.Minute)); err != nil {
|
||||
return err
|
||||
}
|
||||
entered <- scope
|
||||
<-ctx.Done()
|
||||
// Exercise the same cleanup as an engine cancelled while in wait_agent.
|
||||
b.End(true)
|
||||
return ctx.Err()
|
||||
}
|
||||
require.NoError(t, f.svc.startQueued(f.ctx, f.now))
|
||||
var scope collaboration.Scope
|
||||
select {
|
||||
case scope = <-entered:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("run did not start")
|
||||
}
|
||||
_, err = f.svc.CancelRun(f.ctx, queued.ID)
|
||||
require.NoError(t, err)
|
||||
done := f.settleRun(t, queued.ID)
|
||||
require.Equal(t, types.AutomationRunCancelled, done.Status)
|
||||
g, err := delegated.Runtime.Store.Read(f.ctx, scope)
|
||||
require.NoError(t, err)
|
||||
require.NotEqual(t, "active", g.Status)
|
||||
require.Nil(t, g.ParentWait, "cancelled checkpoints must not be recovered")
|
||||
require.True(t, g.Members[0].Runs[0].CancelRequested)
|
||||
require.Equal(t, "cancelled", g.Members[0].Runs[0].State)
|
||||
require.True(t, g.Members[1].Runs[0].CancelRequested)
|
||||
require.ErrorIs(t, delegated.Runtime.CheckOwner(f.ctx, collaboration.Work{
|
||||
Scope: scope, Member: *g.Members[1], Run: *g.Members[1].Runs[0],
|
||||
}), collaboration.ErrConflict, "a remote child must lose permission to execute tools immediately")
|
||||
}
|
||||
|
||||
func mustJSON(t *testing.T, v any) json.RawMessage {
|
||||
t.Helper()
|
||||
data, err := json.Marshal(v)
|
||||
require.NoError(t, err)
|
||||
return data
|
||||
}
|
||||
|
||||
func TestAutomationParentRecoveryDoesNotEscapeScheduler(t *testing.T) {
|
||||
svc, _, messages, ctx := collaborationService(t)
|
||||
scope := collaboration.Scope{TenantID: automationTenant, UserID: "u", SessionID: "parent", ID: "answer"}
|
||||
b, err := svc.Runtime.AcquireParent(ctx, scope)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { b.End(false) })
|
||||
require.NoError(t, messages.UpdateMessage(ctx, &types.Message{ID: scope.ID, SessionID: scope.SessionID}))
|
||||
svc.streams = &collaborationStreams{}
|
||||
svc.ParentEvents = func(
|
||||
context.Context, *types.Session, *types.Message, *event.EventBus,
|
||||
) func(context.Context) error {
|
||||
return func(context.Context) error { return nil }
|
||||
}
|
||||
called := false
|
||||
svc.Bind(
|
||||
&collaborationSessions{
|
||||
run: func(context.Context, *types.QARequest, *event.EventBus) error { called = true; return nil },
|
||||
},
|
||||
)
|
||||
ctx = types.WithPrincipal(
|
||||
ctx,
|
||||
types.Principal{Type: types.PrincipalWebUser, ID: "u", ActingVia: types.AutomationActingVia("a")},
|
||||
)
|
||||
req := resumableAgentRequest(
|
||||
ctx,
|
||||
&types.QARequest{CustomAgent: &types.CustomAgent{ID: "a"}, Mode: types.RunModeDeferred},
|
||||
)
|
||||
snapshot := mustJSON(
|
||||
t,
|
||||
agent.RunSnapshot{Version: agent.RunSnapshotVersion, Request: mustJSON(t, req), CollaborationWait: true},
|
||||
)
|
||||
err = svc.resumeParent(context.Background(), b, snapshot)
|
||||
require.ErrorContains(t, err, "scheduler-owned parent")
|
||||
require.False(t, called)
|
||||
}
|
||||
|
||||
func TestAutomationStartSerializesDifferentRunsAcrossInstances(t *testing.T) {
|
||||
f := newAutomationFixture(t)
|
||||
a := f.create(t, types.AutomationPatch{OverlapPolicy: ptr(types.AutomationPolicyQueue)})
|
||||
rows := []*types.AutomationRun{
|
||||
{
|
||||
AutomationID: a.ID,
|
||||
TenantID: automationTenant,
|
||||
ScheduledAt: f.now.Add(-time.Hour),
|
||||
TriggerKind: types.AutomationRunSchedule,
|
||||
Status: types.AutomationRunQueued,
|
||||
Resume: true,
|
||||
},
|
||||
{
|
||||
AutomationID: a.ID,
|
||||
TenantID: automationTenant,
|
||||
ScheduledAt: f.now,
|
||||
TriggerKind: types.AutomationRunSchedule,
|
||||
Status: types.AutomationRunQueued,
|
||||
},
|
||||
}
|
||||
for _, r := range rows {
|
||||
require.NoError(t, f.db.Create(r).Error)
|
||||
}
|
||||
// A separate connection pool represents another scheduler process.
|
||||
db, err := gorm.Open(
|
||||
sqlite.Open(f.db.Dialector.(*sqlite.Dialector).DSN),
|
||||
&gorm.Config{Logger: logger.Default.LogMode(logger.Silent)},
|
||||
)
|
||||
require.NoError(t, err)
|
||||
raw, err := db.DB()
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = raw.Close() })
|
||||
other := &AutomationService{db: db, instance: "second-instance"}
|
||||
arrived := make(chan struct{}, 2)
|
||||
release := make(chan struct{})
|
||||
for _, db := range []*gorm.DB{f.db, db} {
|
||||
require.NoError(t, db.Callback().Query().After("gorm:query").Register("admission_barrier", func(db *gorm.DB) {
|
||||
if n, ok := db.Statement.Dest.(*int64); ok && *n == 0 && db.Statement.Table == "automation_runs" &&
|
||||
strings.Contains(db.Statement.SQL.String(), "count(*)") {
|
||||
arrived <- struct{}{}
|
||||
<-release
|
||||
}
|
||||
}))
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
started := make([]bool, 2)
|
||||
errs := make([]error, 2)
|
||||
go func() { defer wg.Done(); started[0], errs[0] = f.svc.startRun(f.ctx, rows[0], f.now) }()
|
||||
select {
|
||||
case <-arrived:
|
||||
case <-time.After(5 * time.Second):
|
||||
close(release)
|
||||
t.Fatal("first admission did not reach check")
|
||||
}
|
||||
go func() { defer wg.Done(); started[1], errs[1] = other.startRun(f.ctx, rows[1], f.now) }()
|
||||
// The old implementation lets both count queries observe zero. With the
|
||||
// automation lock the second query cannot run until the first commits.
|
||||
select {
|
||||
case <-arrived:
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
close(release)
|
||||
wg.Wait()
|
||||
require.NoError(t, errs[0])
|
||||
require.NoError(t, errs[1])
|
||||
require.True(t, started[0])
|
||||
require.False(t, started[1])
|
||||
var active int64
|
||||
require.NoError(
|
||||
t,
|
||||
f.db.Model(&types.AutomationRun{}).
|
||||
Where("automation_id = ? AND status = ?", a.ID, types.AutomationRunRunning).
|
||||
Count(&active).
|
||||
Error,
|
||||
)
|
||||
require.EqualValues(t, 1, active)
|
||||
}
|
||||
|
||||
func TestAutomationQueuedRunsSkipBlockedHead(t *testing.T) {
|
||||
f := newAutomationFixture(t)
|
||||
for i := 0; i < 16; i++ {
|
||||
a := f.create(
|
||||
t,
|
||||
types.AutomationPatch{
|
||||
Name: ptr(fmt.Sprintf("blocked%d", i)),
|
||||
OverlapPolicy: ptr(types.AutomationPolicyQueue),
|
||||
},
|
||||
)
|
||||
for j, status := range []string{types.AutomationRunAwaitingApproval, types.AutomationRunQueued} {
|
||||
require.NoError(
|
||||
t,
|
||||
f.db.Create(
|
||||
&types.AutomationRun{
|
||||
AutomationID: a.ID,
|
||||
TenantID: automationTenant,
|
||||
ScheduledAt: f.now.Add(time.Duration(j-2) * time.Hour),
|
||||
TriggerKind: types.AutomationRunSchedule,
|
||||
Status: status,
|
||||
},
|
||||
).Error,
|
||||
)
|
||||
}
|
||||
}
|
||||
a := f.create(t, types.AutomationPatch{Name: ptr("runnable")})
|
||||
r, err := f.svc.RunNow(f.ctx, a.ID)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, f.svc.startQueued(context.Background(), f.now))
|
||||
require.Eventually(t, func() bool {
|
||||
var row types.AutomationRun
|
||||
return f.db.First(&row, "id = ?", r.ID).Error == nil && row.Status == types.AutomationRunSucceeded &&
|
||||
len(f.svc.slots) == 0
|
||||
}, 5*time.Second, 10*time.Millisecond)
|
||||
}
|
||||
|
||||
func TestAutomationApprovalReconciliationVisitsEveryBatch(t *testing.T) {
|
||||
f := newAutomationFixture(t)
|
||||
f.svc.cfg.MaxPerTenant = 200
|
||||
var expired, answered *types.AutomationRun
|
||||
for i := 0; i < 102; i++ {
|
||||
timeout := 10080
|
||||
updated := f.now.Add(-time.Hour)
|
||||
if i == 100 {
|
||||
timeout = 5
|
||||
updated = f.now.Add(-10 * time.Minute)
|
||||
}
|
||||
a := f.create(
|
||||
t,
|
||||
types.AutomationPatch{Name: ptr(fmt.Sprintf("approval%d", i)), ApprovalTimeoutMinutes: ptr(timeout)},
|
||||
)
|
||||
status := types.AgentPromptSuspended
|
||||
if i == 0 || i == 101 {
|
||||
status = types.AgentPromptAnswered
|
||||
}
|
||||
p := &types.AgentPrompt{ID: fmt.Sprintf("p%d", i), Status: status}
|
||||
require.NoError(t, f.broker.Store().Create(f.ctx, p))
|
||||
r := &types.AutomationRun{
|
||||
ID: fmt.Sprintf("run-%03d", i),
|
||||
AutomationID: a.ID,
|
||||
TenantID: automationTenant,
|
||||
ScheduledAt: updated,
|
||||
TriggerKind: types.AutomationRunSchedule,
|
||||
Status: types.AutomationRunAwaitingApproval,
|
||||
PendingPromptID: p.ID,
|
||||
UpdatedAt: updated,
|
||||
}
|
||||
require.NoError(t, f.db.Create(r).Error)
|
||||
if i == 100 {
|
||||
expired = r
|
||||
}
|
||||
if i == 101 {
|
||||
answered = r
|
||||
}
|
||||
}
|
||||
require.NoError(t, f.svc.reconcileAwaiting(f.ctx, f.now))
|
||||
require.NoError(t, f.db.First(expired, "id = ?", expired.ID).Error)
|
||||
require.Equal(t, types.AutomationRunFailed, expired.Status)
|
||||
require.NoError(t, f.db.First(answered, "id = ?", answered.ID).Error)
|
||||
require.Equal(t, types.AutomationRunQueued, answered.Status)
|
||||
require.True(t, answered.Resume)
|
||||
}
|
||||
|
||||
func TestAutomationApprovalResumeUsesRemainingExecutionBudget(t *testing.T) {
|
||||
f := newAutomationFixture(t)
|
||||
f.svc.cfg.DefaultMaxDuration = time.Second
|
||||
a := f.create(t, types.AutomationPatch{})
|
||||
f.suspendOnApproval(t, "op:x")
|
||||
original := f.sessions.run
|
||||
f.sessions.run = func(ctx context.Context, req *types.QARequest, bus *event.EventBus) error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(600 * time.Millisecond):
|
||||
}
|
||||
return original(ctx, req, bus)
|
||||
}
|
||||
queued, err := f.svc.RunNow(f.ctx, a.ID)
|
||||
require.NoError(t, err)
|
||||
suspended := f.settleRun(t, queued.ID)
|
||||
require.Equal(t, types.AutomationRunAwaitingApproval, suspended.Status)
|
||||
// Approval waiting is free, even across service restarts.
|
||||
f.now = f.now.Add(12 * time.Hour)
|
||||
restarted := NewAutomationService(
|
||||
f.db,
|
||||
f.agents,
|
||||
f.users,
|
||||
f.members,
|
||||
f.svc.tenants,
|
||||
f.messages,
|
||||
f.svc.grants,
|
||||
f.svc.notices,
|
||||
)
|
||||
restarted.Configure(AutomationConfig{DefaultMaxDuration: time.Second})
|
||||
restarted.now = func() time.Time { return f.now }
|
||||
restarted.Bind(f.sessions, f.broker, nil, nil)
|
||||
f.svc = restarted
|
||||
ok, err := f.broker.Store().Transition(f.ctx, suspended.PendingPromptID, types.AgentPromptTransition{
|
||||
From: []string{types.AgentPromptSuspended},
|
||||
To: types.AgentPromptAnswered,
|
||||
Response: types.JSON(`{"approved":true}`),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.True(t, ok)
|
||||
resumed := f.settleRun(t, queued.ID)
|
||||
require.Equal(t, types.AutomationRunFailed, resumed.Status)
|
||||
require.Contains(t, resumed.Error, "longer than 1s")
|
||||
require.NotNil(t, f.sessions.last().req.Resume, "waiting time must not exhaust the execution budget")
|
||||
}
|
||||
|
||||
func TestAutomationStaleFinalizerDoesNotCancelResumedDelegation(t *testing.T) {
|
||||
f := newAutomationFixture(t)
|
||||
a := f.create(t, types.AutomationPatch{})
|
||||
r, err := f.svc.RunNow(f.ctx, a.ID)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, f.db.Model(r).Updates(map[string]any{
|
||||
"status": types.AutomationRunRunning, "session_id": "session", "assistant_message_id": "answer",
|
||||
}).Error)
|
||||
stopped := false
|
||||
f.svc.StopDelegation = func(context.Context, uint64, string, string, string) error { stopped = true; return nil }
|
||||
ok, err := f.svc.finishRun(
|
||||
f.ctx,
|
||||
r,
|
||||
[]string{types.AutomationRunAwaitingApproval},
|
||||
types.AutomationRunFailed,
|
||||
"stale timeout",
|
||||
nil,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
require.False(t, ok)
|
||||
require.False(t, stopped)
|
||||
ok, err = f.svc.finishRun(
|
||||
f.ctx,
|
||||
r,
|
||||
[]string{types.AutomationRunRunning},
|
||||
types.AutomationRunNeedsAttention,
|
||||
"executor lost",
|
||||
nil,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
require.True(t, ok)
|
||||
require.True(t, stopped)
|
||||
}
|
||||
|
||||
func TestAutomationCancellationWinsApprovalRace(t *testing.T) {
|
||||
f := newAutomationFixture(t)
|
||||
a := f.create(t, types.AutomationPatch{})
|
||||
f.suspendOnApproval(t, "op:x")
|
||||
queued, err := f.svc.RunNow(f.ctx, a.ID)
|
||||
require.NoError(t, err)
|
||||
r := f.settleRun(t, queued.ID)
|
||||
require.Equal(t, types.AutomationRunAwaitingApproval, r.Status)
|
||||
// A remote cancellation can land after execute checked its context but
|
||||
// before it records the approval suspension. Do not resume that run.
|
||||
require.NoError(t, f.db.Model(r).Update("cancel_requested", true).Error)
|
||||
ok, err := f.broker.Store().Transition(f.ctx, r.PendingPromptID, types.AgentPromptTransition{
|
||||
From: []string{types.AgentPromptSuspended},
|
||||
To: types.AgentPromptAnswered,
|
||||
Response: types.JSON(`{"approved":true}`),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.True(t, ok)
|
||||
require.NoError(t, f.svc.queueResume(f.ctx, r.ID, r.PendingPromptID))
|
||||
require.NoError(t, f.svc.reconcileAwaiting(f.ctx, f.now))
|
||||
require.NoError(t, f.db.First(&r, "id = ?", r.ID).Error)
|
||||
require.Equal(t, types.AutomationRunCancelled, r.Status)
|
||||
}
|
||||
|
|
@ -36,6 +36,7 @@ type runOutcome struct {
|
|||
answer string
|
||||
usage *types.TokenUsage
|
||||
promptID string
|
||||
elapsed time.Duration
|
||||
// pause, when set, pauses the automation with this reason.
|
||||
pause string
|
||||
}
|
||||
|
|
@ -56,6 +57,7 @@ func (s *AutomationService) execute(parent context.Context, run *types.Automatio
|
|||
}()
|
||||
go s.heartbeat(ctx, run.ID, cancel)
|
||||
|
||||
started := time.Now()
|
||||
out := func() (out runOutcome) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
|
|
@ -64,11 +66,15 @@ func (s *AutomationService) execute(parent context.Context, run *types.Automatio
|
|||
}()
|
||||
return s.runOnce(ctx, run)
|
||||
}()
|
||||
if ctx.Err() != nil && out.status != types.AutomationRunAwaitingApproval {
|
||||
out.elapsed = time.Since(started)
|
||||
if ctx.Err() != nil {
|
||||
var row types.AutomationRun
|
||||
if err := s.db.WithContext(context.WithoutCancel(ctx)).Select("cancel_requested").
|
||||
Where("id = ?", run.ID).Take(&row).Error; err == nil && row.CancelRequested {
|
||||
out.status, out.reason = types.AutomationRunCancelled, "cancelled by the owner"
|
||||
if out.promptID != "" {
|
||||
s.closePrompt(context.WithoutCancel(ctx), out.promptID, types.AgentPromptCanceled, out.reason)
|
||||
}
|
||||
}
|
||||
}
|
||||
s.record(ctx, run, out)
|
||||
|
|
@ -109,7 +115,10 @@ func (s *AutomationService) heartbeat(ctx context.Context, runID string, cancel
|
|||
func (s *AutomationService) record(ctx context.Context, run *types.AutomationRun, out runOutcome) {
|
||||
ctx = context.WithoutCancel(ctx)
|
||||
run.Answer = clipRunes(out.answer, maxStoredAnswerRunes)
|
||||
updates := map[string]any{"answer": run.Answer, "updated_at": s.now()}
|
||||
updates := map[string]any{
|
||||
"answer": run.Answer, "updated_at": s.now(),
|
||||
"execution_duration_ms": gorm.Expr("execution_duration_ms + ?", out.elapsed.Milliseconds()),
|
||||
}
|
||||
if out.usage != nil {
|
||||
if data, err := json.Marshal(out.usage); err == nil {
|
||||
updates["usage"] = types.JSON(data)
|
||||
|
|
@ -204,6 +213,7 @@ func (s *AutomationService) ownerContext(ctx context.Context, a *types.Automatio
|
|||
|
||||
// runOnce executes run, or continues it from its answered approval.
|
||||
func (s *AutomationService) runOnce(ctx context.Context, run *types.AutomationRun) runOutcome {
|
||||
started := time.Now()
|
||||
if s.sessions == nil {
|
||||
return failed("the agent executor is unavailable")
|
||||
}
|
||||
|
|
@ -234,7 +244,14 @@ func (s *AutomationService) runOnce(ctx context.Context, run *types.AutomationRu
|
|||
if m := a.Limits.MaxDurationMinutes; m > 0 {
|
||||
limit = time.Duration(m) * time.Minute
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, limit)
|
||||
remaining := limit - time.Duration(run.ExecutionDurationMS)*time.Millisecond - time.Since(started)
|
||||
if remaining <= 0 {
|
||||
if run.Resume && run.PendingPromptID != "" {
|
||||
s.closePrompt(ctx, run.PendingPromptID, types.AgentPromptFailed, "execution time limit exceeded")
|
||||
}
|
||||
return failed(fmt.Sprintf("the run took longer than %s", limit))
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, remaining)
|
||||
defer cancel()
|
||||
|
||||
var resume *types.AgentResume
|
||||
|
|
@ -524,7 +541,10 @@ func (s *AutomationService) agentQA(
|
|||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
answer.IsCompleted = suspended == nil
|
||||
answer.IsCompleted = suspended == nil || err != nil
|
||||
if answer.IsCompleted {
|
||||
answer.RunState = ""
|
||||
}
|
||||
answer.UpdatedAt = s.now()
|
||||
if err != nil && answer.Content == "" {
|
||||
answer.Content = err.Error()
|
||||
|
|
|
|||
|
|
@ -163,6 +163,9 @@ func (s *AutomationService) startQueued(ctx context.Context, now time.Time) erro
|
|||
}
|
||||
var queued []*types.AutomationRun
|
||||
if err := s.db.WithContext(ctx).Where("status = ?", types.AutomationRunQueued).
|
||||
Where(`NOT EXISTS (SELECT 1 FROM automation_runs AS holding
|
||||
WHERE holding.automation_id = automation_runs.automation_id AND holding.status IN ? )`,
|
||||
[]string{types.AutomationRunRunning, types.AutomationRunAwaitingApproval}).
|
||||
Order("resume DESC, scheduled_at").Limit(free * 4).Find(&queued).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -186,16 +189,6 @@ func (s *AutomationService) startQueued(ctx context.Context, now time.Time) erro
|
|||
}
|
||||
|
||||
func (s *AutomationService) startRun(ctx context.Context, run *types.AutomationRun, now time.Time) (bool, error) {
|
||||
var holding int64
|
||||
if err := s.db.WithContext(ctx).Model(&types.AutomationRun{}).
|
||||
Where("automation_id = ? AND id <> ? AND status IN ?", run.AutomationID, run.ID, []string{
|
||||
types.AutomationRunRunning, types.AutomationRunAwaitingApproval,
|
||||
}).Count(&holding).Error; err != nil {
|
||||
return false, err
|
||||
}
|
||||
if holding > 0 {
|
||||
return false, nil
|
||||
}
|
||||
lease := now.Add(s.cfg.Lease)
|
||||
updates := map[string]any{
|
||||
"status": types.AutomationRunRunning, "owner": s.instance, "lease_until": lease, "updated_at": now,
|
||||
|
|
@ -203,10 +196,33 @@ func (s *AutomationService) startRun(ctx context.Context, run *types.AutomationR
|
|||
if run.StartedAt == nil {
|
||||
updates["started_at"] = now
|
||||
}
|
||||
res := s.db.WithContext(ctx).Model(&types.AutomationRun{}).
|
||||
Where("id = ? AND status = ?", run.ID, types.AutomationRunQueued).Updates(updates)
|
||||
if res.Error != nil || res.RowsAffected == 0 {
|
||||
return false, res.Error
|
||||
started := false
|
||||
err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
// Take the automation's write lock before reading its runs. A run-row
|
||||
// CAS alone cannot serialize two different queued runs. Writing first
|
||||
// also avoids a deferred read-to-write transaction upgrade on SQLite.
|
||||
locked := tx.Model(&types.Automation{}).Where("id = ?", run.AutomationID).
|
||||
UpdateColumn("version", gorm.Expr("version + 1"))
|
||||
if locked.Error != nil || locked.RowsAffected == 0 {
|
||||
return locked.Error
|
||||
}
|
||||
var holding int64
|
||||
if err := tx.Model(&types.AutomationRun{}).
|
||||
Where("automation_id = ? AND id <> ? AND status IN ?", run.AutomationID, run.ID, []string{
|
||||
types.AutomationRunRunning, types.AutomationRunAwaitingApproval,
|
||||
}).Count(&holding).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if holding > 0 {
|
||||
return nil
|
||||
}
|
||||
res := tx.Model(&types.AutomationRun{}).
|
||||
Where("id = ? AND status = ?", run.ID, types.AutomationRunQueued).Updates(updates)
|
||||
started = res.Error == nil && res.RowsAffected > 0
|
||||
return res.Error
|
||||
})
|
||||
if err != nil || !started {
|
||||
return false, err
|
||||
}
|
||||
run.Status, run.Owner, run.LeaseUntil = types.AutomationRunRunning, s.instance, &lease
|
||||
if run.StartedAt == nil {
|
||||
|
|
@ -266,20 +282,31 @@ func (s *AutomationService) reconcileAwaiting(ctx context.Context, now time.Time
|
|||
if s.broker == nil {
|
||||
return nil
|
||||
}
|
||||
var waiting []*types.AutomationRun
|
||||
if err := s.db.WithContext(ctx).Where("status = ?", types.AutomationRunAwaitingApproval).
|
||||
Order("updated_at").Limit(automationClaimBatch * 2).Find(&waiting).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for _, run := range waiting {
|
||||
if err := s.reconcileRun(ctx, run, now); err != nil {
|
||||
logger.Warnf(ctx, "Reconcile automation run %s: %v", run.ID, err)
|
||||
// Keyset pagination visits later approvals even when earlier ones stay
|
||||
// open. Offsets would skip rows as reconciled runs leave the waiting set.
|
||||
for cursor := ""; ; {
|
||||
var waiting []*types.AutomationRun
|
||||
if err := s.db.WithContext(ctx).
|
||||
Where("status = ? AND id > ?", types.AutomationRunAwaitingApproval, cursor).
|
||||
Order("id").Limit(automationClaimBatch * 2).Find(&waiting).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for _, run := range waiting {
|
||||
if err := s.reconcileRun(ctx, run, now); err != nil {
|
||||
logger.Warnf(ctx, "Reconcile automation run %s: %v", run.ID, err)
|
||||
}
|
||||
}
|
||||
if len(waiting) < automationClaimBatch*2 {
|
||||
return nil
|
||||
}
|
||||
cursor = waiting[len(waiting)-1].ID
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *AutomationService) reconcileRun(ctx context.Context, run *types.AutomationRun, now time.Time) error {
|
||||
if run.CancelRequested {
|
||||
return s.cancelRun(ctx, run, "cancelled by the owner")
|
||||
}
|
||||
waiting := []string{types.AutomationRunAwaitingApproval}
|
||||
prompt, err := s.broker.Store().Get(ctx, run.PendingPromptID)
|
||||
if err != nil {
|
||||
|
|
@ -324,7 +351,8 @@ func (s *AutomationService) reconcileRun(ctx context.Context, run *types.Automat
|
|||
// queue, to continue from the answer.
|
||||
func (s *AutomationService) queueResume(ctx context.Context, runID, promptID string) error {
|
||||
res := s.db.WithContext(ctx).Model(&types.AutomationRun{}).
|
||||
Where("id = ? AND status = ? AND pending_prompt_id = ?", runID, types.AutomationRunAwaitingApproval, promptID).
|
||||
Where("id = ? AND status = ? AND pending_prompt_id = ? AND cancel_requested = ?",
|
||||
runID, types.AutomationRunAwaitingApproval, promptID, false).
|
||||
Updates(map[string]any{"status": types.AutomationRunQueued, "resume": true, "updated_at": s.now()})
|
||||
if res.Error == nil && res.RowsAffected > 0 {
|
||||
s.Wake()
|
||||
|
|
@ -407,8 +435,15 @@ func (s *AutomationService) finishRun(
|
|||
return false, res.Error
|
||||
}
|
||||
run.Status, run.Error, run.FinishedAt = status, reason, &now
|
||||
// Only the winning finalizer may stop work. A stale approval reaper must
|
||||
// not cancel a run that another scheduler has already resumed.
|
||||
var stopErr error
|
||||
if status == types.AutomationRunFailed || status == types.AutomationRunCancelled ||
|
||||
status == types.AutomationRunNeedsAttention {
|
||||
stopErr = s.stopRunDelegation(context.WithoutCancel(ctx), run)
|
||||
}
|
||||
s.settle(ctx, run)
|
||||
return true, nil
|
||||
return true, stopErr
|
||||
}
|
||||
|
||||
// settle records a finished run on its automation, pauses it after too many
|
||||
|
|
|
|||
|
|
@ -431,6 +431,7 @@ func (s *sessionService) AgentQA(
|
|||
// run elsewhere. Session, attachments and images are reloaded from storage.
|
||||
func resumableAgentRequest(ctx context.Context, req *types.QARequest) types.ResumableAgentRequest {
|
||||
r := types.ResumableAgentRequest{
|
||||
ActingVia: types.ActingViaFromContext(ctx),
|
||||
Query: req.Query,
|
||||
SummaryModelID: req.SummaryModelID,
|
||||
ReasoningEffort: req.ReasoningEffort,
|
||||
|
|
|
|||
|
|
@ -655,6 +655,7 @@ func BuildContainer(container *dig.Container) *dig.Container {
|
|||
}
|
||||
automations.Bind(sessionService, broker, runs, streams)
|
||||
automations.Events = sessions.CollaborationEvents
|
||||
automations.StopDelegation = collaboration.StopOwned
|
||||
broker.SetRunHooks(runs.Hooks())
|
||||
runs.StopDelegation = func(ctx context.Context, p *types.AgentPrompt) error {
|
||||
return collaboration.StopPrompt(ctx, p)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,22 @@
|
|||
package database
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/golang-migrate/migrate/v4/source"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// Check both migration registries without requiring a live PostgreSQL server.
|
||||
// Independently merged migrations must not reuse a version in either dialect.
|
||||
func TestMigrationSourcesHaveUniqueVersions(t *testing.T) {
|
||||
root := sqliteRepoRoot(t)
|
||||
for _, dialect := range []string{"sqlite", "versioned"} {
|
||||
t.Run(dialect, func(t *testing.T) {
|
||||
migrations, err := source.Open("file://" + filepath.Join(root, "migrations", dialect))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, migrations.Close())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
package database
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSQLiteAutomationDurationUpgradePreservesExistingRuns(t *testing.T) {
|
||||
repoRoot := sqliteRepoRoot(t)
|
||||
for _, version := range []int{45, 47} {
|
||||
t.Run(fmt.Sprintf("from-v%d", version), func(t *testing.T) {
|
||||
chdirAndRestore(t, copySQLiteMigrationsThrough(t, repoRoot, version))
|
||||
dbPath := filepath.Join(t.TempDir(), "automations.db")
|
||||
require.NoError(t, RunMigrationsWithOptions("sqlite3://unused", MigrationOptions{SQLiteDBPath: dbPath}))
|
||||
db := openSQLiteDB(t, dbPath)
|
||||
_, err := db.Exec(`INSERT INTO automation_runs
|
||||
(id, automation_id, tenant_id, scheduled_at, trigger_kind, status, answer, created_at, updated_at)
|
||||
VALUES ('existing-run', 'automation', 1, CURRENT_TIMESTAMP, 'schedule',
|
||||
'awaiting_approval', 'saved answer', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`)
|
||||
require.NoError(t, err)
|
||||
|
||||
chdirAndRestore(t, repoRoot)
|
||||
require.NoError(t, RunMigrationsWithOptions("sqlite3://unused", MigrationOptions{SQLiteDBPath: dbPath}))
|
||||
var duration int64
|
||||
var answer, status string
|
||||
require.NoError(t, db.QueryRow(`SELECT execution_duration_ms, answer, status
|
||||
FROM automation_runs WHERE id = 'existing-run'`).Scan(&duration, &answer, &status))
|
||||
require.Zero(t, duration)
|
||||
require.Equal(t, "saved answer", answer)
|
||||
require.Equal(t, "awaiting_approval", status)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -52,6 +52,7 @@ var versionedSQLiteTables = []string{
|
|||
// versionedSQLiteColumns maps each existing table to the columns that the
|
||||
// versioned migrations add and the SQLite baseline was missing.
|
||||
var versionedSQLiteColumns = map[string][]string{
|
||||
"automation_runs": {"execution_duration_ms"}, // 000048
|
||||
"model_catalog_configs": {"version", "overlay", "history", "updated_by", "updated_at"}, // 000031
|
||||
"memory_subjects": {"extraction_state"}, // 000094
|
||||
"memory_items": {"replaces_id"}, // 000094
|
||||
|
|
@ -86,7 +87,7 @@ var versionedSQLiteColumns = map[string][]string{
|
|||
}, // 000028
|
||||
}
|
||||
|
||||
const expectedSQLiteMigrationVersion = 47
|
||||
const expectedSQLiteMigrationVersion = 48
|
||||
|
||||
func TestSQLiteMigrationsCreateVersionedSchema(t *testing.T) {
|
||||
repoRoot := sqliteRepoRoot(t)
|
||||
|
|
@ -262,19 +263,20 @@ func TestSQLiteSandboxAllocationMigrationRollsBack(t *testing.T) {
|
|||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _, _ = m.Close() })
|
||||
|
||||
require.NoError(t, m.Steps(-1))
|
||||
// Target the allocation migration even when newer migrations exist.
|
||||
require.NoError(t, m.Migrate(46))
|
||||
version, dirty = sqliteMigrationState(t, sqlDB)
|
||||
require.Equal(t, expectedSQLiteMigrationVersion-1, version)
|
||||
require.Equal(t, 46, version)
|
||||
require.False(t, dirty)
|
||||
for _, table := range []string{"session_sandbox_assignments", "sandbox_user_machines"} {
|
||||
require.Falsef(t, sqliteTableExists(t, sqlDB, table), "down must drop %s", table)
|
||||
}
|
||||
require.True(t, sqliteColumnExists(t, sqlDB, "sessions", "owner_auth_source"),
|
||||
"rolling back the allocation tables keeps the owner source of 000034")
|
||||
"rolling back the allocation tables keeps the owner source of 000046")
|
||||
|
||||
require.NoError(t, m.Steps(1))
|
||||
require.NoError(t, m.Migrate(47))
|
||||
version, dirty = sqliteMigrationState(t, sqlDB)
|
||||
require.Equal(t, expectedSQLiteMigrationVersion, version)
|
||||
require.Equal(t, 47, version)
|
||||
require.False(t, dirty)
|
||||
require.True(t, sqliteTableExists(t, sqlDB, "session_sandbox_assignments"))
|
||||
require.True(t, sqliteTableExists(t, sqlDB, "sandbox_user_machines"))
|
||||
|
|
|
|||
|
|
@ -168,8 +168,10 @@ func (p *AgentPrompt) TimelineView() *PendingPromptView {
|
|||
// ResumableAgentRequest is the part of a QARequest needed to rebuild the run
|
||||
// on another replica. It holds the original query and durable request settings.
|
||||
type ResumableAgentRequest struct {
|
||||
AgentID string `json:"agent_id"`
|
||||
Query string `json:"query,omitempty"`
|
||||
// ActingVia keeps scheduler-owned runs out of ordinary chat recovery.
|
||||
ActingVia string `json:"acting_via,omitempty"`
|
||||
AgentID string `json:"agent_id"`
|
||||
Query string `json:"query,omitempty"`
|
||||
// AgentTenantID is the agent owner's workspace; it differs from the
|
||||
// session's when the agent is shared.
|
||||
AgentTenantID uint64 `json:"agent_tenant_id,omitempty"`
|
||||
|
|
|
|||
|
|
@ -170,6 +170,9 @@ type AutomationRun struct {
|
|||
Resume bool `json:"-" gorm:"not null;default:false"`
|
||||
// CancelRequested stops the run at its next check.
|
||||
CancelRequested bool `json:"cancel_requested" gorm:"not null;default:false"`
|
||||
// ExecutionDurationMS accumulates execution segments across approvals;
|
||||
// queued time and time spent waiting for an answer do not consume it.
|
||||
ExecutionDurationMS int64 `json:"-" gorm:"not null;default:0"`
|
||||
// Answer is the start of the final answer, for listings.
|
||||
Answer string `json:"answer" gorm:"type:text;not null;default:''" extensions:"x-agent,x-agent-response"`
|
||||
Error string `json:"error" gorm:"type:text;not null;default:''" extensions:"x-agent,x-agent-response"`
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
ALTER TABLE automation_runs DROP COLUMN execution_duration_ms;
|
||||
|
|
@ -0,0 +1 @@
|
|||
ALTER TABLE automation_runs ADD COLUMN execution_duration_ms BIGINT NOT NULL DEFAULT 0;
|
||||
|
|
@ -0,0 +1 @@
|
|||
ALTER TABLE automation_runs DROP COLUMN execution_duration_ms;
|
||||
|
|
@ -0,0 +1 @@
|
|||
ALTER TABLE automation_runs ADD COLUMN execution_duration_ms BIGINT NOT NULL DEFAULT 0;
|
||||
Loading…
Reference in New Issue