WeKnora/internal/database/migration_sqlite_message_ar...

136 lines
5.6 KiB
Go
Raw Normal View History

feat(artifacts): artifacts library page backed by a message_artifacts table (#3384) * feat(artifacts): add an artifacts library with its own message_artifacts table Skill-generated files were only reachable from inside the conversation that produced them. This adds a sidebar entry and a page that lists them across every conversation the user can see. Storage: artifacts move out of the messages.artifacts JSONB column into a new message_artifacts table (PostgreSQL 000103, SQLite 000023), backfilled on upgrade. The table is now the only source of truth; the old column is kept and the down migration writes the table back into it. The message repository loads artifacts for every message it returns and writes them on create, update (when non-nil) and fork, so callers still use Message.Artifacts. mod_time is stored as RFC 3339 text because the collector matches files on nanosecond mtimes, which PostgreSQL timestamps would truncate. API: GET /api/v1/artifacts lists the latest version of each file (grouped by session + source path, with version_count), scoped like the session list, with keyword / file type filters and paging. Downloads keep using the per-session endpoint. Frontend: "产物/Artifacts" sidebar entry (hidden without sandbox support), a library page with type tabs, search, date groups, infinite loading, preview drawer, download and jump-to-conversation; strings in all five locales. The sidebar's top-menu paths are now one shared list. * fix(artifacts): keep the restore hash guard and count versions by URL RecordRestoredArtifactMtime again only stamps rows whose content hash matches the restored content (and ignores empty hashes), and never rewrites the hash, matching MessageArtifacts.WithRestoredMtime. Without the guard a restore of v1's content also moved v2's mtime and relabelled v2 with v1's hash. The library's version_count now counts distinct URLs per (session, source path). A later answer that references an earlier file stores another row with the same URL, which COUNT(*) reported as an extra version. * fix(artifacts): scope the library to web chats and clear the legacy column The library used the session list's default scope, which counts sessions without an owner as legacy tenant-level rows. IM sessions are created without an owner, so every member saw the file names and titles of every IM chat (downloads then 404 for non-admins). The library now applies the same "web" bucket predicate as the home sidebar, extracted from QueryPaged so the two cannot drift: no IM mapping (including cleared ones), no embed marker, no API-key owner. Migrations 000103 / SQLite 000023 now clear messages.artifacts after the backfill. Message loads still SELECT *, so the stale copy was read on every history load. Only rows that carried artifacts are rewritten; the column stays for instances on the old binary during a rolling upgrade, and the down migration still rebuilds it from the table. * fix(migrations): keep the 000103 backfill alive on malformed artifact JSON A single file_size that is not an integer ("12.5", "") or a created_at that does not parse ("garbage", month 13) raised a cast error and failed the whole migration, blocking startup. Casts now go through pg_temp helpers that return NULL on bad input, falling back to 0 and the message's created_at; valid rows are unchanged. SQLite casts never raise; its migration test now covers the same malformed values. * docs(artifacts): describe the web-only library scope and the cleared legacy column
2026-09-18 04:10:19 +00:00
package database
import (
"database/sql"
"encoding/json"
"os"
"path/filepath"
"testing"
"time"
"github.com/stretchr/testify/require"
)
// TestSQLiteMessageArtifactsBackfillAndRollback upgrades a v22 database that
// still keeps artifacts on messages.artifacts, checks 000023 copies them into
// message_artifacts, then replays the down migration and checks the JSON
// column is rebuilt from the table in a shape Go can decode.
func TestSQLiteMessageArtifactsBackfillAndRollback(t *testing.T) {
repoRoot := sqliteRepoRoot(t)
legacyRoot := copySQLiteMigrationsThrough(t, repoRoot, 22)
chdirAndRestore(t, legacyRoot)
dbPath := filepath.Join(t.TempDir(), "artifacts.db")
require.NoError(t, RunMigrationsWithOptions("sqlite3://unused", MigrationOptions{SQLiteDBPath: dbPath}))
db := openSQLiteDB(t, dbPath)
legacy := `[
{"url":"local://a/report.pptx","file_name":"report.pptx","file_type":".pptx","file_size":2048,
"content_hash":"abc","source_path":"/workspace/report.pptx",
"mod_time":"2026-09-01T10:00:00.123456789+08:00","created_at":"2026-09-01T10:00:05+08:00"},
{"url":"local://a/chart.png","file_name":"chart.png","file_type":".png","file_size":10,
"source_path":"/workspace/chart.png","mod_time":"2026-09-01T10:00:00Z","created_at":"2026-09-01T02:00:06Z"}
]`
_, err := db.Exec(
`INSERT INTO messages (id, request_id, session_id, role, content, artifacts, created_at)
VALUES ('msg-1', 'req-1', 'sess-1', 'assistant', 'done', ?, '2026-09-01 02:00:00')`,
legacy,
)
require.NoError(t, err)
_, err = db.Exec(
`INSERT INTO messages (id, request_id, session_id, role, content, artifacts)
VALUES ('msg-2', 'req-2', 'sess-1', 'user', 'hi', '[]')`,
)
require.NoError(t, err)
// Malformed values must fall back instead of failing the migration.
_, err = db.Exec(
`INSERT INTO messages (id, request_id, session_id, role, content, artifacts, created_at)
VALUES ('msg-3', 'req-3', 'sess-1', 'assistant', 'dirty', ?, '2026-09-02 00:00:00')`,
`[{"file_name":"dirty.bin","file_size":"12.5x","created_at":"garbage"}, 7, "x"]`,
)
require.NoError(t, err)
require.NoError(t, db.Close())
chdirAndRestore(t, repoRoot)
require.NoError(t, RunMigrationsWithOptions("sqlite3://unused", MigrationOptions{SQLiteDBPath: dbPath}))
db = openSQLiteDB(t, dbPath)
type row struct {
position int
fileName string
fileSize int64
sourcePath string
modTime string
createdAt time.Time
}
rows, err := db.Query(
`SELECT position, file_name, file_size, source_path, mod_time, created_at
FROM message_artifacts WHERE message_id = 'msg-1' ORDER BY position`,
)
require.NoError(t, err)
var got []row
for rows.Next() {
var r row
require.NoError(t, rows.Scan(&r.position, &r.fileName, &r.fileSize, &r.sourcePath, &r.modTime, &r.createdAt))
got = append(got, r)
}
require.NoError(t, rows.Err())
require.NoError(t, rows.Close())
require.Len(t, got, 2)
require.Equal(t, 0, got[0].position)
require.Equal(t, "report.pptx", got[0].fileName)
require.Equal(t, int64(2048), got[0].fileSize)
require.Equal(t, "/workspace/report.pptx", got[0].sourcePath)
require.Equal(t, "2026-09-01T10:00:00.123456789+08:00", got[0].modTime, "mod_time keeps the exact RFC 3339 text")
wantCreated, _ := time.Parse(time.RFC3339, "2026-09-01T02:00:05Z")
require.True(t, got[0].createdAt.Equal(wantCreated), "created_at normalised to UTC: %v", got[0].createdAt)
require.Equal(t, 1, got[1].position)
require.Equal(t, "chart.png", got[1].fileName)
var dirtySize int64
var dirtyCreated time.Time
require.NoError(t, db.QueryRow(
`SELECT file_size, created_at FROM message_artifacts WHERE message_id = 'msg-3'`,
).Scan(&dirtySize, &dirtyCreated))
require.EqualValues(t, 12, dirtySize, "SQLite CAST keeps the numeric prefix")
require.True(t, dirtyCreated.Equal(time.Date(2026, 9, 2, 0, 0, 0, 0, time.UTC)),
"an unparseable created_at falls back to the message time: %v", dirtyCreated)
var userRows int
require.NoError(t, db.QueryRow(`SELECT COUNT(*) FROM message_artifacts WHERE message_id = 'msg-2'`).Scan(&userRows))
require.Zero(t, userRows)
// The legacy column is cleared once copied, so history loads (SELECT *)
// no longer read a stale duplicate; rows without artifacts are untouched.
var legacy1, legacy2 sql.NullString
require.NoError(t, db.QueryRow(`SELECT artifacts FROM messages WHERE id = 'msg-1'`).Scan(&legacy1))
require.NoError(t, db.QueryRow(`SELECT artifacts FROM messages WHERE id = 'msg-2'`).Scan(&legacy2))
require.False(t, legacy1.Valid, "copied artifacts are cleared from messages.artifacts")
require.Equal(t, "[]", legacy2.String)
// Rollback rebuilds the legacy column from the table.
down, err := os.ReadFile(filepath.Join(repoRoot, "migrations", "sqlite", "000023_message_artifacts_table.down.sql"))
require.NoError(t, err)
_, err = db.Exec(string(down))
require.NoError(t, err)
require.False(t, sqliteTableExists(t, db, "message_artifacts"))
var restored string
require.NoError(t, db.QueryRow(`SELECT artifacts FROM messages WHERE id = 'msg-1'`).Scan(&restored))
var decoded []struct {
FileName string `json:"file_name"`
FileSize int64 `json:"file_size"`
ModTime time.Time `json:"mod_time"`
CreatedAt time.Time `json:"created_at"`
}
wantMod, _ := time.Parse(time.RFC3339Nano, "2026-09-01T10:00:00.123456789+08:00")
require.NoError(t, json.Unmarshal([]byte(restored), &decoded))
require.Len(t, decoded, 2)
require.Equal(t, "report.pptx", decoded[0].FileName)
require.Equal(t, int64(2048), decoded[0].FileSize)
require.True(t, decoded[0].CreatedAt.Equal(wantCreated))
require.True(t, decoded[0].ModTime.Equal(wantMod), "rollback keeps nanosecond mtime")
require.Equal(t, "chart.png", decoded[1].FileName)
}