feat: bound retention cleanup and pin highlighted dashboard models

Author: chrisbbreuer

storage/framework/core/orm/src/define-model.ts

@@ -1328,10 +1328,11 @@ function addStaticHelpers(baseModel: Record<string, unknown>, definition: BQBMod
         const instance = await (make as Function).call(baseModel) as {
           forceFill?: (data: Record<string, unknown>) => unknown
           save?: () => Promise<unknown>
+          saveAsync?: () => Promise<unknown>
         } | null
         if (instance && typeof instance.forceFill === 'function' && typeof instance.save === 'function') {
           instance.forceFill(data)
-          const saved = await instance.save()
+          const saved = await (typeof instance.saveAsync === 'function' ? instance.saveAsync() : instance.save())
           // The wrapped instance, never the builder's raw `this`: that one
           // reads every column as undefined (see keepWrapped).
           return saved && typeof saved === 'object' && saved !== instance && !(saved as { [STACKS_PROXY_TAG]?: boolean })[STACKS_PROXY_TAG]

storage/framework/core/orm/src/gdpr/engine.ts

@@ -57,6 +57,10 @@ export interface GdprPruneOptions extends GdprRunOptions {
   dryRun?: boolean
   /** The moment ages are measured from. @default new Date() */
   now?: Date
+  /** Maximum rows scanned per model in this call. Continue with the returned `next` cursors. */
+  batchSize?: number
+  /** Primary-key cursors from a previous bounded call, with the same `now`. */
+  after?: Record<string, string | number>
 }
 
 export class GdprSubjectNotFoundError extends Error {
@@ -501,10 +505,14 @@ const DAY_MS = 86_400_000
 export async function pruneRetainedData(options: GdprPruneOptions = {}): Promise<GdprPruneResult> {
   const dryRun = options.dryRun === true
   const now = options.now ?? new Date()
+  if (!Number.isFinite(now.getTime())) throw new RangeError('Retention requires a valid date')
+  if (options.batchSize !== undefined && (!Number.isSafeInteger(options.batchSize) || options.batchSize < 1 || options.batchSize > 10000))
+    throw new RangeError('Retention batchSize must be an integer from 1 to 10000')
+  const next: Record<string, string | number> = {}
   const plan = await loadPlan(options.models)
   const policies = plan.models.filter(model => model.retention)
   if (!policies.length)
-    return { dryRun, cutoffs: {}, changes: [], skipped: [] }
+    return { dryRun, cutoffs: {}, changes: [], skipped: [], ...(options.batchSize === undefined ? {} : { next }) }
 
   if (!dryRun)
     await requireAuditTable()
@@ -520,13 +528,14 @@ export async function pruneRetainedData(options: GdprPruneOptions = {}): Promise
     cutoffs[model.model] = cutoff.toISOString()
 
     const expired: Row[] = []
-    let last: unknown
+    let last: unknown = options.after?.[model.model]
     for (;;) {
       const handle: Handle = db
       let query = handle.selectFrom(model.table).selectAll().where(retention.column, '<', bound)
       if (last !== undefined)
         query = query.where(model.primaryKey, '>', last)
-      const page: Row[] = await query.orderBy(model.primaryKey, 'asc').limit(CHUNK).execute()
+      const size = options.batchSize ?? CHUNK
+      const page: Row[] = await query.orderBy(model.primaryKey, 'asc').limit(size).execute()
       if (!page.length)
         break
       for (const row of page) {
@@ -535,6 +544,14 @@ export async function pruneRetainedData(options: GdprPruneOptions = {}): Promise
           expired.push(row)
       }
       last = page[page.length - 1]![model.primaryKey]
+      if (options.batchSize !== undefined) {
+        if (page.length === size) {
+          if (typeof last === 'bigint') last = last.toString()
+          if (typeof last !== 'string' && typeof last !== 'number') throw new TypeError('Retention requires a string or numeric primary key')
+          next[model.model] = last
+        }
+        break
+      }
       if (page.length < CHUNK)
         break
     }
@@ -546,6 +563,10 @@ export async function pruneRetainedData(options: GdprPruneOptions = {}): Promise
           await deleteRows(handle, model, pending)
         else
           await anonymizeRows(handle, model, pending)
+        if (options.batchSize !== undefined) await recordAudit(handle, 'retention', null, options.actor ?? 'scheduler', {
+          changes: [{ model: model.model, action: retention.action, matched: expired.length, changed: pending.length }],
+          cutoffs: { [model.model]: cutoff.toISOString() },
+        })
       })
     }
 
@@ -559,12 +580,12 @@ export async function pruneRetainedData(options: GdprPruneOptions = {}): Promise
     })
   }
 
-  if (!dryRun && changes.some(change => change.changed > 0)) {
+  if (!dryRun && options.batchSize === undefined && changes.some(change => change.changed > 0)) {
     await recordAudit(db, 'retention', null, options.actor ?? 'scheduler', {
       changes: changes.map(({ model, action, matched, changed }) => ({ model, action, matched, changed })),
       cutoffs,
     })
   }
 
-  return { dryRun, cutoffs, changes: changes.sort((a, b) => a.model.localeCompare(b.model)), skipped }
+  return { dryRun, cutoffs, changes: changes.sort((a, b) => a.model.localeCompare(b.model)), skipped, ...(options.batchSize === undefined ? {} : { next }) }
 }

storage/framework/core/orm/src/gdpr/types.ts

@@ -154,6 +154,8 @@ export interface GdprErasureResult {
 }
 
 export interface GdprPruneResult {
+  /** Cursors for models with another bounded page to scan. Reuse the same cutoff date. */
+  next?: Record<string, string | number>
   dryRun: boolean
   cutoffs: Record<string, string>
   changes: GdprModelChange[]

storage/framework/core/orm/tests/force-create-guarded.test.ts

@@ -64,6 +64,19 @@ describe('forceCreate persists guarded columns', () => {
     expect(row).toEqual({ name: 'yelp', api_key: 'secret-key', note: 'set deliberately' })
   })
 
+  it('awaits asynchronous setters exactly once through the trusted create path', async () => {
+    let calls = 0
+    const AsyncCredential = defineModel({
+      name: 'AsyncFcCredential', table: 'fc_credentials',
+      attributes: { name: { fillable: true }, apiKey: { guarded: true }, note: { guarded: true } },
+      set: { note: async (attributes: Record<string, unknown>) => { calls++; return `encoded:${attributes.note}` } },
+    } as const)
+    const created = await AsyncCredential.forceCreate({ name: 'async', apiKey: 'key', note: 'plain' })
+    const row = db.query('SELECT note FROM fc_credentials WHERE id = ?').get(Number(created.id)) as { note: string }
+    expect(row.note).toBe('encoded:plain')
+    expect(calls).toBe(1)
+  })
+
   it('returns a row carrying the guarded values', async () => {
     const created = await (Credential as any).forceCreate({ name: 'google', apiKey: 'another-key' })
     expect(Number(created.id)).toBeGreaterThan(0)

storage/framework/core/orm/tests/gdpr.test.ts

@@ -452,6 +452,48 @@ describe('retention', () => {
     expect(result.skipped).toEqual(['ghosts'])
     expect(result.changes.find(change => change.model === 'Visit')!.changed).toBe(2)
   })
+
+  test('bounded retention advances over unexpired candidates and resumes the same cutoff', async () => {
+    const scoped = { Visit: models.Visit }
+    let after: Record<string, string | number> | undefined
+    let calls = 0
+    let deleted = 0
+    do {
+      const result = await pruneRetainedData({ models: scoped, now, batchSize: 1, after })
+      expect(result.changes[0]!.changed).toBeLessThanOrEqual(1)
+      expect(result.cutoffs.Visit).toBe('2026-05-02T12:00:00.000Z')
+      deleted += result.changes[0]!.changed
+      after = result.next
+      calls++
+      expect(calls).toBeLessThan(6)
+    } while (Object.keys(after ?? {}).length)
+    expect(deleted).toBe(2)
+    expect((await rows('visits')).map(row => row.path)).toEqual(['/edge-inside', '/new', '/undated'])
+    expect(calls).toBe(4)
+    expect((await pruneRetainedData({ models: scoped, now, batchSize: 1 })).changes[0]!.changed).toBe(0)
+  })
+
+  test('bounded previews preserve data and invalid bounds fail before writes', async () => {
+    const before = await rows('visits')
+    const scoped = { Visit: models.Visit }
+    const result = await pruneRetainedData({ models: scoped, now, batchSize: 1, dryRun: true })
+    expect(result.next?.Visit).toBeDefined()
+    expect(await rows('visits')).toEqual(before)
+    for (const batchSize of [0, -1, 1.5, 10001, Number.NaN])
+      await expect(pruneRetainedData({ models: scoped, now, batchSize })).rejects.toThrow('batchSize')
+    await expect(pruneRetainedData({ models: scoped, now: new Date('invalid'), batchSize: 1 })).rejects.toThrow('valid date')
+    expect(await rows('gdpr_requests')).toHaveLength(0)
+  })
+
+  test('bounded deletion rolls back when its audit cannot be saved', async () => {
+    const before = await rows('visits')
+    await db.unsafe("CREATE TRIGGER reject_retention_audit BEFORE INSERT ON gdpr_requests BEGIN SELECT RAISE(ABORT, 'audit unavailable'); END").execute()
+    try {
+      await expect(pruneRetainedData({ models: { Visit: models.Visit }, now, batchSize: 1 })).rejects.toThrow('audit unavailable')
+      expect(await rows('visits')).toEqual(before)
+    }
+    finally { await db.unsafe('DROP TRIGGER reject_retention_audit').execute() }
+  })
 })
 
 describe('processing register', () => {

storage/framework/core/types/src/dashboard.ts

@@ -396,7 +396,7 @@ export interface DashboardModelOptions {
   label?: string
 
   /**
-   * Give this model a prominent row. A number orders the highlighted set;
+   * Pin this model in Data above the collapsed More models group. A number orders the highlighted set;
    * `true` highlights without taking a position.
    */
   highlight?: boolean | number

storage/framework/defaults/ai/skills/stacks-dashboard/SKILL.md

@@ -740,3 +740,11 @@ by itself: configure `createDatabaseErrorReporter()` from
 `Error`, and migrate it. Reporting is bounded, buffered, identity-deduplicated
 by `captureError`, and flushed by `flushErrorReporters`/the logger. The database
 reporter stores allowlisted operational context and excludes bodies and headers.
+
+### Pinned model rows
+
+Declare `dashboard: { highlight: true }` on a model to pin its Data row, or use
+numeric `highlight` values to order pinned rows. Other enabled Data models live
+under the native Sidebar's collapsed More models disclosure. Labels, icons and
+role gates apply to nested rows too. `enabled: false` omits navigation;
+`access: false` excludes generic catalogs and reads as well.

storage/framework/defaults/ai/skills/stacks-models/SKILL.md

@@ -519,3 +519,14 @@ commands, not a project's resolved models.
   of a validator is not a promise that an arbitrary request body is safe.
 - Dashboard-highlighted models (`dashboard: { highlight: true }`) appear
   prominently in the admin UI.
+
+### Bounded retention jobs
+
+`pruneRetainedData({ models: { Request }, now, batchSize: 500, after })` scans at
+most 500 candidates per model. Persist `result.next` in the next queue job and
+reuse the same `now` until no cursors remain, even when a page changed zero rows.
+This handles mixed SQL/ISO timestamps without skipping candidates. Each bounded
+page commits its deletion/anonymization and retention audit together. A dry run
+returns cursors without changing records. Omitting `batchSize` retains the CLI's
+whole-run behavior. Declare the policy on `traits.gdpr.retention`, publish the
+native `GdprRequest` audit model, and run cleanup through durable queue workers.

storage/framework/defaults/resources/functions/dashboard/sidebar.ts

@@ -18,6 +18,7 @@ type ModelCategory = 'userland' | 'data' | 'commerce' | 'content' | 'marketing'
  * full type imports aren't available. Keep in sync with the source.
  */
 interface DashboardModelOptionsLite {
+  highlight?: boolean | number
   access?: boolean
   enabled?: boolean
   label?: string
@@ -102,6 +103,9 @@ export const DEFAULT_TOGGLES: DashboardSectionToggles = {
 }
 
 interface NavItem {
+  id?: string
+  children?: NavItem[]
+  expanded?: boolean
   to: string
   icon: string
   text: string
@@ -413,37 +417,50 @@ export function buildNavSections(
     ...categoryNavItems(discoveredModels, 'system', new Set()),
   ]])
 
-  // Data section: configurable basic rows + ALL userland models + any
-  // root-level framework `data` models. Per-row toggles let projects hide
-  // the basic rows they don't need; userland models always appear.
+  // Data pins model-declared highlights. Other visible models stay accessible
+  // through a collapsed disclosure using the shared Sidebar component.
   // Per-model rows link to `/models/<id>` (dynamic ORM viewer); the
   // built-in dashboard / activity / users / teams / subscribers rows keep
   // their bespoke pages under `/data/<row>`.
   const dataRows = toggles.data ?? DEFAULT_DATA_TOGGLES
+  const dedicatedDataRows: Record<string, keyof DataRowToggles> = { user: 'users', team: 'teams', subscriber: 'subscribers', activity: 'activity' }
   const dataNav: NavItem[] = []
   if (dataRows.dashboard) dataNav.push({ to: '/data/dashboard', icon: 'dashboard', text: 'Dashboard' })
   if (dataRows.activity) dataNav.push({ to: '/data/activity', icon: 'activity', text: 'Activity' })
   if (dataRows.users) dataNav.push({ to: '/data/users', icon: 'users', text: 'Users' })
   if (dataRows.teams) dataNav.push({ to: '/data/teams', icon: 'group', text: 'Teams' })
   if (dataRows.subscribers) dataNav.push({ to: '/data/subscribers', icon: 'mail', text: 'Subscribers' })
   if (dataRows.allModels) dataNav.push({ to: '/models', icon: 'table', text: 'All Models' })
-  for (const m of discoveredModels) {
+  const moreModels: NavItem[] = []
+  const highlighted = (model: DiscoveredModel) => model.dashboard?.highlight === true
+    || (typeof model.dashboard?.highlight === 'number' && Number.isFinite(model.dashboard.highlight))
+  const rank = (model: DiscoveredModel) => typeof model.dashboard?.highlight === 'number' ? model.dashboard.highlight : Number.POSITIVE_INFINITY
+  const dataModels = [...discoveredModels].sort((a, b) => {
+    if (highlighted(a) !== highlighted(b)) return highlighted(a) ? -1 : 1
+    if (highlighted(a)) return rank(a) - rank(b) || a.name.localeCompare(b.name)
+    return a.name.localeCompare(b.name)
+  })
+  for (const m of dataModels) {
     // Skip when:
     //   - explicit `dashboard.enabled === false` (effectiveCategory returns undefined)
     //   - pinned to a different section via `dashboard.section`
     //   - already has a dedicated dashboard page
     const eff = effectiveCategory(m)
     if (eff !== 'userland' && eff !== 'data') continue
-    if (m.hasDedicatedPage) continue
-    dataNav.push({
+    const dedicatedRow = dedicatedDataRows[m.id]
+    if (m.hasDedicatedPage && (!dedicatedRow || dataRows[dedicatedRow])) continue
+    const item: NavItem = {
       to: `/models/${m.id}`,
       icon: m.dashboard?.icon ?? m.icon ?? 'table',
       text: m.dashboard?.label ?? m.name,
       ...(m.dashboard?.roles && m.dashboard.roles.length > 0
         ? { roles: m.dashboard.roles }
         : {}),
-    })
+    }
+    if (highlighted(m)) dataNav.push(item)
+    else moreModels.push(item)
   }
+  if (moreModels.length) dataNav.push({ id: 'models-more', to: '', text: 'More models', icon: 'table', expanded: false, children: moreModels })
   sections.push(['data', 'data', dataNav])
 
   if (toggles.commerce) {
@@ -650,7 +667,9 @@ export interface WebSidebarItem {
   icon: string
   /** macOS system color name or CSS color for the icon tint. */
   iconColor: string
-  href: string
+  href?: string
+  children?: WebSidebarItem[]
+  expanded?: boolean
   roles?: string[]
 }
 
@@ -726,6 +745,15 @@ export function buildWebSidebarSections(): WebSidebarSection[] {
   const appNav = manifest.nav.length > 0 ? manifest.nav : loadAppNavFromConfig()
   const sections = filterSchemaNavSections(buildNavSections(manifest.models, manifest.sections, appNav), manifest.schemaModels)
 
+  const webItem = (item: NavItem): WebSidebarItem => ({
+    id: item.id ?? navItemId(item.to),
+    label: item.text,
+    icon: resolveNavIcon(item.icon),
+    iconColor: 'blue',
+    ...(item.to ? { href: item.to } : {}),
+    ...(item.roles?.length ? { roles: item.roles } : {}),
+    ...(item.children ? { children: item.children.map(webItem), expanded: item.expanded ?? false } : {}),
+  })
   return [
     {
       id: 'top',
@@ -735,14 +763,20 @@ export function buildWebSidebarSections(): WebSidebarSection[] {
     ...sections.map(([key, label, items]) => ({
       id: key,
       label: titleCase(label),
-      items: items.map(item => ({
-        id: navItemId(item.to),
-        label: item.text,
-        icon: resolveNavIcon(item.icon),
-        iconColor: 'blue',
-        href: item.to,
-        ...(item.roles && item.roles.length > 0 ? { roles: item.roles } : {}),
-      })),
+      items: items.map(webItem),
     })),
   ]
 }
+
+/** Role metadata follows nested model rows as well as pinned rows. */
+export function webSidebarRoleMap(sections: WebSidebarSection[]): Record<string, string[]> {
+  const roles: Record<string, string[]> = Object.create(null)
+  const visit = (items: WebSidebarItem[]) => {
+    for (const item of items) {
+      if (item.roles?.length) roles[item.id] = item.roles
+      if (item.children) visit(item.children)
+    }
+  }
+  for (const section of sections) visit(section.items)
+  return roles
+}

storage/framework/defaults/views/dashboard/layouts/default.stx

@@ -16,7 +16,7 @@
 // vendored or linked. `@stacksjs/defaults/*` is mapped onto the vendored tree in
 // `storage/framework/tsconfig.app.json` and falls back to the installed
 // package when there is none, so it resolves in both layouts.
-import { buildWebSidebarSections } from '@stacksjs/defaults/resources/functions/dashboard/sidebar'
+import { buildWebSidebarSections, webSidebarRoleMap } from '@stacksjs/defaults/resources/functions/dashboard/sidebar'
 
 export const sidebarSections = buildWebSidebarSections()
 
@@ -31,15 +31,7 @@ export const sidebarSections = buildWebSidebarSections()
 // then ran `Object.entries` over that string, which yields index/character
 // pairs, and called `.some()` on a single character. Every dashboard load
 // threw inside applyRoleFilter before it could filter anything.
-export const sidebarRoleMap: Record<string, string[]> = Object.fromEntries(
-  sidebarSections.flatMap(section =>
-    section.items.flatMap(item =>
-      item.roles && item.roles.length > 0
-        ? [[item.id, item.roles] as const]
-        : [],
-    ),
-  ),
-)
+export const sidebarRoleMap: Record<string, string[]> = webSidebarRoleMap(sidebarSections)
 
 // Declared, so the client block below gets the map typed rather than inferred
 // off the implicit bridge - which is what let the mismatch above go unseen.