This commit is contained in:
Nathan Anderson 2026-05-31 13:11:09 -05:00
parent eb5f57361c
commit 6615dc5d17
25 changed files with 239 additions and 136 deletions

View File

@ -14,7 +14,7 @@ Build an **admin-only** area in the Flutter app that shows a **live audit log**
- **Historical backfill** (`kind=backfill`) — ended **4-hour UTC slots** (`4Hour` bars) into `market_data_snapshots`; open slot not fetched yet - **Historical backfill** (`kind=backfill`) — ended **4-hour UTC slots** (`4Hour` bars) into `market_data_snapshots`; open slot not fetched yet
- **Retention / cleanup** (`kind=cleanup`) — prune (or archive-then-delete) expired rows - **Retention / cleanup** (`kind=cleanup`) — prune (or archive-then-delete) expired rows
Operators use it to confirm the rolling 7-day window is healthy, see when cleanup ran, and **notice Alpaca/API failures or rate limits immediately** without reading server logs or SQL. Operators use it to confirm the rolling 5-trading-day window is healthy, see when cleanup ran, and **notice Alpaca/API failures or rate limits immediately** without reading server logs or SQL.
This plan **replaces the need for §8 shell CLIs** for day-to-day product use. CLIs remain optional for CI/SRE (see §12). This plan **replaces the need for §8 shell CLIs** for day-to-day product use. CLIs remain optional for CI/SRE (see §12).

View File

@ -15,15 +15,15 @@ class MarketHistoryAdminConfig {
if (json == null) { if (json == null) {
return const MarketHistoryAdminConfig( return const MarketHistoryAdminConfig(
archiveEnabled: false, archiveEnabled: false,
windowDays: 7, windowDays: 5,
retentionDays: 7, retentionDays: 5,
syncEnabled: false, syncEnabled: false,
); );
} }
return MarketHistoryAdminConfig( return MarketHistoryAdminConfig(
archiveEnabled: json['archiveEnabled'] as bool? ?? false, archiveEnabled: json['archiveEnabled'] as bool? ?? false,
windowDays: (json['windowDays'] as num?)?.toInt() ?? 7, windowDays: (json['windowDays'] as num?)?.toInt() ?? 5,
retentionDays: (json['retentionDays'] as num?)?.toInt() ?? 7, retentionDays: (json['retentionDays'] as num?)?.toInt() ?? 5,
syncEnabled: json['syncEnabled'] as bool? ?? false, syncEnabled: json['syncEnabled'] as bool? ?? false,
); );
} }

View File

@ -80,7 +80,7 @@ class MarketHistoryWeekCoverageReport {
json['days'] as List<dynamic>? ?? <dynamic>[]; json['days'] as List<dynamic>? ?? <dynamic>[];
return MarketHistoryWeekCoverageReport( return MarketHistoryWeekCoverageReport(
asOf: DateTime.parse(json['asOf'] as String).toUtc(), asOf: DateTime.parse(json['asOf'] as String).toUtc(),
windowDays: (json['windowDays'] as num?)?.toInt() ?? 7, windowDays: (json['windowDays'] as num?)?.toInt() ?? 5,
slotsPerDay: (json['slotsPerDay'] as num?)?.toInt() ?? 2, slotsPerDay: (json['slotsPerDay'] as num?)?.toInt() ?? 2,
symbolCount: (json['symbolCount'] as num?)?.toInt() ?? 0, symbolCount: (json['symbolCount'] as num?)?.toInt() ?? 0,
isConsistent: json['isConsistent'] as bool? ?? false, isConsistent: json['isConsistent'] as bool? ?? false,

View File

@ -178,8 +178,8 @@ class SyncRunLogPage {
required this.nextBefore, required this.nextBefore,
this.config = const MarketHistoryAdminConfig( this.config = const MarketHistoryAdminConfig(
archiveEnabled: false, archiveEnabled: false,
windowDays: 7, windowDays: 5,
retentionDays: 7, retentionDays: 5,
syncEnabled: false, syncEnabled: false,
), ),
}); });

View File

@ -25,8 +25,8 @@ class SyncRunLogRepositoryState {
required this.errorMessage, required this.errorMessage,
this.config = const MarketHistoryAdminConfig( this.config = const MarketHistoryAdminConfig(
archiveEnabled: false, archiveEnabled: false,
windowDays: 7, windowDays: 5,
retentionDays: 7, retentionDays: 5,
syncEnabled: false, syncEnabled: false,
), ),
}); });
@ -59,8 +59,8 @@ class SyncRunLogRepository implements SyncRunLogController {
String? _errorMessage; String? _errorMessage;
MarketHistoryAdminConfig _config = const MarketHistoryAdminConfig( MarketHistoryAdminConfig _config = const MarketHistoryAdminConfig(
archiveEnabled: false, archiveEnabled: false,
windowDays: 7, windowDays: 5,
retentionDays: 7, retentionDays: 5,
syncEnabled: false, syncEnabled: false,
); );

View File

@ -294,7 +294,7 @@ class _MarketHistoryLogScreenState extends State<MarketHistoryLogScreen> {
), ),
IconButton( IconButton(
key: const Key('week-coverage-button'), key: const Key('week-coverage-button'),
tooltip: '7-day sync calendar', tooltip: 'Trading-week sync calendar',
onPressed: _weekCoverageLoading ? null : _showWeekCoverage, onPressed: _weekCoverageLoading ? null : _showWeekCoverage,
icon: const Icon(Icons.calendar_month_outlined), icon: const Icon(Icons.calendar_month_outlined),
), ),

View File

@ -13,7 +13,7 @@ const List<String> _weekdayLabels = <String>[
'Sun', 'Sun',
]; ];
/// Mini 7-day Eastern week view showing session-half slot sync health. /// Rolling US trading-week view (session-half slot sync health per weekday).
class MarketHistoryWeekCoverageSheet extends StatelessWidget { class MarketHistoryWeekCoverageSheet extends StatelessWidget {
const MarketHistoryWeekCoverageSheet({ const MarketHistoryWeekCoverageSheet({
super.key, super.key,
@ -53,10 +53,10 @@ class MarketHistoryWeekCoverageSheet extends StatelessWidget {
children: <Widget>[ children: <Widget>[
const Icon(Icons.calendar_month, color: AppColors.accent, size: 22), const Icon(Icons.calendar_month, color: AppColors.accent, size: 22),
const SizedBox(width: 8), const SizedBox(width: 8),
const Expanded( Expanded(
child: Text( child: Text(
'7-day sync health', '${report.windowDays}-day sync health',
style: TextStyle( style: const TextStyle(
fontSize: 18, fontSize: 18,
fontWeight: FontWeight.w700, fontWeight: FontWeight.w700,
color: AppColors.textPrimary, color: AppColors.textPrimary,
@ -98,7 +98,7 @@ class MarketHistoryWeekCoverageSheet extends StatelessWidget {
const SizedBox(height: 12), const SizedBox(height: 12),
Text( Text(
'ET · ${report.slotsPerDay} slots per trading day · ' 'ET · ${report.slotsPerDay} slots per trading day · '
'${report.windowDays}-day window', '${report.days.length} trading days · ${report.windowDays}-day sync window',
textAlign: TextAlign.center, textAlign: TextAlign.center,
style: const TextStyle( style: const TextStyle(
fontSize: 11, fontSize: 11,

View File

@ -131,7 +131,7 @@ curl -s -X POST http://localhost:3000/v1/me/incoming-question \
Alpaca **`1Min`** bars aggregated into two **US regular-session** half-days per trading day Alpaca **`1Min`** bars aggregated into two **US regular-session** half-days per trading day
(morning **9:3012:45 ET**, afternoon **12:4516:00 ET**, ~195 minutes each). (morning **9:3012:45 ET**, afternoon **12:4516:00 ET**, ~195 minutes each).
Stored as `metric=bar`, `timeframe=sessionHalf`. Rolling window: `MARKET_HISTORY_WINDOW_DAYS` (default 7). Stored as `metric=bar`, `timeframe=sessionHalf`. Rolling window: `MARKET_HISTORY_WINDOW_DAYS` (default **5 US trading days**, two session halves each; weekends/holidays skipped).
**Backfill** (`kind=backfill`): fetches each **ended** slot still missing in DB; skips the open slot. **Backfill** (`kind=backfill`): fetches each **ended** slot still missing in DB; skips the open slot.
Throttled to `MARKET_HISTORY_API_REQUESTS_PER_MINUTE` (default 200/min). On HTTP 429: wait 1 minute, retry once; if still limited, save partial rows and resume next tick. Throttled to `MARKET_HISTORY_API_REQUESTS_PER_MINUTE` (default 200/min). On HTTP 429: wait 1 minute, retry once; if still limited, save partial rows and resume next tick.
@ -152,8 +152,8 @@ Requires `TRADING_ENABLED=true` when `MARKET_HISTORY_SYNC_ENABLED=true`.
| Env var | Default | Purpose | | Env var | Default | Purpose |
|---------|---------|---------| |---------|---------|---------|
| `MARKET_HISTORY_SYNC_ENABLED` | `false` | Worker: universe → backfill → cleanup | | `MARKET_HISTORY_SYNC_ENABLED` | `false` | Worker: universe → backfill → cleanup |
| `MARKET_HISTORY_WINDOW_DAYS` | `7` | Oldest slot start to retain / backfill | | `MARKET_HISTORY_WINDOW_DAYS` | `5` | Oldest slot start to retain / backfill |
| `MARKET_HISTORY_RETENTION_DAYS` | `7` | Delete `as_of` older than this | | `MARKET_HISTORY_RETENTION_DAYS` | `5` | Delete `as_of` older than this |
| `MARKET_HISTORY_ARCHIVE_ENABLED` | `false` | Archive before delete | | `MARKET_HISTORY_ARCHIVE_ENABLED` | `false` | Archive before delete |
| `MARKET_UNIVERSE_REFRESH_HOURS` | `24` | Min hours between universe syncs | | `MARKET_UNIVERSE_REFRESH_HOURS` | `24` | Min hours between universe syncs |
| `MARKET_HISTORY_SYNC_HOURS` | `24` | Unused for backfill (slot-gated); fallback if no slot gate | | `MARKET_HISTORY_SYNC_HOURS` | `24` | Unused for backfill (slot-gated); fallback if no slot gate |
@ -191,7 +191,7 @@ Scheduled worker sync also requires `MARKET_HISTORY_SYNC_ENABLED=true`.
`sync-runs` includes optional `config` when the server has market-history env loaded: `sync-runs` includes optional `config` when the server has market-history env loaded:
```json ```json
{ "archiveEnabled": true, "windowDays": 7, "retentionDays": 7, "syncEnabled": true } { "archiveEnabled": true, "windowDays": 5, "retentionDays": 5, "syncEnabled": true }
``` ```
Flutter uses `config.archiveEnabled` to show the archive checkbox in the cleanup Flutter uses `config.archiveEnabled` to show the archive checkbox in the cleanup

View File

@ -143,12 +143,15 @@ Handler marketHistoryAdminHandler({
try { try {
final int windowDays = final int windowDays =
portalConfig?.windowDays ?? MarketHistoryConfig.windowDays; portalConfig?.windowDays ?? MarketHistoryConfig.windowDays;
final DateTime now =
_parseBefore(request.requestedUri.queryParameters['asOf']) ??
DateTime.now().toUtc();
final MarketHistoryWeekCoverage coverage = MarketHistoryWeekCoverage( final MarketHistoryWeekCoverage coverage = MarketHistoryWeekCoverage(
connection: connection, connection: connection,
tradableAssetsDb: TradableAssetsDb(connection), tradableAssetsDb: TradableAssetsDb(connection),
windowDays: windowDays, windowDays: windowDays,
); );
final MarketHistoryWeekCoverageReport report = await coverage.compute(); final MarketHistoryWeekCoverageReport report = await coverage.compute(now: now);
return _jsonResponse(200, report.toJson()); return _jsonResponse(200, report.toJson());
} catch (e, st) { } catch (e, st) {
stderr.writeln('market history admin week-coverage failed: $e\n$st'); stderr.writeln('market history admin week-coverage failed: $e\n$st');

View File

@ -38,10 +38,10 @@ class MarketHistoryEnv {
final bool syncEnabled = final bool syncEnabled =
(env['MARKET_HISTORY_SYNC_ENABLED'] ?? 'false').toLowerCase() == 'true'; (env['MARKET_HISTORY_SYNC_ENABLED'] ?? 'false').toLowerCase() == 'true';
final int windowDays = final int windowDays =
_positiveInt(env['MARKET_HISTORY_WINDOW_DAYS'], defaultValue: 7, name: 'MARKET_HISTORY_WINDOW_DAYS'); _positiveInt(env['MARKET_HISTORY_WINDOW_DAYS'], defaultValue: 5, name: 'MARKET_HISTORY_WINDOW_DAYS');
final int retentionDays = _positiveInt( final int retentionDays = _positiveInt(
env['MARKET_HISTORY_RETENTION_DAYS'], env['MARKET_HISTORY_RETENTION_DAYS'],
defaultValue: 7, defaultValue: 5,
name: 'MARKET_HISTORY_RETENTION_DAYS', name: 'MARKET_HISTORY_RETENTION_DAYS',
); );
final bool archiveEnabled = final bool archiveEnabled =

View File

@ -1,6 +1,7 @@
import 'package:postgres/postgres.dart'; import 'package:postgres/postgres.dart';
import 'market_history_config.dart'; import 'market_history_config.dart';
import 'market_history_session_slot.dart';
import 'sync_run_recorder.dart'; import 'sync_run_recorder.dart';
/// Outcome of a [MarketDataRetention] cleanup pass. /// Outcome of a [MarketDataRetention] cleanup pass.
@ -44,12 +45,12 @@ class MarketDataRetention {
static const String kind = 'cleanup'; static const String kind = 'cleanup';
/// Hard-delete rows with `as_of` older than [windowDays]. /// Hard-delete rows older than the [windowDays] trading-day sync window.
Future<MarketDataRetentionResult> runCleanup({DateTime? now}) { Future<MarketDataRetentionResult> runCleanup({DateTime? now}) {
return run(archive: false, now: now); return run(archive: false, now: now);
} }
/// Archive-then-delete for rows older than [windowDays]. /// Archive-then-delete for rows older than the [windowDays] trading-day window.
Future<MarketDataRetentionResult> runArchiveAndCleanup({DateTime? now}) { Future<MarketDataRetentionResult> runArchiveAndCleanup({DateTime? now}) {
return run(archive: true, now: now); return run(archive: true, now: now);
} }
@ -86,7 +87,8 @@ class MarketDataRetention {
required int windowDays, required int windowDays,
required bool archive, required bool archive,
}) async { }) async {
final DateTime cutoff = now.subtract(Duration(days: windowDays)); final DateTime cutoff =
MarketHistorySessionSlot.windowFirstSlotStart(now, windowDays);
int totalRemoved = 0; int totalRemoved = 0;
while (true) { while (true) {

View File

@ -1,8 +1,8 @@
/// Defaults for RTH session-half market history ([MarketHistorySessionSlot]). /// Defaults for RTH session-half market history ([MarketHistorySessionSlot]).
/// Env overrides via [MarketHistoryEnv] in [ServerEnv.load]. /// Env overrides via [MarketHistoryEnv] in [ServerEnv.load].
abstract final class MarketHistoryConfig { abstract final class MarketHistoryConfig {
/// Rolling window length in calendar days (UTC). /// Rolling window length in US trading days (MonFri; holidays skipped).
static const int windowDays = 7; static const int windowDays = 5;
/// Stored bar timeframe (two aggregates per US trading day). /// Stored bar timeframe (two aggregates per US trading day).
static const String barTimeframe = 'sessionHalf'; static const String barTimeframe = 'sessionHalf';

View File

@ -5,6 +5,7 @@ import 'package:postgres/postgres.dart';
import 'market_data_db.dart' show MarketDataDb; import 'market_data_db.dart' show MarketDataDb;
import 'market_history_bar_placeholder.dart'; import 'market_history_bar_placeholder.dart';
import 'market_history_config.dart'; import 'market_history_config.dart';
import 'market_history_session_slot.dart';
import 'tradable_assets_db.dart'; import 'tradable_assets_db.dart';
/// One symbol's rolling-window 4-hour bar summary for the guessing game. /// One symbol's rolling-window 4-hour bar summary for the guessing game.
@ -49,7 +50,7 @@ class MarketHistoryQuery {
}) async { }) async {
final DateTime until = asOf.toUtc(); final DateTime until = asOf.toUtc();
final DateTime since = final DateTime since =
until.subtract(Duration(days: windowDays)); MarketHistorySessionSlot.windowFirstSlotStart(until, windowDays);
final DateTime freshSince = final DateTime freshSince =
until.subtract(Duration(days: maxStalenessDays)); until.subtract(Duration(days: maxStalenessDays));

View File

@ -119,16 +119,28 @@ abstract final class MarketHistorySessionSlot {
return _morningStartUtc(cursor.year, cursor.month, cursor.day); return _morningStartUtc(cursor.year, cursor.month, cursor.day);
} }
/// Morning open of the earliest US trading day in the rolling window.
///
/// [windowDays] counts **NYSE trading days** (skips weekends/holidays), anchored
/// on the Eastern calendar day of [lastCompletedSlotStart] as of [now].
static DateTime windowFirstSlotStart(DateTime now, int windowDays) { static DateTime windowFirstSlotStart(DateTime now, int windowDays) {
final tz.TZDateTime ny = tz.TZDateTime.from(now.toUtc(), _eastern); if (windowDays < 1) {
var cursor = ny.subtract(Duration(days: windowDays)); throw ArgumentError.value(windowDays, 'windowDays', 'must be >= 1');
for (var i = 0; i < windowDays + 14; i++) { }
final DateTime last = lastCompletedSlotStart(now);
final tz.TZDateTime lastNy = tz.TZDateTime.from(last.toUtc(), _eastern);
var cursor = tz.TZDateTime(_eastern, lastNy.year, lastNy.month, lastNy.day);
var tradingDaysCounted = 0;
for (var i = 0; i < 366; i++) {
if (_isTradingDayEt(cursor.year, cursor.month, cursor.day)) { if (_isTradingDayEt(cursor.year, cursor.month, cursor.day)) {
tradingDaysCounted++;
if (tradingDaysCounted >= windowDays) {
return _morningStartUtc(cursor.year, cursor.month, cursor.day); return _morningStartUtc(cursor.year, cursor.month, cursor.day);
} }
cursor = cursor.add(const Duration(days: 1));
} }
return _morningStartUtc(ny.year, ny.month, ny.day); cursor = cursor.subtract(const Duration(days: 1));
}
return _morningStartUtc(cursor.year, cursor.month, cursor.day);
} }
static List<DateTime> completedSlotStartsInWindow( static List<DateTime> completedSlotStartsInWindow(

View File

@ -107,7 +107,7 @@ class MarketHistoryWeekCoverage {
final int symbolCount = symbols.length; final int symbolCount = symbols.length;
final List<(int, int, int)> calendarDays = final List<(int, int, int)> calendarDays =
calendarDaysEndingTodayEt(tick, windowDays); tradingDaysInSyncWindow(tick, windowDays);
final Map<String, Set<String>> symbolsBySlot = final Map<String, Set<String>> symbolsBySlot =
await _loadSyncedSymbolsBySlot(tick, symbols); await _loadSyncedSymbolsBySlot(tick, symbols);
@ -208,7 +208,7 @@ class MarketHistoryWeekCoverage {
windowDays, windowDays,
); );
final DateTime until = MarketHistorySessionSlot.endExclusive( final DateTime until = MarketHistorySessionSlot.endExclusive(
MarketHistorySessionSlot.slotStartContaining(now), MarketHistorySessionSlot.lastCompletedSlotStart(now),
); );
final Result rows = await _connection.execute( final Result rows = await _connection.execute(
@ -268,6 +268,33 @@ class MarketHistoryWeekCoverage {
return MarketHistorySessionSlot.slotStartContaining(asOf); return MarketHistorySessionSlot.slotStartContaining(asOf);
} }
/// US trading-day ET dates from [windowDays] sync window (oldest newest).
static List<(int, int, int)> tradingDaysInSyncWindow(
DateTime now,
int windowDays,
) {
ensureMarketHistoryTimezonesInitialized();
final tz.Location eastern = tz.getLocation('America/New_York');
final DateTime first =
MarketHistorySessionSlot.windowFirstSlotStart(now, windowDays);
final DateTime last =
MarketHistorySessionSlot.lastCompletedSlotStart(now);
final tz.TZDateTime firstNy = tz.TZDateTime.from(first.toUtc(), eastern);
var cursor = tz.TZDateTime(eastern, firstNy.year, firstNy.month, firstNy.day);
final tz.TZDateTime endDay = tz.TZDateTime.from(last.toUtc(), eastern);
final tz.TZDateTime endDate =
tz.TZDateTime(eastern, endDay.year, endDay.month, endDay.day);
final List<(int, int, int)> days = <(int, int, int)>[];
while (!cursor.isAfter(endDate)) {
if (_slotStartsForEtDay(cursor.year, cursor.month, cursor.day).isNotEmpty) {
days.add((cursor.year, cursor.month, cursor.day));
}
cursor = cursor.add(const Duration(days: 1));
}
return days;
}
/// Eastern calendar dates (y, m, d) for [windowDays] ending on today's ET date. /// Eastern calendar dates (y, m, d) for [windowDays] ending on today's ET date.
static List<(int, int, int)> calendarDaysEndingTodayEt( static List<(int, int, int)> calendarDaysEndingTodayEt(
DateTime now, DateTime now,

View File

@ -1,6 +1,6 @@
-- 005_market_history.sql -- 005_market_history.sql
-- --
-- Adds the rolling 7-day market-history substrate: -- Adds the rolling market-history substrate (window length from env):
-- * timeframe column + idempotent unique observation key on -- * timeframe column + idempotent unique observation key on
-- market_data_snapshots -- market_data_snapshots
-- * tradable_assets cache for the daily Alpaca asset universe -- * tradable_assets cache for the daily Alpaca asset universe

View File

@ -7,8 +7,8 @@ void main() {
final MarketHistoryEnv env = MarketHistoryEnv.fromMap(<String, String>{}); final MarketHistoryEnv env = MarketHistoryEnv.fromMap(<String, String>{});
expect(env.syncEnabled, isFalse); expect(env.syncEnabled, isFalse);
expect(env.windowDays, 7); expect(env.windowDays, 5);
expect(env.retentionDays, 7); expect(env.retentionDays, 5);
expect(env.archiveEnabled, isFalse); expect(env.archiveEnabled, isFalse);
expect(env.universeRefreshHours, 24); expect(env.universeRefreshHours, 24);
expect(env.historySyncHours, 24); expect(env.historySyncHours, 24);

View File

@ -3,6 +3,7 @@ library;
import 'package:cyberhybridhub_server/trading/market_data_db.dart'; import 'package:cyberhybridhub_server/trading/market_data_db.dart';
import 'package:cyberhybridhub_server/trading/market_data_retention.dart'; import 'package:cyberhybridhub_server/trading/market_data_retention.dart';
import 'package:cyberhybridhub_server/trading/market_history_session_slot.dart';
import 'package:postgres/postgres.dart'; import 'package:postgres/postgres.dart';
import 'package:test/test.dart'; import 'package:test/test.dart';
@ -12,9 +13,13 @@ void main() {
TestDb? testDb; TestDb? testDb;
setUpAll(() async { setUpAll(() async {
ensureMarketHistoryTimezonesInitialized();
testDb = await TestDb.open(); testDb = await TestDb.open();
}); });
/// After US close so [windowFirstSlotStart] is stable in tests.
final DateTime retentionNow = DateTime.utc(2026, 5, 26, 21, 0);
tearDown(() async { tearDown(() async {
if (testDb != null) { if (testDb != null) {
await testDb!.connection.execute('TRUNCATE TABLE market_data_archive'); await testDb!.connection.execute('TRUNCATE TABLE market_data_archive');
@ -44,40 +49,43 @@ void main() {
} }
final MarketDataDb db = testDb!.marketDataDb; final MarketDataDb db = testDb!.marketDataDb;
final DateTime now = DateTime.utc(2026, 5, 26, 12); final DateTime now = retentionNow;
final DateTime cutoff = now.subtract(const Duration(days: 7)); final DateTime cutoff =
MarketHistorySessionSlot.windowFirstSlotStart(now, 7);
int removedCount = 0;
int keptCount = 0;
for (int day = 0; day < 10; day++) {
final DateTime asOf = now.subtract(Duration(days: 14 - day));
await db.upsertSnapshot( await db.upsertSnapshot(
symbol: 'SPY', symbol: 'SPY',
metric: 'bar', metric: 'bar',
timeframe: '1Day', timeframe: 'sessionHalf',
asOf: asOf, asOf: cutoff.subtract(const Duration(days: 2)),
price: 490 + day, price: 480,
);
await db.upsertSnapshot(
symbol: 'SPY',
metric: 'bar',
timeframe: 'sessionHalf',
asOf: cutoff,
price: 490,
);
await db.upsertSnapshot(
symbol: 'SPY',
metric: 'bar',
timeframe: 'sessionHalf',
asOf: cutoff.add(const Duration(hours: 3, minutes: 15)),
price: 500,
); );
if (asOf.isBefore(cutoff)) {
removedCount++;
} else {
keptCount++;
}
}
final MarketDataRetentionResult result = await retention().runCleanup( final MarketDataRetentionResult result = await retention().runCleanup(
now: now, now: now,
); );
expect(result.error, isNull); expect(result.error, isNull);
expect(result.rowsRemoved, removedCount); expect(result.rowsRemoved, 1);
expect(keptCount, greaterThan(0));
final Result remaining = await testDb!.connection.execute( final Result remaining = await testDb!.connection.execute(
'SELECT COUNT(*)::int FROM market_data_snapshots', 'SELECT COUNT(*)::int FROM market_data_snapshots',
); );
expect((remaining.first[0]! as num).toInt(), keptCount); expect((remaining.first[0]! as num).toInt(), 2);
}); });
test('empty table returns rowsRemoved 0 without throwing', () async { test('empty table returns rowsRemoved 0 without throwing', () async {
@ -89,7 +97,7 @@ void main() {
} }
final MarketDataRetentionResult result = await retention().runCleanup( final MarketDataRetentionResult result = await retention().runCleanup(
now: DateTime.utc(2026, 5, 26, 12), now: retentionNow,
); );
expect(result.rowsRemoved, 0); expect(result.rowsRemoved, 0);
@ -107,8 +115,11 @@ void main() {
const int rowCount = 5000; const int rowCount = 5000;
const int batchSize = 1000; const int batchSize = 1000;
final DateTime now = DateTime.utc(2026, 5, 26, 12); final DateTime now = retentionNow;
final DateTime oldAsOf = now.subtract(const Duration(days: 14)); final DateTime oldAsOf = MarketHistorySessionSlot.windowFirstSlotStart(
now,
7,
).subtract(const Duration(days: 2));
await testDb!.connection.execute( await testDb!.connection.execute(
Sql.named( Sql.named(
@ -148,12 +159,14 @@ void main() {
return; return;
} }
final DateTime now = DateTime.utc(2026, 5, 26, 12); final DateTime now = retentionNow;
final DateTime cutoff =
MarketHistorySessionSlot.windowFirstSlotStart(now, 7);
await testDb!.marketDataDb.upsertSnapshot( await testDb!.marketDataDb.upsertSnapshot(
symbol: 'SPY', symbol: 'SPY',
metric: 'bar', metric: 'bar',
timeframe: '1Day', timeframe: 'sessionHalf',
asOf: now.subtract(const Duration(days: 10)), asOf: cutoff.subtract(const Duration(days: 2)),
price: 480, price: 480,
); );
@ -183,20 +196,22 @@ void main() {
} }
final MarketDataDb db = testDb!.marketDataDb; final MarketDataDb db = testDb!.marketDataDb;
final DateTime now = DateTime.utc(2026, 5, 26, 12); final DateTime now = retentionNow;
final DateTime cutoff =
MarketHistorySessionSlot.windowFirstSlotStart(now, 7);
final MarketDataSnapshot kept = await db.upsertSnapshot( final MarketDataSnapshot kept = await db.upsertSnapshot(
symbol: 'SPY', symbol: 'SPY',
metric: 'bar', metric: 'bar',
timeframe: '1Day', timeframe: 'sessionHalf',
asOf: now.subtract(const Duration(days: 2)), asOf: cutoff.add(const Duration(hours: 3, minutes: 15)),
price: 500, price: 500,
); );
await db.upsertSnapshot( await db.upsertSnapshot(
symbol: 'SPY', symbol: 'SPY',
metric: 'bar', metric: 'bar',
timeframe: '1Day', timeframe: 'sessionHalf',
asOf: now.subtract(const Duration(days: 10)), asOf: cutoff.subtract(const Duration(days: 2)),
price: 480, price: 480,
); );
@ -219,12 +234,14 @@ void main() {
return; return;
} }
final DateTime now = DateTime.utc(2026, 5, 26, 12); final DateTime now = retentionNow;
final DateTime cutoff =
MarketHistorySessionSlot.windowFirstSlotStart(now, 7);
await testDb!.marketDataDb.upsertSnapshot( await testDb!.marketDataDb.upsertSnapshot(
symbol: 'SPY', symbol: 'SPY',
metric: 'bar', metric: 'bar',
timeframe: '1Day', timeframe: 'sessionHalf',
asOf: now.subtract(const Duration(days: 10)), asOf: cutoff.subtract(const Duration(days: 2)),
price: 480, price: 480,
volume: 1000, volume: 1000,
raw: <String, dynamic>{'c': 480}, raw: <String, dynamic>{'c': 480},
@ -232,8 +249,8 @@ void main() {
await testDb!.marketDataDb.upsertSnapshot( await testDb!.marketDataDb.upsertSnapshot(
symbol: 'SPY', symbol: 'SPY',
metric: 'bar', metric: 'bar',
timeframe: '1Day', timeframe: 'sessionHalf',
asOf: now.subtract(const Duration(days: 2)), asOf: cutoff.add(const Duration(hours: 3, minutes: 15)),
price: 500, price: 500,
); );
@ -262,12 +279,14 @@ void main() {
return; return;
} }
final DateTime now = DateTime.utc(2026, 5, 26, 12); final DateTime now = retentionNow;
final DateTime cutoff =
MarketHistorySessionSlot.windowFirstSlotStart(now, 7);
await testDb!.marketDataDb.upsertSnapshot( await testDb!.marketDataDb.upsertSnapshot(
symbol: 'SPY', symbol: 'SPY',
metric: 'bar', metric: 'bar',
timeframe: '1Day', timeframe: 'sessionHalf',
asOf: now.subtract(const Duration(days: 10)), asOf: cutoff.subtract(const Duration(days: 2)),
price: 480, price: 480,
); );
@ -299,12 +318,14 @@ void main() {
return; return;
} }
final DateTime now = DateTime.utc(2026, 5, 26, 12); final DateTime now = retentionNow;
final DateTime cutoff =
MarketHistorySessionSlot.windowFirstSlotStart(now, 7);
await testDb!.marketDataDb.upsertSnapshot( await testDb!.marketDataDb.upsertSnapshot(
symbol: 'SPY', symbol: 'SPY',
metric: 'bar', metric: 'bar',
timeframe: '1Day', timeframe: 'sessionHalf',
asOf: now.subtract(const Duration(days: 10)), asOf: cutoff.subtract(const Duration(days: 2)),
price: 480, price: 480,
); );

View File

@ -425,7 +425,7 @@ void main() {
adminFirebaseUids: <String>{'admin-uid'}, adminFirebaseUids: <String>{'admin-uid'},
portalConfig: const MarketHistoryAdminPortalConfig( portalConfig: const MarketHistoryAdminPortalConfig(
archiveEnabled: true, archiveEnabled: true,
windowDays: 7, windowDays: 5,
retentionDays: 14, retentionDays: 14,
syncEnabled: true, syncEnabled: true,
), ),
@ -442,7 +442,7 @@ void main() {
final Map<String, dynamic> config = final Map<String, dynamic> config =
body['config'] as Map<String, dynamic>; body['config'] as Map<String, dynamic>;
expect(config['archiveEnabled'], isTrue); expect(config['archiveEnabled'], isTrue);
expect(config['windowDays'], 7); expect(config['windowDays'], 5);
expect(config['retentionDays'], 14); expect(config['retentionDays'], 14);
expect(config['syncEnabled'], isTrue); expect(config['syncEnabled'], isTrue);
}); });
@ -576,8 +576,8 @@ void main() {
adminFirebaseUids: <String>{'admin-uid'}, adminFirebaseUids: <String>{'admin-uid'},
portalConfig: const MarketHistoryAdminPortalConfig( portalConfig: const MarketHistoryAdminPortalConfig(
archiveEnabled: false, archiveEnabled: false,
windowDays: 7, windowDays: 5,
retentionDays: 7, retentionDays: 5,
syncEnabled: false, syncEnabled: false,
), ),
); );
@ -591,7 +591,7 @@ void main() {
final Map<String, dynamic> body = final Map<String, dynamic> body =
jsonDecode(await response.readAsString()) as Map<String, dynamic>; jsonDecode(await response.readAsString()) as Map<String, dynamic>;
expect(body['windowDays'], 7); expect(body['windowDays'], 5);
expect(body['canStepNewer'], isFalse); expect(body['canStepNewer'], isFalse);
expect(body['canStepOlder'], isTrue); expect(body['canStepOlder'], isTrue);
expect( expect(
@ -728,7 +728,7 @@ void main() {
return; return;
} }
final DateTime now = DateTime.now().toUtc(); final DateTime now = DateTime.utc(2026, 5, 31, 21);
final DateTime slotStart = final DateTime slotStart =
MarketHistorySessionSlot.lastCompletedSlotStart(now); MarketHistorySessionSlot.lastCompletedSlotStart(now);
@ -736,7 +736,9 @@ void main() {
Sql.named( Sql.named(
''' '''
INSERT INTO tradable_assets (symbol, asset_class, status, tradable, refreshed_at) INSERT INTO tradable_assets (symbol, asset_class, status, tradable, refreshed_at)
VALUES ('AAA', 'us_equity', 'active', true, @refreshed_at) VALUES
('AAA', 'us_equity', 'active', true, @refreshed_at),
('BBB', 'us_equity', 'active', true, @refreshed_at)
''', ''',
), ),
parameters: <String, dynamic>{'refreshed_at': now}, parameters: <String, dynamic>{'refreshed_at': now},
@ -767,41 +769,49 @@ void main() {
adminFirebaseUids: <String>{'admin-uid'}, adminFirebaseUids: <String>{'admin-uid'},
portalConfig: const MarketHistoryAdminPortalConfig( portalConfig: const MarketHistoryAdminPortalConfig(
archiveEnabled: false, archiveEnabled: false,
windowDays: 7, windowDays: 5,
retentionDays: 7, retentionDays: 5,
syncEnabled: false, syncEnabled: false,
), ),
); );
final Response response = await _get( final Response response = await _get(
handler, handler,
path: '/v1/admin/market-history/week-coverage', path: '/v1/admin/market-history/week-coverage?asOf=${Uri.encodeComponent(now.toIso8601String())}',
bearer: 'admin-uid', bearer: 'admin-uid',
); );
expect(response.statusCode, 200); expect(response.statusCode, 200);
final Map<String, dynamic> body = final Map<String, dynamic> body =
jsonDecode(await response.readAsString()) as Map<String, dynamic>; jsonDecode(await response.readAsString()) as Map<String, dynamic>;
expect(body['windowDays'], 7); expect(body['windowDays'], 5);
expect(body['slotsPerDay'], 2); expect(body['slotsPerDay'], 2);
expect(body['symbolCount'], 1); expect(body['symbolCount'], 2);
expect(body['isConsistent'], isFalse); expect(body['isConsistent'], isFalse);
final List<dynamic> days = body['days'] as List<dynamic>; final List<dynamic> days = body['days'] as List<dynamic>;
expect(days, hasLength(7)); expect(days, hasLength(5));
final tz.TZDateTime slotDayEt = tz.TZDateTime.from(
slotStart, final String slotWire = MarketHistorySessionSlot.slotStartWire(slotStart);
tz.getLocation('America/New_York'), Map<String, dynamic>? matchingSlot;
); for (final Map<String, dynamic> day in days.cast<Map<String, dynamic>>()) {
final String slotDayWire = for (final Map<String, dynamic> slot
'${slotDayEt.year.toString().padLeft(4, '0')}-' in (day['slots'] as List<dynamic>).cast<Map<String, dynamic>>()) {
'${slotDayEt.month.toString().padLeft(2, '0')}-' if (MarketHistorySessionSlot.slotStartWire(
'${slotDayEt.day.toString().padLeft(2, '0')}'; DateTime.parse(slot['slotStart'] as String).toUtc(),
final Map<String, dynamic> slotDay = days.cast<Map<String, dynamic>>().firstWhere( ) ==
(Map<String, dynamic> d) => d['date'] == slotDayWire, slotWire) {
); matchingSlot = slot;
expect(slotDay['fullySyncedSlots'], 1); break;
expect(slotDay['completedSlots'], greaterThan(0)); }
}
if (matchingSlot != null) {
break;
}
}
expect(matchingSlot, isNotNull);
expect(matchingSlot!['fullySynced'], isFalse);
expect((matchingSlot['syncedSymbolCount'] as num).toInt(), 1);
}); });
test('resync returns 503 when sync is disabled in portal config', () async { test('resync returns 503 when sync is disabled in portal config', () async {
@ -817,8 +827,8 @@ void main() {
actions: null, actions: null,
portalConfig: const MarketHistoryAdminPortalConfig( portalConfig: const MarketHistoryAdminPortalConfig(
archiveEnabled: false, archiveEnabled: false,
windowDays: 7, windowDays: 5,
retentionDays: 7, retentionDays: 5,
syncEnabled: false, syncEnabled: false,
), ),
); );

View File

@ -67,6 +67,23 @@ void main() {
); );
}); });
test('windowFirstSlotStart counts trading days not calendar days', () {
final DateTime now = DateTime.utc(2026, 5, 31, 21, 0);
expect(
MarketHistorySessionSlot.windowFirstSlotStart(now, 7),
DateTime.utc(2026, 5, 20, 13, 30),
);
});
test('completedSlotStartsInWindow spans seven trading days', () {
final DateTime now = DateTime.utc(2026, 5, 31, 21, 0);
final List<DateTime> slots =
MarketHistorySessionSlot.completedSlotStartsInWindow(now, 7);
expect(slots, hasLength(14));
expect(slots.first, DateTime.utc(2026, 5, 20, 13, 30));
expect(slots.last, DateTime.utc(2026, 5, 29, 16, 45));
});
test('previousSlotStart walks afternoon to morning', () { test('previousSlotStart walks afternoon to morning', () {
final DateTime afternoon = DateTime.utc(2026, 6, 2, 16, 45); final DateTime afternoon = DateTime.utc(2026, 6, 2, 16, 45);
expect( expect(

View File

@ -6,12 +6,22 @@ void main() {
setUp(ensureMarketHistoryTimezonesInitialized); setUp(ensureMarketHistoryTimezonesInitialized);
group('MarketHistoryWeekCoverage calendar days', () { group('MarketHistoryWeekCoverage calendar days', () {
test('returns windowDays Eastern dates ending on today', () { test('tradingDaysInSyncWindow lists five trading days for window', () {
final DateTime now = DateTime.utc(2026, 5, 31, 21);
final List<(int, int, int)> days =
MarketHistoryWeekCoverage.tradingDaysInSyncWindow(now, 5);
expect(days, hasLength(5));
expect(days.first, (2026, 5, 22));
expect(days.last, (2026, 5, 29));
});
test('calendarDaysEndingTodayEt still spans five calendar days', () {
final DateTime now = DateTime.utc(2026, 6, 2, 21); final DateTime now = DateTime.utc(2026, 6, 2, 21);
final List<(int, int, int)> days = final List<(int, int, int)> days =
MarketHistoryWeekCoverage.calendarDaysEndingTodayEt(now, 7); MarketHistoryWeekCoverage.calendarDaysEndingTodayEt(now, 5);
expect(days, hasLength(7)); expect(days, hasLength(5));
expect(days.last, (2026, 6, 2)); expect(days.last, (2026, 6, 2));
}); });
}); });

View File

@ -202,8 +202,8 @@ void main() {
errorMessage: null, errorMessage: null,
config: const MarketHistoryAdminConfig( config: const MarketHistoryAdminConfig(
archiveEnabled: true, archiveEnabled: true,
windowDays: 7, windowDays: 5,
retentionDays: 7, retentionDays: 5,
syncEnabled: true, syncEnabled: true,
), ),
), ),

View File

@ -133,7 +133,7 @@ class _FakeCoverageApi extends MarketHistoryAdminApi {
compareUntil: DateTime.utc(2026, 5, 30, 16), compareUntil: DateTime.utc(2026, 5, 30, 16),
newerSlotStart: DateTime.utc(2026, 5, 30, 12), newerSlotStart: DateTime.utc(2026, 5, 30, 12),
olderSlotStart: DateTime.utc(2026, 5, 30, 8), olderSlotStart: DateTime.utc(2026, 5, 30, 8),
windowDays: 7, windowDays: 5,
canStepOlder: false, canStepOlder: false,
canStepNewer: false, canStepNewer: false,
assets: const <QuestionAuditAsset>[], assets: const <QuestionAuditAsset>[],
@ -144,14 +144,14 @@ MarketHistoryWeekCoverageReport _sampleWeekReport() {
final DateTime day = DateTime.utc(2026, 5, 30); final DateTime day = DateTime.utc(2026, 5, 30);
return MarketHistoryWeekCoverageReport( return MarketHistoryWeekCoverageReport(
asOf: DateTime.utc(2026, 5, 30, 15), asOf: DateTime.utc(2026, 5, 30, 15),
windowDays: 7, windowDays: 5,
slotsPerDay: 6, slotsPerDay: 6,
symbolCount: 10, symbolCount: 10,
isConsistent: true, isConsistent: true,
days: List<MarketHistoryDayCoverage>.generate( days: List<MarketHistoryDayCoverage>.generate(
7, 5,
(int index) { (int index) {
final DateTime date = day.subtract(Duration(days: 6 - index)); final DateTime date = day.subtract(Duration(days: 4 - index));
return MarketHistoryDayCoverage( return MarketHistoryDayCoverage(
date: date, date: date,
slotsPerDay: 6, slotsPerDay: 6,
@ -425,8 +425,8 @@ void main() {
errorMessage: null, errorMessage: null,
config: const MarketHistoryAdminConfig( config: const MarketHistoryAdminConfig(
archiveEnabled: false, archiveEnabled: false,
windowDays: 7, windowDays: 5,
retentionDays: 7, retentionDays: 5,
syncEnabled: true, syncEnabled: true,
), ),
), ),
@ -466,8 +466,8 @@ void main() {
errorMessage: null, errorMessage: null,
config: const MarketHistoryAdminConfig( config: const MarketHistoryAdminConfig(
archiveEnabled: false, archiveEnabled: false,
windowDays: 7, windowDays: 5,
retentionDays: 7, retentionDays: 5,
syncEnabled: true, syncEnabled: true,
), ),
), ),
@ -535,7 +535,7 @@ void main() {
await tester.pumpAndSettle(); await tester.pumpAndSettle();
expect(find.byKey(const Key('week-coverage-dialog')), findsOneWidget); expect(find.byKey(const Key('week-coverage-dialog')), findsOneWidget);
expect(find.text('7-day sync health'), findsOneWidget); expect(find.text('5-day sync health'), findsOneWidget);
expect(find.textContaining('6/6'), findsWidgets); expect(find.textContaining('6/6'), findsWidgets);
await tester.tap(find.byKey(const Key('week-coverage-close'))); await tester.tap(find.byKey(const Key('week-coverage-close')));

View File

@ -136,7 +136,7 @@ void main() {
'pinned': <dynamic>[], 'pinned': <dynamic>[],
'config': <String, dynamic>{ 'config': <String, dynamic>{
'archiveEnabled': true, 'archiveEnabled': true,
'windowDays': 7, 'windowDays': 5,
'retentionDays': 14, 'retentionDays': 14,
'syncEnabled': true, 'syncEnabled': true,
}, },
@ -160,7 +160,7 @@ void main() {
return http.Response( return http.Response(
jsonEncode(<String, dynamic>{ jsonEncode(<String, dynamic>{
'asOf': '2026-05-30T15:30:00Z', 'asOf': '2026-05-30T15:30:00Z',
'windowDays': 7, 'windowDays': 5,
'slotsPerDay': 6, 'slotsPerDay': 6,
'symbolCount': 2, 'symbolCount': 2,
'isConsistent': false, 'isConsistent': false,
@ -205,7 +205,7 @@ void main() {
'compareUntil': '2026-05-30T16:00:00Z', 'compareUntil': '2026-05-30T16:00:00Z',
'newerSlotStart': '2026-05-30T12:00:00Z', 'newerSlotStart': '2026-05-30T12:00:00Z',
'olderSlotStart': '2026-05-30T08:00:00Z', 'olderSlotStart': '2026-05-30T08:00:00Z',
'windowDays': 7, 'windowDays': 5,
'canStepOlder': true, 'canStepOlder': true,
'canStepNewer': false, 'canStepNewer': false,
'stepOlderCompareUntil': '2026-05-30T12:00:00Z', 'stepOlderCompareUntil': '2026-05-30T12:00:00Z',
@ -242,7 +242,7 @@ void main() {
); );
final QuestionAuditReport report = await api.fetchQuestionAudit(); final QuestionAuditReport report = await api.fetchQuestionAudit();
expect(report.windowDays, 7); expect(report.windowDays, 5);
expect(report.canStepOlder, isTrue); expect(report.canStepOlder, isTrue);
expect(report.canStepNewer, isFalse); expect(report.canStepNewer, isFalse);
expect(report.stepOlderCompareUntil, DateTime.utc(2026, 5, 30, 12)); expect(report.stepOlderCompareUntil, DateTime.utc(2026, 5, 30, 12));

View File

@ -46,7 +46,7 @@ QuestionAuditReport _sampleReport({
compareUntil: until, compareUntil: until,
newerSlotStart: newer, newerSlotStart: newer,
olderSlotStart: older, olderSlotStart: older,
windowDays: 7, windowDays: 5,
canStepOlder: canStepOlder, canStepOlder: canStepOlder,
canStepNewer: canStepNewer, canStepNewer: canStepNewer,
stepOlderCompareUntil: stepOlderCompareUntil, stepOlderCompareUntil: stepOlderCompareUntil,