From 6615dc5d17208e67fcafb0d686df1e36a9a9be58 Mon Sep 17 00:00:00 2001 From: Nathan Anderson Date: Sun, 31 May 2026 13:11:09 -0500 Subject: [PATCH] 5 day --- FLUTTER-ADMIN-PORTAL.md | 2 +- .../models/market_history_admin_config.dart | 8 +- .../models/market_history_week_coverage.dart | 2 +- lib/admin/models/sync_run_event.dart | 4 +- .../repositories/sync_run_log_repository.dart | 8 +- .../screens/market_history_log_screen.dart | 2 +- .../market_history_week_coverage_sheet.dart | 10 +- server/README.md | 8 +- .../market_history_admin_handler.dart | 5 +- server/lib/market_history_env.dart | 4 +- server/lib/trading/market_data_retention.dart | 8 +- server/lib/trading/market_history_config.dart | 4 +- server/lib/trading/market_history_query.dart | 3 +- .../trading/market_history_session_slot.dart | 28 +++-- .../trading/market_history_week_coverage.dart | 31 ++++- server/migrations/005_market_history.sql | 2 +- server/test/env/market_history_env_test.dart | 4 +- .../market_data_retention_test.dart | 111 +++++++++++------- .../market_history_admin_handler_test.dart | 66 ++++++----- .../market_history_session_slot_test.dart | 17 +++ .../market_history_week_coverage_test.dart | 16 ++- .../admin_portal_acceptance_test.dart | 4 +- .../market_history_log_screen_test.dart | 18 +-- .../market_history_admin_api_test.dart | 8 +- ...ket_history_question_audit_sheet_test.dart | 2 +- 25 files changed, 239 insertions(+), 136 deletions(-) diff --git a/FLUTTER-ADMIN-PORTAL.md b/FLUTTER-ADMIN-PORTAL.md index ed27837..5b6ba10 100644 --- a/FLUTTER-ADMIN-PORTAL.md +++ b/FLUTTER-ADMIN-PORTAL.md @@ -14,7 +14,7 @@ Build an **admin-only** area in the Flutter app that shows a **live audit log** - **Historical backfill** (`kind=backfill`) — ended **4-hour UTC slots** (`4Hour` bars) into `market_data_snapshots`; open slot not fetched yet - **Retention / cleanup** (`kind=cleanup`) — prune (or archive-then-delete) expired rows -Operators use it to confirm the rolling 7-day window is healthy, see when cleanup ran, and **notice Alpaca/API failures or rate limits immediately** without reading server logs or SQL. +Operators use it to confirm the rolling 5-trading-day window is healthy, see when cleanup ran, and **notice Alpaca/API failures or rate limits immediately** without reading server logs or SQL. This plan **replaces the need for §8 shell CLIs** for day-to-day product use. CLIs remain optional for CI/SRE (see §12). diff --git a/lib/admin/models/market_history_admin_config.dart b/lib/admin/models/market_history_admin_config.dart index 5b62ba0..b3c4d5f 100644 --- a/lib/admin/models/market_history_admin_config.dart +++ b/lib/admin/models/market_history_admin_config.dart @@ -15,15 +15,15 @@ class MarketHistoryAdminConfig { if (json == null) { return const MarketHistoryAdminConfig( archiveEnabled: false, - windowDays: 7, - retentionDays: 7, + windowDays: 5, + retentionDays: 5, syncEnabled: false, ); } return MarketHistoryAdminConfig( archiveEnabled: json['archiveEnabled'] as bool? ?? false, - windowDays: (json['windowDays'] as num?)?.toInt() ?? 7, - retentionDays: (json['retentionDays'] as num?)?.toInt() ?? 7, + windowDays: (json['windowDays'] as num?)?.toInt() ?? 5, + retentionDays: (json['retentionDays'] as num?)?.toInt() ?? 5, syncEnabled: json['syncEnabled'] as bool? ?? false, ); } diff --git a/lib/admin/models/market_history_week_coverage.dart b/lib/admin/models/market_history_week_coverage.dart index 11bea75..0030227 100644 --- a/lib/admin/models/market_history_week_coverage.dart +++ b/lib/admin/models/market_history_week_coverage.dart @@ -80,7 +80,7 @@ class MarketHistoryWeekCoverageReport { json['days'] as List? ?? []; return MarketHistoryWeekCoverageReport( asOf: DateTime.parse(json['asOf'] as String).toUtc(), - windowDays: (json['windowDays'] as num?)?.toInt() ?? 7, + windowDays: (json['windowDays'] as num?)?.toInt() ?? 5, slotsPerDay: (json['slotsPerDay'] as num?)?.toInt() ?? 2, symbolCount: (json['symbolCount'] as num?)?.toInt() ?? 0, isConsistent: json['isConsistent'] as bool? ?? false, diff --git a/lib/admin/models/sync_run_event.dart b/lib/admin/models/sync_run_event.dart index 7da34bb..9ecdfbd 100644 --- a/lib/admin/models/sync_run_event.dart +++ b/lib/admin/models/sync_run_event.dart @@ -178,8 +178,8 @@ class SyncRunLogPage { required this.nextBefore, this.config = const MarketHistoryAdminConfig( archiveEnabled: false, - windowDays: 7, - retentionDays: 7, + windowDays: 5, + retentionDays: 5, syncEnabled: false, ), }); diff --git a/lib/admin/repositories/sync_run_log_repository.dart b/lib/admin/repositories/sync_run_log_repository.dart index 0ea558c..9ecc3e8 100644 --- a/lib/admin/repositories/sync_run_log_repository.dart +++ b/lib/admin/repositories/sync_run_log_repository.dart @@ -25,8 +25,8 @@ class SyncRunLogRepositoryState { required this.errorMessage, this.config = const MarketHistoryAdminConfig( archiveEnabled: false, - windowDays: 7, - retentionDays: 7, + windowDays: 5, + retentionDays: 5, syncEnabled: false, ), }); @@ -59,8 +59,8 @@ class SyncRunLogRepository implements SyncRunLogController { String? _errorMessage; MarketHistoryAdminConfig _config = const MarketHistoryAdminConfig( archiveEnabled: false, - windowDays: 7, - retentionDays: 7, + windowDays: 5, + retentionDays: 5, syncEnabled: false, ); diff --git a/lib/admin/screens/market_history_log_screen.dart b/lib/admin/screens/market_history_log_screen.dart index 9f7de30..cf4a4f5 100644 --- a/lib/admin/screens/market_history_log_screen.dart +++ b/lib/admin/screens/market_history_log_screen.dart @@ -294,7 +294,7 @@ class _MarketHistoryLogScreenState extends State { ), IconButton( key: const Key('week-coverage-button'), - tooltip: '7-day sync calendar', + tooltip: 'Trading-week sync calendar', onPressed: _weekCoverageLoading ? null : _showWeekCoverage, icon: const Icon(Icons.calendar_month_outlined), ), diff --git a/lib/admin/widgets/market_history_week_coverage_sheet.dart b/lib/admin/widgets/market_history_week_coverage_sheet.dart index 7f19285..3ba1086 100644 --- a/lib/admin/widgets/market_history_week_coverage_sheet.dart +++ b/lib/admin/widgets/market_history_week_coverage_sheet.dart @@ -13,7 +13,7 @@ const List _weekdayLabels = [ 'Sun', ]; -/// Mini 7-day Eastern week view showing session-half slot sync health. +/// Rolling US trading-week view (session-half slot sync health per weekday). class MarketHistoryWeekCoverageSheet extends StatelessWidget { const MarketHistoryWeekCoverageSheet({ super.key, @@ -53,10 +53,10 @@ class MarketHistoryWeekCoverageSheet extends StatelessWidget { children: [ const Icon(Icons.calendar_month, color: AppColors.accent, size: 22), const SizedBox(width: 8), - const Expanded( + Expanded( child: Text( - '7-day sync health', - style: TextStyle( + '${report.windowDays}-day sync health', + style: const TextStyle( fontSize: 18, fontWeight: FontWeight.w700, color: AppColors.textPrimary, @@ -98,7 +98,7 @@ class MarketHistoryWeekCoverageSheet extends StatelessWidget { const SizedBox(height: 12), Text( 'ET · ${report.slotsPerDay} slots per trading day · ' - '${report.windowDays}-day window', + '${report.days.length} trading days · ${report.windowDays}-day sync window', textAlign: TextAlign.center, style: const TextStyle( fontSize: 11, diff --git a/server/README.md b/server/README.md index f0e07be..93341c2 100644 --- a/server/README.md +++ b/server/README.md @@ -131,7 +131,7 @@ curl -s -X POST http://localhost:3000/v1/me/incoming-question \ Alpaca **`1Min`** bars aggregated into two **US regular-session** half-days per trading day (morning **9:30–12:45 ET**, afternoon **12:45–16:00 ET**, ~195 minutes each). -Stored as `metric=bar`, `timeframe=sessionHalf`. Rolling window: `MARKET_HISTORY_WINDOW_DAYS` (default 7). +Stored as `metric=bar`, `timeframe=sessionHalf`. Rolling window: `MARKET_HISTORY_WINDOW_DAYS` (default **5 US trading days**, two session halves each; weekends/holidays skipped). **Backfill** (`kind=backfill`): fetches each **ended** slot still missing in DB; skips the open slot. Throttled to `MARKET_HISTORY_API_REQUESTS_PER_MINUTE` (default 200/min). On HTTP 429: wait 1 minute, retry once; if still limited, save partial rows and resume next tick. @@ -152,8 +152,8 @@ Requires `TRADING_ENABLED=true` when `MARKET_HISTORY_SYNC_ENABLED=true`. | Env var | Default | Purpose | |---------|---------|---------| | `MARKET_HISTORY_SYNC_ENABLED` | `false` | Worker: universe → backfill → cleanup | -| `MARKET_HISTORY_WINDOW_DAYS` | `7` | Oldest slot start to retain / backfill | -| `MARKET_HISTORY_RETENTION_DAYS` | `7` | Delete `as_of` older than this | +| `MARKET_HISTORY_WINDOW_DAYS` | `5` | Oldest slot start to retain / backfill | +| `MARKET_HISTORY_RETENTION_DAYS` | `5` | Delete `as_of` older than this | | `MARKET_HISTORY_ARCHIVE_ENABLED` | `false` | Archive before delete | | `MARKET_UNIVERSE_REFRESH_HOURS` | `24` | Min hours between universe syncs | | `MARKET_HISTORY_SYNC_HOURS` | `24` | Unused for backfill (slot-gated); fallback if no slot gate | @@ -191,7 +191,7 @@ Scheduled worker sync also requires `MARKET_HISTORY_SYNC_ENABLED=true`. `sync-runs` includes optional `config` when the server has market-history env loaded: ```json -{ "archiveEnabled": true, "windowDays": 7, "retentionDays": 7, "syncEnabled": true } +{ "archiveEnabled": true, "windowDays": 5, "retentionDays": 5, "syncEnabled": true } ``` Flutter uses `config.archiveEnabled` to show the archive checkbox in the cleanup diff --git a/server/lib/handlers/market_history_admin_handler.dart b/server/lib/handlers/market_history_admin_handler.dart index 7611364..c3bbe90 100644 --- a/server/lib/handlers/market_history_admin_handler.dart +++ b/server/lib/handlers/market_history_admin_handler.dart @@ -143,12 +143,15 @@ Handler marketHistoryAdminHandler({ try { final int windowDays = portalConfig?.windowDays ?? MarketHistoryConfig.windowDays; + final DateTime now = + _parseBefore(request.requestedUri.queryParameters['asOf']) ?? + DateTime.now().toUtc(); final MarketHistoryWeekCoverage coverage = MarketHistoryWeekCoverage( connection: connection, tradableAssetsDb: TradableAssetsDb(connection), windowDays: windowDays, ); - final MarketHistoryWeekCoverageReport report = await coverage.compute(); + final MarketHistoryWeekCoverageReport report = await coverage.compute(now: now); return _jsonResponse(200, report.toJson()); } catch (e, st) { stderr.writeln('market history admin week-coverage failed: $e\n$st'); diff --git a/server/lib/market_history_env.dart b/server/lib/market_history_env.dart index 5d57af0..2f0f586 100644 --- a/server/lib/market_history_env.dart +++ b/server/lib/market_history_env.dart @@ -38,10 +38,10 @@ class MarketHistoryEnv { final bool syncEnabled = (env['MARKET_HISTORY_SYNC_ENABLED'] ?? 'false').toLowerCase() == 'true'; final int windowDays = - _positiveInt(env['MARKET_HISTORY_WINDOW_DAYS'], defaultValue: 7, name: 'MARKET_HISTORY_WINDOW_DAYS'); + _positiveInt(env['MARKET_HISTORY_WINDOW_DAYS'], defaultValue: 5, name: 'MARKET_HISTORY_WINDOW_DAYS'); final int retentionDays = _positiveInt( env['MARKET_HISTORY_RETENTION_DAYS'], - defaultValue: 7, + defaultValue: 5, name: 'MARKET_HISTORY_RETENTION_DAYS', ); final bool archiveEnabled = diff --git a/server/lib/trading/market_data_retention.dart b/server/lib/trading/market_data_retention.dart index cba077b..04480bc 100644 --- a/server/lib/trading/market_data_retention.dart +++ b/server/lib/trading/market_data_retention.dart @@ -1,6 +1,7 @@ import 'package:postgres/postgres.dart'; import 'market_history_config.dart'; +import 'market_history_session_slot.dart'; import 'sync_run_recorder.dart'; /// Outcome of a [MarketDataRetention] cleanup pass. @@ -44,12 +45,12 @@ class MarketDataRetention { static const String kind = 'cleanup'; - /// Hard-delete rows with `as_of` older than [windowDays]. + /// Hard-delete rows older than the [windowDays] trading-day sync window. Future runCleanup({DateTime? now}) { return run(archive: false, now: now); } - /// Archive-then-delete for rows older than [windowDays]. + /// Archive-then-delete for rows older than the [windowDays] trading-day window. Future runArchiveAndCleanup({DateTime? now}) { return run(archive: true, now: now); } @@ -86,7 +87,8 @@ class MarketDataRetention { required int windowDays, required bool archive, }) async { - final DateTime cutoff = now.subtract(Duration(days: windowDays)); + final DateTime cutoff = + MarketHistorySessionSlot.windowFirstSlotStart(now, windowDays); int totalRemoved = 0; while (true) { diff --git a/server/lib/trading/market_history_config.dart b/server/lib/trading/market_history_config.dart index 6f10e26..309783e 100644 --- a/server/lib/trading/market_history_config.dart +++ b/server/lib/trading/market_history_config.dart @@ -1,8 +1,8 @@ /// Defaults for RTH session-half market history ([MarketHistorySessionSlot]). /// Env overrides via [MarketHistoryEnv] in [ServerEnv.load]. abstract final class MarketHistoryConfig { - /// Rolling window length in calendar days (UTC). - static const int windowDays = 7; + /// Rolling window length in US trading days (Mon–Fri; holidays skipped). + static const int windowDays = 5; /// Stored bar timeframe (two aggregates per US trading day). static const String barTimeframe = 'sessionHalf'; diff --git a/server/lib/trading/market_history_query.dart b/server/lib/trading/market_history_query.dart index 660b005..fea68a2 100644 --- a/server/lib/trading/market_history_query.dart +++ b/server/lib/trading/market_history_query.dart @@ -5,6 +5,7 @@ import 'package:postgres/postgres.dart'; import 'market_data_db.dart' show MarketDataDb; import 'market_history_bar_placeholder.dart'; import 'market_history_config.dart'; +import 'market_history_session_slot.dart'; import 'tradable_assets_db.dart'; /// One symbol's rolling-window 4-hour bar summary for the guessing game. @@ -49,7 +50,7 @@ class MarketHistoryQuery { }) async { final DateTime until = asOf.toUtc(); final DateTime since = - until.subtract(Duration(days: windowDays)); + MarketHistorySessionSlot.windowFirstSlotStart(until, windowDays); final DateTime freshSince = until.subtract(Duration(days: maxStalenessDays)); diff --git a/server/lib/trading/market_history_session_slot.dart b/server/lib/trading/market_history_session_slot.dart index c6a49a3..ad36e8d 100644 --- a/server/lib/trading/market_history_session_slot.dart +++ b/server/lib/trading/market_history_session_slot.dart @@ -119,16 +119,28 @@ abstract final class MarketHistorySessionSlot { return _morningStartUtc(cursor.year, cursor.month, cursor.day); } + /// Morning open of the earliest US trading day in the rolling window. + /// + /// [windowDays] counts **NYSE trading days** (skips weekends/holidays), anchored + /// on the Eastern calendar day of [lastCompletedSlotStart] as of [now]. static DateTime windowFirstSlotStart(DateTime now, int windowDays) { - final tz.TZDateTime ny = tz.TZDateTime.from(now.toUtc(), _eastern); - var cursor = ny.subtract(Duration(days: windowDays)); - for (var i = 0; i < windowDays + 14; i++) { - if (_isTradingDayEt(cursor.year, cursor.month, cursor.day)) { - return _morningStartUtc(cursor.year, cursor.month, cursor.day); - } - cursor = cursor.add(const Duration(days: 1)); + if (windowDays < 1) { + throw ArgumentError.value(windowDays, 'windowDays', 'must be >= 1'); } - return _morningStartUtc(ny.year, ny.month, ny.day); + final DateTime last = lastCompletedSlotStart(now); + final tz.TZDateTime lastNy = tz.TZDateTime.from(last.toUtc(), _eastern); + var cursor = tz.TZDateTime(_eastern, lastNy.year, lastNy.month, lastNy.day); + var tradingDaysCounted = 0; + for (var i = 0; i < 366; i++) { + if (_isTradingDayEt(cursor.year, cursor.month, cursor.day)) { + tradingDaysCounted++; + if (tradingDaysCounted >= windowDays) { + return _morningStartUtc(cursor.year, cursor.month, cursor.day); + } + } + cursor = cursor.subtract(const Duration(days: 1)); + } + return _morningStartUtc(cursor.year, cursor.month, cursor.day); } static List completedSlotStartsInWindow( diff --git a/server/lib/trading/market_history_week_coverage.dart b/server/lib/trading/market_history_week_coverage.dart index 7ff4df5..6715945 100644 --- a/server/lib/trading/market_history_week_coverage.dart +++ b/server/lib/trading/market_history_week_coverage.dart @@ -107,7 +107,7 @@ class MarketHistoryWeekCoverage { final int symbolCount = symbols.length; final List<(int, int, int)> calendarDays = - calendarDaysEndingTodayEt(tick, windowDays); + tradingDaysInSyncWindow(tick, windowDays); final Map> symbolsBySlot = await _loadSyncedSymbolsBySlot(tick, symbols); @@ -208,7 +208,7 @@ class MarketHistoryWeekCoverage { windowDays, ); final DateTime until = MarketHistorySessionSlot.endExclusive( - MarketHistorySessionSlot.slotStartContaining(now), + MarketHistorySessionSlot.lastCompletedSlotStart(now), ); final Result rows = await _connection.execute( @@ -268,6 +268,33 @@ class MarketHistoryWeekCoverage { return MarketHistorySessionSlot.slotStartContaining(asOf); } + /// US trading-day ET dates from [windowDays] sync window (oldest → newest). + static List<(int, int, int)> tradingDaysInSyncWindow( + DateTime now, + int windowDays, + ) { + ensureMarketHistoryTimezonesInitialized(); + final tz.Location eastern = tz.getLocation('America/New_York'); + final DateTime first = + MarketHistorySessionSlot.windowFirstSlotStart(now, windowDays); + final DateTime last = + MarketHistorySessionSlot.lastCompletedSlotStart(now); + final tz.TZDateTime firstNy = tz.TZDateTime.from(first.toUtc(), eastern); + var cursor = tz.TZDateTime(eastern, firstNy.year, firstNy.month, firstNy.day); + final tz.TZDateTime endDay = tz.TZDateTime.from(last.toUtc(), eastern); + final tz.TZDateTime endDate = + tz.TZDateTime(eastern, endDay.year, endDay.month, endDay.day); + + final List<(int, int, int)> days = <(int, int, int)>[]; + while (!cursor.isAfter(endDate)) { + if (_slotStartsForEtDay(cursor.year, cursor.month, cursor.day).isNotEmpty) { + days.add((cursor.year, cursor.month, cursor.day)); + } + cursor = cursor.add(const Duration(days: 1)); + } + return days; + } + /// Eastern calendar dates (y, m, d) for [windowDays] ending on today's ET date. static List<(int, int, int)> calendarDaysEndingTodayEt( DateTime now, diff --git a/server/migrations/005_market_history.sql b/server/migrations/005_market_history.sql index f4fc392..b06d044 100644 --- a/server/migrations/005_market_history.sql +++ b/server/migrations/005_market_history.sql @@ -1,6 +1,6 @@ -- 005_market_history.sql -- --- Adds the rolling 7-day market-history substrate: +-- Adds the rolling market-history substrate (window length from env): -- * timeframe column + idempotent unique observation key on -- market_data_snapshots -- * tradable_assets cache for the daily Alpaca asset universe diff --git a/server/test/env/market_history_env_test.dart b/server/test/env/market_history_env_test.dart index 23f1d7a..cde91c2 100644 --- a/server/test/env/market_history_env_test.dart +++ b/server/test/env/market_history_env_test.dart @@ -7,8 +7,8 @@ void main() { final MarketHistoryEnv env = MarketHistoryEnv.fromMap({}); expect(env.syncEnabled, isFalse); - expect(env.windowDays, 7); - expect(env.retentionDays, 7); + expect(env.windowDays, 5); + expect(env.retentionDays, 5); expect(env.archiveEnabled, isFalse); expect(env.universeRefreshHours, 24); expect(env.historySyncHours, 24); diff --git a/server/test/integration/market_data_retention_test.dart b/server/test/integration/market_data_retention_test.dart index 1714be6..a3b6923 100644 --- a/server/test/integration/market_data_retention_test.dart +++ b/server/test/integration/market_data_retention_test.dart @@ -3,6 +3,7 @@ library; import 'package:cyberhybridhub_server/trading/market_data_db.dart'; import 'package:cyberhybridhub_server/trading/market_data_retention.dart'; +import 'package:cyberhybridhub_server/trading/market_history_session_slot.dart'; import 'package:postgres/postgres.dart'; import 'package:test/test.dart'; @@ -12,9 +13,13 @@ void main() { TestDb? testDb; setUpAll(() async { + ensureMarketHistoryTimezonesInitialized(); testDb = await TestDb.open(); }); + /// After US close so [windowFirstSlotStart] is stable in tests. + final DateTime retentionNow = DateTime.utc(2026, 5, 26, 21, 0); + tearDown(() async { if (testDb != null) { await testDb!.connection.execute('TRUNCATE TABLE market_data_archive'); @@ -44,40 +49,43 @@ void main() { } final MarketDataDb db = testDb!.marketDataDb; - final DateTime now = DateTime.utc(2026, 5, 26, 12); - final DateTime cutoff = now.subtract(const Duration(days: 7)); + final DateTime now = retentionNow; + final DateTime cutoff = + MarketHistorySessionSlot.windowFirstSlotStart(now, 7); - int removedCount = 0; - int keptCount = 0; - - for (int day = 0; day < 10; day++) { - final DateTime asOf = now.subtract(Duration(days: 14 - day)); - await db.upsertSnapshot( - symbol: 'SPY', - metric: 'bar', - timeframe: '1Day', - asOf: asOf, - price: 490 + day, - ); - if (asOf.isBefore(cutoff)) { - removedCount++; - } else { - keptCount++; - } - } + await db.upsertSnapshot( + symbol: 'SPY', + metric: 'bar', + timeframe: 'sessionHalf', + asOf: cutoff.subtract(const Duration(days: 2)), + price: 480, + ); + await db.upsertSnapshot( + symbol: 'SPY', + metric: 'bar', + timeframe: 'sessionHalf', + asOf: cutoff, + price: 490, + ); + await db.upsertSnapshot( + symbol: 'SPY', + metric: 'bar', + timeframe: 'sessionHalf', + asOf: cutoff.add(const Duration(hours: 3, minutes: 15)), + price: 500, + ); final MarketDataRetentionResult result = await retention().runCleanup( now: now, ); expect(result.error, isNull); - expect(result.rowsRemoved, removedCount); - expect(keptCount, greaterThan(0)); + expect(result.rowsRemoved, 1); final Result remaining = await testDb!.connection.execute( 'SELECT COUNT(*)::int FROM market_data_snapshots', ); - expect((remaining.first[0]! as num).toInt(), keptCount); + expect((remaining.first[0]! as num).toInt(), 2); }); test('empty table returns rowsRemoved 0 without throwing', () async { @@ -89,7 +97,7 @@ void main() { } final MarketDataRetentionResult result = await retention().runCleanup( - now: DateTime.utc(2026, 5, 26, 12), + now: retentionNow, ); expect(result.rowsRemoved, 0); @@ -107,8 +115,11 @@ void main() { const int rowCount = 5000; const int batchSize = 1000; - final DateTime now = DateTime.utc(2026, 5, 26, 12); - final DateTime oldAsOf = now.subtract(const Duration(days: 14)); + final DateTime now = retentionNow; + final DateTime oldAsOf = MarketHistorySessionSlot.windowFirstSlotStart( + now, + 7, + ).subtract(const Duration(days: 2)); await testDb!.connection.execute( Sql.named( @@ -148,12 +159,14 @@ void main() { return; } - final DateTime now = DateTime.utc(2026, 5, 26, 12); + final DateTime now = retentionNow; + final DateTime cutoff = + MarketHistorySessionSlot.windowFirstSlotStart(now, 7); await testDb!.marketDataDb.upsertSnapshot( symbol: 'SPY', metric: 'bar', - timeframe: '1Day', - asOf: now.subtract(const Duration(days: 10)), + timeframe: 'sessionHalf', + asOf: cutoff.subtract(const Duration(days: 2)), price: 480, ); @@ -183,20 +196,22 @@ void main() { } final MarketDataDb db = testDb!.marketDataDb; - final DateTime now = DateTime.utc(2026, 5, 26, 12); + final DateTime now = retentionNow; + final DateTime cutoff = + MarketHistorySessionSlot.windowFirstSlotStart(now, 7); final MarketDataSnapshot kept = await db.upsertSnapshot( symbol: 'SPY', metric: 'bar', - timeframe: '1Day', - asOf: now.subtract(const Duration(days: 2)), + timeframe: 'sessionHalf', + asOf: cutoff.add(const Duration(hours: 3, minutes: 15)), price: 500, ); await db.upsertSnapshot( symbol: 'SPY', metric: 'bar', - timeframe: '1Day', - asOf: now.subtract(const Duration(days: 10)), + timeframe: 'sessionHalf', + asOf: cutoff.subtract(const Duration(days: 2)), price: 480, ); @@ -219,12 +234,14 @@ void main() { return; } - final DateTime now = DateTime.utc(2026, 5, 26, 12); + final DateTime now = retentionNow; + final DateTime cutoff = + MarketHistorySessionSlot.windowFirstSlotStart(now, 7); await testDb!.marketDataDb.upsertSnapshot( symbol: 'SPY', metric: 'bar', - timeframe: '1Day', - asOf: now.subtract(const Duration(days: 10)), + timeframe: 'sessionHalf', + asOf: cutoff.subtract(const Duration(days: 2)), price: 480, volume: 1000, raw: {'c': 480}, @@ -232,8 +249,8 @@ void main() { await testDb!.marketDataDb.upsertSnapshot( symbol: 'SPY', metric: 'bar', - timeframe: '1Day', - asOf: now.subtract(const Duration(days: 2)), + timeframe: 'sessionHalf', + asOf: cutoff.add(const Duration(hours: 3, minutes: 15)), price: 500, ); @@ -262,12 +279,14 @@ void main() { return; } - final DateTime now = DateTime.utc(2026, 5, 26, 12); + final DateTime now = retentionNow; + final DateTime cutoff = + MarketHistorySessionSlot.windowFirstSlotStart(now, 7); await testDb!.marketDataDb.upsertSnapshot( symbol: 'SPY', metric: 'bar', - timeframe: '1Day', - asOf: now.subtract(const Duration(days: 10)), + timeframe: 'sessionHalf', + asOf: cutoff.subtract(const Duration(days: 2)), price: 480, ); @@ -299,12 +318,14 @@ void main() { return; } - final DateTime now = DateTime.utc(2026, 5, 26, 12); + final DateTime now = retentionNow; + final DateTime cutoff = + MarketHistorySessionSlot.windowFirstSlotStart(now, 7); await testDb!.marketDataDb.upsertSnapshot( symbol: 'SPY', metric: 'bar', - timeframe: '1Day', - asOf: now.subtract(const Duration(days: 10)), + timeframe: 'sessionHalf', + asOf: cutoff.subtract(const Duration(days: 2)), price: 480, ); diff --git a/server/test/integration/market_history_admin_handler_test.dart b/server/test/integration/market_history_admin_handler_test.dart index b06ecaa..36dc521 100644 --- a/server/test/integration/market_history_admin_handler_test.dart +++ b/server/test/integration/market_history_admin_handler_test.dart @@ -425,7 +425,7 @@ void main() { adminFirebaseUids: {'admin-uid'}, portalConfig: const MarketHistoryAdminPortalConfig( archiveEnabled: true, - windowDays: 7, + windowDays: 5, retentionDays: 14, syncEnabled: true, ), @@ -442,7 +442,7 @@ void main() { final Map config = body['config'] as Map; expect(config['archiveEnabled'], isTrue); - expect(config['windowDays'], 7); + expect(config['windowDays'], 5); expect(config['retentionDays'], 14); expect(config['syncEnabled'], isTrue); }); @@ -576,8 +576,8 @@ void main() { adminFirebaseUids: {'admin-uid'}, portalConfig: const MarketHistoryAdminPortalConfig( archiveEnabled: false, - windowDays: 7, - retentionDays: 7, + windowDays: 5, + retentionDays: 5, syncEnabled: false, ), ); @@ -591,7 +591,7 @@ void main() { final Map body = jsonDecode(await response.readAsString()) as Map; - expect(body['windowDays'], 7); + expect(body['windowDays'], 5); expect(body['canStepNewer'], isFalse); expect(body['canStepOlder'], isTrue); expect( @@ -728,7 +728,7 @@ void main() { return; } - final DateTime now = DateTime.now().toUtc(); + final DateTime now = DateTime.utc(2026, 5, 31, 21); final DateTime slotStart = MarketHistorySessionSlot.lastCompletedSlotStart(now); @@ -736,7 +736,9 @@ void main() { Sql.named( ''' INSERT INTO tradable_assets (symbol, asset_class, status, tradable, refreshed_at) - VALUES ('AAA', 'us_equity', 'active', true, @refreshed_at) + VALUES + ('AAA', 'us_equity', 'active', true, @refreshed_at), + ('BBB', 'us_equity', 'active', true, @refreshed_at) ''', ), parameters: {'refreshed_at': now}, @@ -767,41 +769,49 @@ void main() { adminFirebaseUids: {'admin-uid'}, portalConfig: const MarketHistoryAdminPortalConfig( archiveEnabled: false, - windowDays: 7, - retentionDays: 7, + windowDays: 5, + retentionDays: 5, syncEnabled: false, ), ); final Response response = await _get( handler, - path: '/v1/admin/market-history/week-coverage', + path: '/v1/admin/market-history/week-coverage?asOf=${Uri.encodeComponent(now.toIso8601String())}', bearer: 'admin-uid', ); expect(response.statusCode, 200); final Map body = jsonDecode(await response.readAsString()) as Map; - expect(body['windowDays'], 7); + expect(body['windowDays'], 5); expect(body['slotsPerDay'], 2); - expect(body['symbolCount'], 1); + expect(body['symbolCount'], 2); expect(body['isConsistent'], isFalse); final List days = body['days'] as List; - expect(days, hasLength(7)); - final tz.TZDateTime slotDayEt = tz.TZDateTime.from( - slotStart, - tz.getLocation('America/New_York'), - ); - final String slotDayWire = - '${slotDayEt.year.toString().padLeft(4, '0')}-' - '${slotDayEt.month.toString().padLeft(2, '0')}-' - '${slotDayEt.day.toString().padLeft(2, '0')}'; - final Map slotDay = days.cast>().firstWhere( - (Map d) => d['date'] == slotDayWire, - ); - expect(slotDay['fullySyncedSlots'], 1); - expect(slotDay['completedSlots'], greaterThan(0)); + expect(days, hasLength(5)); + + final String slotWire = MarketHistorySessionSlot.slotStartWire(slotStart); + Map? matchingSlot; + for (final Map day in days.cast>()) { + for (final Map slot + in (day['slots'] as List).cast>()) { + if (MarketHistorySessionSlot.slotStartWire( + DateTime.parse(slot['slotStart'] as String).toUtc(), + ) == + slotWire) { + matchingSlot = slot; + break; + } + } + if (matchingSlot != null) { + break; + } + } + expect(matchingSlot, isNotNull); + expect(matchingSlot!['fullySynced'], isFalse); + expect((matchingSlot['syncedSymbolCount'] as num).toInt(), 1); }); test('resync returns 503 when sync is disabled in portal config', () async { @@ -817,8 +827,8 @@ void main() { actions: null, portalConfig: const MarketHistoryAdminPortalConfig( archiveEnabled: false, - windowDays: 7, - retentionDays: 7, + windowDays: 5, + retentionDays: 5, syncEnabled: false, ), ); diff --git a/server/test/trading/market_history_session_slot_test.dart b/server/test/trading/market_history_session_slot_test.dart index aef79e2..ef92f13 100644 --- a/server/test/trading/market_history_session_slot_test.dart +++ b/server/test/trading/market_history_session_slot_test.dart @@ -67,6 +67,23 @@ void main() { ); }); + test('windowFirstSlotStart counts trading days not calendar days', () { + final DateTime now = DateTime.utc(2026, 5, 31, 21, 0); + expect( + MarketHistorySessionSlot.windowFirstSlotStart(now, 7), + DateTime.utc(2026, 5, 20, 13, 30), + ); + }); + + test('completedSlotStartsInWindow spans seven trading days', () { + final DateTime now = DateTime.utc(2026, 5, 31, 21, 0); + final List slots = + MarketHistorySessionSlot.completedSlotStartsInWindow(now, 7); + expect(slots, hasLength(14)); + expect(slots.first, DateTime.utc(2026, 5, 20, 13, 30)); + expect(slots.last, DateTime.utc(2026, 5, 29, 16, 45)); + }); + test('previousSlotStart walks afternoon to morning', () { final DateTime afternoon = DateTime.utc(2026, 6, 2, 16, 45); expect( diff --git a/server/test/trading/market_history_week_coverage_test.dart b/server/test/trading/market_history_week_coverage_test.dart index f1c867a..45bc1c2 100644 --- a/server/test/trading/market_history_week_coverage_test.dart +++ b/server/test/trading/market_history_week_coverage_test.dart @@ -6,12 +6,22 @@ void main() { setUp(ensureMarketHistoryTimezonesInitialized); group('MarketHistoryWeekCoverage calendar days', () { - test('returns windowDays Eastern dates ending on today', () { + test('tradingDaysInSyncWindow lists five trading days for window', () { + final DateTime now = DateTime.utc(2026, 5, 31, 21); + final List<(int, int, int)> days = + MarketHistoryWeekCoverage.tradingDaysInSyncWindow(now, 5); + + expect(days, hasLength(5)); + expect(days.first, (2026, 5, 22)); + expect(days.last, (2026, 5, 29)); + }); + + test('calendarDaysEndingTodayEt still spans five calendar days', () { final DateTime now = DateTime.utc(2026, 6, 2, 21); final List<(int, int, int)> days = - MarketHistoryWeekCoverage.calendarDaysEndingTodayEt(now, 7); + MarketHistoryWeekCoverage.calendarDaysEndingTodayEt(now, 5); - expect(days, hasLength(7)); + expect(days, hasLength(5)); expect(days.last, (2026, 6, 2)); }); }); diff --git a/test/admin/acceptance/admin_portal_acceptance_test.dart b/test/admin/acceptance/admin_portal_acceptance_test.dart index 9ed5370..d66fe8b 100644 --- a/test/admin/acceptance/admin_portal_acceptance_test.dart +++ b/test/admin/acceptance/admin_portal_acceptance_test.dart @@ -202,8 +202,8 @@ void main() { errorMessage: null, config: const MarketHistoryAdminConfig( archiveEnabled: true, - windowDays: 7, - retentionDays: 7, + windowDays: 5, + retentionDays: 5, syncEnabled: true, ), ), diff --git a/test/admin/screens/market_history_log_screen_test.dart b/test/admin/screens/market_history_log_screen_test.dart index 72821e9..33b9050 100644 --- a/test/admin/screens/market_history_log_screen_test.dart +++ b/test/admin/screens/market_history_log_screen_test.dart @@ -133,7 +133,7 @@ class _FakeCoverageApi extends MarketHistoryAdminApi { compareUntil: DateTime.utc(2026, 5, 30, 16), newerSlotStart: DateTime.utc(2026, 5, 30, 12), olderSlotStart: DateTime.utc(2026, 5, 30, 8), - windowDays: 7, + windowDays: 5, canStepOlder: false, canStepNewer: false, assets: const [], @@ -144,14 +144,14 @@ MarketHistoryWeekCoverageReport _sampleWeekReport() { final DateTime day = DateTime.utc(2026, 5, 30); return MarketHistoryWeekCoverageReport( asOf: DateTime.utc(2026, 5, 30, 15), - windowDays: 7, + windowDays: 5, slotsPerDay: 6, symbolCount: 10, isConsistent: true, days: List.generate( - 7, + 5, (int index) { - final DateTime date = day.subtract(Duration(days: 6 - index)); + final DateTime date = day.subtract(Duration(days: 4 - index)); return MarketHistoryDayCoverage( date: date, slotsPerDay: 6, @@ -425,8 +425,8 @@ void main() { errorMessage: null, config: const MarketHistoryAdminConfig( archiveEnabled: false, - windowDays: 7, - retentionDays: 7, + windowDays: 5, + retentionDays: 5, syncEnabled: true, ), ), @@ -466,8 +466,8 @@ void main() { errorMessage: null, config: const MarketHistoryAdminConfig( archiveEnabled: false, - windowDays: 7, - retentionDays: 7, + windowDays: 5, + retentionDays: 5, syncEnabled: true, ), ), @@ -535,7 +535,7 @@ void main() { await tester.pumpAndSettle(); expect(find.byKey(const Key('week-coverage-dialog')), findsOneWidget); - expect(find.text('7-day sync health'), findsOneWidget); + expect(find.text('5-day sync health'), findsOneWidget); expect(find.textContaining('6/6'), findsWidgets); await tester.tap(find.byKey(const Key('week-coverage-close'))); diff --git a/test/admin/services/market_history_admin_api_test.dart b/test/admin/services/market_history_admin_api_test.dart index 1ad326b..0cac9b8 100644 --- a/test/admin/services/market_history_admin_api_test.dart +++ b/test/admin/services/market_history_admin_api_test.dart @@ -136,7 +136,7 @@ void main() { 'pinned': [], 'config': { 'archiveEnabled': true, - 'windowDays': 7, + 'windowDays': 5, 'retentionDays': 14, 'syncEnabled': true, }, @@ -160,7 +160,7 @@ void main() { return http.Response( jsonEncode({ 'asOf': '2026-05-30T15:30:00Z', - 'windowDays': 7, + 'windowDays': 5, 'slotsPerDay': 6, 'symbolCount': 2, 'isConsistent': false, @@ -205,7 +205,7 @@ void main() { 'compareUntil': '2026-05-30T16:00:00Z', 'newerSlotStart': '2026-05-30T12:00:00Z', 'olderSlotStart': '2026-05-30T08:00:00Z', - 'windowDays': 7, + 'windowDays': 5, 'canStepOlder': true, 'canStepNewer': false, 'stepOlderCompareUntil': '2026-05-30T12:00:00Z', @@ -242,7 +242,7 @@ void main() { ); final QuestionAuditReport report = await api.fetchQuestionAudit(); - expect(report.windowDays, 7); + expect(report.windowDays, 5); expect(report.canStepOlder, isTrue); expect(report.canStepNewer, isFalse); expect(report.stepOlderCompareUntil, DateTime.utc(2026, 5, 30, 12)); diff --git a/test/admin/widgets/market_history_question_audit_sheet_test.dart b/test/admin/widgets/market_history_question_audit_sheet_test.dart index f08c853..39539ac 100644 --- a/test/admin/widgets/market_history_question_audit_sheet_test.dart +++ b/test/admin/widgets/market_history_question_audit_sheet_test.dart @@ -46,7 +46,7 @@ QuestionAuditReport _sampleReport({ compareUntil: until, newerSlotStart: newer, olderSlotStart: older, - windowDays: 7, + windowDays: 5, canStepOlder: canStepOlder, canStepNewer: canStepNewer, stepOlderCompareUntil: stepOlderCompareUntil,