diff --git a/.github/workflows/admin-portal.yml b/.github/workflows/admin-portal.yml new file mode 100644 index 0000000..5433097 --- /dev/null +++ b/.github/workflows/admin-portal.yml @@ -0,0 +1,65 @@ +name: Admin portal tests + +on: + push: + paths: + - 'lib/admin/**' + - 'test/admin/**' + - 'server/lib/handlers/market_history_admin_handler.dart' + - 'server/lib/trading/market_history_admin_*.dart' + - 'server/test/trading/market_history_admin_*.dart' + - 'server/test/integration/market_history_admin_risk_test.dart' + - 'server/test/helpers/admin_sync_run_fixtures.dart' + - 'scripts/test-admin-portal.sh' + - '.github/workflows/admin-portal.yml' + pull_request: + paths: + - 'lib/admin/**' + - 'test/admin/**' + - 'server/lib/handlers/market_history_admin_handler.dart' + - 'server/lib/trading/market_history_admin_*.dart' + - 'server/test/trading/market_history_admin_*.dart' + - 'server/test/integration/market_history_admin_risk_test.dart' + - 'server/test/helpers/admin_sync_run_fixtures.dart' + - 'scripts/test-admin-portal.sh' + - '.github/workflows/admin-portal.yml' + +jobs: + test: + runs-on: ubuntu-latest + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + env: + TEST_DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres + + steps: + - uses: actions/checkout@v4 + + - uses: dart-lang/setup-dart@v1 + + - uses: subosito/flutter-action@v2 + with: + channel: stable + + - name: Run admin portal test suite + env: + TZ: UTC + run: bash scripts/test-admin-portal.sh + + - name: Enforce admin portal coverage thresholds + env: + TZ: UTC + run: bash scripts/check-admin-portal-coverage.sh diff --git a/FLUTTER-ADMIN-PORTAL.md b/FLUTTER-ADMIN-PORTAL.md new file mode 100644 index 0000000..ed27837 --- /dev/null +++ b/FLUTTER-ADMIN-PORTAL.md @@ -0,0 +1,461 @@ +# Flutter Admin Portal — Development Plan + +**Status:** Planning +**Related:** `TODO.md` §8 (operational CLIs), §8.5 (admin API), rolling market-history pipeline (§1–§7) +**Audience:** Mobile/web Flutter developers, API maintainers, operators + +--- + +## 1. Purpose + +Build an **admin-only** area in the Flutter app that shows a **live audit log** of periodic market-history operations: + +- **Universe sync** (`kind=universe`) — tradable asset refresh +- **Historical backfill** (`kind=backfill`) — ended **4-hour UTC slots** (`4Hour` bars) into `market_data_snapshots`; open slot not fetched yet +- **Retention / cleanup** (`kind=cleanup`) — prune (or archive-then-delete) expired rows + +Operators use it to confirm the rolling 7-day window is healthy, see when cleanup ran, and **notice Alpaca/API failures or rate limits immediately** without reading server logs or SQL. + +This plan **replaces the need for §8 shell CLIs** for day-to-day product use. CLIs remain optional for CI/SRE (see §12). + +--- + +## 2. Goals and non-goals + +### Goals + +| # | Goal | +|---|------| +| G1 | Chronological log: **newest events at the top**; scroll down for older history | +| G2 | **Pin unresolved failures** (API errors, rate limits, partial batch failures) at the top until a **successful retry** for the same `kind` (and optionally same error “thread”) | +| G3 | **Expandable list rows** — collapsed summary; expanded shows structured detail **without** raw JSON payloads (no bar blobs, no `tradable_assets.raw`) | +| G4 | For API failures, expanded view shows **full operator-relevant error text** (HTTP status, Alpaca message body summary, batch scope) | +| G5 | Optional: **on-demand** “Run sync” / “Run cleanup” from the portal (API-triggered), with new rows appearing in the log | +| G6 | Reuse existing auth (Firebase ID token → API) and app theme/navigation patterns | + +### Non-goals (v1) + +- End-user (non-admin) access to sync logs +- Displaying raw `market_data_snapshots` or per-symbol bar tables +- Full log export / SIEM integration (future) +- Replacing the background worker; scheduler still runs automatically when `MARKET_HISTORY_SYNC_ENABLED=true` +- Building §8 `bin/sync_market_history.dart` / `bin/cleanup_market_history.dart` (defer unless ops requests them) + +--- + +## 3. Source of truth (already in Postgres) + +All events are recorded in **`market_data_sync_runs`** (migration `005_market_history.sql`): + +| Column | Meaning | +|--------|---------| +| `id` | Stable row id | +| `kind` | `universe` \| `backfill` \| `cleanup` | +| `started_at` | UTC start | +| `finished_at` | UTC end (null if crash mid-run — treat as in-progress) | +| `rows_written` | Upserted rows (universe/backfill) | +| `rows_removed` | Deleted rows (cleanup) | +| `error` | `NULL` = success; non-null = failure or **partial** failure message | + +**Kinds map to UI labels:** + +| `kind` | UI title | Typical success summary | +|--------|----------|-------------------------| +| `universe` | Asset universe sync | N assets refreshed | +| `backfill` | 4-hour slot backfill (ended slots only) | N bar rows written | +| `cleanup` | Data retention cleanup | N rows removed | + +**Error shapes today (no schema change required for v1):** + +- Thrown exception → `error` is `e.toString()` (e.g. `AlpacaMarketDataException: getBarsRange rate limited: 429 …`) +- Partial backfill → `error` may list batch failures while `rows_written > 0` (`SyncRunCounts.error` in `market_data_history.dart`) + +**v1 recommendation:** Parse `error` text client-side for keywords (`429`, `rate`, `Alpaca`, `batch`) to set severity and pin rules. **v2 (optional):** add `error_code`, `http_status`, `severity` columns or a JSON `detail` object on insert (server change). + +--- + +## 4. List ordering and “pin until resolved” + +### 4.1 Default sort + +Primary: **`started_at` DESC** (most recent at top). + +### 4.2 Pin rule (failure priority) + +An event is **pinned** when: + +1. `error IS NOT NULL` **or** `finished_at IS NULL` (stuck in-progress > N minutes → show as warning), **and** +2. There is **no later successful run** of the **same `kind`** with `error IS NULL` and `finished_at IS NOT NULL` **after** this row’s `started_at`. + +**Pinned block** appears **above** the chronological list (fixed section or visually distinct header: “Needs attention”). + +When a new successful run for that `kind` completes, the older failed row **unpins** and appears only in normal history order. + +### 4.3 Rate limits and API errors + +Treat as **high severity** when `error` matches (case-insensitive): + +- `429`, `rate limit`, `rate limited` +- `AlpacaMarketDataException`, `AlpacaAssetsException`, `AlpacaTradingException` +- HTTP `5xx` patterns in message + +Pinned rate-limit rows use a **warning/error** color and icon until the pin rule clears. + +### 4.4 Partial success (backfill) + +If `rows_written > 0` **and** `error != null`: + +- Collapsed: **“Partial success”** badge + counts +- Expanded: show `rows_written`, full `error` text (batch list), **do not** show raw bar JSON +- Pin until next **fully successful** backfill (`error` null) **or** product decision: pin until error null even if partial (stricter — **recommended for ops**) + +--- + +## 5. Backend API (prerequisite) + +No admin market-history endpoints exist today. Implement **before** Flutter UI (or in parallel with mock API). + +### 5.1 Auth model + +| Approach | Recommendation | +|----------|----------------| +| Firebase + allowlist | `ADMIN_FIREBASE_UIDS=uid1,uid2` in server env; middleware rejects others with `403` | +| Custom claim | `admin: true` on Firebase token; server verifies claim | +| Separate admin API key | Avoid for mobile; poor UX | + +Match existing pattern: `Authorization: Bearer ` like `QuestionsApiService`. + +**Do not** expose these under `/v1/me/...` without admin checks — use `/v1/admin/...` prefix. + +### 5.2 Endpoints (v1) + +#### `GET /v1/admin/market-history/sync-runs` + +Query params: + +| Param | Default | Description | +|-------|---------|-------------| +| `limit` | `50` | Page size | +| `before` | — | Cursor: `id` or `started_at` of oldest item in current page (for infinite scroll) | +| `kind` | — | Optional filter: `universe`, `backfill`, `cleanup` | + +Response: + +```json +{ + "runs": [ + { + "id": 123, + "kind": "cleanup", + "startedAt": "2026-05-26T10:00:00Z", + "finishedAt": "2026-05-26T10:00:05Z", + "rowsWritten": 0, + "rowsRemoved": 4200, + "error": null, + "severity": "ok", + "status": "success", + "durationMs": 5000, + "summary": "Removed 4,200 expired snapshots" + } + ], + "pinned": [ + { + "id": 120, + "kind": "backfill", + "error": "getBarsRange rate limited: 429 ...", + "severity": "rate_limit", + "status": "failed", + "summary": "Backfill partial: 12,000 rows; rate limited" + } + ], + "nextBefore": "2026-05-25T08:00:00Z" +} +``` + +Server computes `pinned[]` with the rule in §4.2. +Server computes `severity`: `ok` \| `warning` \| `error` \| `rate_limit` from `error` text + `finished_at`. +**Never** include `raw` JSON from snapshots or assets in this API. + +#### `GET /v1/admin/market-history/sync-runs/{id}` + +Single run for expanded row refresh (optional if list payload is enough). + +#### `POST /v1/admin/market-data/resync` (optional v1.1) + +Runs universe + slot backfill (ended `4Hour` slots in window, not the open slot). Returns `{ "runIds": […] }` (202). `windowDays` on cleanup only today. + +#### `POST /v1/admin/market-data/cleanup` (optional v1.1) + +Query: `windowDays`, `archive=true|false`. +Runs `MarketDataRetention.run(archive: …)` once. + +### 5.3 Implementation notes (server) + +- New handler: `server/lib/handlers/market_history_admin_handler.dart` +- Mount only when `ADMIN_PORTAL_ENABLED=true` (or reuse `TRADING_DEV_ENDPOINTS_ENABLED` — **prefer dedicated flag**) +- Reuse `TradableAssetsSync`, `MarketDataHistorySync`, `MarketDataRetention` — same as `bin/server.dart` scheduler wiring +- List query: indexed `ORDER BY started_at DESC` (index exists: `market_data_sync_runs_kind_started_idx`) + +### 5.4 Error detail for API failures (v2 enhancement) + +When recording sync failures, optionally persist structured fields: + +```json +{ + "httpStatus": 429, + "endpoint": "GET /v2/stocks/bars", + "message": "...", + "batchSymbols": ["AAPL", "MSFT"] +} +``` + +Store in new `detail JSONB` column or append to `error` as delimited text for v1. + +--- + +## 6. Flutter architecture + +### 6.1 Entry and routing + +| Item | Proposal | +|------|----------| +| Route | `/admin/market-history` or dedicated `AdminPortalScreen` pushed from a hidden entry | +| Gate | `AdminGate` widget: calls `GET /v1/admin/market-history/sync-runs?limit=1` — `403` → “Not authorized” | +| Discovery | Long-press app logo, `kDebugMode` menu item, or profile flag `isAdmin` from `GET /v1/me/profile` extension | + +Do **not** show admin nav to all signed-in users. + +### 6.2 Layering (match existing app) + +``` +lib/ + admin/ + models/ + sync_run_event.dart # DTO from API + services/ + market_history_admin_api.dart # HTTP + auth headers (mirror QuestionsApiService) + repositories/ + sync_run_log_repository.dart # paging, pin merge, refresh + screens/ + market_history_log_screen.dart + widgets/ + sync_run_expansion_tile.dart + pinned_alerts_section.dart + sync_run_status_chip.dart +``` + +- **API service:** `apiBaseUrl` from `lib/config/api_config.dart`, token via `AuthService.instance.getIdToken()` +- **State:** `Listenable`/`ChangeNotifier` or `flutter_riverpod` if project adopts it later; v1 can use `StatefulWidget` + repository like profile sync + +### 6.3 `SyncRunEvent` model (client) + +Fields from API plus derived: + +- `isPinned` (from `pinned` array or client recompute) +- `Severity` enum: `ok`, `warning`, `error`, `rateLimit` +- `displayTitle` from `kind` +- `collapsedSubtitle` — one line: time ago + summary +- `expandedSections` — `List` title/body pairs (no raw JSON) + +### 6.4 Expanded row content (templates) + +**Success — cleanup** + +- Started / finished (local timezone) +- Duration +- Rows removed +- Archive mode on/off (if API adds flag later) + +**Success — backfill** + +- Rows written +- Window days (from env or API meta) +- “No errors” + +**Failure / rate limit** + +- Full `error` string (selectable text) +- Parsed HTTP status if regex finds it +- Kind + run id (for support tickets) +- Hint: “Will retry on next scheduled run” or button **Retry now** if POST endpoint exists + +**Partial backfill** + +- Rows written (success count) +- Error block (batch failures) +- Do **not** render `raw` from bars + +**In progress** + +- Started at; spinner if `finished_at == null` and started < 30m ago +- Stale run warning if > 30m without `finished_at` + +--- + +## 7. UI specification + +### 7.1 Screen layout + +``` +┌─────────────────────────────────────┐ +│ ← Market history log [↻] │ AppBar: refresh +├─────────────────────────────────────┤ +│ NEEDS ATTENTION (0–n) │ Pinned section (hidden if empty) +│ ⚠ Backfill — Rate limited 2h ago │ ExpansionTile, error styling +│ ✗ Cleanup — failed 1d ago │ +├─────────────────────────────────────┤ +│ HISTORY │ Section header +│ ✓ Cleanup — 4.2k removed 10:00 │ Newest first +│ ✓ Backfill — 98k written 09:00 │ +│ ✓ Universe — 8.2k assets 08:00 │ +│ ... │ Infinite scroll +└─────────────────────────────────────┘ +``` + +### 7.2 Collapsed row + +- Leading icon by `severity` / `kind` +- Title: `{Kind label} — {short status}` +- Trailing: relative time (`2h ago`) +- Optional chips: `rows_written`, `rows_removed` + +### 7.3 Expanded row + +- `ExpansionTile` children: labeled rows (not monospace dumps) +- **Selectable** `SelectableText` for full error message +- Copy-to-clipboard button for error block + +### 7.4 Refresh and pagination + +| Action | Behavior | +|--------|----------| +| Pull-to-refresh | Reload first page + recompute pins | +| Scroll end | `before=` cursor load next page, **append** to history (not pinned) | +| Auto-refresh | Optional 60s timer while screen visible | + +### 7.5 On-demand actions (v1.1) + +AppBar overflow menu: + +- **Run backfill now** → `POST resync` → show in-progress row → refresh +- **Run cleanup now** → `POST cleanup` → confirm dialog (destructive) → refresh + +Disable buttons while any `kind` has in-progress run (`finished_at == null`). + +--- + +## 8. Phased delivery + +### Phase A — Read-only log (MVP) + +| Track | Work | +|-------|------| +| Server | `GET sync-runs` + admin auth + `pinned` computation | +| Flutter | Models, API service, log screen, expansion tiles, pull-to-refresh | +| QA | Seed `market_data_sync_runs` in test DB; verify pin/unpin when new success inserted | + +**Exit:** Operator can view retention/cleanup/backfill history on device/emulator. + +### Phase B — Failure UX polish + +- Severity icons/colors, rate-limit copy, partial-success template +- Copy error, relative timestamps, empty/loading/error states +- Widget tests for sort/pin merge logic + +### Phase C — On-demand triggers + +- `POST resync`, `POST cleanup` +- Flutter buttons + confirmation + in-progress state + +### Phase D — Structured errors (optional) + +- Server writes `detail JSONB`; Flutter renders HTTP status, endpoint, batch symbols without parsing free text + +--- + +## 9. Testing + +### Server + +| Test | Description | +|------|-------------| +| Integration | `test/integration/market_history_admin_handler_test.dart` — 403 non-admin, 200 list shape, pinned logic | +| Fixture rows | Success cleanup, failed backfill 429, partial backfill | + +### Flutter + +| Test | Description | +|------|-------------| +| Unit | `sync_run_log_repository_test.dart` — merge pinned + chronological, pin clearance | +| Unit | `sync_run_event_test.dart` — severity from error string | +| Widget | Expansion tile shows/hides detail; pinned section visibility | +| Integration | Mock HTTP → screen goldens or pump tests | + +### Manual + +1. Enable worker sync; wait for 3 kinds in log +2. Simulate 429 (mock Alpaca or test env throttle) +3. Confirm pin → run successful backfill → pin clears + +--- + +## 10. Security and compliance + +- Admin routes **only** with verified admin identity +- Logs may contain Alpaca error bodies — **no PII**, but treat as internal ops data +- Do not log API keys; ensure `error` column never stores secrets (review `SyncRunRecorder` callers) +- CORS: same as existing API (`apiCorsHeaders`) +- Production: `ADMIN_PORTAL_ENABLED=false` by default + +--- + +## 11. Configuration (align with §7 env) + +Document in `server/README.md` when implemented: + +| Env var | Purpose | +|---------|---------| +| `ADMIN_PORTAL_ENABLED` | Mount `/v1/admin/market-history/*` | +| `ADMIN_FIREBASE_UIDS` | Comma-separated allowlist | +| `MARKET_HISTORY_*` | Window, sync cadence (worker); shown in expanded row as read-only meta | + +--- + +## 12. Relationship to §8 CLIs + +| Capability | Admin portal | §8 CLI | +|------------|----------------|--------| +| View sync/cleanup log | Yes (primary) | No | +| On-demand sync/cleanup | Yes (Phase C API) | Yes (planned) | +| CI / headless / no UI | No | Yes | +| Ops SSH without app | No | Yes | + +**Recommendation:** Implement **portal + admin API** first; defer §8 CLIs unless SRE requests them. + +--- + +## 13. Open decisions + +| # | Question | Default proposal | +|---|----------|------------------| +| 1 | Admin discovery in app? | Debug-only entry + allowlist | +| 2 | Pin partial backfill with `rows_written > 0`? | Yes, until `error` null | +| 3 | Separate web-only admin build? | Single app, gated route (v1) | +| 4 | Real-time updates? | Pull-to-refresh only (v1); WebSocket later | +| 5 | `finished_at` null stale threshold? | 30 minutes → warning row | + +--- + +## 14. Acceptance criteria + +- [ ] Admin user sees **newest** history events at top of main list +- [ ] Failed/rate-limited runs appear in **Needs attention** until same `kind` succeeds later +- [ ] Expand row shows operational detail **without** raw market-data JSON +- [ ] API error expanded view shows **complete** `error` text from server +- [ ] Non-admin receives 403 and no data +- [ ] Pull-to-refresh and pagination load additional older events +- [ ] (Phase C) On-demand cleanup/sync creates visible new log entries + +--- + +*Document version: 1.0 — Flutter admin portal for `market_data_sync_runs` audit log.* diff --git a/FLUTTER-TDD-PLAN.md b/FLUTTER-TDD-PLAN.md new file mode 100644 index 0000000..973991f --- /dev/null +++ b/FLUTTER-TDD-PLAN.md @@ -0,0 +1,307 @@ +# Flutter Admin Portal TDD Plan + +**Scope:** Validate the functionality defined in `FLUTTER-ADMIN-PORTAL.md` using test-first development across server API + Flutter UI/repository layers. + +**Primary objective:** Ship an admin portal that reliably shows retention/cleanup/backfill history, prioritizes unresolved failures/rate limits, and presents concise expandable details. + +--- + +## 1. Test strategy at a glance + +Use a layered pyramid so core behavior is proven at the cheapest layer first: + +| Layer | Purpose | Tools | +|---|---|---| +| Unit (server) | pin/severity/status logic, query mapping | `dart test` | +| Integration (server) | auth, DB query shape/order, endpoint behavior | `dart test` + Postgres test DB | +| Unit (Flutter) | DTO mapping, repository merge, parsing, formatting | `flutter test` | +| Widget (Flutter) | pinned section, newest-first list, expansion details, pagination interactions | `flutter test` | +| Optional e2e/manual | confidence pass with real API + seeded rows | local runbook | + +--- + +## 2. Coverage targets (reasonable) + +Aim for practical coverage without over-testing visual polish: + +- **Server admin handler package:** >= **85% line coverage** +- **Server pin/severity helper logic:** >= **95% line coverage** +- **Flutter admin repository/models:** >= **90% line coverage** +- **Flutter admin widgets/screens:** >= **75% line coverage** +- **Critical behavior cases:** **100% scenario coverage** for: + - unresolved errors pinned + - unpin after successful retry by same `kind` + - newest-first ordering + - expanded full error text display + - no raw data fields rendered + +--- + +## 3. Test data model fixtures + +Create reusable fixture builders for `market_data_sync_runs` rows: + +```text +run(id, kind, startedAt, finishedAt, rowsWritten, rowsRemoved, error) +``` + +Named fixture sets: + +1. `all_success_recent_first` +2. `rate_limit_unresolved` +3. `failed_then_success_same_kind` +4. `partial_backfill_error` +5. `in_progress_stale` +6. `mixed_kinds_mixed_outcomes` + +Keep fixtures concise; avoid raw bar payloads by design. + +--- + +## 4. Server TDD plan + +Target files (planned): + +- `server/lib/handlers/market_history_admin_handler.dart` +- `server/lib/trading/market_history_admin_query.dart` (or equivalent helper) +- `server/test/integration/market_history_admin_handler_test.dart` +- `server/test/trading/market_history_admin_logic_test.dart` + +### 4.1 RED: pin/severity/status logic unit tests + +Write tests before implementation: + +1. **Pinned unresolved failure** + - failed `backfill` with no later success => pinned +2. **Unpin after retry success** + - failed `backfill` then later successful `backfill` => old failure not pinned +3. **Pinned is per-kind** + - `cleanup` success does not clear `backfill` failure pin +4. **Rate limit classification** + - `error` containing `429` or `rate limited` => `severity=rate_limit` +5. **Partial success classification** + - `rows_written>0 && error!=null` => `status=partial` +6. **In-progress classification** + - `finished_at==null` recent => `status=in_progress` + - stale threshold exceeded => `severity=warning/error` (per chosen policy) +7. **Newest-first ordering** + - runs sorted by `started_at DESC` + +### 4.2 GREEN: minimal implementation + +Implement pure helper functions first: + +- `deriveSeverity(error, finishedAt, startedAt, now)` +- `deriveStatus(error, finishedAt, rowsWritten, rowsRemoved)` +- `computePinned(runs)` +- `toSummary(run)` + +Keep these deterministic and injectable with `clock` for testability. + +### 4.3 RED: handler integration tests + +`market_history_admin_handler_test.dart`: + +1. **403 for non-admin token** +2. **200 + shape for admin** +3. **`runs` newest-first** +4. **`pinned` contains unresolved rate-limit row** +5. **`pinned` clears when later success seeded** +6. **`limit` respected** +7. **pagination via `before`** +8. **`kind` filter** +9. **response excludes raw fields** (no `raw`, no snapshot payload) + +### 4.4 GREEN: handler + routing + +Implement route: + +- `GET /v1/admin/market-history/sync-runs` + +Wire auth middleware with admin allowlist/claim gate. + +### 4.5 REFACTOR + +- Move query + transform logic out of handler into service/query class. +- Ensure all SQL is parameterized. +- Add response DTO class to reduce map-shape drift. + +--- + +## 5. Flutter TDD plan + +Planned files: + +- `lib/admin/models/sync_run_event.dart` +- `lib/admin/services/market_history_admin_api.dart` +- `lib/admin/repositories/sync_run_log_repository.dart` +- `lib/admin/screens/market_history_log_screen.dart` +- `lib/admin/widgets/sync_run_expansion_tile.dart` + +Tests: + +- `test/admin/models/sync_run_event_test.dart` +- `test/admin/repositories/sync_run_log_repository_test.dart` +- `test/admin/services/market_history_admin_api_test.dart` +- `test/admin/widgets/sync_run_expansion_tile_test.dart` +- `test/admin/screens/market_history_log_screen_test.dart` + +### 5.1 RED: model + repository unit tests + +1. Parse API response into domain model +2. Severity/status mapping from API and fallback parsing +3. Merge pinned + history sections +4. Keep pinned visible above chronological list +5. Append paged results without reordering newer rows +6. Deduplicate by `id` during refresh +7. Error state handling (`401/403/500/network`) + +### 5.2 GREEN: implement model/repository + +- Implement immutable model objects +- Implement repository APIs: + - `loadInitial()` + - `refresh()` + - `loadMore()` +- Include simple in-memory cache for current session + +### 5.3 RED: service HTTP tests + +Using mocked HTTP client: + +1. sends bearer token header +2. sends `limit`, `before`, `kind` query params +3. handles non-200 responses +4. parse malformed payload guardrails + +### 5.4 GREEN: service implementation + +Mirror existing app API style (`apiBaseUrl`, auth token retrieval, robust parse). + +### 5.5 RED: widget/screen tests + +1. **Pinned section shown** when pinned rows exist +2. **Pinned section hidden** when none exist +3. **Newest row at top** in history list +4. **Expansion tile details** + - shows full message text + - does not show raw data fields +5. **Severity styling** (icon/chip text present) +6. **Pull-to-refresh triggers repository refresh** +7. **Scroll-to-end triggers `loadMore()`** +8. **Loading, empty, and error states render correctly** + +### 5.6 GREEN: implement UI + +- Build screen with two sections: + - `Needs attention` (pinned) + - `History` (paged, newest-first) +- Use `ExpansionTile` rows with concise collapsed summary. + +### 5.7 REFACTOR + +- Extract common chips/icons/date formatters +- Keep widget tree shallow for testability +- Avoid business logic in widget build methods + +--- + +## 6. API-trigger actions (optional phase C) TDD + +If adding on-demand actions: + +Server tests: + +- `POST /v1/admin/market-data/resync` returns `202` and creates run rows +- `POST /v1/admin/market-data/cleanup` with `archive` flag routes correctly +- auth + invalid input tests + +Flutter tests: + +- retry button visible for pinned failures +- confirm dialog for cleanup +- action triggers refresh and shows new in-progress/success rows + +This phase can be deferred without affecting MVP log validation. + +--- + +## 7. Quality gates and CI commands + +### Server + +```bash +cd server && dart test test/trading/market_history_admin_logic_test.dart +cd server && dart test test/integration/market_history_admin_handler_test.dart +``` + +### Flutter + +```bash +flutter test test/admin/models/ +flutter test test/admin/repositories/ +flutter test test/admin/services/ +flutter test test/admin/widgets/ +flutter test test/admin/screens/ +``` + +### Coverage checks (recommended) + +Server: + +```bash +cd server && dart test --coverage=coverage +``` + +Flutter: + +```bash +flutter test --coverage +``` + +Add CI thresholds for the targets in Section 2. + +--- + +## 8. Risk-driven test checklist + +High-priority cases to lock down early: + +- [x] Pin logic regression when multiple kinds interleave +- [x] Rate-limit text variants (`429`, `rate limit`, capitalization differences) +- [x] Very long error message expansion rendering +- [x] Pagination duplicates/holes on refresh + load-more race +- [x] Timezone display mismatch (`started_at` UTC vs local display) +- [x] Empty payload safety (no crashes) + +--- + +## 9. Definition of done + +The admin portal feature is considered validated when: + +1. All MVP tests listed above pass. +2. Coverage targets in Section 2 are met. +3. Automated acceptance tests (`test/admin/acceptance/`) cover: + - newest-first order, + - pinned unresolved issues, + - expandable full error detail, + - no raw payload details in UI, + - cleanup archive toggle when enabled. +4. CI includes these test suites and fails on regression. + +Manual sanity pass (live server + seeded rows) remains recommended before production rollout. + +--- + +## 10. Suggested implementation order + +1. Server pure logic tests + implementation ✅ +2. Server handler integration tests + endpoint ✅ +3. Flutter models/repository/service tests + implementation ✅ +4. Flutter widget/screen tests + UI implementation ✅ +5. Optional trigger actions phase ✅ +6. Risk checklist (§8) + acceptance tests (§9) ✅ + +This order keeps risky logic proven first and gives UI teams a stable API contract early. + diff --git a/lib/admin/admin_routes.dart b/lib/admin/admin_routes.dart new file mode 100644 index 0000000..a895f2f --- /dev/null +++ b/lib/admin/admin_routes.dart @@ -0,0 +1,3 @@ +abstract final class AdminRoutes { + static const String marketHistoryLog = '/admin/market-history'; +} diff --git a/lib/admin/models/backfill_sync_item.dart b/lib/admin/models/backfill_sync_item.dart new file mode 100644 index 0000000..d1c38c1 --- /dev/null +++ b/lib/admin/models/backfill_sync_item.dart @@ -0,0 +1,30 @@ +import '../utils/sync_run_formatters.dart'; + +class BackfillSyncItem { + const BackfillSyncItem({ + required this.slotStart, + required this.symbols, + this.slotStartWire, + }); + + final DateTime slotStart; + final List symbols; + + /// Exact `slotStart` string from the sync-run API (Alpaca `start` param). + final String? slotStartWire; + + /// Wire form for DB spot checks: `raw->>'slot_start'` and Alpaca `start`. + String get fetchSlotStartWire => + slotStartWire ?? formatMarketHistorySlotWire(slotStart); + + factory BackfillSyncItem.fromJson(Map json) { + final String slotStartRaw = json['slotStart'] as String; + return BackfillSyncItem( + slotStart: DateTime.parse(slotStartRaw).toUtc(), + slotStartWire: slotStartRaw, + symbols: (json['symbols'] as List? ?? []) + .map((dynamic value) => value.toString()) + .toList(growable: false), + ); + } +} diff --git a/lib/admin/models/market_history_admin_config.dart b/lib/admin/models/market_history_admin_config.dart new file mode 100644 index 0000000..5b62ba0 --- /dev/null +++ b/lib/admin/models/market_history_admin_config.dart @@ -0,0 +1,30 @@ +class MarketHistoryAdminConfig { + const MarketHistoryAdminConfig({ + required this.archiveEnabled, + required this.windowDays, + required this.retentionDays, + required this.syncEnabled, + }); + + final bool archiveEnabled; + final int windowDays; + final int retentionDays; + final bool syncEnabled; + + factory MarketHistoryAdminConfig.fromJson(Map? json) { + if (json == null) { + return const MarketHistoryAdminConfig( + archiveEnabled: false, + windowDays: 7, + retentionDays: 7, + syncEnabled: false, + ); + } + return MarketHistoryAdminConfig( + archiveEnabled: json['archiveEnabled'] as bool? ?? false, + windowDays: (json['windowDays'] as num?)?.toInt() ?? 7, + retentionDays: (json['retentionDays'] as num?)?.toInt() ?? 7, + syncEnabled: json['syncEnabled'] as bool? ?? false, + ); + } +} diff --git a/lib/admin/models/market_history_week_coverage.dart b/lib/admin/models/market_history_week_coverage.dart new file mode 100644 index 0000000..a4ea6f8 --- /dev/null +++ b/lib/admin/models/market_history_week_coverage.dart @@ -0,0 +1,96 @@ +class MarketHistorySlotCoverage { + const MarketHistorySlotCoverage({ + required this.slotStart, + required this.completed, + required this.fullySynced, + required this.syncedSymbolCount, + required this.expectedSymbolCount, + }); + + final DateTime slotStart; + final bool completed; + final bool fullySynced; + final int syncedSymbolCount; + final int expectedSymbolCount; + + factory MarketHistorySlotCoverage.fromJson(Map json) { + return MarketHistorySlotCoverage( + slotStart: DateTime.parse(json['slotStart'] as String).toUtc(), + completed: json['completed'] as bool? ?? false, + fullySynced: json['fullySynced'] as bool? ?? false, + syncedSymbolCount: (json['syncedSymbolCount'] as num?)?.toInt() ?? 0, + expectedSymbolCount: (json['expectedSymbolCount'] as num?)?.toInt() ?? 0, + ); + } +} + +class MarketHistoryDayCoverage { + const MarketHistoryDayCoverage({ + required this.date, + required this.slotsPerDay, + required this.completedSlots, + required this.fullySyncedSlots, + required this.slots, + }); + + final DateTime date; + final int slotsPerDay; + final int completedSlots; + final int fullySyncedSlots; + final List slots; + + factory MarketHistoryDayCoverage.fromJson(Map json) { + final List rawSlots = + json['slots'] as List? ?? []; + return MarketHistoryDayCoverage( + date: DateTime.parse('${json['date']}T00:00:00Z').toUtc(), + slotsPerDay: (json['slotsPerDay'] as num?)?.toInt() ?? 6, + completedSlots: (json['completedSlots'] as num?)?.toInt() ?? 0, + fullySyncedSlots: (json['fullySyncedSlots'] as num?)?.toInt() ?? 0, + slots: rawSlots + .map( + (dynamic item) => MarketHistorySlotCoverage.fromJson( + item as Map, + ), + ) + .toList(growable: false), + ); + } +} + +class MarketHistoryWeekCoverageReport { + const MarketHistoryWeekCoverageReport({ + required this.asOf, + required this.windowDays, + required this.slotsPerDay, + required this.symbolCount, + required this.isConsistent, + required this.days, + }); + + final DateTime asOf; + final int windowDays; + final int slotsPerDay; + final int symbolCount; + final bool isConsistent; + final List days; + + factory MarketHistoryWeekCoverageReport.fromJson(Map json) { + final List rawDays = + json['days'] as List? ?? []; + return MarketHistoryWeekCoverageReport( + asOf: DateTime.parse(json['asOf'] as String).toUtc(), + windowDays: (json['windowDays'] as num?)?.toInt() ?? 7, + slotsPerDay: (json['slotsPerDay'] as num?)?.toInt() ?? 6, + symbolCount: (json['symbolCount'] as num?)?.toInt() ?? 0, + isConsistent: json['isConsistent'] as bool? ?? false, + days: rawDays + .map( + (dynamic item) => MarketHistoryDayCoverage.fromJson( + item as Map, + ), + ) + .toList(growable: false), + ); + } +} diff --git a/lib/admin/models/question_audit_asset.dart b/lib/admin/models/question_audit_asset.dart new file mode 100644 index 0000000..e80daae --- /dev/null +++ b/lib/admin/models/question_audit_asset.dart @@ -0,0 +1,119 @@ +class QuestionAuditBarSlot { + const QuestionAuditBarSlot({ + required this.asOf, + required this.avgPrice, + required this.volume, + this.open, + this.high, + this.low, + this.close, + }); + + final DateTime asOf; + final num? open; + final num? high; + final num? low; + final num? close; + final num avgPrice; + final num volume; + + factory QuestionAuditBarSlot.fromJson(Map json) { + return QuestionAuditBarSlot( + asOf: DateTime.parse(json['asOf']! as String).toUtc(), + open: json['open'] as num?, + high: json['high'] as num?, + low: json['low'] as num?, + close: json['close'] as num?, + avgPrice: json['avgPrice'] as num, + volume: json['volume'] as num, + ); + } +} + +class QuestionAuditAsset { + const QuestionAuditAsset({ + required this.symbol, + required this.priceDelta, + required this.volumeDelta, + required this.olderSlot, + required this.newerSlot, + }); + + final String symbol; + final num priceDelta; + final num volumeDelta; + final QuestionAuditBarSlot olderSlot; + final QuestionAuditBarSlot newerSlot; + + factory QuestionAuditAsset.fromJson(Map json) { + return QuestionAuditAsset( + symbol: json['symbol']! as String, + priceDelta: json['priceDelta'] as num, + volumeDelta: json['volumeDelta'] as num, + olderSlot: QuestionAuditBarSlot.fromJson( + json['olderSlot']! as Map, + ), + newerSlot: QuestionAuditBarSlot.fromJson( + json['newerSlot']! as Map, + ), + ); + } +} + +class QuestionAuditReport { + const QuestionAuditReport({ + required this.compareUntil, + required this.newerSlotStart, + required this.olderSlotStart, + required this.windowDays, + required this.assets, + required this.canStepOlder, + required this.canStepNewer, + this.stepOlderCompareUntil, + this.stepNewerCompareUntil, + }); + + final DateTime compareUntil; + final DateTime newerSlotStart; + final DateTime olderSlotStart; + final int windowDays; + final List assets; + final bool canStepOlder; + final bool canStepNewer; + final DateTime? stepOlderCompareUntil; + final DateTime? stepNewerCompareUntil; + + factory QuestionAuditReport.fromJson(Map json) { + final List raw = json['assets'] as List? ?? []; + final DateTime compareUntil = DateTime.parse( + (json['compareUntil'] ?? json['asOf'])! as String, + ).toUtc(); + return QuestionAuditReport( + compareUntil: compareUntil, + newerSlotStart: json['newerSlotStart'] == null + ? compareUntil.subtract(const Duration(hours: 4)) + : DateTime.parse(json['newerSlotStart']! as String).toUtc(), + olderSlotStart: json['olderSlotStart'] == null + ? compareUntil.subtract(const Duration(hours: 8)) + : DateTime.parse(json['olderSlotStart']! as String).toUtc(), + windowDays: (json['windowDays'] as num).toInt(), + canStepOlder: json['canStepOlder'] as bool? ?? false, + canStepNewer: json['canStepNewer'] as bool? ?? false, + stepOlderCompareUntil: _parseOptionalUtc(json['stepOlderCompareUntil']), + stepNewerCompareUntil: _parseOptionalUtc(json['stepNewerCompareUntil']), + assets: raw + .map( + (dynamic item) => + QuestionAuditAsset.fromJson(item as Map), + ) + .toList(growable: false), + ); + } +} + +DateTime? _parseOptionalUtc(dynamic raw) { + if (raw == null) { + return null; + } + return DateTime.parse(raw as String).toUtc(); +} diff --git a/lib/admin/models/sync_run_event.dart b/lib/admin/models/sync_run_event.dart new file mode 100644 index 0000000..7da34bb --- /dev/null +++ b/lib/admin/models/sync_run_event.dart @@ -0,0 +1,191 @@ +import 'market_history_admin_config.dart'; +import 'backfill_sync_item.dart'; + +enum SyncRunSeverity { ok, warning, error, rateLimit } + +enum SyncRunStatus { success, failed, partial, inProgress } + +class SyncRunEvent { + const SyncRunEvent({ + required this.id, + required this.kind, + required this.startedAt, + required this.finishedAt, + required this.rowsWritten, + required this.rowsRemoved, + required this.error, + required this.severity, + required this.status, + required this.durationMs, + required this.summary, + this.backfillItems = const [], + }); + + final int id; + final String kind; + final DateTime startedAt; + final DateTime? finishedAt; + final int rowsWritten; + final int rowsRemoved; + final String? error; + final SyncRunSeverity severity; + final SyncRunStatus status; + final int? durationMs; + final String summary; + final List backfillItems; + + String get displayTitle { + switch (kind) { + case 'universe': + return 'Asset universe sync'; + case 'backfill': + return 'Market history backfill'; + case 'cleanup': + return 'Data retention cleanup'; + default: + return kind; + } + } + + factory SyncRunEvent.fromJson(Map json) { + return SyncRunEvent( + id: (json['id'] as num).toInt(), + kind: json['kind'] as String, + startedAt: DateTime.parse(json['startedAt'] as String).toUtc(), + finishedAt: (json['finishedAt'] as String?) == null + ? null + : DateTime.parse(json['finishedAt'] as String).toUtc(), + rowsWritten: (json['rowsWritten'] as num?)?.toInt() ?? 0, + rowsRemoved: (json['rowsRemoved'] as num?)?.toInt() ?? 0, + error: json['error'] as String?, + severity: _parseSeverity( + json['severity'] as String?, + json['error'] as String?, + (json['finishedAt'] as String?) == null, + ), + status: _parseStatus( + json['status'] as String?, + json['error'] as String?, + (json['finishedAt'] as String?) == null, + (json['rowsWritten'] as num?)?.toInt() ?? 0, + ), + durationMs: (json['durationMs'] as num?)?.toInt(), + summary: (json['summary'] as String?)?.trim().isNotEmpty == true + ? (json['summary'] as String) + : _fallbackSummary( + kind: json['kind'] as String, + rowsWritten: (json['rowsWritten'] as num?)?.toInt() ?? 0, + rowsRemoved: (json['rowsRemoved'] as num?)?.toInt() ?? 0, + error: json['error'] as String?, + ), + backfillItems: _parseBackfillItems(json['backfillItems']), + ); + } + + static List _parseBackfillItems(dynamic raw) { + if (raw is! List) { + return const []; + } + return raw + .whereType>() + .map(BackfillSyncItem.fromJson) + .toList(growable: false); + } + + static SyncRunSeverity _parseSeverity( + String? wire, + String? error, + bool inProgress, + ) { + switch (wire) { + case 'ok': + return SyncRunSeverity.ok; + case 'warning': + return SyncRunSeverity.warning; + case 'error': + return SyncRunSeverity.error; + case 'rate_limit': + return SyncRunSeverity.rateLimit; + default: + final String normalized = (error ?? '').toLowerCase(); + if (normalized.contains('429') || + normalized.contains('rate limit') || + normalized.contains('rate limited')) { + return SyncRunSeverity.rateLimit; + } + if (error != null && error.trim().isNotEmpty) { + return SyncRunSeverity.error; + } + if (inProgress) { + return SyncRunSeverity.warning; + } + return SyncRunSeverity.ok; + } + } + + static SyncRunStatus _parseStatus( + String? wire, + String? error, + bool inProgress, + int rowsWritten, + ) { + switch (wire) { + case 'success': + return SyncRunStatus.success; + case 'failed': + return SyncRunStatus.failed; + case 'partial': + return SyncRunStatus.partial; + case 'in_progress': + return SyncRunStatus.inProgress; + default: + if (inProgress) { + return SyncRunStatus.inProgress; + } + if (error != null && error.trim().isNotEmpty) { + return rowsWritten > 0 ? SyncRunStatus.partial : SyncRunStatus.failed; + } + return SyncRunStatus.success; + } + } + + static String _fallbackSummary({ + required String kind, + required int rowsWritten, + required int rowsRemoved, + required String? error, + }) { + if (error != null && error.trim().isNotEmpty && rowsWritten > 0) { + return 'Partial success: $rowsWritten rows written'; + } + if (error != null && error.trim().isNotEmpty) { + return 'Run failed'; + } + if (kind == 'cleanup') { + return '$rowsRemoved rows removed'; + } + if (kind == 'universe') { + return '$rowsWritten assets refreshed'; + } + return '$rowsWritten bar rows written'; + } +} + +class SyncRunLogPage { + const SyncRunLogPage({ + required this.runs, + required this.pinned, + required this.nextBefore, + this.config = const MarketHistoryAdminConfig( + archiveEnabled: false, + windowDays: 7, + retentionDays: 7, + syncEnabled: false, + ), + }); + + final List runs; + final List pinned; + final DateTime? nextBefore; + final MarketHistoryAdminConfig config; +} diff --git a/lib/admin/repositories/sync_run_log_repository.dart b/lib/admin/repositories/sync_run_log_repository.dart new file mode 100644 index 0000000..0ea558c --- /dev/null +++ b/lib/admin/repositories/sync_run_log_repository.dart @@ -0,0 +1,179 @@ +import '../models/market_history_admin_config.dart'; +import '../models/sync_run_event.dart'; +import '../services/market_history_admin_api.dart'; + +abstract class SyncRunLogController { + SyncRunLogRepositoryState get state; + + Future loadInitial({int limit = 50}); + + Future refresh({int limit = 50}); + + Future loadMore({int limit = 50}); + + Future triggerResync(); + + Future triggerCleanup({bool archive = false}); +} + +class SyncRunLogRepositoryState { + const SyncRunLogRepositoryState({ + required this.pinned, + required this.history, + required this.nextBefore, + required this.isLoading, + required this.errorMessage, + this.config = const MarketHistoryAdminConfig( + archiveEnabled: false, + windowDays: 7, + retentionDays: 7, + syncEnabled: false, + ), + }); + + final List pinned; + final List history; + final DateTime? nextBefore; + final bool isLoading; + final String? errorMessage; + final MarketHistoryAdminConfig config; + + bool get hasMore => nextBefore != null; + + bool get hasInProgressRun { + bool inProgress(SyncRunEvent event) => + event.status == SyncRunStatus.inProgress; + return pinned.any(inProgress) || history.any(inProgress); + } +} + +class SyncRunLogRepository implements SyncRunLogController { + SyncRunLogRepository({required MarketHistoryAdminApi api}) : _api = api; + + final MarketHistoryAdminApi _api; + + List _pinned = []; + List _history = []; + DateTime? _nextBefore; + bool _isLoading = false; + String? _errorMessage; + MarketHistoryAdminConfig _config = const MarketHistoryAdminConfig( + archiveEnabled: false, + windowDays: 7, + retentionDays: 7, + syncEnabled: false, + ); + + SyncRunLogRepositoryState get state => SyncRunLogRepositoryState( + pinned: List.unmodifiable(_pinned), + history: List.unmodifiable(_history), + nextBefore: _nextBefore, + isLoading: _isLoading, + errorMessage: _errorMessage, + config: _config, + ); + + Future loadInitial({int limit = 50}) async { + _isLoading = true; + _errorMessage = null; + try { + final SyncRunLogPage page = await _api.fetchSyncRuns(limit: limit); + _pinned = _dedupeById(page.pinned); + _history = _dedupeById(_sortedNewestFirst(page.runs)); + _nextBefore = page.nextBefore; + _config = page.config; + } on MarketHistoryAdminApiException catch (e) { + _errorMessage = e.message; + } catch (e) { + _errorMessage = e.toString(); + } finally { + _isLoading = false; + } + return state; + } + + Future refresh({int limit = 50}) async { + return loadInitial(limit: limit); + } + + Future loadMore({int limit = 50}) async { + if (_isLoading || _nextBefore == null) { + return state; + } + _isLoading = true; + _errorMessage = null; + try { + final SyncRunLogPage page = await _api.fetchSyncRuns( + limit: limit, + before: _nextBefore, + ); + _pinned = _dedupeById(page.pinned); + final List merged = [ + ..._history, + ...page.runs, + ]; + _history = _dedupeById(_sortedNewestFirst(merged)); + _nextBefore = page.nextBefore; + _config = page.config; + } on MarketHistoryAdminApiException catch (e) { + _errorMessage = e.message; + } catch (e) { + _errorMessage = e.toString(); + } finally { + _isLoading = false; + } + return state; + } + + @override + Future triggerResync() async { + _isLoading = true; + _errorMessage = null; + try { + await _api.triggerResync(); + return refresh(); + } on MarketHistoryAdminApiException catch (e) { + _errorMessage = e.message; + _isLoading = false; + return state; + } catch (e) { + _errorMessage = e.toString(); + _isLoading = false; + return state; + } + } + + @override + Future triggerCleanup({bool archive = false}) async { + _isLoading = true; + _errorMessage = null; + try { + await _api.triggerCleanup(archive: archive); + return refresh(); + } on MarketHistoryAdminApiException catch (e) { + _errorMessage = e.message; + _isLoading = false; + return state; + } catch (e) { + _errorMessage = e.toString(); + _isLoading = false; + return state; + } + } + + static List _dedupeById(List events) { + final Map byId = {}; + for (final SyncRunEvent event in events) { + byId[event.id] = event; + } + return byId.values.toList(); + } + + static List _sortedNewestFirst(List events) { + final List sorted = List.from(events); + sorted.sort( + (SyncRunEvent a, SyncRunEvent b) => b.startedAt.compareTo(a.startedAt), + ); + return sorted; + } +} diff --git a/lib/admin/screens/market_history_log_screen.dart b/lib/admin/screens/market_history_log_screen.dart new file mode 100644 index 0000000..9f7de30 --- /dev/null +++ b/lib/admin/screens/market_history_log_screen.dart @@ -0,0 +1,501 @@ +import 'dart:async'; + +import 'package:flutter/material.dart'; + +import '../../theme/app_theme.dart'; +import '../models/sync_run_event.dart'; +import '../repositories/sync_run_log_repository.dart'; +import '../services/market_history_admin_api.dart'; +import '../widgets/market_history_question_audit_sheet.dart'; +import '../widgets/market_history_week_coverage_sheet.dart'; +import '../widgets/sync_run_expansion_tile.dart'; + +class MarketHistoryLogScreen extends StatefulWidget { + const MarketHistoryLogScreen({ + super.key, + required this.controller, + this.coverageApi, + this.now, + this.autoRefreshInterval = const Duration(seconds: 60), + }); + + final SyncRunLogController controller; + final MarketHistoryAdminApi? coverageApi; + final DateTime? now; + final Duration? autoRefreshInterval; + + @override + State createState() => _MarketHistoryLogScreenState(); +} + +class _MarketHistoryLogScreenState extends State { + final ScrollController _scrollController = ScrollController(); + Timer? _autoRefreshTimer; + + SyncRunLogRepositoryState _state = const SyncRunLogRepositoryState( + pinned: [], + history: [], + nextBefore: null, + isLoading: true, + errorMessage: null, + ); + + bool _initialLoaded = false; + bool _loadingMore = false; + bool _triggerInFlight = false; + bool _weekCoverageLoading = false; + bool _questionAuditLoading = false; + + MarketHistoryAdminApi get _coverageApi => + widget.coverageApi ?? MarketHistoryAdminApi(); + + @override + void initState() { + super.initState(); + _scrollController.addListener(_onScroll); + _loadInitial(); + _startAutoRefresh(); + } + + void _startAutoRefresh() { + final Duration? interval = widget.autoRefreshInterval; + if (interval == null) { + return; + } + _autoRefreshTimer = Timer.periodic(interval, (_) { + if (!mounted || _state.isLoading || _triggerInFlight) { + return; + } + _refresh(); + }); + } + + @override + void dispose() { + _autoRefreshTimer?.cancel(); + _scrollController.dispose(); + super.dispose(); + } + + Future _loadInitial() async { + final SyncRunLogRepositoryState next = + await widget.controller.loadInitial(); + if (!mounted) { + return; + } + setState(() { + _state = next; + _initialLoaded = true; + }); + } + + Future _refresh() async { + final SyncRunLogRepositoryState next = await widget.controller.refresh(); + if (!mounted) { + return; + } + setState(() => _state = next); + } + + Future _loadMore() async { + if (_loadingMore || !_state.hasMore || _state.isLoading) { + return; + } + _loadingMore = true; + final SyncRunLogRepositoryState next = await widget.controller.loadMore(); + if (!mounted) { + return; + } + setState(() { + _state = next; + _loadingMore = false; + }); + } + + Future _runResync() async { + if (_triggerInFlight || _state.hasInProgressRun) { + return; + } + setState(() => _triggerInFlight = true); + final SyncRunLogRepositoryState next = + await widget.controller.triggerResync(); + if (!mounted) { + return; + } + setState(() { + _state = next; + _triggerInFlight = false; + }); + if (next.errorMessage != null) { + _showSnack(next.errorMessage!); + } + } + + Future _confirmAndRunCleanup() async { + if (_triggerInFlight || _state.hasInProgressRun) { + return; + } + bool archive = false; + final bool? confirmed = await showDialog( + context: context, + builder: (BuildContext context) { + return StatefulBuilder( + builder: (BuildContext context, StateSetter setDialogState) { + return AlertDialog( + key: const Key('cleanup-confirm-dialog'), + title: const Text('Run cleanup now?'), + content: Column( + mainAxisSize: MainAxisSize.min, + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text( + 'This removes expired market history snapshots older than ' + '${_state.config.retentionDays} days. Continue?', + ), + if (_state.config.archiveEnabled) ...[ + const SizedBox(height: 12), + CheckboxListTile( + key: const Key('cleanup-archive-toggle'), + contentPadding: EdgeInsets.zero, + title: const Text('Archive before delete'), + value: archive, + onChanged: (bool? value) { + setDialogState(() => archive = value ?? false); + }, + controlAffinity: ListTileControlAffinity.leading, + ), + ], + ], + ), + actions: [ + TextButton( + key: const Key('cleanup-cancel'), + onPressed: () => Navigator.of(context).pop(false), + child: const Text('Cancel'), + ), + FilledButton( + key: const Key('cleanup-confirm'), + onPressed: () => Navigator.of(context).pop(true), + child: const Text('Run cleanup'), + ), + ], + ); + }, + ); + }, + ); + if (confirmed != true || !mounted) { + return; + } + + setState(() => _triggerInFlight = true); + final SyncRunLogRepositoryState next = + await widget.controller.triggerCleanup(archive: archive); + if (!mounted) { + return; + } + setState(() { + _state = next; + _triggerInFlight = false; + }); + if (next.errorMessage != null) { + _showSnack(next.errorMessage!); + } + } + + void _showSnack(String message) { + ScaffoldMessenger.of(context).showSnackBar( + SnackBar(content: Text(message)), + ); + } + + Future _showQuestionAudit() async { + if (_questionAuditLoading) { + return; + } + setState(() => _questionAuditLoading = true); + try { + if (!mounted) { + return; + } + await MarketHistoryQuestionAuditSheet.show( + context, + api: _coverageApi, + ); + } on MarketHistoryAdminApiException catch (e) { + if (mounted) { + _showSnack('Could not load question audit (${e.statusCode ?? 'error'})'); + } + } catch (e) { + if (mounted) { + _showSnack('Could not load question audit'); + } + } finally { + if (mounted) { + setState(() => _questionAuditLoading = false); + } + } + } + + Future _showWeekCoverage() async { + if (_weekCoverageLoading) { + return; + } + setState(() => _weekCoverageLoading = true); + try { + final report = await _coverageApi.fetchWeekCoverage(); + if (!mounted) { + return; + } + await MarketHistoryWeekCoverageSheet.show(context, report: report); + } on MarketHistoryAdminApiException catch (e) { + if (mounted) { + _showSnack('Could not load week coverage (${e.statusCode ?? 'error'})'); + } + } catch (e) { + if (mounted) { + _showSnack('Could not load week coverage'); + } + } finally { + if (mounted) { + setState(() => _weekCoverageLoading = false); + } + } + } + + void _onScroll() { + if (!_scrollController.hasClients) { + return; + } + const double threshold = 120; + final ScrollPosition position = _scrollController.position; + if (position.pixels >= position.maxScrollExtent - threshold) { + _loadMore(); + } + } + + bool get _actionsDisabled => + !_state.config.syncEnabled || + _state.isLoading || + _triggerInFlight || + _state.hasInProgressRun; + + @override + Widget build(BuildContext context) { + return Scaffold( + appBar: AppBar( + title: const Text('Market history log'), + actions: [ + IconButton( + key: const Key('question-audit-button'), + tooltip: 'Audit prospective question data', + onPressed: _questionAuditLoading ? null : _showQuestionAudit, + icon: const Icon(Icons.help_outline), + ), + IconButton( + key: const Key('week-coverage-button'), + tooltip: '7-day sync calendar', + onPressed: _weekCoverageLoading ? null : _showWeekCoverage, + icon: const Icon(Icons.calendar_month_outlined), + ), + PopupMenuButton( + key: const Key('actions-menu'), + enabled: !_actionsDisabled, + onSelected: (String value) { + switch (value) { + case 'resync': + _runResync(); + case 'cleanup': + _confirmAndRunCleanup(); + } + }, + itemBuilder: (BuildContext context) => >[ + const PopupMenuItem( + key: Key('action-resync'), + value: 'resync', + child: Text('Run backfill now'), + ), + const PopupMenuItem( + key: Key('action-cleanup'), + value: 'cleanup', + child: Text('Run cleanup now'), + ), + ], + ), + IconButton( + key: const Key('refresh-button'), + tooltip: 'Refresh', + onPressed: _state.isLoading ? null : _refresh, + icon: const Icon(Icons.refresh), + ), + ], + ), + body: _buildBody(), + ); + } + + Widget _buildBody() { + if (!_initialLoaded && _state.isLoading) { + return const Center( + key: Key('loading-indicator'), + child: CircularProgressIndicator(color: AppColors.accent), + ); + } + + if (_state.errorMessage != null && + _state.pinned.isEmpty && + _state.history.isEmpty) { + return Center( + key: const Key('error-state'), + child: Padding( + padding: const EdgeInsets.all(24), + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + const Icon(Icons.error_outline, color: Colors.redAccent, size: 40), + const SizedBox(height: 12), + Text( + _state.errorMessage!, + textAlign: TextAlign.center, + style: const TextStyle(color: AppColors.textSecondary), + ), + const SizedBox(height: 12), + FilledButton(onPressed: _refresh, child: const Text('Retry')), + ], + ), + ), + ); + } + + if (_state.pinned.isEmpty && _state.history.isEmpty) { + return Column( + children: [ + if (!_state.config.syncEnabled) const _SyncDisabledBanner(), + const Expanded( + child: Center( + key: Key('empty-state'), + child: Text( + 'No sync runs yet.', + style: TextStyle(color: AppColors.textSecondary), + ), + ), + ), + ], + ); + } + + return RefreshIndicator( + key: const Key('refresh-indicator'), + onRefresh: _refresh, + color: AppColors.accent, + child: CustomScrollView( + controller: _scrollController, + physics: const AlwaysScrollableScrollPhysics(), + slivers: [ + if (!_state.config.syncEnabled) + const SliverToBoxAdapter(child: _SyncDisabledBanner()), + if (_state.pinned.isNotEmpty) ...[ + const SliverToBoxAdapter( + child: Padding( + key: Key('pinned-section'), + padding: EdgeInsets.fromLTRB(16, 16, 16, 8), + child: Text( + 'NEEDS ATTENTION', + style: TextStyle( + fontSize: 12, + fontWeight: FontWeight.w700, + letterSpacing: 0.8, + color: Colors.amber, + ), + ), + ), + ), + SliverList( + delegate: SliverChildBuilderDelegate( + (BuildContext context, int index) { + final SyncRunEvent event = _state.pinned[index]; + return SyncRunExpansionTile( + event: event, + now: widget.now, + emphasizeError: true, + onRetry: _actionsDisabled ? null : _runResync, + ); + }, + childCount: _state.pinned.length, + ), + ), + ], + const SliverToBoxAdapter( + child: Padding( + key: Key('history-section'), + padding: EdgeInsets.fromLTRB(16, 16, 16, 8), + child: Text( + 'HISTORY', + style: TextStyle( + fontSize: 12, + fontWeight: FontWeight.w700, + letterSpacing: 0.8, + color: AppColors.textSecondary, + ), + ), + ), + ), + SliverList( + delegate: SliverChildBuilderDelegate( + (BuildContext context, int index) { + final SyncRunEvent event = _state.history[index]; + return SyncRunExpansionTile( + event: event, + now: widget.now, + ); + }, + childCount: _state.history.length, + ), + ), + if (_state.isLoading || _loadingMore || _triggerInFlight) + const SliverToBoxAdapter( + child: Padding( + padding: EdgeInsets.all(16), + child: Center( + child: CircularProgressIndicator(color: AppColors.accent), + ), + ), + ), + ], + ), + ); + } +} + +/// Convenience factory for production use. +MarketHistoryLogScreen marketHistoryLogScreen() { + return MarketHistoryLogScreen( + controller: SyncRunLogRepository( + api: MarketHistoryAdminApi(), + ), + ); +} + +class _SyncDisabledBanner extends StatelessWidget { + const _SyncDisabledBanner(); + + @override + Widget build(BuildContext context) { + return Container( + key: const Key('sync-disabled-banner'), + width: double.infinity, + margin: const EdgeInsets.fromLTRB(16, 12, 16, 0), + padding: const EdgeInsets.symmetric(horizontal: 12, vertical: 10), + decoration: BoxDecoration( + color: Colors.amber.withValues(alpha: 0.12), + borderRadius: BorderRadius.circular(8), + border: Border.all(color: Colors.amber.withValues(alpha: 0.45)), + ), + child: const Text( + 'Market history sync is disabled (MARKET_HISTORY_SYNC_ENABLED=false). ' + 'Log and calendar views are read-only.', + style: TextStyle(fontSize: 12, color: AppColors.textSecondary), + ), + ); + } +} diff --git a/lib/admin/services/admin_access_service.dart b/lib/admin/services/admin_access_service.dart new file mode 100644 index 0000000..bd34ddc --- /dev/null +++ b/lib/admin/services/admin_access_service.dart @@ -0,0 +1,42 @@ +import 'package:flutter/foundation.dart'; + +import 'market_history_admin_api.dart'; + +enum AdminAccessStatus { + unknown, + loading, + authorized, + forbidden, + unauthorized, + error, +} + +/// Probes admin API access for home-screen discovery (allowlist-backed). +class AdminAccessService { + AdminAccessService._(); + + static final AdminAccessService instance = AdminAccessService._(); + + final ValueNotifier status = + ValueNotifier(AdminAccessStatus.unknown); + + Future refresh({MarketHistoryAdminApi? api}) async { + status.value = AdminAccessStatus.loading; + try { + await (api ?? MarketHistoryAdminApi()).fetchSyncRuns(limit: 1); + status.value = AdminAccessStatus.authorized; + } on MarketHistoryAdminApiException catch (e) { + status.value = switch (e.statusCode) { + 401 => AdminAccessStatus.unauthorized, + 403 => AdminAccessStatus.forbidden, + _ => AdminAccessStatus.error, + }; + } catch (_) { + status.value = AdminAccessStatus.error; + } + } + + void reset() { + status.value = AdminAccessStatus.unknown; + } +} diff --git a/lib/admin/services/market_history_admin_api.dart b/lib/admin/services/market_history_admin_api.dart new file mode 100644 index 0000000..31c098a --- /dev/null +++ b/lib/admin/services/market_history_admin_api.dart @@ -0,0 +1,233 @@ +import 'dart:convert'; + +import 'package:http/http.dart' as http; + +import '../../config/api_config.dart'; +import '../../services/auth_service.dart'; +import '../models/market_history_admin_config.dart'; +import '../models/market_history_week_coverage.dart'; +import '../models/question_audit_asset.dart'; +import '../models/sync_run_event.dart'; + +typedef AdminTokenProvider = Future Function(); + +class MarketHistoryAdminApiException implements Exception { + MarketHistoryAdminApiException(this.message, {this.statusCode}); + + final String message; + final int? statusCode; + + @override + String toString() => 'MarketHistoryAdminApiException($statusCode): $message'; +} + +class AdminTriggerResponse { + const AdminTriggerResponse({required this.runIds}); + + final List runIds; + + factory AdminTriggerResponse.fromJson(Map json) { + final List raw = json['runIds'] as List? ?? []; + return AdminTriggerResponse( + runIds: raw.map((dynamic id) => (id as num).toInt()).toList(), + ); + } +} + +class MarketHistoryAdminApi { + MarketHistoryAdminApi({ + http.Client? client, + String? baseUrl, + AdminTokenProvider? tokenProvider, + }) : _client = client ?? http.Client(), + _baseUrl = baseUrl ?? apiBaseUrl, + _tokenProvider = tokenProvider ?? AuthService.instance.getIdToken; + + final http.Client _client; + final String _baseUrl; + final AdminTokenProvider _tokenProvider; + + Future fetchSyncRuns({ + int limit = 50, + DateTime? before, + String? kind, + }) async { + final String? token = await _tokenProvider(); + if (token == null || token.isEmpty) { + throw MarketHistoryAdminApiException('Missing auth token'); + } + + final Map query = {'limit': '$limit'}; + if (before != null) { + query['before'] = before.toUtc().toIso8601String(); + } + if (kind != null && kind.isNotEmpty) { + query['kind'] = kind; + } + final Uri uri = Uri.parse('$_baseUrl/v1/admin/market-history/sync-runs') + .replace(queryParameters: query); + final http.Response response = await _client.get( + uri, + headers: { + 'Authorization': 'Bearer $token', + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }, + ); + + if (response.statusCode != 200) { + throw MarketHistoryAdminApiException( + response.body, + statusCode: response.statusCode, + ); + } + + final Map body = + jsonDecode(response.body) as Map; + final List runs = _parseEventList(body['runs'], 'runs'); + final List pinned = _parseEventList(body['pinned'], 'pinned'); + final String? nextBeforeRaw = body['nextBefore'] as String?; + final MarketHistoryAdminConfig config = MarketHistoryAdminConfig.fromJson( + body['config'] as Map?, + ); + + return SyncRunLogPage( + runs: runs, + pinned: pinned, + nextBefore: nextBeforeRaw == null ? null : DateTime.parse(nextBeforeRaw).toUtc(), + config: config, + ); + } + + Future fetchQuestionAudit({DateTime? asOf}) async { + final String? token = await _tokenProvider(); + if (token == null || token.isEmpty) { + throw MarketHistoryAdminApiException('Missing auth token'); + } + + final Map query = {}; + if (asOf != null) { + query['asOf'] = asOf.toUtc().toIso8601String(); + } + final Uri uri = Uri.parse('$_baseUrl/v1/admin/market-history/question-audit') + .replace(queryParameters: query.isEmpty ? null : query); + final http.Response response = await _client.get( + uri, + headers: { + 'Authorization': 'Bearer $token', + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }, + ); + + if (response.statusCode != 200) { + throw MarketHistoryAdminApiException( + response.body, + statusCode: response.statusCode, + ); + } + + final Map body = + jsonDecode(response.body) as Map; + return QuestionAuditReport.fromJson(body); + } + + Future fetchWeekCoverage() async { + final String? token = await _tokenProvider(); + if (token == null || token.isEmpty) { + throw MarketHistoryAdminApiException('Missing auth token'); + } + + final Uri uri = + Uri.parse('$_baseUrl/v1/admin/market-history/week-coverage'); + final http.Response response = await _client.get( + uri, + headers: { + 'Authorization': 'Bearer $token', + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }, + ); + + if (response.statusCode != 200) { + throw MarketHistoryAdminApiException( + response.body, + statusCode: response.statusCode, + ); + } + + final Map body = + jsonDecode(response.body) as Map; + return MarketHistoryWeekCoverageReport.fromJson(body); + } + + static List _parseEventList(dynamic raw, String field) { + if (raw == null) { + return []; + } + if (raw is! List) { + throw MarketHistoryAdminApiException( + 'Invalid sync-runs payload: $field must be a list', + ); + } + final List events = []; + for (int i = 0; i < raw.length; i++) { + final dynamic item = raw[i]; + if (item is! Map) { + throw MarketHistoryAdminApiException( + 'Invalid sync-runs payload: $field[$i] must be an object', + ); + } + try { + events.add(SyncRunEvent.fromJson(item)); + } catch (e) { + throw MarketHistoryAdminApiException( + 'Invalid sync-runs payload: $field[$i] $e', + ); + } + } + return events; + } + + Future triggerResync() async { + return _postTrigger('/v1/admin/market-data/resync'); + } + + Future triggerCleanup({bool archive = false}) async { + final Uri uri = Uri.parse('$_baseUrl/v1/admin/market-data/cleanup').replace( + queryParameters: archive ? {'archive': 'true'} : null, + ); + return _postTriggerUri(uri); + } + + Future _postTrigger(String path) { + return _postTriggerUri(Uri.parse('$_baseUrl$path')); + } + + Future _postTriggerUri(Uri uri) async { + final String? token = await _tokenProvider(); + if (token == null || token.isEmpty) { + throw MarketHistoryAdminApiException('Missing auth token'); + } + + final http.Response response = await _client.post( + uri, + headers: { + 'Authorization': 'Bearer $token', + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }, + ); + + if (response.statusCode != 202) { + throw MarketHistoryAdminApiException( + response.body, + statusCode: response.statusCode, + ); + } + + final Map body = + jsonDecode(response.body) as Map; + return AdminTriggerResponse.fromJson(body); + } +} diff --git a/lib/admin/utils/sync_run_formatters.dart b/lib/admin/utils/sync_run_formatters.dart new file mode 100644 index 0000000..37c2596 --- /dev/null +++ b/lib/admin/utils/sync_run_formatters.dart @@ -0,0 +1,101 @@ +import 'package:flutter/material.dart'; + +import '../../theme/app_theme.dart'; +import '../models/sync_run_event.dart'; + +String formatRelativeTime(DateTime startedAt, {DateTime? now}) { + final DateTime reference = (now ?? DateTime.now()).toUtc(); + final Duration delta = reference.difference(startedAt.toUtc()); + if (delta.inMinutes < 1) { + return 'just now'; + } + if (delta.inHours < 1) { + return '${delta.inMinutes}m ago'; + } + if (delta.inDays < 1) { + return '${delta.inHours}h ago'; + } + return '${delta.inDays}d ago'; +} + +String formatMarketHistorySlotWire(DateTime value) { + final DateTime utc = value.toUtc(); + final int slotHour = (utc.hour ~/ 4) * 4; + final DateTime slotStart = DateTime.utc( + utc.year, + utc.month, + utc.day, + slotHour, + ); + String two(int n) => n.toString().padLeft(2, '0'); + return '${slotStart.year.toString().padLeft(4, '0')}-' + '${two(slotStart.month)}-${two(slotStart.day)}T' + '${two(slotStart.hour)}:${two(slotStart.minute)}:${two(slotStart.second)}Z'; +} + +String formatUtcTimestamp(DateTime? value) { + if (value == null) { + return '—'; + } + final DateTime utc = value.toUtc(); + final String hour = utc.hour.toString().padLeft(2, '0'); + final String minute = utc.minute.toString().padLeft(2, '0'); + return '${utc.year}-${utc.month.toString().padLeft(2, '0')}-' + '${utc.day.toString().padLeft(2, '0')} $hour:$minute UTC'; +} + +String formatLocalTimestamp(DateTime? value) { + if (value == null) { + return '—'; + } + final DateTime local = value.toLocal(); + final String hour = local.hour.toString().padLeft(2, '0'); + final String minute = local.minute.toString().padLeft(2, '0'); + return '${local.year}-${local.month.toString().padLeft(2, '0')}-' + '${local.day.toString().padLeft(2, '0')} $hour:$minute'; +} + +String formatDurationMs(int? durationMs) { + if (durationMs == null) { + return '—'; + } + if (durationMs < 1000) { + return '${durationMs}ms'; + } + return '${(durationMs / 1000).toStringAsFixed(1)}s'; +} + +String shortStatusLabel(SyncRunEvent event) { + return switch (event.status) { + SyncRunStatus.success => 'Success', + SyncRunStatus.failed => 'Failed', + SyncRunStatus.partial => 'Partial success', + SyncRunStatus.inProgress => 'In progress', + }; +} + +IconData severityIcon(SyncRunSeverity severity) { + return switch (severity) { + SyncRunSeverity.ok => Icons.check_circle_outline, + SyncRunSeverity.warning => Icons.schedule, + SyncRunSeverity.error => Icons.error_outline, + SyncRunSeverity.rateLimit => Icons.speed, + }; +} + +Color severityColor(SyncRunSeverity severity) { + return switch (severity) { + SyncRunSeverity.ok => AppColors.success, + SyncRunSeverity.warning => Colors.orange, + SyncRunSeverity.error => const Color(0xFFF87171), + SyncRunSeverity.rateLimit => Colors.amber, + }; +} + +String? parseHttpStatus(String? error) { + if (error == null || error.trim().isEmpty) { + return null; + } + final RegExpMatch? match = RegExp(r'\b(\d{3})\b').firstMatch(error); + return match?.group(1); +} diff --git a/lib/admin/widgets/admin_app_bar_action.dart b/lib/admin/widgets/admin_app_bar_action.dart new file mode 100644 index 0000000..2f5b478 --- /dev/null +++ b/lib/admin/widgets/admin_app_bar_action.dart @@ -0,0 +1,43 @@ +import 'package:flutter/material.dart'; + +import '../admin_routes.dart'; +import '../services/admin_access_service.dart'; +import '../services/market_history_admin_api.dart'; + +/// App bar icon shown when the signed-in user's Firebase UID is on the admin allowlist. +class AdminAppBarAction extends StatefulWidget { + const AdminAppBarAction({super.key, this.api}); + + final MarketHistoryAdminApi? api; + + @override + State createState() => _AdminAppBarActionState(); +} + +class _AdminAppBarActionState extends State { + @override + void initState() { + super.initState(); + AdminAccessService.instance.refresh(api: widget.api); + } + + @override + Widget build(BuildContext context) { + return ValueListenableBuilder( + valueListenable: AdminAccessService.instance.status, + builder: (BuildContext context, AdminAccessStatus access, Widget? child) { + if (access != AdminAccessStatus.authorized) { + return const SizedBox.shrink(); + } + return IconButton( + key: const Key('admin-app-bar-action'), + tooltip: 'Admin portal', + onPressed: () { + Navigator.of(context).pushNamed(AdminRoutes.marketHistoryLog); + }, + icon: const Icon(Icons.admin_panel_settings_outlined), + ); + }, + ); + } +} diff --git a/lib/admin/widgets/admin_gate.dart b/lib/admin/widgets/admin_gate.dart new file mode 100644 index 0000000..7149f2d --- /dev/null +++ b/lib/admin/widgets/admin_gate.dart @@ -0,0 +1,144 @@ +import 'package:flutter/material.dart'; + +import '../../theme/app_theme.dart'; +import '../services/market_history_admin_api.dart'; + +enum AdminGateStatus { loading, authorized, forbidden, unauthorized, error } + +/// Probes admin API access before showing [child]. +/// +/// Calls `GET /v1/admin/market-history/sync-runs?limit=1`. +/// Returns 403 → not authorized UI; 200 → [child]. +class AdminGate extends StatefulWidget { + const AdminGate({ + super.key, + required this.child, + this.api, + }); + + final Widget child; + final MarketHistoryAdminApi? api; + + @override + State createState() => _AdminGateState(); +} + +class _AdminGateState extends State { + AdminGateStatus _status = AdminGateStatus.loading; + String? _errorMessage; + + @override + void initState() { + super.initState(); + _probe(); + } + + Future _probe() async { + final MarketHistoryAdminApi api = widget.api ?? MarketHistoryAdminApi(); + try { + await api.fetchSyncRuns(limit: 1); + if (!mounted) { + return; + } + setState(() => _status = AdminGateStatus.authorized); + } on MarketHistoryAdminApiException catch (e) { + if (!mounted) { + return; + } + setState(() { + _errorMessage = e.message; + _status = switch (e.statusCode) { + 401 => AdminGateStatus.unauthorized, + 403 => AdminGateStatus.forbidden, + _ => AdminGateStatus.error, + }; + }); + } catch (e) { + if (!mounted) { + return; + } + setState(() { + _status = AdminGateStatus.error; + _errorMessage = e.toString(); + }); + } + } + + @override + Widget build(BuildContext context) { + return switch (_status) { + AdminGateStatus.loading => const Scaffold( + key: Key('admin-gate-loading'), + body: Center( + child: CircularProgressIndicator(color: AppColors.accent), + ), + ), + AdminGateStatus.authorized => widget.child, + AdminGateStatus.forbidden => _messageScaffold( + key: const Key('admin-gate-forbidden'), + title: 'Not authorized', + message: 'Your account does not have admin access.', + icon: Icons.lock_outline, + ), + AdminGateStatus.unauthorized => _messageScaffold( + key: const Key('admin-gate-unauthorized'), + title: 'Sign in required', + message: 'Sign in again to access the admin portal.', + icon: Icons.login, + ), + AdminGateStatus.error => _messageScaffold( + key: const Key('admin-gate-error'), + title: 'Admin check failed', + message: _errorMessage ?? 'Unknown error', + icon: Icons.error_outline, + showRetry: true, + ), + }; + } + + Widget _messageScaffold({ + required Key key, + required String title, + required String message, + required IconData icon, + bool showRetry = false, + }) { + return Scaffold( + key: key, + appBar: AppBar(title: const Text('Admin portal')), + body: Center( + child: Padding( + padding: const EdgeInsets.all(24), + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + Icon(icon, size: 40, color: AppColors.textSecondary), + const SizedBox(height: 12), + Text( + title, + style: Theme.of(context).textTheme.titleLarge, + textAlign: TextAlign.center, + ), + const SizedBox(height: 8), + Text( + message, + style: const TextStyle(color: AppColors.textSecondary), + textAlign: TextAlign.center, + ), + if (showRetry) ...[ + const SizedBox(height: 16), + FilledButton( + onPressed: () { + setState(() => _status = AdminGateStatus.loading); + _probe(); + }, + child: const Text('Retry'), + ), + ], + ], + ), + ), + ), + ); + } +} diff --git a/lib/admin/widgets/market_history_question_audit_sheet.dart b/lib/admin/widgets/market_history_question_audit_sheet.dart new file mode 100644 index 0000000..7dbca45 --- /dev/null +++ b/lib/admin/widgets/market_history_question_audit_sheet.dart @@ -0,0 +1,573 @@ +import 'package:flutter/material.dart'; + +import '../../theme/app_theme.dart'; +import '../models/question_audit_asset.dart'; +import '../services/market_history_admin_api.dart'; + +/// Scrollable audit of last-two 4-hour bar price and volume deltas per symbol. +class MarketHistoryQuestionAuditSheet extends StatefulWidget { + const MarketHistoryQuestionAuditSheet({ + super.key, + required this.api, + }); + + final MarketHistoryAdminApi api; + + static Future show( + BuildContext context, { + required MarketHistoryAdminApi api, + }) { + return showDialog( + context: context, + builder: (BuildContext context) { + return Dialog( + key: const Key('question-audit-dialog'), + insetPadding: const EdgeInsets.symmetric(horizontal: 16, vertical: 24), + child: ConstrainedBox( + constraints: BoxConstraints( + maxWidth: 480, + maxHeight: MediaQuery.sizeOf(context).height * 0.85, + ), + child: MarketHistoryQuestionAuditSheet(api: api), + ), + ); + }, + ); + } + + @override + State createState() => + _MarketHistoryQuestionAuditSheetState(); +} + +class _MarketHistoryQuestionAuditSheetState + extends State { + QuestionAuditReport? _report; + bool _loading = true; + String? _errorMessage; + + @override + void initState() { + super.initState(); + _load(); + } + + Future _load({DateTime? compareUntil}) async { + setState(() { + _loading = true; + _errorMessage = null; + }); + try { + final QuestionAuditReport report = + await widget.api.fetchQuestionAudit(asOf: compareUntil); + if (!mounted) { + return; + } + setState(() { + _report = report; + _loading = false; + }); + } on MarketHistoryAdminApiException catch (e) { + if (!mounted) { + return; + } + setState(() { + _loading = false; + _errorMessage = e.message; + }); + } catch (e) { + if (!mounted) { + return; + } + setState(() { + _loading = false; + _errorMessage = e.toString(); + }); + } + } + + Future _stepOlder() async { + final QuestionAuditReport? report = _report; + final DateTime? next = report?.stepOlderCompareUntil; + if (report == null || !report.canStepOlder || next == null || _loading) { + return; + } + await _load(compareUntil: next); + } + + Future _stepNewer() async { + final QuestionAuditReport? report = _report; + final DateTime? next = report?.stepNewerCompareUntil; + if (report == null || !report.canStepNewer || next == null || _loading) { + return; + } + await _load(compareUntil: next); + } + + @override + Widget build(BuildContext context) { + final QuestionAuditReport? report = _report; + + return Padding( + padding: const EdgeInsets.fromLTRB(16, 12, 8, 16), + child: Column( + crossAxisAlignment: CrossAxisAlignment.stretch, + children: [ + Row( + children: [ + const Icon(Icons.help_outline, color: AppColors.accent, size: 22), + const SizedBox(width: 8), + const Expanded( + child: Text( + 'Prospective question data', + style: TextStyle( + fontSize: 18, + fontWeight: FontWeight.w700, + color: AppColors.textPrimary, + ), + ), + ), + IconButton( + key: const Key('question-audit-close'), + tooltip: 'Close', + onPressed: () => Navigator.of(context).pop(), + icon: const Icon(Icons.close), + ), + ], + ), + _SymbolCountNav( + newerSlotStart: report?.newerSlotStart, + olderSlotStart: report?.olderSlotStart, + symbolCount: report?.assets.length ?? 0, + canStepOlder: report?.canStepOlder ?? false, + canStepNewer: report?.canStepNewer ?? false, + loading: _loading, + onStepOlder: _stepOlder, + onStepNewer: _stepNewer, + ), + const SizedBox(height: 12), + Expanded(child: _buildBody(report)), + ], + ), + ); + } + + Widget _buildBody(QuestionAuditReport? report) { + if (_loading && report == null) { + return const Center( + key: Key('question-audit-loading'), + child: CircularProgressIndicator(color: AppColors.accent), + ); + } + + if (_errorMessage != null && report == null) { + return Center( + key: const Key('question-audit-error'), + child: Padding( + padding: const EdgeInsets.all(16), + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + Text( + _errorMessage!, + textAlign: TextAlign.center, + style: const TextStyle(color: AppColors.textSecondary), + ), + const SizedBox(height: 12), + FilledButton( + onPressed: _load, + child: const Text('Retry'), + ), + ], + ), + ), + ); + } + + if (report == null) { + return const SizedBox.shrink(); + } + + if (report.assets.isEmpty) { + return const Center( + child: Text( + 'No symbols with bars for both slots in this range.', + textAlign: TextAlign.center, + style: TextStyle(color: AppColors.textSecondary), + ), + ); + } + + return Stack( + children: [ + ListView.separated( + key: ValueKey( + 'question-audit-list-${report.compareUntil.toIso8601String()}', + ), + itemCount: report.assets.length, + separatorBuilder: (_, __) => const SizedBox(height: 8), + itemBuilder: (BuildContext context, int index) { + return _AssetTile(asset: report.assets[index]); + }, + ), + if (_loading) + const Positioned( + top: 0, + left: 0, + right: 0, + child: LinearProgressIndicator( + color: AppColors.accent, + minHeight: 2, + ), + ), + ], + ); + } +} + +class _SymbolCountNav extends StatelessWidget { + const _SymbolCountNav({ + required this.newerSlotStart, + required this.olderSlotStart, + required this.symbolCount, + required this.canStepOlder, + required this.canStepNewer, + required this.loading, + required this.onStepOlder, + required this.onStepNewer, + }); + + final DateTime? newerSlotStart; + final DateTime? olderSlotStart; + final int symbolCount; + final bool canStepOlder; + final bool canStepNewer; + final bool loading; + final VoidCallback onStepOlder; + final VoidCallback onStepNewer; + + @override + Widget build(BuildContext context) { + final String label = symbolCount == 1 ? '1 symbol' : '$symbolCount symbols'; + final String? slotRange = newerSlotStart == null || olderSlotStart == null + ? null + : _AuditFormat.compareSlotRange( + older: olderSlotStart!, + newer: newerSlotStart!, + ); + + return Row( + children: [ + IconButton( + key: const Key('question-audit-step-older'), + tooltip: 'Earlier 4-hour slot', + onPressed: canStepOlder && !loading ? onStepOlder : null, + icon: const Icon(Icons.chevron_left), + ), + Expanded( + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + if (slotRange != null) + Text( + slotRange, + key: const Key('question-audit-slot-range'), + textAlign: TextAlign.center, + style: const TextStyle( + fontSize: 12, + fontWeight: FontWeight.w600, + color: AppColors.accent, + ), + ), + if (slotRange != null) const SizedBox(height: 2), + Text( + label, + key: const Key('question-audit-symbol-count'), + textAlign: TextAlign.center, + style: const TextStyle( + fontSize: 15, + fontWeight: FontWeight.w600, + color: AppColors.textPrimary, + ), + ), + ], + ), + ), + IconButton( + key: const Key('question-audit-step-newer'), + tooltip: 'Later 4-hour slot', + onPressed: canStepNewer && !loading ? onStepNewer : null, + icon: const Icon(Icons.chevron_right), + ), + ], + ); + } +} + +class _AssetTile extends StatefulWidget { + const _AssetTile({required this.asset}); + + final QuestionAuditAsset asset; + + @override + State<_AssetTile> createState() => _AssetTileState(); +} + +class _AssetTileState extends State<_AssetTile> { + bool _expanded = false; + + QuestionAuditAsset get asset => widget.asset; + + @override + Widget build(BuildContext context) { + return Material( + key: Key('question-audit-tile-${asset.symbol}'), + color: AppColors.surfaceElevated, + shape: RoundedRectangleBorder( + borderRadius: BorderRadius.circular(8), + side: BorderSide(color: AppColors.textSecondary.withValues(alpha: 0.2)), + ), + clipBehavior: Clip.antiAlias, + child: InkWell( + onTap: () => setState(() => _expanded = !_expanded), + child: Padding( + padding: const EdgeInsets.symmetric(horizontal: 12, vertical: 10), + child: Column( + crossAxisAlignment: CrossAxisAlignment.stretch, + children: [ + Stack( + alignment: Alignment.center, + children: [ + Align( + alignment: Alignment.centerLeft, + child: Row( + mainAxisSize: MainAxisSize.min, + children: [ + Icon( + _expanded + ? Icons.expand_less + : Icons.expand_more, + size: 18, + color: AppColors.textSecondary, + ), + const SizedBox(width: 4), + Text( + asset.symbol, + style: const TextStyle( + fontSize: 14, + fontWeight: FontWeight.w700, + color: AppColors.textPrimary, + ), + ), + ], + ), + ), + Column( + mainAxisSize: MainAxisSize.min, + children: [ + Text( + 'P:${_AuditFormat.delta(asset.priceDelta)}', + style: TextStyle( + fontSize: 15, + fontWeight: FontWeight.w600, + color: _AuditFormat.deltaColor(asset.priceDelta), + ), + ), + const SizedBox(height: 2), + Text( + 'V:${_AuditFormat.delta(asset.volumeDelta)}', + style: TextStyle( + fontSize: 15, + fontWeight: FontWeight.w600, + color: _AuditFormat.deltaColor(asset.volumeDelta), + ), + ), + ], + ), + ], + ), + if (_expanded) ...[ + const SizedBox(height: 12), + const Divider(height: 1), + const SizedBox(height: 10), + IntrinsicHeight( + child: Row( + crossAxisAlignment: CrossAxisAlignment.stretch, + children: [ + Expanded( + child: _SlotPanel( + key: Key('question-audit-slot-older-${asset.symbol}'), + label: 'Older', + slot: asset.olderSlot, + ), + ), + const SizedBox(width: 8), + Container( + width: 1, + color: AppColors.textSecondary.withValues(alpha: 0.25), + ), + const SizedBox(width: 8), + Expanded( + child: _SlotPanel( + key: Key('question-audit-slot-newer-${asset.symbol}'), + label: 'Newer', + slot: asset.newerSlot, + ), + ), + ], + ), + ), + ], + ], + ), + ), + ), + ); + } +} + +class _SlotPanel extends StatelessWidget { + const _SlotPanel({ + super.key, + required this.label, + required this.slot, + }); + + final String label; + final QuestionAuditBarSlot slot; + + @override + Widget build(BuildContext context) { + return Column( + crossAxisAlignment: CrossAxisAlignment.stretch, + children: [ + Text( + label, + textAlign: TextAlign.center, + style: const TextStyle( + fontSize: 11, + fontWeight: FontWeight.w700, + letterSpacing: 0.6, + color: AppColors.accent, + ), + ), + const SizedBox(height: 4), + Text( + _AuditFormat.slotTime(slot.asOf), + textAlign: TextAlign.center, + style: const TextStyle( + fontSize: 10, + color: AppColors.textSecondary, + ), + ), + const SizedBox(height: 8), + _SlotRow(label: 'O', value: slot.open), + _SlotRow(label: 'H', value: slot.high), + _SlotRow(label: 'L', value: slot.low), + _SlotRow(label: 'C', value: slot.close), + _SlotRow(label: 'Avg', value: slot.avgPrice, emphasize: true), + _SlotRow(label: 'Vol', value: slot.volume), + ], + ); + } +} + +class _SlotRow extends StatelessWidget { + const _SlotRow({ + required this.label, + required this.value, + this.emphasize = false, + }); + + final String label; + final num? value; + final bool emphasize; + + @override + Widget build(BuildContext context) { + return Padding( + padding: const EdgeInsets.symmetric(vertical: 2), + child: Row( + children: [ + SizedBox( + width: 28, + child: Text( + label, + style: const TextStyle( + fontSize: 11, + color: AppColors.textSecondary, + ), + ), + ), + Expanded( + child: Text( + value == null ? '—' : _AuditFormat.value(value!), + textAlign: TextAlign.right, + style: TextStyle( + fontSize: emphasize ? 13 : 12, + fontWeight: emphasize ? FontWeight.w700 : FontWeight.w500, + color: AppColors.textPrimary, + ), + ), + ), + ], + ), + ); + } +} + +abstract final class _AuditFormat { + /// Older → newer UTC 4-hour slot starts being compared. + static String compareSlotRange({ + required DateTime older, + required DateTime newer, + }) { + return '${_slotLabel(older)} – ${_slotLabel(newer)} UTC'; + } + + static String _slotLabel(DateTime asOf) { + final DateTime utc = asOf.toUtc(); + final String month = utc.month.toString().padLeft(2, '0'); + final String day = utc.day.toString().padLeft(2, '0'); + final String hour = utc.hour.toString().padLeft(2, '0'); + return '$month/$day $hour:00'; + } + + static String slotTime(DateTime asOf) { + final DateTime utc = asOf.toUtc(); + final String month = utc.month.toString().padLeft(2, '0'); + final String day = utc.day.toString().padLeft(2, '0'); + final String hour = utc.hour.toString().padLeft(2, '0'); + return '$month/$day ${hour}:00 UTC'; + } + + static String value(num n) { + if (n == n.roundToDouble() && n.abs() < 1e12) { + return n.round().toString(); + } + return n.toStringAsFixed(2); + } + + static Color deltaColor(num delta) { + if (delta > 0) { + return AppColors.success; + } + if (delta < 0) { + return const Color(0xFFF87171); + } + return AppColors.textPrimary; + } + + static String delta(num value) { + final num rounded = value.abs() >= 1000 + ? (value * 100).round() / 100 + : (value * 10000).round() / 10000; + final String text = rounded == rounded.roundToDouble() + ? rounded.round().toString() + : rounded.toStringAsFixed(2); + if (value > 0) { + return '+$text'; + } + return text; + } +} diff --git a/lib/admin/widgets/market_history_week_coverage_sheet.dart b/lib/admin/widgets/market_history_week_coverage_sheet.dart new file mode 100644 index 0000000..fb6f70c --- /dev/null +++ b/lib/admin/widgets/market_history_week_coverage_sheet.dart @@ -0,0 +1,208 @@ +import 'package:flutter/material.dart'; + +import '../../theme/app_theme.dart'; +import '../models/market_history_week_coverage.dart'; + +const List _weekdayLabels = [ + 'Mon', + 'Tue', + 'Wed', + 'Thu', + 'Fri', + 'Sat', + 'Sun', +]; + +/// Mini 7-day UTC week view showing 4-hour slot sync health. +class MarketHistoryWeekCoverageSheet extends StatelessWidget { + const MarketHistoryWeekCoverageSheet({ + super.key, + required this.report, + }); + + final MarketHistoryWeekCoverageReport report; + + static Future show( + BuildContext context, { + required MarketHistoryWeekCoverageReport report, + }) { + return showDialog( + context: context, + builder: (BuildContext context) { + return Dialog( + key: const Key('week-coverage-dialog'), + insetPadding: const EdgeInsets.symmetric(horizontal: 16, vertical: 24), + child: ConstrainedBox( + constraints: const BoxConstraints(maxWidth: 560), + child: MarketHistoryWeekCoverageSheet(report: report), + ), + ); + }, + ); + } + + @override + Widget build(BuildContext context) { + return Padding( + padding: const EdgeInsets.fromLTRB(16, 12, 8, 16), + child: Column( + mainAxisSize: MainAxisSize.min, + crossAxisAlignment: CrossAxisAlignment.stretch, + children: [ + Row( + children: [ + const Icon(Icons.calendar_month, color: AppColors.accent, size: 22), + const SizedBox(width: 8), + const Expanded( + child: Text( + '7-day sync health', + style: TextStyle( + fontSize: 18, + fontWeight: FontWeight.w700, + color: AppColors.textPrimary, + ), + ), + ), + IconButton( + key: const Key('week-coverage-close'), + tooltip: 'Close', + onPressed: () => Navigator.of(context).pop(), + icon: const Icon(Icons.close), + ), + ], + ), + Text( + report.symbolCount == 0 + ? 'No active tradable symbols to validate.' + : report.isConsistent + ? 'All completed 4-hour slots are fully synced across ' + '${report.symbolCount} symbols (bars or no-data placeholders).' + : 'Some completed slots are missing a bar or no-data placeholder ' + 'for one or more symbols.', + style: const TextStyle( + fontSize: 13, + color: AppColors.textSecondary, + ), + ), + const SizedBox(height: 16), + Row( + crossAxisAlignment: CrossAxisAlignment.start, + children: report.days + .map( + (MarketHistoryDayCoverage day) => Expanded( + child: _DayColumn(day: day), + ), + ) + .toList(growable: false), + ), + const SizedBox(height: 12), + Text( + 'UTC · ${report.slotsPerDay} slots per day · ' + '${report.windowDays}-day window', + textAlign: TextAlign.center, + style: const TextStyle( + fontSize: 11, + color: AppColors.textSecondary, + ), + ), + ], + ), + ); + } +} + +class _DayColumn extends StatelessWidget { + const _DayColumn({required this.day}); + + final MarketHistoryDayCoverage day; + + @override + Widget build(BuildContext context) { + final int weekdayIndex = day.date.weekday - DateTime.monday; + final String weekday = _weekdayLabels[weekdayIndex]; + final String dateLabel = '${day.date.month}/${day.date.day}'; + final int denominator = + day.completedSlots > 0 ? day.completedSlots : day.slotsPerDay; + + return Padding( + padding: const EdgeInsets.symmetric(horizontal: 2), + child: Column( + children: [ + Text( + weekday, + style: const TextStyle( + fontSize: 11, + fontWeight: FontWeight.w600, + color: AppColors.textSecondary, + ), + ), + Text( + dateLabel, + style: const TextStyle( + fontSize: 10, + color: AppColors.textSecondary, + ), + ), + const SizedBox(height: 8), + ...day.slots.map(_SlotDot.new), + const SizedBox(height: 8), + Text( + '${day.fullySyncedSlots}/$denominator', + style: TextStyle( + fontSize: 11, + fontWeight: FontWeight.w700, + color: day.fullySyncedSlots == denominator && day.completedSlots > 0 + ? AppColors.success + : day.fullySyncedSlots < day.completedSlots + ? Colors.amber + : AppColors.textPrimary, + ), + ), + const SizedBox(height: 2), + Text( + 'synced', + style: const TextStyle( + fontSize: 9, + color: AppColors.textSecondary, + ), + ), + ], + ), + ); + } +} + +class _SlotDot extends StatelessWidget { + const _SlotDot(this.slot); + + final MarketHistorySlotCoverage slot; + + @override + Widget build(BuildContext context) { + final Color color; + if (!slot.completed) { + color = AppColors.surface; + } else if (slot.fullySynced) { + color = AppColors.success; + } else if (slot.syncedSymbolCount > 0) { + color = Colors.amber; + } else { + color = Colors.redAccent; + } + + return Container( + key: Key('slot-${slot.slotStart.toIso8601String()}'), + width: 10, + height: 10, + margin: const EdgeInsets.symmetric(vertical: 2), + decoration: BoxDecoration( + color: slot.completed ? color : Colors.transparent, + shape: BoxShape.circle, + border: Border.all( + color: slot.completed ? color : AppColors.textSecondary.withValues(alpha: 0.35), + width: 1.2, + ), + ), + ); + } +} diff --git a/lib/admin/widgets/sync_run_expansion_tile.dart b/lib/admin/widgets/sync_run_expansion_tile.dart new file mode 100644 index 0000000..6b6ab8b --- /dev/null +++ b/lib/admin/widgets/sync_run_expansion_tile.dart @@ -0,0 +1,261 @@ +import 'package:flutter/material.dart'; +import 'package:flutter/services.dart'; + +import '../../theme/app_theme.dart'; +import '../models/backfill_sync_item.dart'; +import '../models/sync_run_event.dart'; +import '../utils/sync_run_formatters.dart'; +import 'sync_run_status_chip.dart'; + +class SyncRunExpansionTile extends StatelessWidget { + const SyncRunExpansionTile({ + super.key, + required this.event, + this.now, + this.emphasizeError = false, + this.onRetry, + }); + + final SyncRunEvent event; + final DateTime? now; + final bool emphasizeError; + final VoidCallback? onRetry; + + @override + Widget build(BuildContext context) { + final Color iconColor = severityColor(event.severity); + return Card( + key: Key('sync-run-${event.id}'), + color: emphasizeError + ? AppColors.surfaceElevated + : AppColors.surface, + margin: const EdgeInsets.symmetric(horizontal: 16, vertical: 4), + child: Theme( + data: Theme.of(context).copyWith(dividerColor: Colors.transparent), + child: ExpansionTile( + leading: Icon(severityIcon(event.severity), color: iconColor), + title: Text( + '${event.displayTitle} — ${shortStatusLabel(event)}', + style: const TextStyle( + fontSize: 14, + fontWeight: FontWeight.w600, + color: AppColors.textPrimary, + ), + ), + subtitle: Text( + event.summary, + style: const TextStyle(fontSize: 12, color: AppColors.textSecondary), + ), + trailing: Text( + formatRelativeTime(event.startedAt, now: now), + style: const TextStyle(fontSize: 12, color: AppColors.textSecondary), + ), + children: [ + Padding( + padding: const EdgeInsets.fromLTRB(16, 0, 16, 16), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + SyncRunStatusChip(event: event), + const SizedBox(height: 12), + ..._detailRows(context), + ], + ), + ), + ], + ), + ), + ); + } + + List _detailRows(BuildContext context) { + final List rows = [ + _detailRow('Run ID', '${event.id}'), + _detailRow('Kind', event.kind), + _detailRow('Started', formatLocalTimestamp(event.startedAt)), + _detailRow('Finished', formatLocalTimestamp(event.finishedAt)), + _detailRow('Duration', formatDurationMs(event.durationMs)), + ]; + + if (event.rowsWritten > 0) { + rows.add(_detailRow('Rows written', '${event.rowsWritten}')); + } + if (event.rowsRemoved > 0) { + rows.add(_detailRow('Rows removed', '${event.rowsRemoved}')); + } + + if (event.kind == 'backfill' && event.backfillItems.isNotEmpty) { + rows.add( + const Padding( + padding: EdgeInsets.only(top: 8, bottom: 4), + child: Text( + 'Backfill fetches (Alpaca start / raw.slot_start)', + style: TextStyle( + fontSize: 12, + fontWeight: FontWeight.w600, + color: AppColors.textSecondary, + ), + ), + ), + ); + for (final BackfillSyncItem item in event.backfillItems) { + rows.add(_backfillFetchRow(item)); + } + } + + if (event.status == SyncRunStatus.success && event.error == null) { + rows.add(_detailRow('Errors', 'No errors')); + } + + if (event.error != null && event.error!.trim().isNotEmpty) { + final String? httpStatus = parseHttpStatus(event.error); + if (httpStatus != null) { + rows.add(_detailRow('HTTP status', httpStatus)); + } + rows.add( + Padding( + padding: const EdgeInsets.only(top: 8), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + const Text( + 'Error', + style: TextStyle( + fontSize: 12, + fontWeight: FontWeight.w600, + color: AppColors.textSecondary, + ), + ), + const SizedBox(height: 4), + ConstrainedBox( + constraints: const BoxConstraints(maxHeight: 240), + child: SingleChildScrollView( + child: SelectableText( + event.error!, + key: Key('sync-run-error-${event.id}'), + style: const TextStyle( + fontSize: 13, + color: AppColors.textPrimary, + ), + ), + ), + ), + TextButton.icon( + onPressed: () { + Clipboard.setData(ClipboardData(text: event.error!)); + ScaffoldMessenger.of(context).showSnackBar( + const SnackBar(content: Text('Error copied')), + ); + }, + icon: const Icon(Icons.copy, size: 16), + label: const Text('Copy error'), + ), + if (event.severity == SyncRunSeverity.rateLimit || + event.status == SyncRunStatus.failed) + const Text( + 'Will retry on next scheduled run.', + style: TextStyle( + fontSize: 12, + color: AppColors.textSecondary, + ), + ), + if (onRetry != null && + (event.status == SyncRunStatus.failed || + event.status == SyncRunStatus.partial || + event.severity == SyncRunSeverity.rateLimit)) + TextButton.icon( + key: Key('retry-run-${event.id}'), + onPressed: onRetry, + icon: const Icon(Icons.replay, size: 16), + label: const Text('Retry now'), + ), + ], + ), + ), + ); + } + + if (event.status == SyncRunStatus.inProgress) { + rows.add( + const Padding( + padding: EdgeInsets.only(top: 8), + child: Row( + children: [ + SizedBox( + width: 16, + height: 16, + child: CircularProgressIndicator(strokeWidth: 2), + ), + SizedBox(width: 8), + Text( + 'Run in progress…', + style: TextStyle(fontSize: 12, color: AppColors.textSecondary), + ), + ], + ), + ), + ); + } + + return rows; + } + + Widget _backfillFetchRow(BackfillSyncItem item) { + final String wire = item.fetchSlotStartWire; + return Padding( + padding: const EdgeInsets.only(bottom: 8), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + SelectableText( + wire, + key: Key('backfill-slot-$wire'), + style: const TextStyle( + fontSize: 12, + fontFamily: 'monospace', + color: AppColors.textPrimary, + ), + ), + const SizedBox(height: 2), + Text( + '${item.symbols.length} assets: ${item.symbols.join(', ')}', + style: const TextStyle( + fontSize: 12, + color: AppColors.textSecondary, + ), + ), + ], + ), + ); + } + + Widget _detailRow(String label, String value) { + return Padding( + padding: const EdgeInsets.only(bottom: 6), + child: Row( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + SizedBox( + width: 110, + child: Text( + label, + style: const TextStyle( + fontSize: 12, + color: AppColors.textSecondary, + ), + ), + ), + Expanded( + child: Text( + value, + style: const TextStyle( + fontSize: 12, + color: AppColors.textPrimary, + ), + ), + ), + ], + ), + ); + } +} diff --git a/lib/admin/widgets/sync_run_status_chip.dart b/lib/admin/widgets/sync_run_status_chip.dart new file mode 100644 index 0000000..348fd9c --- /dev/null +++ b/lib/admin/widgets/sync_run_status_chip.dart @@ -0,0 +1,27 @@ +import 'package:flutter/material.dart'; + +import '../models/sync_run_event.dart'; +import '../utils/sync_run_formatters.dart'; + +class SyncRunStatusChip extends StatelessWidget { + const SyncRunStatusChip({super.key, required this.event}); + + final SyncRunEvent event; + + @override + Widget build(BuildContext context) { + final Color color = severityColor(event.severity); + return Container( + padding: const EdgeInsets.symmetric(horizontal: 8, vertical: 2), + decoration: BoxDecoration( + color: color.withValues(alpha: 0.15), + borderRadius: BorderRadius.circular(12), + border: Border.all(color: color.withValues(alpha: 0.5)), + ), + child: Text( + shortStatusLabel(event), + style: TextStyle(color: color, fontSize: 12, fontWeight: FontWeight.w600), + ), + ); + } +} diff --git a/lib/main.dart b/lib/main.dart index 5fdd7c5..da6ad27 100644 --- a/lib/main.dart +++ b/lib/main.dart @@ -1,5 +1,8 @@ import 'package:flutter/material.dart'; +import 'admin/admin_routes.dart'; +import 'admin/screens/market_history_log_screen.dart'; +import 'admin/widgets/admin_gate.dart'; import 'bootstrap.dart'; import 'models/app_user.dart'; import 'screens/home_screen.dart'; @@ -24,6 +27,17 @@ class CyberHybridHubApp extends StatelessWidget { debugShowCheckedModeBanner: false, theme: buildAppTheme(), home: const AuthGate(), + onGenerateRoute: (RouteSettings settings) { + if (settings.name == AdminRoutes.marketHistoryLog) { + return MaterialPageRoute( + settings: settings, + builder: (BuildContext context) => AdminGate( + child: marketHistoryLogScreen(), + ), + ); + } + return null; + }, ); } } diff --git a/lib/screens/home_screen.dart b/lib/screens/home_screen.dart index 8f588c7..6b7f696 100644 --- a/lib/screens/home_screen.dart +++ b/lib/screens/home_screen.dart @@ -1,5 +1,6 @@ import 'package:flutter/material.dart'; +import '../admin/widgets/admin_app_bar_action.dart'; import '../models/app_user.dart'; import '../models/incoming_question.dart'; import '../models/sync_result.dart'; @@ -8,6 +9,7 @@ import '../repositories/user_profile_repository.dart'; import '../services/auth_service.dart'; import '../services/questions_hub_service.dart'; import '../theme/app_theme.dart'; +import '../widgets/profile_avatar.dart'; import '../widgets/swipe_question_tile.dart'; class HomeScreen extends StatelessWidget { @@ -63,6 +65,7 @@ class HomeScreen extends StatelessWidget { }, ), actions: [ + const AdminAppBarAction(), IconButton( onPressed: () => UserProfileRepository.instance.sync(), tooltip: 'Sync profile', @@ -155,16 +158,10 @@ class HomeScreen extends StatelessWidget { ), child: Column( children: [ - if (photoUrl != null) - CircleAvatar( - radius: 36, - backgroundImage: NetworkImage(photoUrl), - ) - else - const CircleAvatar( - radius: 36, - child: Icon(Icons.person, size: 36), - ), + ProfileAvatar( + photoUrl: photoUrl, + radius: 36, + ), const SizedBox(height: 16), Text( 'Welcome, $displayName', diff --git a/lib/services/auth_service.dart b/lib/services/auth_service.dart index 1363594..2ed83f8 100644 --- a/lib/services/auth_service.dart +++ b/lib/services/auth_service.dart @@ -1,6 +1,7 @@ import 'package:flutter/foundation.dart' show TargetPlatform, defaultTargetPlatform, kIsWeb; +import '../admin/services/admin_access_service.dart'; import '../models/app_user.dart'; import 'auth_service_firebase.dart'; import 'auth_service_linux.dart'; @@ -52,6 +53,7 @@ class AuthService { } else { await AuthServiceFirebase.instance.signOut(); } + AdminAccessService.instance.reset(); } /// Firebase ID token for API requests. Returns null when signed out. diff --git a/lib/widgets/profile_avatar.dart b/lib/widgets/profile_avatar.dart new file mode 100644 index 0000000..68f8d68 --- /dev/null +++ b/lib/widgets/profile_avatar.dart @@ -0,0 +1,81 @@ +import 'package:flutter/material.dart'; + +import '../theme/app_theme.dart'; + +/// User profile photo with a graceful fallback when the URL fails (e.g. +/// Google `lh3.googleusercontent.com` returning HTTP 429). +class ProfileAvatar extends StatelessWidget { + const ProfileAvatar({ + super.key, + this.photoUrl, + this.radius = 36, + }); + + final String? photoUrl; + final double radius; + + @override + Widget build(BuildContext context) { + final String? url = photoUrl?.trim(); + if (url == null || url.isEmpty) { + return _fallbackAvatar(); + } + + final double size = radius * 2; + return ClipOval( + child: SizedBox( + width: size, + height: size, + child: Image.network( + url, + width: size, + height: size, + fit: BoxFit.cover, + // Prevents NetworkImageLoadException from bubbling to the console + // when Google rate-limits profile photo URLs. + errorBuilder: (_, Object error, StackTrace? stackTrace) { + return _fallbackContents(); + }, + loadingBuilder: ( + BuildContext context, + Widget child, + ImageChunkEvent? progress, + ) { + if (progress == null) { + return child; + } + return ColoredBox( + color: AppColors.surfaceElevated, + child: Center( + child: SizedBox( + width: radius * 0.55, + height: radius * 0.55, + child: CircularProgressIndicator( + strokeWidth: 2, + color: AppColors.accent.withValues(alpha: 0.7), + ), + ), + ), + ); + }, + ), + ), + ); + } + + Widget _fallbackAvatar() { + return CircleAvatar( + radius: radius, + backgroundColor: AppColors.surfaceElevated, + child: _fallbackContents(), + ); + } + + Widget _fallbackContents() { + return Icon( + Icons.person, + size: radius, + color: AppColors.accent.withValues(alpha: 0.85), + ); + } +} diff --git a/lib/widgets/swipe_question_tile.dart b/lib/widgets/swipe_question_tile.dart index dd4668a..b1b0be9 100644 --- a/lib/widgets/swipe_question_tile.dart +++ b/lib/widgets/swipe_question_tile.dart @@ -1,6 +1,8 @@ import 'dart:async'; +import 'dart:math' as math; import 'package:flutter/material.dart'; +import 'package:flutter/services.dart'; import '../guid/guid_glyph_shape.dart'; import '../theme/app_theme.dart'; @@ -26,19 +28,66 @@ class SwipeQuestionTile extends StatefulWidget { State createState() => _SwipeQuestionTileState(); } -class _SwipeQuestionTileState extends State { +class _SwipeQuestionTileState extends State + with SingleTickerProviderStateMixin { double _dragOffset = 0; double _verticalOffset = 0; bool _acting = false; + int _lastSnappedValue = 0; + late final AnimationController _snapController; static const double _swipeThreshold = 96; - static const double _maxVerticalDrag = 120; - static const double _sliderMin = -10; - static const double _sliderMax = 10; + static const double _glyphSize = 80; + static const double _trackEdgeInset = 8; + static const int _sliderMin = -10; + static const int _sliderMax = 10; - /// Swipe up → +10, swipe down → -10 (linear between). - double get _sliderValue => - (_verticalOffset / _maxVerticalDrag * _sliderMax).clamp(_sliderMin, _sliderMax); + /// Updated each build from the tile height so ±10 reaches near the track edges. + double _maxVerticalDrag = 120; + + @override + void initState() { + super.initState(); + _snapController = AnimationController( + vsync: this, + duration: const Duration(milliseconds: 140), + ); + } + + @override + void dispose() { + _snapController.dispose(); + super.dispose(); + } + + /// Swipe up → +10, swipe down → -10; snaps to whole numbers. + int get _snappedSliderValue => + (_clampedVerticalOffset / _maxVerticalDrag * _sliderMax) + .round() + .clamp(_sliderMin, _sliderMax); + + double get _clampedVerticalOffset => + _verticalOffset.clamp(-_maxVerticalDrag, _maxVerticalDrag); + + double get _snappedVerticalOffset => + _snappedSliderValue / _sliderMax * _maxVerticalDrag; + + void _maybeTriggerSnapFeedback(int snapped) { + if (snapped == _lastSnappedValue) { + return; + } + _lastSnappedValue = snapped; + unawaited(_snapController.forward(from: 0)); + HapticFeedback.selectionClick(); + } + + ({double shakeX, double scale}) _snapMotion(double t) { + final double damp = 1 - t; + return ( + shakeX: math.sin(t * math.pi * 6) * 5 * damp, + scale: 1 + 0.035 * math.sin(t * math.pi) * damp, + ); + } Future _releaseDrag() async { if (_acting || widget.busy) { @@ -51,7 +100,7 @@ class _SwipeQuestionTileState extends State { _acting = true; _dragOffset = MediaQuery.sizeOf(context).width; }); - await widget.onSwipeRight(_sliderValue); + await widget.onSwipeRight(_snappedSliderValue); } else if (_dragOffset < -_swipeThreshold) { setState(() { _acting = true; @@ -76,6 +125,18 @@ class _SwipeQuestionTileState extends State { return LayoutBuilder( builder: (BuildContext context, BoxConstraints constraints) { + // Container vertical padding (24×2) + track insets (8×2). + const double outerVerticalPadding = 48; + const double trackVerticalInset = 16; + final double innerHeight = + math.max(constraints.maxHeight - outerVerticalPadding, 172); + final double trackHeight = + math.max(innerHeight - trackVerticalInset, _glyphSize); + _maxVerticalDrag = math.max( + (trackHeight / 2) - (_glyphSize / 2) - _trackEdgeInset, + 40, + ); + return Stack( alignment: Alignment.center, children: [ @@ -101,82 +162,101 @@ class _SwipeQuestionTileState extends State { ), Transform.translate( offset: Offset(_dragOffset, 0), - child: GestureDetector( - onHorizontalDragUpdate: widget.busy || _acting - ? null - : (DragUpdateDetails details) { - setState(() { - _dragOffset += details.delta.dx; - _dragOffset = _dragOffset.clamp(-width * 0.55, width * 0.55); - }); - }, - onHorizontalDragEnd: widget.busy || _acting - ? null - : (_) => unawaited(_releaseDrag()), - child: Material( - color: AppColors.surfaceElevated, - elevation: 4, - shadowColor: Colors.black45, - borderRadius: BorderRadius.circular(16), - child: Container( - width: constraints.maxWidth, - constraints: const BoxConstraints(minHeight: 220), - padding: const EdgeInsets.symmetric( - horizontal: 16, - vertical: 24, + child: AnimatedBuilder( + animation: _snapController, + builder: (BuildContext context, Widget? child) { + final ({double shakeX, double scale}) motion = + _snapMotion(_snapController.value); + return Transform.translate( + offset: Offset(motion.shakeX, 0), + child: Transform.scale( + scale: motion.scale, + child: child, ), - decoration: BoxDecoration( - borderRadius: BorderRadius.circular(16), - ), - child: Stack( - alignment: Alignment.center, - children: [ - Positioned( - top: 20, - bottom: 20, - left: constraints.maxWidth * 0.22, - right: constraints.maxWidth * 0.22, - child: DecoratedBox( - decoration: BoxDecoration( - borderRadius: BorderRadius.circular(14), - color: Color.lerp( - AppColors.surfaceElevated, - AppColors.surface, - 0.45, - ), - ), - ), - ), - Positioned( - top: 24, - bottom: 24, - left: constraints.maxWidth * 0.22, - right: constraints.maxWidth * 0.22, - child: GestureDetector( - behavior: HitTestBehavior.opaque, - onVerticalDragUpdate: widget.busy || _acting - ? null - : (DragUpdateDetails details) { - setState(() { - _verticalOffset -= details.delta.dy; - _verticalOffset = _verticalOffset.clamp( - -_maxVerticalDrag, - _maxVerticalDrag, - ); - }); - }, - child: Center( - child: Transform.translate( - offset: Offset(0, -_verticalOffset), - child: QuestionGuidGlyph( - guid: widget.questionId, - displayValue: _sliderValue, + ); + }, + child: GestureDetector( + onHorizontalDragUpdate: widget.busy || _acting + ? null + : (DragUpdateDetails details) { + setState(() { + _dragOffset += details.delta.dx; + _dragOffset = + _dragOffset.clamp(-width * 0.55, width * 0.55); + }); + }, + onHorizontalDragEnd: widget.busy || _acting + ? null + : (_) => unawaited(_releaseDrag()), + child: Material( + color: AppColors.surfaceElevated, + elevation: 4, + shadowColor: Colors.black45, + borderRadius: BorderRadius.circular(16), + child: Container( + width: constraints.maxWidth, + constraints: const BoxConstraints(minHeight: 220), + padding: const EdgeInsets.symmetric( + horizontal: 16, + vertical: 24, + ), + decoration: BoxDecoration( + borderRadius: BorderRadius.circular(16), + ), + child: Stack( + alignment: Alignment.center, + children: [ + Positioned( + top: 8, + bottom: 8, + left: constraints.maxWidth * 0.22, + right: constraints.maxWidth * 0.22, + child: DecoratedBox( + decoration: BoxDecoration( + borderRadius: BorderRadius.circular(14), + color: Color.lerp( + AppColors.surfaceElevated, + AppColors.surface, + 0.45, ), ), ), ), - ), - ], + Positioned( + top: 8, + bottom: 8, + left: constraints.maxWidth * 0.22, + right: constraints.maxWidth * 0.22, + child: GestureDetector( + behavior: HitTestBehavior.opaque, + onVerticalDragUpdate: widget.busy || _acting + ? null + : (DragUpdateDetails details) { + setState(() { + _verticalOffset -= details.delta.dy; + _verticalOffset = _verticalOffset.clamp( + -_maxVerticalDrag, + _maxVerticalDrag, + ); + _maybeTriggerSnapFeedback( + _snappedSliderValue, + ); + }); + }, + child: Center( + child: Transform.translate( + offset: Offset(0, -_snappedVerticalOffset), + child: QuestionGuidGlyph( + guid: widget.questionId, + size: _glyphSize, + displayValue: _snappedSliderValue, + ), + ), + ), + ), + ), + ], + ), ), ), ), diff --git a/scripts/admin-portal-coverage.sh b/scripts/admin-portal-coverage.sh new file mode 100755 index 0000000..26ced1b --- /dev/null +++ b/scripts/admin-portal-coverage.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Collect coverage for admin portal packages (Section 2 targets are advisory). +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +echo "==> Server coverage" +cd "$ROOT/server" +rm -rf coverage +dart test \ + test/trading/market_history_admin_logic_test.dart \ + test/trading/market_history_admin_actions_test.dart \ + test/integration/market_history_admin_handler_test.dart \ + --coverage=coverage +dart pub global activate coverage 2>/dev/null || true +if command -v format_coverage >/dev/null 2>&1; then + format_coverage --lcov --in=coverage --out=coverage/lcov.info --report-on=lib + echo "Server lcov: server/coverage/lcov.info" +else + echo "Install coverage package for lcov: dart pub global activate coverage" +fi + +echo "==> Flutter coverage" +cd "$ROOT" +rm -rf coverage +flutter test test/admin/ --coverage +echo "Flutter lcov: coverage/lcov.info" + +echo "Review lcov files against FLUTTER-TDD-PLAN.md Section 2 targets." diff --git a/scripts/check-admin-portal-coverage.sh b/scripts/check-admin-portal-coverage.sh new file mode 100755 index 0000000..f286e6c --- /dev/null +++ b/scripts/check-admin-portal-coverage.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Generate coverage and enforce FLUTTER-TDD-PLAN.md Section 2 thresholds. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +bash "$ROOT/scripts/admin-portal-coverage.sh" +dart "$ROOT/scripts/check_admin_portal_coverage.dart" diff --git a/scripts/check_admin_portal_coverage.dart b/scripts/check_admin_portal_coverage.dart new file mode 100644 index 0000000..c36ec09 --- /dev/null +++ b/scripts/check_admin_portal_coverage.dart @@ -0,0 +1,131 @@ +import 'dart:io'; + +/// Enforces FLUTTER-TDD-PLAN.md Section 2 line-coverage targets for admin portal files. +void main() { + final String root = _repoRoot(); + final List<_CoverageGate> gates = <_CoverageGate>[ + _CoverageGate( + label: 'server admin logic', + lcovPath: '$root/server/coverage/lcov.info', + pathContains: 'market_history_admin_logic.dart', + minLinePercent: 95, + ), + _CoverageGate( + label: 'server admin handler', + lcovPath: '$root/server/coverage/lcov.info', + pathContains: 'market_history_admin_handler.dart', + minLinePercent: 85, + ), + _CoverageGate( + label: 'server admin actions', + lcovPath: '$root/server/coverage/lcov.info', + pathContains: 'market_history_admin_actions.dart', + minLinePercent: 85, + ), + _CoverageGate( + label: 'flutter admin models', + lcovPath: '$root/coverage/lcov.info', + pathContains: 'lib/admin/models/', + minLinePercent: 90, + ), + _CoverageGate( + label: 'flutter admin repositories', + lcovPath: '$root/coverage/lcov.info', + pathContains: 'lib/admin/repositories/', + minLinePercent: 90, + ), + _CoverageGate( + label: 'flutter admin widgets', + lcovPath: '$root/coverage/lcov.info', + pathContains: 'lib/admin/widgets/', + minLinePercent: 75, + ), + _CoverageGate( + label: 'flutter admin screens', + lcovPath: '$root/coverage/lcov.info', + pathContains: 'lib/admin/screens/', + minLinePercent: 75, + ), + ]; + + final List failures = []; + for (final _CoverageGate gate in gates) { + final File file = File(gate.lcovPath); + if (!file.existsSync()) { + failures.add('${gate.label}: missing ${gate.lcovPath}'); + continue; + } + final double pct = _lineCoveragePercent( + file.readAsLinesSync(), + gate.pathContains, + ); + stdout.writeln( + '${gate.label}: ${pct.toStringAsFixed(1)}% (min ${gate.minLinePercent}%)', + ); + if (pct + 0.05 < gate.minLinePercent) { + failures.add( + '${gate.label}: ${pct.toStringAsFixed(1)}% < ${gate.minLinePercent}%', + ); + } + } + + if (failures.isNotEmpty) { + stderr.writeln('Admin portal coverage gates failed:'); + for (final String failure in failures) { + stderr.writeln(' - $failure'); + } + exit(1); + } +} + +String _repoRoot() { + final String script = Platform.script.toFilePath(); + return File(script).parent.parent.path; +} + +double _lineCoveragePercent(List lines, String pathContains) { + double total = 0; + double hit = 0; + String? currentFile; + for (final String line in lines) { + if (line.startsWith('SF:')) { + currentFile = line.substring(3); + continue; + } + if (line == 'end_of_record') { + currentFile = null; + continue; + } + if (currentFile == null || !currentFile.contains(pathContains)) { + continue; + } + if (line.startsWith('DA:')) { + final List parts = line.substring(3).split(','); + if (parts.length != 2) { + continue; + } + total++; + if (int.tryParse(parts[1]) != 0) { + hit++; + } + } + } + if (total == 0) { + return 0; + } + return hit * 100 / total; +} + +class _CoverageGate { + const _CoverageGate({ + required this.label, + required this.lcovPath, + required this.pathContains, + required this.minLinePercent, + }); + + final String label; + final String lcovPath; + final String pathContains; + final int minLinePercent; +} diff --git a/scripts/copy_flutter_js_source_map.sh b/scripts/copy_flutter_js_source_map.sh new file mode 100755 index 0000000..c858712 --- /dev/null +++ b/scripts/copy_flutter_js_source_map.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Copies flutter.js.map next to build/web/flutter_bootstrap.js so Firefox/ +# Chrome DevTools stop 404ing on "flutter.js.map" after `flutter build web`. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +OUT_DIR="${1:-$ROOT/build/web}" + +if [ ! -d "$OUT_DIR" ]; then + echo "copy_flutter_js_source_map: output dir not found: $OUT_DIR" >&2 + exit 1 +fi + +if ! command -v flutter >/dev/null 2>&1; then + echo "copy_flutter_js_source_map: flutter not on PATH" >&2 + exit 1 +fi + +FLUTTER_BIN="$(command -v flutter)" +SDK_ROOT="$(cd "$(dirname "$FLUTTER_BIN")/.." && pwd)" +MAP_SRC="$SDK_ROOT/bin/cache/flutter_web_sdk/flutter_js/flutter.js.map" +MAP_DST="$OUT_DIR/flutter.js.map" + +if [ ! -f "$MAP_SRC" ]; then + echo "copy_flutter_js_source_map: source map missing at $MAP_SRC" >&2 + exit 1 +fi + +cp "$MAP_SRC" "$MAP_DST" +echo "copy_flutter_js_source_map: installed $MAP_DST" diff --git a/scripts/test-admin-portal.sh b/scripts/test-admin-portal.sh new file mode 100755 index 0000000..bb658bb --- /dev/null +++ b/scripts/test-admin-portal.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Run Flutter Admin Portal test suites (server + Flutter). +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +echo "==> Server admin portal tests" +cd "$ROOT/server" +if [[ -f .env ]]; then + set -a + # shellcheck source=/dev/null + source .env + set +a +fi +dart pub get +dart test \ + test/trading/market_history_admin_logic_test.dart \ + test/trading/market_history_admin_actions_test.dart \ + test/integration/market_history_admin_handler_test.dart \ + test/integration/market_history_admin_risk_test.dart + +echo "==> Flutter admin portal tests" +cd "$ROOT" +flutter test test/admin/ + +echo "Admin portal tests passed." diff --git a/server/README.md b/server/README.md index b611acb..c9d777a 100644 --- a/server/README.md +++ b/server/README.md @@ -43,7 +43,7 @@ dart pub get dart test ``` -Integration tests apply migrations `001`–`004` on `cyberhybridhub_test` and truncate +Integration tests apply migrations `001`–`009` on `cyberhybridhub_test` and truncate trading tables between cases. Optional override: `TEST_DATABASE_URL`. ## Endpoints @@ -127,6 +127,84 @@ curl -s -X POST http://localhost:3000/v1/me/incoming-question \ -d '{"text":"What is your preferred contact method?"}' ``` +## Market history + +Alpaca **`4Hour`** bars in six **UTC slots** per day (`00`, `04`, `08`, `12`, `16`, `20`). +Stored as `metric=bar`, `timeframe=4Hour`. Rolling window: `MARKET_HISTORY_WINDOW_DAYS` (default 7). + +**Backfill** (`kind=backfill`): fetches each **ended** slot still missing in DB; skips the open slot. +Throttled to `MARKET_HISTORY_API_REQUESTS_PER_MINUTE` (default 200/min). On HTTP 429: wait 1 minute, retry once; if still limited, save partial rows and resume next tick. +Runs when `hasPendingSlots` is true (worker tick). One bars request per slot × symbol batch. +Before each worker or admin pipeline, orphaned `market_data_sync_runs` rows with +`finished_at IS NULL` are closed (stale rows first, then any remaining orphans) so a +crashed prior sync cannot block new work. +**Universe** / **cleanup** keep hour-based cadence. `guess_weekly_move` reads these bars. + +Requires `TRADING_ENABLED=true` when `MARKET_HISTORY_SYNC_ENABLED=true`. + +**Migration `008`:** drops legacy `1Day` history bars, adds `timeframe` check (`4Hour` allowed), partial index on `4Hour` bars, `market_data_sync_runs.slots_synced`. + +**Migration `009`:** adds `market_data_sync_runs.backfill_items` JSONB — per-slot UTC start + symbol list for each backfill run. + +| Env var | Default | Purpose | +|---------|---------|---------| +| `MARKET_HISTORY_SYNC_ENABLED` | `false` | Worker: universe → backfill → cleanup | +| `MARKET_HISTORY_WINDOW_DAYS` | `7` | Oldest slot start to retain / backfill | +| `MARKET_HISTORY_RETENTION_DAYS` | `7` | Delete `as_of` older than this | +| `MARKET_HISTORY_ARCHIVE_ENABLED` | `false` | Archive before delete | +| `MARKET_UNIVERSE_REFRESH_HOURS` | `24` | Min hours between universe syncs | +| `MARKET_HISTORY_SYNC_HOURS` | `24` | Unused for backfill (slot-gated); fallback if no slot gate | +| `MARKET_HISTORY_CLEANUP_HOURS` | `24` | Min hours between cleanups | +| `MARKET_HISTORY_SYNC_HOUR_UTC` | *(unset)* | UTC hour floor; same-day cap applies to universe/cleanup only | +| `HISTORY_SYNC_BATCH_SIZE` | `50` | Symbols per bars request | +| `HISTORY_SYNC_MAX_SYMBOLS` | `2000` | Max symbols per backfill run | +| `MARKET_HISTORY_API_REQUESTS_PER_MINUTE` | `200` | Max Alpaca HTTP calls per rolling minute during backfill | +| `MARKET_HISTORY_SYNC_STALE_MINUTES` | `30` | Abort in-progress sync rows older than this before a new pipeline | +| `MIN_BARS_FOR_GUESS` | `5` | Min 4-hour bars for guess eligibility | +| `GUESS_COOLDOWN_HOURS` | `24` | Per-symbol guess cooldown | + +## Admin portal (market history log) + +Read-only audit log and on-demand triggers for universe sync, backfill, and +retention cleanup. Mounted when `ADMIN_PORTAL_ENABLED=true`. + +| Env var | Default | Purpose | +|---------|---------|---------| +| `ADMIN_PORTAL_ENABLED` | `false` | Mount `/v1/admin/market-history/*` and `/v1/admin/market-data/*` | +| `ADMIN_FIREBASE_UIDS` | *(empty)* | Comma-separated Firebase UIDs allowed to call admin routes | + +Requires `TRADING_ENABLED=true`, Alpaca credentials, and +`QUESTION_PIPELINE_TEST_MODE=false` for on-demand resync/cleanup when +`MARKET_HISTORY_SYNC_ENABLED=true`. When sync is disabled, the admin log and +week-coverage calendar remain available read-only; resync/cleanup return `503`. +Scheduled worker sync also requires `MARKET_HISTORY_SYNC_ENABLED=true`. + +| Method | Path | Auth | +|--------|------|------| +| `GET` | `/v1/admin/market-history/sync-runs` | Admin Firebase UID | +| `POST` | `/v1/admin/market-data/resync` | Admin Firebase UID | +| `POST` | `/v1/admin/market-data/cleanup?archive=true\|false` | Admin Firebase UID | + +`sync-runs` includes optional `config` when the server has market-history env loaded: + +```json +{ "archiveEnabled": true, "windowDays": 7, "retentionDays": 7, "syncEnabled": true } +``` + +Flutter uses `config.archiveEnabled` to show the archive checkbox in the cleanup +confirm dialog. + +### Admin portal tests + +From the repo root: + +```bash +./scripts/test-admin-portal.sh # server + Flutter admin suites +./scripts/check-admin-portal-coverage.sh # coverage + Section 2 thresholds +``` + +CI: `.github/workflows/admin-portal.yml` runs both on admin-related changes. + ## Flutter client Run the app with the API URL (defaults to `http://localhost:3000`): diff --git a/server/bin/server.dart b/server/bin/server.dart index d0b025b..5d9e0e5 100644 --- a/server/bin/server.dart +++ b/server/bin/server.dart @@ -3,12 +3,15 @@ import 'dart:io'; import 'package:shelf/shelf.dart'; import 'package:shelf/shelf_io.dart' as shelf_io; +import '../lib/alpaca/alpaca_assets_client.dart'; import '../lib/alpaca/alpaca_market_data_client.dart'; import '../lib/alpaca/alpaca_trading_client.dart'; import '../lib/db.dart'; import '../lib/env.dart'; +import '../lib/market_history_env.dart'; import '../lib/firebase_auth.dart'; import '../lib/handlers/incoming_question_handler.dart'; +import '../lib/handlers/market_history_admin_handler.dart'; import '../lib/handlers/profile_handler.dart'; import '../lib/handlers/questions_handler.dart'; import '../lib/handlers/questions_hub_handler.dart'; @@ -18,7 +21,14 @@ import '../lib/question_service.dart'; import '../lib/questions_db.dart'; import '../lib/trading/guardrails.dart'; import '../lib/trading/market_data_db.dart'; +import '../lib/trading/market_data_history.dart'; +import '../lib/trading/market_history_api_rate_limiter.dart'; +import '../lib/trading/market_history_query.dart'; import '../lib/trading/market_data_ingest.dart'; +import '../lib/trading/market_data_retention.dart'; +import '../lib/trading/market_history_admin_actions.dart'; +import '../lib/trading/tradable_assets_db.dart'; +import '../lib/trading/tradable_assets_sync.dart'; import '../lib/trading/trade_actuator.dart'; import '../lib/trading/trade_orders_db.dart'; import '../lib/trading/trading_config_db.dart'; @@ -26,6 +36,8 @@ import '../lib/trading/trading_dev_actions.dart'; import '../lib/trading/trading_orchestrator.dart'; import '../lib/trading/trading_pipeline.dart'; import '../lib/trading/user_trading_state_db.dart'; +import '../lib/workers/market_history_scheduler.dart'; +import '../lib/workers/market_history_scheduler_config.dart'; import '../lib/workers/question_background_worker.dart'; Future main() async { @@ -57,6 +69,8 @@ Future main() async { TradingPipeline? tradingPipeline; TradingOrchestrator? tradingOrchestrator; TradingDevActions? tradingDevActions; + MarketHistoryScheduler? marketHistoryScheduler; + MarketHistoryAdminActions? marketHistoryAdminActions; AlpacaMarketDataClient? alpacaMarketDataClient; AlpacaTradingClient? alpacaTradingClient; if (env.tradingEnabled) { @@ -66,12 +80,15 @@ Future main() async { final UserTradingStateDb tradingStateDb = UserTradingStateDb(db.connection); + final MarketHistoryEnv mh = env.marketHistory; tradingPipeline = TradingPipeline( questionsDb: questionsDb, questionService: questionService, marketDataDb: marketDataDb, tradingConfigDb: tradingConfigDb, tradingStateDb: tradingStateDb, + marketHistoryQuery: MarketHistoryQuery(connection: db.connection), + marketHistoryEnv: mh, guardrails: Guardrails(allowLive: env.alpaca.allowLive), ); @@ -118,6 +135,79 @@ Future main() async { tradingPipeline: tradingPipeline, ); } + + if (useRealAlpaca && + (env.marketHistorySyncEnabled || env.adminPortalEnabled)) { + final TradableAssetsDb tradableAssetsDb = + TradableAssetsDb(db.connection); + final AlpacaAssetsClient assetsClient = AlpacaAssetsClient(env: env.alpaca); + final MarketHistoryApiRateLimiter historyRateLimiter = + MarketHistoryApiRateLimiter( + requestsPerMinute: mh.apiRequestsPerMinute, + ); + final AlpacaMarketDataClient historyMarketDataClient = + AlpacaMarketDataClient(env: env.alpaca); + final TradableAssetsSync tradableAssetsSync = TradableAssetsSync( + assetsClient: assetsClient, + assetsDb: tradableAssetsDb, + connection: db.connection, + ); + final MarketDataHistorySync historySync = MarketDataHistorySync( + marketDataClient: historyMarketDataClient, + apiRequestsPerMinute: mh.apiRequestsPerMinute, + tradableAssetsDb: tradableAssetsDb, + marketDataDb: marketDataDb, + connection: db.connection, + batchSize: mh.historySyncBatchSize, + maxSymbols: mh.historySyncMaxSymbols, + windowDays: mh.windowDays, + ); + final MarketDataRetention retention = MarketDataRetention( + connection: db.connection, + windowDays: mh.retentionDays, + ); + if (env.marketHistorySyncEnabled) { + marketHistoryScheduler = MarketHistoryScheduler( + connection: db.connection, + config: MarketHistorySchedulerConfig( + universeRefreshHours: mh.universeRefreshHours, + historySyncHours: mh.historySyncHours, + cleanupHours: mh.cleanupHours, + syncHourUtc: mh.syncHourUtc, + staleSyncRunMinutes: mh.staleSyncRunMinutes, + ), + runUniverse: (DateTime now) => tradableAssetsSync.runOnce(now: now), + runBackfill: (DateTime now) => historySync.runOnce(now: now), + backfillIsDue: historySync.hasPendingSlots, + runCleanup: (DateTime now) => + retention.run(archive: mh.archiveEnabled, now: now), + ); + } + if (env.adminPortalEnabled) { + if (env.marketHistorySyncEnabled) { + marketHistoryAdminActions = MarketHistoryAdminActions( + connection: db.connection, + runUniverse: (DateTime now) => tradableAssetsSync.runOnce(now: now), + runBackfill: (DateTime now) => historySync.runOnce(now: now), + runCleanup: (DateTime now, bool archive, int windowDays) => + retention.run(archive: archive, now: now, windowDays: windowDays), + defaultArchiveEnabled: mh.archiveEnabled, + defaultWindowDays: mh.windowDays, + ); + } else { + stderr.writeln( + 'Admin portal on-demand sync/cleanup disabled: ' + 'MARKET_HISTORY_SYNC_ENABLED=false.', + ); + } + } + } else if (env.adminPortalEnabled) { + stderr.writeln( + 'Admin portal on-demand sync/cleanup unavailable: requires ' + 'TRADING_ENABLED=true, Alpaca credentials, and ' + 'QUESTION_PIPELINE_TEST_MODE=false.', + ); + } } final QuestionPipeline questionPipeline = QuestionPipeline( @@ -132,6 +222,7 @@ Future main() async { pipeline: questionPipeline, interval: Duration(seconds: env.questionWorkerIntervalSeconds), tradingOrchestrator: tradingOrchestrator, + marketHistoryScheduler: marketHistoryScheduler, ); backgroundWorker.start(); } @@ -153,6 +244,20 @@ Future main() async { final Handler? tradingDev = tradingDevActions == null ? null : tradingDevHandler(auth: auth, devActions: tradingDevActions); + final Handler? marketHistoryAdmin = env.adminPortalEnabled + ? marketHistoryAdminHandler( + auth: auth, + connection: db.connection, + adminFirebaseUids: env.adminFirebaseUids, + actions: marketHistoryAdminActions, + portalConfig: MarketHistoryAdminPortalConfig( + archiveEnabled: env.marketHistory.archiveEnabled, + windowDays: env.marketHistory.windowDays, + retentionDays: env.marketHistory.retentionDays, + syncEnabled: env.marketHistorySyncEnabled, + ), + ) + : null; final Handler handler = Pipeline() .addMiddleware(logRequests()) @@ -167,6 +272,9 @@ Future main() async { if (tradingDev != null && path.startsWith(tradingDevBasePath)) { return tradingDev(request); } + if (marketHistoryAdmin != null && path.startsWith('/v1/admin')) { + return marketHistoryAdmin(request); + } if (path.startsWith(questionsBasePath)) { return questions(request); } diff --git a/server/lib/alpaca/alpaca_assets_client.dart b/server/lib/alpaca/alpaca_assets_client.dart new file mode 100644 index 0000000..a95c314 --- /dev/null +++ b/server/lib/alpaca/alpaca_assets_client.dart @@ -0,0 +1,82 @@ +import 'dart:convert'; + +import 'package:http/http.dart' as http; + +import 'alpaca_env.dart'; +import 'alpaca_models.dart'; + +/// REST client for Alpaca's Trading-API `/v2/assets` endpoint. +/// +/// The asset universe lives behind the trading host, NOT the data host — +/// see [AlpacaEnv.tradingBaseUrl]. We treat it as read-only here; the +/// tradable_assets sync (§2.2) is the only writer to the DB. +class AlpacaAssetsClient { + AlpacaAssetsClient({ + required AlpacaEnv env, + http.Client? httpClient, + }) : _env = env, + _client = httpClient ?? http.Client(); + + final AlpacaEnv _env; + final http.Client _client; + + /// `GET ${tradingBaseUrl}/v2/assets?status=active&asset_class=us_equity`. + /// + /// Filters to `tradable=true` are applied **server-side** by the caller + /// (the asset-universe sync) so the client stays a thin wrapper and the + /// audit trail in [tradable_assets] still records inactive symbols when + /// they later disappear. + Future> listActiveTradable() async { + _env.requireCredentials(); + + final Uri uri = Uri.parse('${_env.tradingBaseUrl}/v2/assets').replace( + queryParameters: { + 'status': 'active', + 'asset_class': 'us_equity', + }, + ); + + final http.Response response; + try { + response = await _client.get(uri, headers: _env.authHeaders); + } on http.ClientException catch (e) { + throw AlpacaAssetsException( + 'GET /v2/assets transport error: ${e.message}', + ); + } + + if (response.statusCode != 200) { + throw AlpacaAssetsException( + 'GET /v2/assets failed: ' + '${response.statusCode} ${response.body}', + ); + } + + final dynamic decoded = jsonDecode(response.body); + if (decoded is! List) { + throw AlpacaAssetsException( + 'GET /v2/assets returned non-list body: ${response.body}', + ); + } + return decoded + .whereType() + .map((Map m) => + AlpacaAsset.fromJson(Map.from(m))) + .toList(growable: false); + } + + void close() => _client.close(); +} + +/// Thrown by [AlpacaAssetsClient] when an upstream HTTP call fails. +/// +/// The message always includes the HTTP status code (when applicable) and +/// the response body so the caller's audit row can capture it verbatim. +class AlpacaAssetsException implements Exception { + AlpacaAssetsException(this.message); + + final String message; + + @override + String toString() => message; +} diff --git a/server/lib/alpaca/alpaca_env.dart b/server/lib/alpaca/alpaca_env.dart index a0cdfb6..23eb38a 100644 --- a/server/lib/alpaca/alpaca_env.dart +++ b/server/lib/alpaca/alpaca_env.dart @@ -24,6 +24,16 @@ class AlpacaEnv { bool get hasCredentials => apiKeyId.isNotEmpty && apiSecretKey.isNotEmpty; + /// HTTP headers that authenticate every Alpaca REST call. + /// + /// Shared across `AlpacaMarketDataClient`, `AlpacaTradingClient`, and + /// `AlpacaAssetsClient`; spread it (`...env.authHeaders`) and add + /// content-type/accept where the request body needs them. + Map get authHeaders => { + 'APCA-API-KEY-ID': apiKeyId, + 'APCA-API-SECRET-KEY': apiSecretKey, + }; + bool get isPaperUrl => tradingBaseUrl.contains('paper-api') || !tradingBaseUrl.contains(liveTradingHost); diff --git a/server/lib/alpaca/alpaca_market_data_client.dart b/server/lib/alpaca/alpaca_market_data_client.dart index 2cfc564..1f32c95 100644 --- a/server/lib/alpaca/alpaca_market_data_client.dart +++ b/server/lib/alpaca/alpaca_market_data_client.dart @@ -4,22 +4,28 @@ import 'package:http/http.dart' as http; import 'alpaca_env.dart'; import 'alpaca_models.dart'; +import '../trading/market_history_four_hour_slot.dart'; /// REST client for Alpaca Market Data API v2 (IEX feed on Basic plan). class AlpacaMarketDataClient { AlpacaMarketDataClient({ required AlpacaEnv env, http.Client? httpClient, + Future Function()? beforeHttpRequest, }) : _env = env, - _client = httpClient ?? http.Client(); + _client = httpClient ?? http.Client(), + _beforeHttpRequest = beforeHttpRequest; final AlpacaEnv _env; final http.Client _client; + final Future Function()? _beforeHttpRequest; - Map get _headers => { - 'APCA-API-KEY-ID': _env.apiKeyId, - 'APCA-API-SECRET-KEY': _env.apiSecretKey, - }; + Future _throttle() async { + final Future Function()? hook = _beforeHttpRequest; + if (hook != null) { + await hook(); + } + } /// `GET /v2/stocks/{symbol}/trades/latest` Future getLatestTrade(String symbol) async { @@ -28,7 +34,9 @@ class AlpacaMarketDataClient { '${_env.dataBaseUrl}/v2/stocks/${Uri.encodeComponent(symbol)}/trades/latest', ).replace(queryParameters: {'feed': _env.dataFeed}); - final http.Response response = await _client.get(uri, headers: _headers); + await _throttle(); + final http.Response response = + await _client.get(uri, headers: _env.authHeaders); if (response.statusCode != 200) { throw AlpacaMarketDataException( 'getLatestTrade($symbol) failed: ${response.statusCode} ${response.body}', @@ -58,7 +66,9 @@ class AlpacaMarketDataClient { }, ); - final http.Response response = await _client.get(uri, headers: _headers); + await _throttle(); + final http.Response response = + await _client.get(uri, headers: _env.authHeaders); if (response.statusCode != 200) { throw AlpacaMarketDataException( 'getDailyBars failed: ${response.statusCode} ${response.body}', @@ -69,13 +79,89 @@ class AlpacaMarketDataClient { ); } + /// `GET /v2/stocks/bars` over a time range with pagination. + /// + /// Follows `next_page_token` until exhausted or [maxPages] is reached. + /// HTTP 429 responses throw [AlpacaMarketDataException] with `rate` in + /// the message so callers can back off. + Future getBarsRange({ + required List symbols, + required String timeframe, + required DateTime start, + required DateTime end, + int maxPages = 20, + int limit = 10000, + }) async { + _env.requireCredentials(); + if (symbols.isEmpty) { + return AlpacaBarsResponse(barsBySymbol: >{}); + } + + AlpacaBarsResponse merged = + AlpacaBarsResponse(barsBySymbol: >{}); + String? pageToken; + int pagesFetched = 0; + + while (pagesFetched < maxPages) { + final Map query = { + 'symbols': symbols.join(','), + 'timeframe': timeframe, + 'start': MarketHistoryFourHourSlot.wireUtc(start), + 'end': MarketHistoryFourHourSlot.wireUtc(end), + 'feed': _env.dataFeed, + 'limit': limit.toString(), + if (pageToken != null && pageToken.isNotEmpty) 'page_token': pageToken, + }; + + final Uri uri = Uri.parse('${_env.dataBaseUrl}/v2/stocks/bars') + .replace(queryParameters: query); + + await _throttle(); + final http.Response response = + await _client.get(uri, headers: _env.authHeaders); + + if (response.statusCode == 429) { + throw AlpacaMarketDataException.rateLimited( + 'getBarsRange rate limited: ${response.statusCode} ${response.body}', + ); + } + if (response.statusCode != 200) { + throw AlpacaMarketDataException( + 'getBarsRange failed: ${response.statusCode} ${response.body}', + ); + } + + final Map decoded = + jsonDecode(response.body) as Map; + final AlpacaBarsResponse page = AlpacaBarsResponse.fromJson(decoded); + merged = merged.merge(page); + pagesFetched++; + + pageToken = page.nextPageToken; + if (pageToken == null || pageToken.isEmpty) { + break; + } + } + + return merged; + } + void close() => _client.close(); } class AlpacaMarketDataException implements Exception { - AlpacaMarketDataException(this.message); + AlpacaMarketDataException(this.message, {this.statusCode}); + + AlpacaMarketDataException.rateLimited(this.message) + : statusCode = 429; final String message; + final int? statusCode; + + bool get isRateLimited => + statusCode == 429 || + message.toLowerCase().contains('rate limited') || + message.contains('429'); @override String toString() => message; diff --git a/server/lib/alpaca/alpaca_models.dart b/server/lib/alpaca/alpaca_models.dart index fea68c5..17f0be2 100644 --- a/server/lib/alpaca/alpaca_models.dart +++ b/server/lib/alpaca/alpaca_models.dart @@ -1,3 +1,46 @@ +/// Alpaca `/v2/assets` row — the catalog entry for one tradable instrument. +/// +/// The API is documented at +/// https://docs.alpaca.markets/reference/get-v2-assets-1; we keep [raw] so +/// downstream code can persist the full payload as JSONB without losing +/// any fields the model doesn't yet expose. +class AlpacaAsset { + AlpacaAsset({ + required this.symbol, + required this.assetClass, + required this.status, + required this.tradable, + required this.fractionable, + this.exchange, + this.name, + this.raw, + }); + + final String symbol; + final String assetClass; + final String? exchange; + final String? name; + final String status; + final bool tradable; + final bool fractionable; + final Map? raw; + + factory AlpacaAsset.fromJson(Map json) { + return AlpacaAsset( + symbol: json['symbol']! as String, + // Alpaca's payload calls the field `class` (a Dart reserved word), + // so we have to read it dynamically rather than via a typed getter. + assetClass: (json['class'] as String?) ?? 'us_equity', + exchange: json['exchange'] as String?, + name: json['name'] as String?, + status: (json['status'] as String?) ?? 'active', + tradable: (json['tradable'] as bool?) ?? false, + fractionable: (json['fractionable'] as bool?) ?? false, + raw: json, + ); + } +} + /// Alpaca v2 latest-trade response: `{ "symbol", "trade" }`. class AlpacaLatestTradeResponse { AlpacaLatestTradeResponse({required this.symbol, required this.trade}); @@ -74,9 +117,13 @@ class AlpacaBar { /// Multi-symbol bars response: `{ "bars": { "SPY": [ ... ] } }`. class AlpacaBarsResponse { - AlpacaBarsResponse({required this.barsBySymbol}); + AlpacaBarsResponse({ + required this.barsBySymbol, + this.nextPageToken, + }); final Map> barsBySymbol; + final String? nextPageToken; factory AlpacaBarsResponse.fromJson(Map json) { final Map rawBars = @@ -90,7 +137,28 @@ class AlpacaBarsResponse { AlpacaBar.fromJson(Map.from(m))) .toList(); } - return AlpacaBarsResponse(barsBySymbol: parsed); + return AlpacaBarsResponse( + barsBySymbol: parsed, + nextPageToken: json['next_page_token'] as String?, + ); + } + + /// Combines [other] into this response, appending bars per symbol and + /// sorting each symbol's series by timestamp ascending. + AlpacaBarsResponse merge(AlpacaBarsResponse other) { + final Map> merged = + Map>.from(barsBySymbol); + for (final MapEntry> entry + in other.barsBySymbol.entries) { + final List existing = + List.from(merged[entry.key] ?? []); + existing.addAll(entry.value); + existing.sort( + (AlpacaBar a, AlpacaBar b) => a.timestamp.compareTo(b.timestamp), + ); + merged[entry.key] = existing; + } + return AlpacaBarsResponse(barsBySymbol: merged); } AlpacaBar? latestBar(String symbol) { diff --git a/server/lib/alpaca/alpaca_trading_client.dart b/server/lib/alpaca/alpaca_trading_client.dart index bc29516..1b43c9c 100644 --- a/server/lib/alpaca/alpaca_trading_client.dart +++ b/server/lib/alpaca/alpaca_trading_client.dart @@ -20,8 +20,7 @@ class AlpacaTradingClient { final http.Client _client; Map get _headers => { - 'APCA-API-KEY-ID': _env.apiKeyId, - 'APCA-API-SECRET-KEY': _env.apiSecretKey, + ..._env.authHeaders, 'Content-Type': 'application/json', 'Accept': 'application/json', }; diff --git a/server/lib/env.dart b/server/lib/env.dart index dd39040..a3ba9cd 100644 --- a/server/lib/env.dart +++ b/server/lib/env.dart @@ -1,6 +1,7 @@ import 'dart:io'; import 'alpaca/alpaca_env.dart'; +import 'market_history_env.dart'; import 'package:dotenv/dotenv.dart'; class ServerEnv { @@ -15,6 +16,9 @@ class ServerEnv { required this.tradingWorkerIngestEnabled, required this.tradingWorkerEvalEnabled, required this.tradingDevEndpointsEnabled, + required this.adminPortalEnabled, + required this.adminFirebaseUids, + required this.marketHistory, required this.alpaca, }); @@ -31,8 +35,22 @@ class ServerEnv { /// Mounts dev-only endpoints under `/v1/me/trading/dev/*` (e.g. `force-fire`). /// Default false — never enable in production. final bool tradingDevEndpointsEnabled; + final bool adminPortalEnabled; + final Set adminFirebaseUids; + + /// Rolling market-history sync, retention, and guess-rule settings. + final MarketHistoryEnv marketHistory; + + /// Shorthand for [MarketHistoryEnv.syncEnabled]. + bool get marketHistorySyncEnabled => marketHistory.syncEnabled; + final AlpacaEnv alpaca; + /// Validates cross-flag constraints after [load]. + void assertConsistent() { + marketHistory.assertConsistent(tradingEnabled: tradingEnabled); + } + static ServerEnv load() { final DotEnv env = DotEnv(includePlatformEnvironment: true) ..load(['.env']); @@ -80,10 +98,41 @@ class ServerEnv { final bool tradingDevEndpointsEnabled = (env['TRADING_DEV_ENDPOINTS_ENABLED'] ?? 'false').toLowerCase() == 'true'; + final bool adminPortalEnabled = + (env['ADMIN_PORTAL_ENABLED'] ?? 'false').toLowerCase() == 'true'; + final Set adminFirebaseUids = (env['ADMIN_FIREBASE_UIDS'] ?? '') + .split(',') + .map((String value) => value.trim()) + .where((String value) => value.isNotEmpty) + .toSet(); + final Map stringEnv = + Map.from(Platform.environment); + const List marketHistoryKeys = [ + 'MARKET_HISTORY_SYNC_ENABLED', + 'MARKET_HISTORY_WINDOW_DAYS', + 'MARKET_HISTORY_RETENTION_DAYS', + 'MARKET_HISTORY_ARCHIVE_ENABLED', + 'MARKET_UNIVERSE_REFRESH_HOURS', + 'MARKET_HISTORY_SYNC_HOURS', + 'MARKET_HISTORY_CLEANUP_HOURS', + 'MARKET_HISTORY_SYNC_HOUR_UTC', + 'HISTORY_SYNC_BATCH_SIZE', + 'HISTORY_SYNC_MAX_SYMBOLS', + 'MIN_BARS_FOR_GUESS', + 'GUESS_COOLDOWN_HOURS', + ]; + for (final String key in marketHistoryKeys) { + final String? value = env[key]; + if (value != null && value.isNotEmpty) { + stringEnv[key] = value; + } + } + final MarketHistoryEnv marketHistory = + MarketHistoryEnv.fromMap(stringEnv); final AlpacaEnv alpaca = AlpacaEnv.fromMap(envMap)..assertPaperOnly(); - return ServerEnv._( + final ServerEnv loaded = ServerEnv._( databaseUrl: databaseUrl, port: port, firebaseWebApiKey: apiKey, @@ -94,7 +143,12 @@ class ServerEnv { tradingWorkerIngestEnabled: tradingWorkerIngestEnabled, tradingWorkerEvalEnabled: tradingWorkerEvalEnabled, tradingDevEndpointsEnabled: tradingDevEndpointsEnabled, + adminPortalEnabled: adminPortalEnabled, + adminFirebaseUids: adminFirebaseUids, + marketHistory: marketHistory, alpaca: alpaca, ); + loaded.assertConsistent(); + return loaded; } } diff --git a/server/lib/handlers/market_history_admin_handler.dart b/server/lib/handlers/market_history_admin_handler.dart new file mode 100644 index 0000000..7611364 --- /dev/null +++ b/server/lib/handlers/market_history_admin_handler.dart @@ -0,0 +1,369 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:postgres/postgres.dart'; +import 'package:shelf/shelf.dart'; +import 'package:shelf_router/shelf_router.dart'; + +import '../cors_headers.dart'; +import '../firebase_auth.dart'; +import '../trading/backfill_sync_item.dart'; +import '../trading/market_history_admin_actions.dart'; +import '../trading/market_history_admin_logic.dart'; +import '../trading/market_history_config.dart'; +import '../trading/market_history_question_audit.dart'; +import '../trading/market_history_week_coverage.dart'; +import '../trading/tradable_assets_db.dart'; + +const String marketHistoryAdminBasePath = '/v1/admin/market-history'; +const String marketHistoryAdminDataBasePath = '/v1/admin/market-data'; + +class MarketHistoryAdminPortalConfig { + const MarketHistoryAdminPortalConfig({ + required this.archiveEnabled, + required this.windowDays, + required this.retentionDays, + required this.syncEnabled, + }); + + final bool archiveEnabled; + final int windowDays; + final int retentionDays; + final bool syncEnabled; + + Map toJson() => { + 'archiveEnabled': archiveEnabled, + 'windowDays': windowDays, + 'retentionDays': retentionDays, + 'syncEnabled': syncEnabled, + }; +} + +Handler marketHistoryAdminHandler({ + required FirebaseAuthVerifier auth, + required Connection connection, + required Set adminFirebaseUids, + MarketHistoryAdminActions? actions, + MarketHistoryAdminPortalConfig? portalConfig, +}) { + final Router router = Router(); + + router.get('$marketHistoryAdminBasePath/sync-runs', (Request request) async { + final String? firebaseUid = await _verifyAdmin(auth, request, adminFirebaseUids); + if (firebaseUid == null) { + return _authFailure(auth, request, adminFirebaseUids); + } + + final Uri uri = request.requestedUri; + final int limit = _parseLimit(uri.queryParameters['limit']); + final DateTime? before = _parseBefore(uri.queryParameters['before']); + final String? kind = _parseKind(uri.queryParameters['kind']); + + try { + final StringBuffer sql = StringBuffer( + ''' + SELECT id, kind, started_at, finished_at, rows_written, rows_removed, + slots_synced, backfill_items, error + FROM market_data_sync_runs + WHERE 1=1 + ''', + ); + final Map params = {'limit': limit}; + if (kind != null) { + sql.write(' AND kind = @kind'); + params['kind'] = kind; + } + if (before != null) { + sql.write(' AND started_at < @before'); + params['before'] = before.toUtc(); + } + sql.write(' ORDER BY started_at DESC, id DESC LIMIT @limit'); + + final Result result = await connection.execute( + Sql.named(sql.toString()), + parameters: params, + ); + + final List allRuns = result.map(_toRecord).toList(); + final List pinned = computePinned(allRuns); + final Set pinnedIds = pinned.map((AdminSyncRunRecord r) => r.id).toSet(); + final List history = allRuns + .where((AdminSyncRunRecord r) => !pinnedIds.contains(r.id)) + .toList(); + + final DateTime now = DateTime.now().toUtc(); + final DateTime? nextBefore = + allRuns.isEmpty ? null : allRuns.last.startedAt.toUtc(); + + return _jsonResponse(200, { + 'runs': history.map((AdminSyncRunRecord run) => _toJson(run, now)).toList(), + 'pinned': pinned.map((AdminSyncRunRecord run) => _toJson(run, now)).toList(), + 'nextBefore': nextBefore?.toIso8601String(), + if (portalConfig != null) 'config': portalConfig.toJson(), + }); + } catch (e, st) { + stderr.writeln('market history admin sync-runs failed: $e\n$st'); + return _jsonResponse(500, {'error': 'Internal error'}); + } + }); + + router.get('$marketHistoryAdminBasePath/question-audit', (Request request) async { + final String? firebaseUid = await _verifyAdmin(auth, request, adminFirebaseUids); + if (firebaseUid == null) { + return _authFailure(auth, request, adminFirebaseUids); + } + + try { + final int windowDays = + portalConfig?.windowDays ?? MarketHistoryConfig.windowDays; + final MarketHistoryQuestionAudit audit = MarketHistoryQuestionAudit( + connection: connection, + ); + final DateTime now = DateTime.now().toUtc(); + final DateTime? compareUntil = + _parseBefore(request.requestedUri.queryParameters['asOf']); + final QuestionAuditPage page = await audit.page( + now: now, + compareUntil: compareUntil, + windowDays: windowDays, + ); + return _jsonResponse(200, page.toJson()); + } catch (e, st) { + stderr.writeln('market history admin question-audit failed: $e\n$st'); + return _jsonResponse(500, {'error': 'Internal error'}); + } + }); + + router.get('$marketHistoryAdminBasePath/week-coverage', (Request request) async { + final String? firebaseUid = await _verifyAdmin(auth, request, adminFirebaseUids); + if (firebaseUid == null) { + return _authFailure(auth, request, adminFirebaseUids); + } + + try { + final int windowDays = + portalConfig?.windowDays ?? MarketHistoryConfig.windowDays; + final MarketHistoryWeekCoverage coverage = MarketHistoryWeekCoverage( + connection: connection, + tradableAssetsDb: TradableAssetsDb(connection), + windowDays: windowDays, + ); + final MarketHistoryWeekCoverageReport report = await coverage.compute(); + return _jsonResponse(200, report.toJson()); + } catch (e, st) { + stderr.writeln('market history admin week-coverage failed: $e\n$st'); + return _jsonResponse(500, {'error': 'Internal error'}); + } + }); + + router.post('$marketHistoryAdminDataBasePath/resync', (Request request) async { + final String? firebaseUid = await _verifyAdmin(auth, request, adminFirebaseUids); + if (firebaseUid == null) { + return _authFailure(auth, request, adminFirebaseUids); + } + if (actions == null) { + return _jsonResponse( + 503, + { + 'error': portalConfig?.syncEnabled == false + ? 'Market history sync is disabled (MARKET_HISTORY_SYNC_ENABLED=false)' + : 'Market history actions unavailable', + }, + ); + } + + try { + final AdminTriggerResult result = await actions.resync(); + return _jsonResponse(202, result.toJson()); + } on StateError catch (e) { + return _jsonResponse(409, {'error': e.message}); + } catch (e, st) { + stderr.writeln('market history admin resync failed: $e\n$st'); + return _jsonResponse(500, {'error': 'Internal error'}); + } + }); + + router.post('$marketHistoryAdminDataBasePath/cleanup', (Request request) async { + final String? firebaseUid = await _verifyAdmin(auth, request, adminFirebaseUids); + if (firebaseUid == null) { + return _authFailure(auth, request, adminFirebaseUids); + } + if (actions == null) { + return _jsonResponse( + 503, + { + 'error': portalConfig?.syncEnabled == false + ? 'Market history sync is disabled (MARKET_HISTORY_SYNC_ENABLED=false)' + : 'Market history actions unavailable', + }, + ); + } + + final Uri uri = request.requestedUri; + final bool archive = _parseBool(uri.queryParameters['archive']); + final int? windowDays = _parseOptionalPositiveInt(uri.queryParameters['windowDays']); + + try { + final AdminTriggerResult result = await actions.cleanup( + archive: archive, + windowDays: windowDays, + ); + return _jsonResponse(202, result.toJson()); + } on StateError catch (e) { + return _jsonResponse(409, {'error': e.message}); + } catch (e, st) { + stderr.writeln('market history admin cleanup failed: $e\n$st'); + return _jsonResponse(500, {'error': 'Internal error'}); + } + }); + + return (Request request) async { + if (request.method == 'OPTIONS' && request.requestedUri.path.startsWith('/v1/admin')) { + return Response.ok('', headers: apiCorsHeaders()); + } + final Response response = await router.call(request); + if (response.statusCode != 404) { + return response; + } + return _jsonResponse(404, {'error': 'Not found'}); + }; +} + +Future _authFailure( + FirebaseAuthVerifier auth, + Request request, + Set adminFirebaseUids, +) async { + final String? firebaseUid = await _verify(auth, request); + if (firebaseUid == null) { + return _jsonResponse(401, {'error': 'Unauthorized'}); + } + return _jsonResponse(403, {'error': 'Forbidden'}); +} + +Future _verifyAdmin( + FirebaseAuthVerifier auth, + Request request, + Set adminFirebaseUids, +) async { + final String? firebaseUid = await _verify(auth, request); + if (firebaseUid == null || !adminFirebaseUids.contains(firebaseUid)) { + return null; + } + return firebaseUid; +} + +AdminSyncRunRecord _toRecord(ResultRow row) { + return AdminSyncRunRecord( + id: (row[0]! as num).toInt(), + kind: row[1]! as String, + startedAt: (row[2]! as DateTime).toUtc(), + finishedAt: (row[3] as DateTime?)?.toUtc(), + rowsWritten: (row[4]! as num).toInt(), + rowsRemoved: (row[5]! as num).toInt(), + slotsSynced: (row[6]! as num).toInt(), + backfillItems: BackfillSyncItem.listFromJson(row[7]), + error: row[8] as String?, + ); +} + +Map _toJson(AdminSyncRunRecord run, DateTime now) { + final AdminRunSeverity severity = deriveSeverity( + error: run.error, + startedAt: run.startedAt, + finishedAt: run.finishedAt, + now: now, + rowsWritten: run.rowsWritten, + ); + final AdminRunStatus status = deriveStatus( + error: run.error, + finishedAt: run.finishedAt, + rowsWritten: run.rowsWritten, + rowsRemoved: run.rowsRemoved, + ); + final String? displayError = effectiveRunError(run); + return { + 'id': run.id, + 'kind': run.kind, + 'startedAt': run.startedAt.toIso8601String(), + 'finishedAt': run.finishedAt?.toIso8601String(), + 'rowsWritten': run.rowsWritten, + 'rowsRemoved': run.rowsRemoved, + 'slotsSynced': run.slotsSynced, + if (run.backfillItems.isNotEmpty) + 'backfillItems': + run.backfillItems.map((BackfillSyncItem item) => item.toJson()).toList(), + 'error': displayError, + 'severity': severity.wireValue, + 'status': status.wireValue, + 'durationMs': run.finishedAt == null + ? null + : run.finishedAt!.difference(run.startedAt).inMilliseconds, + 'summary': toSummary(run), + }; +} + +int _parseLimit(String? raw) { + final int parsed = int.tryParse(raw ?? '') ?? 50; + if (parsed < 1) { + return 1; + } + if (parsed > 200) { + return 200; + } + return parsed; +} + +DateTime? _parseBefore(String? raw) { + if (raw == null || raw.trim().isEmpty) { + return null; + } + return DateTime.tryParse(raw)?.toUtc(); +} + +String? _parseKind(String? raw) { + if (raw == null || raw.trim().isEmpty) { + return null; + } + const Set allowed = {'universe', 'backfill', 'cleanup'}; + final String kind = raw.trim(); + if (!allowed.contains(kind)) { + return null; + } + return kind; +} + +bool _parseBool(String? raw) { + if (raw == null || raw.trim().isEmpty) { + return false; + } + return raw.toLowerCase() == 'true'; +} + +int? _parseOptionalPositiveInt(String? raw) { + if (raw == null || raw.trim().isEmpty) { + return null; + } + final int? parsed = int.tryParse(raw); + if (parsed == null || parsed <= 0) { + return null; + } + return parsed; +} + +Future _verify(FirebaseAuthVerifier auth, Request request) { + return auth.verifyBearerToken( + request.headers['Authorization'] ?? request.headers['authorization'], + ); +} + +Response _jsonResponse(int status, Map body) { + return Response( + status, + body: jsonEncode(body), + headers: { + ...apiCorsHeaders(), + 'Content-Type': 'application/json', + }, + ); +} diff --git a/server/lib/market_history_env.dart b/server/lib/market_history_env.dart new file mode 100644 index 0000000..5d57af0 --- /dev/null +++ b/server/lib/market_history_env.dart @@ -0,0 +1,153 @@ +/// Market-history pipeline settings loaded from env (§7). +class MarketHistoryEnv { + MarketHistoryEnv({ + required this.syncEnabled, + required this.windowDays, + required this.retentionDays, + required this.archiveEnabled, + required this.universeRefreshHours, + required this.historySyncHours, + required this.cleanupHours, + required this.syncHourUtc, + required this.historySyncBatchSize, + required this.historySyncMaxSymbols, + required this.minBarsForGuess, + required this.guessCooldownHours, + required this.apiRequestsPerMinute, + required this.staleSyncRunMinutes, + }); + + final bool syncEnabled; + final int windowDays; + final int retentionDays; + final bool archiveEnabled; + final int universeRefreshHours; + final int historySyncHours; + final int cleanupHours; + + /// UTC hour (0–23) for optional daily alignment, or `null` when unset. + final int? syncHourUtc; + final int historySyncBatchSize; + final int historySyncMaxSymbols; + final int minBarsForGuess; + final int guessCooldownHours; + final int apiRequestsPerMinute; + final int staleSyncRunMinutes; + + static MarketHistoryEnv fromMap(Map env) { + final bool syncEnabled = + (env['MARKET_HISTORY_SYNC_ENABLED'] ?? 'false').toLowerCase() == 'true'; + final int windowDays = + _positiveInt(env['MARKET_HISTORY_WINDOW_DAYS'], defaultValue: 7, name: 'MARKET_HISTORY_WINDOW_DAYS'); + final int retentionDays = _positiveInt( + env['MARKET_HISTORY_RETENTION_DAYS'], + defaultValue: 7, + name: 'MARKET_HISTORY_RETENTION_DAYS', + ); + final bool archiveEnabled = + (env['MARKET_HISTORY_ARCHIVE_ENABLED'] ?? 'false').toLowerCase() == + 'true'; + final int universeRefreshHours = _positiveInt( + env['MARKET_UNIVERSE_REFRESH_HOURS'], + defaultValue: 24, + name: 'MARKET_UNIVERSE_REFRESH_HOURS', + ); + final int historySyncHours = _positiveInt( + env['MARKET_HISTORY_SYNC_HOURS'], + defaultValue: 24, + name: 'MARKET_HISTORY_SYNC_HOURS', + ); + final int cleanupHours = _positiveInt( + env['MARKET_HISTORY_CLEANUP_HOURS'], + defaultValue: 24, + name: 'MARKET_HISTORY_CLEANUP_HOURS', + ); + final int? syncHourUtc = _optionalSyncHourUtc(env['MARKET_HISTORY_SYNC_HOUR_UTC']); + final int historySyncBatchSize = _positiveInt( + env['HISTORY_SYNC_BATCH_SIZE'], + defaultValue: 50, + name: 'HISTORY_SYNC_BATCH_SIZE', + ); + final int historySyncMaxSymbols = _positiveInt( + env['HISTORY_SYNC_MAX_SYMBOLS'], + defaultValue: 2000, + name: 'HISTORY_SYNC_MAX_SYMBOLS', + ); + final int minBarsForGuess = _positiveInt( + env['MIN_BARS_FOR_GUESS'], + defaultValue: 5, + name: 'MIN_BARS_FOR_GUESS', + ); + final int guessCooldownHours = _positiveInt( + env['GUESS_COOLDOWN_HOURS'], + defaultValue: 24, + name: 'GUESS_COOLDOWN_HOURS', + ); + final int apiRequestsPerMinute = _positiveInt( + env['MARKET_HISTORY_API_REQUESTS_PER_MINUTE'], + defaultValue: 200, + name: 'MARKET_HISTORY_API_REQUESTS_PER_MINUTE', + ); + final int staleSyncRunMinutes = _positiveInt( + env['MARKET_HISTORY_SYNC_STALE_MINUTES'], + defaultValue: 30, + name: 'MARKET_HISTORY_SYNC_STALE_MINUTES', + ); + + return MarketHistoryEnv( + syncEnabled: syncEnabled, + windowDays: windowDays, + retentionDays: retentionDays, + archiveEnabled: archiveEnabled, + universeRefreshHours: universeRefreshHours, + historySyncHours: historySyncHours, + cleanupHours: cleanupHours, + syncHourUtc: syncHourUtc, + historySyncBatchSize: historySyncBatchSize, + historySyncMaxSymbols: historySyncMaxSymbols, + minBarsForGuess: minBarsForGuess, + guessCooldownHours: guessCooldownHours, + apiRequestsPerMinute: apiRequestsPerMinute, + staleSyncRunMinutes: staleSyncRunMinutes, + ); + } + + /// Fails fast when market-history flags conflict with trading gates. + void assertConsistent({required bool tradingEnabled}) { + if (syncEnabled && !tradingEnabled) { + throw StateError( + 'MARKET_HISTORY_SYNC_ENABLED=true requires TRADING_ENABLED=true', + ); + } + } + + static int _positiveInt( + String? raw, { + required int defaultValue, + required String name, + }) { + if (raw == null || raw.trim().isEmpty) { + return defaultValue; + } + final int? parsed = int.tryParse(raw.trim()); + if (parsed == null || parsed <= 0) { + throw ArgumentError.value(raw, name, 'must be a positive integer'); + } + return parsed; + } + + static int? _optionalSyncHourUtc(String? raw) { + if (raw == null || raw.trim().isEmpty) { + return null; + } + final int? parsed = int.tryParse(raw.trim()); + if (parsed == null || parsed < 0 || parsed > 23) { + throw ArgumentError.value( + raw, + 'MARKET_HISTORY_SYNC_HOUR_UTC', + 'must be an integer from 0 to 23', + ); + } + return parsed; + } +} diff --git a/server/lib/pipeline/question_pipeline.dart b/server/lib/pipeline/question_pipeline.dart index fbc2f52..fbf6153 100644 --- a/server/lib/pipeline/question_pipeline.dart +++ b/server/lib/pipeline/question_pipeline.dart @@ -46,6 +46,9 @@ abstract final class TradingPhases { /// User said +10; order staged in pending_orders for the actuator. static const String submitOrder = 'submit_order'; + /// Guess-the-move question awaiting +10/-10 (scoring only, no order). + static const String awaitAnswer = 'await_answer'; + /// Outcome recorded; rule returns to [idle] after cooldown rolls off. static const String done = 'done'; } diff --git a/server/lib/question_service.dart b/server/lib/question_service.dart index 28d03b7..680ca7a 100644 --- a/server/lib/question_service.dart +++ b/server/lib/question_service.dart @@ -39,6 +39,7 @@ class QuestionService { String? sourceTag, String? pipelineKey, String? pipelineStep, + Map? metadata, }) async { final Map question = await _questionsDb.createQuestion( assignedUserId: assignedUserId, @@ -47,6 +48,7 @@ class QuestionService { sourceTag: sourceTag, pipelineKey: pipelineKey, pipelineStep: pipelineStep, + metadata: metadata, ); final int unansweredCount = await _questionsDb.countUnansweredQuestions(assignedUserId); diff --git a/server/lib/questions_db.dart b/server/lib/questions_db.dart index 5b78d11..d7da70c 100644 --- a/server/lib/questions_db.dart +++ b/server/lib/questions_db.dart @@ -33,7 +33,8 @@ class QuestionsDb { Sql.named( ''' SELECT id, assigned_user_id, question_text, user_response, correct_answer, - created_at, modified_at, source_tag, pipeline_key, pipeline_step + created_at, modified_at, source_tag, pipeline_key, pipeline_step, + metadata FROM questions WHERE assigned_user_id = @uid AND user_response IS NULL ORDER BY created_at ASC @@ -56,7 +57,8 @@ class QuestionsDb { Sql.named( ''' SELECT id, assigned_user_id, question_text, user_response, correct_answer, - created_at, modified_at, source_tag, pipeline_key, pipeline_step + created_at, modified_at, source_tag, pipeline_key, pipeline_step, + metadata FROM questions WHERE assigned_user_id = @uid AND user_response IS NULL ORDER BY created_at ASC @@ -83,7 +85,8 @@ class QuestionsDb { AND assigned_user_id = @uid AND user_response IS NULL RETURNING id, assigned_user_id, question_text, user_response, correct_answer, - created_at, modified_at, source_tag, pipeline_key, pipeline_step + created_at, modified_at, source_tag, pipeline_key, pipeline_step, + metadata ''', ), parameters: { @@ -192,7 +195,7 @@ class QuestionsDb { AND q.user_response IS NULL RETURNING q.id, q.assigned_user_id, q.question_text, q.user_response, q.correct_answer, q.created_at, q.modified_at, - q.source_tag, q.pipeline_key, q.pipeline_step + q.source_tag, q.pipeline_key, q.pipeline_step, q.metadata ''', ), parameters: { @@ -283,6 +286,7 @@ class QuestionsDb { String? sourceTag, String? pipelineKey, String? pipelineStep, + Map? metadata, }) async { await ensureUserExists(assignedUserId); final String id = _uuid.v4(); @@ -293,10 +297,12 @@ class QuestionsDb { ''' INSERT INTO questions ( id, assigned_user_id, question_text, user_response, correct_answer, - created_at, modified_at, source_tag, pipeline_key, pipeline_step + created_at, modified_at, source_tag, pipeline_key, pipeline_step, + metadata ) VALUES ( @id::uuid, @assigned_user_id, @question_text, NULL, @correct_answer, - @created_at, @modified_at, @source_tag, @pipeline_key, @pipeline_step + @created_at, @modified_at, @source_tag, @pipeline_key, @pipeline_step, + @metadata::jsonb ) ''', ), @@ -310,6 +316,7 @@ class QuestionsDb { 'source_tag': sourceTag, 'pipeline_key': pipelineKey, 'pipeline_step': pipelineStep, + 'metadata': jsonEncode(metadata ?? {}), }, ); @@ -324,6 +331,7 @@ class QuestionsDb { sourceTag: sourceTag, pipelineKey: pipelineKey, pipelineStep: pipelineStep, + metadata: metadata ?? {}, ); } @@ -357,9 +365,23 @@ class QuestionsDb { sourceTag: row.length > 7 ? row[7] as String? : null, pipelineKey: row.length > 8 ? row[8] as String? : null, pipelineStep: row.length > 9 ? row[9] as String? : null, + metadata: _readJsonMap(row.length > 10 ? row[10] : null), ); } + Map _readJsonMap(Object? value) { + if (value is Map) { + return value; + } + if (value is Map) { + return Map.from(value); + } + if (value == null) { + return {}; + } + return jsonDecode(value.toString()) as Map; + } + /// Postgres NUMERIC columns may decode as [String] or [num]. static num _readNumeric(Object? value) { if (value == null) { @@ -392,6 +414,7 @@ class QuestionsDb { String? sourceTag, String? pipelineKey, String? pipelineStep, + Map? metadata, }) { return { 'id': id, @@ -404,6 +427,7 @@ class QuestionsDb { if (sourceTag != null) 'sourceTag': sourceTag, if (pipelineKey != null) 'pipelineKey': pipelineKey, if (pipelineStep != null) 'pipelineStep': pipelineStep, + if (metadata != null && metadata.isNotEmpty) 'metadata': metadata, }; } } diff --git a/server/lib/trading/backfill_sync_item.dart b/server/lib/trading/backfill_sync_item.dart new file mode 100644 index 0000000..0e8963a --- /dev/null +++ b/server/lib/trading/backfill_sync_item.dart @@ -0,0 +1,59 @@ +import 'market_history_four_hour_slot.dart'; + +/// One Alpaca backfill request bucket: a UTC 4-hour slot and its symbols. +class BackfillSyncItem { + const BackfillSyncItem({ + required this.slotStart, + required this.symbols, + }); + + final DateTime slotStart; + final List symbols; + + Map toJson() => { + 'slotStart': MarketHistoryFourHourSlot.slotStartWire(slotStart), + 'symbols': symbols, + }; + + static BackfillSyncItem? tryFromJson(dynamic raw) { + if (raw is! Map) { + return null; + } + final String? slotStartRaw = raw['slotStart'] as String?; + if (slotStartRaw == null) { + return null; + } + final DateTime? slotStart = DateTime.tryParse(slotStartRaw)?.toUtc(); + if (slotStart == null) { + return null; + } + final List symbolRaw = + raw['symbols'] as List? ?? []; + final List symbols = symbolRaw + .map((dynamic value) => value.toString()) + .where((String value) => value.isNotEmpty) + .toList(growable: false); + return BackfillSyncItem(slotStart: slotStart, symbols: symbols); + } + + static List listFromJson(dynamic raw) { + if (raw == null) { + return []; + } + if (raw is! List) { + return []; + } + final List items = []; + for (final dynamic entry in raw) { + final BackfillSyncItem? item = tryFromJson(entry); + if (item != null) { + items.add(item); + } + } + return items; + } + + static List> encodeList(List items) { + return items.map((BackfillSyncItem item) => item.toJson()).toList(); + } +} diff --git a/server/lib/trading/market_data_db.dart b/server/lib/trading/market_data_db.dart index 8837f85..680ec87 100644 --- a/server/lib/trading/market_data_db.dart +++ b/server/lib/trading/market_data_db.dart @@ -2,6 +2,9 @@ import 'dart:convert'; import 'package:postgres/postgres.dart'; +import 'market_history_bar_placeholder.dart'; +import 'market_history_four_hour_slot.dart'; + /// Normalized market data row persisted for rule evaluation. class MarketDataSnapshot { MarketDataSnapshot({ @@ -41,6 +44,32 @@ class MarketDataDb { required DateTime asOf, String assetClass = 'us_equity', String feed = 'iex', + String timeframe = 'tick', + num? price, + num? volume, + Map? raw, + }) { + return upsertSnapshot( + symbol: symbol, + metric: metric, + asOf: asOf, + assetClass: assetClass, + feed: feed, + timeframe: timeframe, + price: price, + volume: volume, + raw: raw, + ); + } + + /// Idempotent write keyed by `(symbol, metric, timeframe, as_of)`. + Future upsertSnapshot({ + required String symbol, + required String metric, + required DateTime asOf, + String assetClass = 'us_equity', + String feed = 'iex', + String timeframe = 'tick', num? price, num? volume, Map? raw, @@ -49,10 +78,15 @@ class MarketDataDb { Sql.named( ''' INSERT INTO market_data_snapshots ( - symbol, asset_class, feed, metric, price, volume, as_of, raw + symbol, asset_class, feed, metric, timeframe, price, volume, as_of, raw ) VALUES ( - @symbol, @asset_class, @feed, @metric, @price, @volume, @as_of, @raw::jsonb + @symbol, @asset_class, @feed, @metric, @timeframe, + @price, @volume, @as_of, @raw::jsonb ) + ON CONFLICT (symbol, metric, timeframe, as_of) DO UPDATE SET + price = EXCLUDED.price, + volume = EXCLUDED.volume, + raw = EXCLUDED.raw RETURNING id, symbol, asset_class, feed, metric, price, volume, as_of, raw, created_at ''', ), @@ -61,6 +95,7 @@ class MarketDataDb { 'asset_class': assetClass, 'feed': feed, 'metric': metric, + 'timeframe': timeframe, 'price': price, 'volume': volume, 'as_of': asOf.toUtc(), @@ -70,6 +105,162 @@ class MarketDataDb { return _rowToSnapshot(result.first); } + /// Tombstone when Alpaca has no 4Hour bar for [symbol] at [slotStart]. + /// + /// Counts toward backfill gap checks but not game/calendar bar coverage. + Future upsertNoDataBarPlaceholder({ + required String symbol, + required DateTime slotStart, + required String timeframe, + required DateTime checkedAt, + String assetClass = 'us_equity', + String feed = 'iex', + String source = MarketHistoryBarPlaceholder.sourceAlpacaEmpty, + }) async { + final DateTime slot = + MarketHistoryFourHourSlot.slotStartContaining(slotStart); + final String slotWire = MarketHistoryFourHourSlot.slotStartWire(slot); + return upsertSnapshot( + symbol: symbol, + metric: 'bar', + timeframe: timeframe, + asOf: slot, + assetClass: assetClass, + feed: feed, + raw: { + 'slot_start': slotWire, + MarketHistoryBarPlaceholder.rawKey: true, + 'source': source, + 'checked_at': MarketHistoryFourHourSlot.wireUtc(checkedAt), + }, + ); + } + + /// Daily (or intraday) bars for [symbol] in [`since`, `until`). + Future> barsForSymbol({ + required String symbol, + required String timeframe, + required DateTime since, + required DateTime until, + String metric = 'bar', + }) async { + final Result result = await _connection.execute( + Sql.named( + ''' + SELECT id, symbol, asset_class, feed, metric, price, volume, as_of, raw, created_at + FROM market_data_snapshots + WHERE symbol = @symbol + AND metric = @metric + AND timeframe = @timeframe + AND as_of >= @since + AND as_of < @until + ORDER BY as_of ASC + ''', + ), + parameters: { + 'symbol': symbol, + 'metric': metric, + 'timeframe': timeframe, + 'since': since.toUtc(), + 'until': until.toUtc(), + }, + ); + if (result.isEmpty) { + return []; + } + return result.map(_rowToSnapshot).toList(growable: false); + } + + /// Newest `as_of` for historical bars, or `null` on cold start. + Future latestSyncedAsOf( + String symbol, + String timeframe, { + String metric = 'bar', + }) async { + final Result result = await _connection.execute( + Sql.named( + ''' + SELECT as_of + FROM market_data_snapshots + WHERE symbol = @symbol + AND metric = @metric + AND timeframe = @timeframe + ORDER BY as_of DESC + LIMIT 1 + ''', + ), + parameters: { + 'symbol': symbol, + 'metric': metric, + 'timeframe': timeframe, + }, + ); + if (result.isEmpty) { + return null; + } + return (result.first[0]! as DateTime).toUtc(); + } + + /// Symbols from [symbols] that already have a bar for the UTC slot [slotStart]. + /// + /// A row counts when [raw.slot_start] matches the canonical wire form, when + /// [raw.slot_start] or [as_of] bucket to the same UTC 4-hour boundary as + /// [slotStart] (same rule as [MarketHistoryFourHourSlot.slotStartContaining]). + Future> symbolsWithBarForSlot({ + required List symbols, + required DateTime slotStart, + required String timeframe, + String metric = 'bar', + }) async { + if (symbols.isEmpty) { + return {}; + } + final DateTime start = + MarketHistoryFourHourSlot.slotStartContaining(slotStart); + final String slotStartWire = MarketHistoryFourHourSlot.slotStartWire(start); + final Result result = await _connection.execute( + Sql.named( + ''' + SELECT DISTINCT symbol + FROM market_data_snapshots + WHERE metric = @metric + AND timeframe = @timeframe + AND symbol = ANY(@symbols) + AND ( + raw->>'slot_start' = @slot_start_wire + OR ( + raw->>'slot_start' IS NOT NULL + AND ${_slotStartBucketSql('(raw->>\'slot_start\')::timestamptz')} + = @slot_start + ) + OR ${_slotStartBucketSql('as_of')} = @slot_start + ) + ''', + ), + parameters: { + 'metric': metric, + 'timeframe': timeframe, + 'symbols': symbols, + 'slot_start_wire': slotStartWire, + 'slot_start': start, + }, + ); + return result + .map((ResultRow row) => row[0]! as String) + .toSet(); + } + + /// UTC 4-hour slot left edge for [timestampExpr] (timestamptz SQL expression). + static String _slotStartBucketSql(String timestampExpr) { + return ''' +( + date_trunc('day', $timestampExpr AT TIME ZONE 'UTC') + + (div(extract(hour from $timestampExpr AT TIME ZONE 'UTC')::int, 4) * 4) + * interval '1 hour' +) AT TIME ZONE 'UTC' +'''; + } + /// Newest snapshot for [symbol] and [metric] by [as_of]. Future latestForSymbol( String symbol, @@ -111,15 +302,15 @@ class MarketDataDb { assetClass: row[2]! as String, feed: row[3]! as String, metric: row[4]! as String, - price: _readOptionalNumeric(row[5]), - volume: _readOptionalNumeric(row[6]), + price: readOptionalNumeric(row[5]), + volume: readOptionalNumeric(row[6]), asOf: (row[7]! as DateTime).toUtc(), raw: raw, createdAt: (row[9]! as DateTime).toUtc(), ); } - static num? _readOptionalNumeric(Object? value) { + static num? readOptionalNumeric(Object? value) { if (value == null) { return null; } diff --git a/server/lib/trading/market_data_history.dart b/server/lib/trading/market_data_history.dart new file mode 100644 index 0000000..2dff86e --- /dev/null +++ b/server/lib/trading/market_data_history.dart @@ -0,0 +1,518 @@ +import 'package:postgres/postgres.dart'; + +import '../alpaca/alpaca_market_data_client.dart'; +import '../alpaca/alpaca_models.dart'; +import 'backfill_sync_item.dart'; +import 'market_data_db.dart'; +import 'market_history_api_rate_limiter.dart'; +import 'market_history_bar_placeholder.dart'; +import 'market_history_config.dart'; +import 'market_history_four_hour_slot.dart'; +import 'market_history_trading_calendar.dart'; +import 'sync_run_recorder.dart'; +import 'tradable_assets_db.dart'; + +/// Result of persisting one Alpaca bars batch for a single slot. +class PersistBarsResult { + const PersistBarsResult({ + required this.written, + required this.symbolsWritten, + required this.notInResponse, + required this.emptyInResponse, + required this.wrongSlotBarTimes, + }); + + final int written; + final Set symbolsWritten; + final List notInResponse; + final List emptyInResponse; + + /// Symbol → comma-separated bar [t] values outside the requested slot. + final Map wrongSlotBarTimes; + + /// Full error when [requested] symbols were fetched but not all persisted + /// and [placeholdersWritten] did not cover the remainder. + String? errorIfIncomplete({ + required List requested, + required DateTime slotStart, + required String timeframe, + int placeholdersWritten = 0, + }) { + if (requested.isEmpty) { + return null; + } + final List unpersisted = requested + .where((String symbol) => !symbolsWritten.contains(symbol)) + .toList(growable: false); + if (unpersisted.isEmpty) { + return null; + } + if (placeholdersWritten >= unpersisted.length) { + return null; + } + + final String slotWire = MarketHistoryFourHourSlot.slotStartWire(slotStart); + final List parts = [ + 'Alpaca returned no persistable $timeframe bars', + 'slot=$slotWire', + 'requested=${requested.join(",")}', + 'unpersisted=${unpersisted.join(",")}', + ]; + if (notInResponse.isNotEmpty) { + parts.add('missing_from_response=${notInResponse.join(",")}'); + } + if (emptyInResponse.isNotEmpty) { + parts.add('empty_bar_series=${emptyInResponse.join(",")}'); + } + if (wrongSlotBarTimes.isNotEmpty) { + parts.add( + 'wrong_slot_bars=${wrongSlotBarTimes.entries.map((MapEntry e) => '${e.key}:${e.value}').join("|")}', + ); + } + if (written == 0) { + parts.add('rows_written=0'); + } + return parts.join('; '); + } +} + +/// Outcome of one [MarketDataHistorySync.runOnce] invocation. +class MarketDataHistorySyncResult { + MarketDataHistorySyncResult({ + required this.rowsWritten, + required this.startedAt, + required this.finishedAt, + this.error, + this.slotsSynced = 0, + }); + + final int rowsWritten; + final DateTime startedAt; + final DateTime finishedAt; + final String? error; + + /// Number of completed 4-hour slots written in this run. + final int slotsSynced; + + bool get succeeded => error == null; +} + +/// One ended UTC slot and the symbols still missing a bar for that slot. +class MarketHistorySlotFetchPlan { + const MarketHistorySlotFetchPlan({ + required this.slotStart, + required this.symbols, + }); + + final DateTime slotStart; + final List symbols; +} + +/// Backfill: one Alpaca `4Hour` request per ended UTC slot × symbol batch. +/// +/// Chooses work from the most recently completed slot backward until the rolling +/// window is full. Only symbols missing that specific slot are requested. +/// +/// Throttles to [apiRequestsPerMinute]. On HTTP 429, waits [rateLimitCooldown], +/// retries once, then stops the run so partial rows are kept for the next tick. +class MarketDataHistorySync { + MarketDataHistorySync({ + required AlpacaMarketDataClient marketDataClient, + required TradableAssetsDb tradableAssetsDb, + required MarketDataDb marketDataDb, + required Connection connection, + this.batchSize = MarketHistoryConfig.historySyncBatchSize, + this.maxSymbols = MarketHistoryConfig.historySyncMaxSymbols, + this.windowDays = MarketHistoryConfig.windowDays, + this.timeframe = MarketHistoryConfig.barTimeframe, + this.feed = 'iex', + int? apiRequestsPerMinute, + MarketHistoryApiRateLimiter? rateLimiter, + Duration rateLimitCooldown = MarketHistoryConfig.rateLimitCooldown, + Future Function(Duration delay)? sleep, + }) : _marketDataClient = marketDataClient, + _tradableAssetsDb = tradableAssetsDb, + _marketDataDb = marketDataDb, + _recorder = SyncRunRecorder(connection), + _rateLimiter = rateLimiter ?? + MarketHistoryApiRateLimiter( + requestsPerMinute: + apiRequestsPerMinute ?? MarketHistoryConfig.apiRequestsPerMinute, + ), + _rateLimitCooldown = rateLimitCooldown, + _sleep = sleep ?? Future.delayed; + + final AlpacaMarketDataClient _marketDataClient; + final TradableAssetsDb _tradableAssetsDb; + final MarketDataDb _marketDataDb; + final SyncRunRecorder _recorder; + final MarketHistoryApiRateLimiter _rateLimiter; + final Duration _rateLimitCooldown; + final Future Function(Duration delay) _sleep; + + final int batchSize; + final int maxSymbols; + final int windowDays; + final String timeframe; + final String feed; + + static const String kind = 'backfill'; + + Future runOnce({DateTime? now}) async { + final DateTime tick = (now ?? DateTime.now()).toUtc(); + + final SyncRunOutcome outcome = await _recorder.record( + kind, + () => _syncBody(tick), + now: tick, + ); + + return MarketDataHistorySyncResult( + rowsWritten: outcome.rowsWritten, + startedAt: outcome.startedAt, + finishedAt: outcome.finishedAt, + error: outcome.error, + slotsSynced: outcome.slotsSynced ?? 0, + ); + } + + /// Whether any symbol is missing a completed slot in the rolling window. + Future hasPendingSlots(DateTime now) async { + final List symbols = await _activeSymbols(); + if (symbols.isEmpty) { + return false; + } + final List plans = + await _pendingSlotFetchPlans(now, symbols); + return plans.isNotEmpty; + } + + Future _syncBody(DateTime now) async { + List symbols = await _activeSymbols(); + if (symbols.isEmpty) { + return const SyncRunCounts(); + } + + final List fetchPlans = + await _pendingSlotFetchPlans(now, symbols); + if (fetchPlans.isEmpty) { + return const SyncRunCounts(); + } + + int rowsWritten = 0; + int slotsSynced = 0; + final List batchErrors = []; + final Map> backfillItemsBySlot = >{}; + bool stopForRateLimit = false; + + for (final MarketHistorySlotFetchPlan plan in fetchPlans) { + if (stopForRateLimit) { + break; + } + + final DateTime slotStart = plan.slotStart; + final DateTime slotEnd = + MarketHistoryFourHourSlot.endInclusive(slotStart); + bool slotWrote = false; + + for (final List batch in chunkList(plan.symbols, batchSize)) { + if (stopForRateLimit) { + break; + } + + final Set alreadySynced = + await _marketDataDb.symbolsWithBarForSlot( + symbols: batch, + slotStart: slotStart, + timeframe: timeframe, + ); + final List toFetch = batch + .where((String symbol) => !alreadySynced.contains(symbol)) + .toList(growable: false); + if (toFetch.isEmpty) { + continue; + } + + backfillItemsBySlot + .putIfAbsent(slotStart, () => {}) + .addAll(toFetch); + + try { + final AlpacaBarsResponse response = await _fetchBarsWithRateLimitRetry( + symbols: toFetch, + slotStart: slotStart, + slotEnd: slotEnd, + ); + + final PersistBarsResult persisted = await _persistBars( + response: response, + batch: toFetch, + slotStart: slotStart, + ); + rowsWritten += persisted.written; + final int placeholders = await _writeNoDataPlaceholders( + symbols: toFetch, + persisted: persisted, + slotStart: slotStart, + checkedAt: now, + ); + rowsWritten += placeholders; + if (persisted.written > 0 || placeholders > 0) { + slotWrote = true; + } + final String? emptyDataError = persisted.errorIfIncomplete( + requested: toFetch, + slotStart: slotStart, + timeframe: timeframe, + placeholdersWritten: placeholders, + ); + if (emptyDataError != null && + !_suppressEmptyMarketError( + slotStart: slotStart, + placeholdersWritten: placeholders, + )) { + batchErrors.add(emptyDataError); + } + } on AlpacaMarketDataException catch (e) { + if (e.isRateLimited) { + batchErrors.add( + 'rate limited after ${_rateLimitCooldown.inSeconds}s cooldown; ' + 'partial sync saved ($rowsWritten rows); resume on next run', + ); + stopForRateLimit = true; + break; + } + batchErrors.add( + 'slot ${slotStart.toIso8601String()} batch ${toFetch.join(",")}: $e', + ); + } on Object catch (e) { + batchErrors.add( + 'slot ${slotStart.toIso8601String()} batch ${toFetch.join(",")}: $e', + ); + } + } + + if (slotWrote) { + slotsSynced++; + } + } + + return SyncRunCounts( + rowsWritten: rowsWritten, + error: batchErrors.isEmpty ? null : batchErrors.join('; '), + slotsSynced: slotsSynced, + backfillItems: _backfillItemsFromMap(backfillItemsBySlot), + ); + } + + static List _backfillItemsFromMap( + Map> itemsBySlot, + ) { + if (itemsBySlot.isEmpty) { + return []; + } + final List slotStarts = itemsBySlot.keys.toList() + ..sort((DateTime a, DateTime b) => b.compareTo(a)); + return slotStarts + .map( + (DateTime slotStart) => BackfillSyncItem( + slotStart: slotStart, + symbols: itemsBySlot[slotStart]!.toList()..sort(), + ), + ) + .toList(growable: false); + } + + Future _fetchBarsWithRateLimitRetry({ + required List symbols, + required DateTime slotStart, + required DateTime slotEnd, + }) async { + for (int attempt = 0; attempt < 2; attempt++) { + await _rateLimiter.acquire(); + try { + return await _marketDataClient.getBarsRange( + symbols: symbols, + timeframe: timeframe, + start: slotStart, + end: slotEnd, + ); + } on AlpacaMarketDataException catch (e) { + if (!e.isRateLimited || attempt == 1) { + rethrow; + } + await _sleep(_rateLimitCooldown); + } + } + throw StateError('unreachable fetch retry loop'); + } + + Future _persistBars({ + required AlpacaBarsResponse response, + required List batch, + required DateTime slotStart, + }) async { + int written = 0; + final Set batchSymbols = batch.toSet(); + final Set symbolsWritten = {}; + final List notInResponse = []; + final List emptyInResponse = []; + final Map wrongSlotBarTimes = {}; + final DateTime plannedSlot = + MarketHistoryFourHourSlot.slotStartContaining(slotStart); + final String slotWire = MarketHistoryFourHourSlot.slotStartWire(plannedSlot); + + for (final String symbol in batch) { + if (!response.barsBySymbol.containsKey(symbol)) { + notInResponse.add(symbol); + continue; + } + final List bars = response.barsBySymbol[symbol]!; + if (bars.isEmpty) { + emptyInResponse.add(symbol); + continue; + } + } + + for (final MapEntry> entry + in response.barsBySymbol.entries) { + if (!batchSymbols.contains(entry.key)) { + continue; + } + final List rejectedTimes = []; + for (final AlpacaBar bar in entry.value) { + final DateTime barAt = bar.timestamp.toUtc(); + final DateTime barSlot = + MarketHistoryFourHourSlot.slotStartContaining(barAt); + if (!barSlot.isAtSameMomentAs(plannedSlot)) { + rejectedTimes.add(MarketHistoryFourHourSlot.wireUtc(barAt)); + continue; + } + await _marketDataDb.upsertSnapshot( + symbol: entry.key, + metric: 'bar', + timeframe: timeframe, + feed: feed, + price: bar.close, + volume: bar.volume, + asOf: barSlot, + raw: { + 'o': bar.open, + 'h': bar.high, + 'l': bar.low, + 'c': bar.close, + 'v': bar.volume, + 't': MarketHistoryFourHourSlot.wireUtc(barAt), + 'slot_start': slotWire, + }, + ); + written++; + symbolsWritten.add(entry.key); + } + if (rejectedTimes.isNotEmpty && !symbolsWritten.contains(entry.key)) { + wrongSlotBarTimes[entry.key] = rejectedTimes.join(','); + } + } + + return PersistBarsResult( + written: written, + symbolsWritten: symbolsWritten, + notInResponse: notInResponse, + emptyInResponse: emptyInResponse, + wrongSlotBarTimes: wrongSlotBarTimes, + ); + } + + Future _writeNoDataPlaceholders({ + required List symbols, + required PersistBarsResult persisted, + required DateTime slotStart, + required DateTime checkedAt, + }) async { + final List needPlaceholder = symbols + .where((String symbol) => !persisted.symbolsWritten.contains(symbol)) + .toList(growable: false); + if (needPlaceholder.isEmpty) { + return 0; + } + for (final String symbol in needPlaceholder) { + await _marketDataDb.upsertNoDataBarPlaceholder( + symbol: symbol, + slotStart: slotStart, + timeframe: timeframe, + feed: feed, + checkedAt: checkedAt, + source: MarketHistoryTradingCalendar.isLikelyNoRegularSession(slotStart) + ? MarketHistoryBarPlaceholder.sourceMarketClosed + : MarketHistoryBarPlaceholder.sourceAlpacaEmpty, + ); + } + return needPlaceholder.length; + } + + /// Suppress error on weekends/holidays when no-data placeholders were stored. + static bool _suppressEmptyMarketError({ + required DateTime slotStart, + required int placeholdersWritten, + }) { + return placeholdersWritten > 0 && + MarketHistoryTradingCalendar.isLikelyNoRegularSession(slotStart); + } + + Future> _activeSymbols() async { + List symbols = await _tradableAssetsDb.listActiveTradableSymbols(); + if (symbols.length > maxSymbols) { + symbols = symbols.sublist(0, maxSymbols); + } + return symbols; + } + + /// Completed slots newest-first; each plan lists symbols missing that slot. + Future> _pendingSlotFetchPlans( + DateTime now, + List symbols, + ) async { + final List completed = + MarketHistoryFourHourSlot.completedSlotStartsInWindow(now, windowDays); + if (completed.isEmpty) { + return []; + } + + final List plans = []; + for (final DateTime slotStart in completed.reversed) { + final Set synced = await _marketDataDb.symbolsWithBarForSlot( + symbols: symbols, + slotStart: slotStart, + timeframe: timeframe, + ); + final List missing = symbols + .where((String symbol) => !synced.contains(symbol)) + .toList(growable: false); + if (missing.isNotEmpty) { + plans.add( + MarketHistorySlotFetchPlan( + slotStart: slotStart, + symbols: missing, + ), + ); + } + } + return plans; + } +} + +/// Splits [items] into consecutive chunks of at most [size]. +List> chunkList(List items, int size) { + if (size <= 0) { + throw ArgumentError.value(size, 'size', 'must be positive'); + } + if (items.isEmpty) { + return >[]; + } + final List> chunks = >[]; + for (int i = 0; i < items.length; i += size) { + final int end = i + size > items.length ? items.length : i + size; + chunks.add(items.sublist(i, end)); + } + return chunks; +} diff --git a/server/lib/trading/market_data_ingest.dart b/server/lib/trading/market_data_ingest.dart index 7061d13..2298679 100644 --- a/server/lib/trading/market_data_ingest.dart +++ b/server/lib/trading/market_data_ingest.dart @@ -92,11 +92,12 @@ class MarketDataIngest { final AlpacaLatestTradeResponse latest = await _alpacaClient.getLatestTrade(symbol); _httpRequests++; - await _marketDataDb.insertSnapshot( + await _marketDataDb.upsertSnapshot( symbol: symbol, assetClass: input.assetClass, feed: input.feed, metric: 'last_trade', + timeframe: 'tick', price: latest.trade.price, volume: latest.trade.size, asOf: latest.trade.timestamp, @@ -115,11 +116,12 @@ class MarketDataIngest { if (input.metrics.contains('daily_bar')) { final AlpacaBar? bar = barsResponse.latestBar(symbol); if (bar != null) { - await _marketDataDb.insertSnapshot( + await _marketDataDb.upsertSnapshot( symbol: symbol, assetClass: input.assetClass, feed: input.feed, metric: 'daily_bar', + timeframe: 'tick', price: bar.close, volume: bar.volume, asOf: bar.timestamp, @@ -135,11 +137,12 @@ class MarketDataIngest { if (input.metrics.contains('prev_close')) { final AlpacaBar? prev = barsResponse.previousDailyBar(symbol); if (prev != null) { - await _marketDataDb.insertSnapshot( + await _marketDataDb.upsertSnapshot( symbol: symbol, assetClass: input.assetClass, feed: input.feed, metric: 'prev_close', + timeframe: 'tick', price: prev.close, volume: prev.volume, asOf: prev.timestamp, diff --git a/server/lib/trading/market_data_retention.dart b/server/lib/trading/market_data_retention.dart new file mode 100644 index 0000000..cba077b --- /dev/null +++ b/server/lib/trading/market_data_retention.dart @@ -0,0 +1,166 @@ +import 'package:postgres/postgres.dart'; + +import 'market_history_config.dart'; +import 'sync_run_recorder.dart'; + +/// Outcome of a [MarketDataRetention] cleanup pass. +class MarketDataRetentionResult { + MarketDataRetentionResult({ + required this.rowsRemoved, + required this.startedAt, + required this.finishedAt, + this.error, + }); + + final int rowsRemoved; + final DateTime startedAt; + final DateTime finishedAt; + final String? error; + + bool get succeeded => error == null; +} + +/// Prunes [market_data_snapshots] older than the rolling window. +/// +/// Phase 1 ([runCleanup]): hard-delete in batches. +/// Phase 2 ([runArchiveAndCleanup]): copy expired rows into +/// [market_data_archive] inside the same transaction, then delete. +class MarketDataRetention { + MarketDataRetention({ + required Connection connection, + this.windowDays = MarketHistoryConfig.windowDays, + this.batchSize = MarketHistoryConfig.retentionBatchSize, + void Function(String sql)? onExecute, + }) : _connection = connection, + _recorder = SyncRunRecorder(connection), + _onExecute = onExecute; + + final Connection _connection; + final SyncRunRecorder _recorder; + final void Function(String sql)? _onExecute; + + final int windowDays; + final int batchSize; + + static const String kind = 'cleanup'; + + /// Hard-delete rows with `as_of` older than [windowDays]. + Future runCleanup({DateTime? now}) { + return run(archive: false, now: now); + } + + /// Archive-then-delete for rows older than [windowDays]. + Future runArchiveAndCleanup({DateTime? now}) { + return run(archive: true, now: now); + } + + /// Dispatches to hard-delete or archive mode. + Future run({ + bool archive = false, + DateTime? now, + int? windowDays, + }) async { + final DateTime tick = (now ?? DateTime.now()).toUtc(); + final int effectiveWindow = windowDays ?? this.windowDays; + + final SyncRunOutcome outcome = await _recorder.record( + kind, + () => _cleanupBody( + now: tick, + windowDays: effectiveWindow, + archive: archive, + ), + now: tick, + ); + + return MarketDataRetentionResult( + rowsRemoved: outcome.rowsRemoved, + startedAt: outcome.startedAt, + finishedAt: outcome.finishedAt, + error: outcome.error, + ); + } + + Future _cleanupBody({ + required DateTime now, + required int windowDays, + required bool archive, + }) async { + final DateTime cutoff = now.subtract(Duration(days: windowDays)); + int totalRemoved = 0; + + while (true) { + final int removed = archive + ? await _archiveAndDeleteBatch(cutoff) + : await _deleteBatch(cutoff); + if (removed == 0) { + break; + } + totalRemoved += removed; + } + + return SyncRunCounts(rowsRemoved: totalRemoved); + } + + Future _deleteBatch(DateTime cutoff) async { + const String sql = ''' + WITH doomed AS ( + SELECT id + FROM market_data_snapshots + WHERE as_of < @cutoff + LIMIT @batch_size + ) + DELETE FROM market_data_snapshots + WHERE id IN (SELECT id FROM doomed) + RETURNING id + '''; + _onExecute?.call(sql); + final Result result = await _connection.execute( + Sql.named(sql), + parameters: { + 'cutoff': cutoff, + 'batch_size': batchSize, + }, + ); + return result.length; + } + + Future _archiveAndDeleteBatch(DateTime cutoff) async { + const String sql = ''' + WITH doomed AS ( + SELECT id, symbol, asset_class, feed, metric, timeframe, + price, volume, as_of, raw + FROM market_data_snapshots + WHERE as_of < @cutoff + LIMIT @batch_size + ), + inserted AS ( + INSERT INTO market_data_archive ( + symbol, asset_class, feed, metric, timeframe, + price, volume, as_of, raw, archived_at + ) + SELECT symbol, asset_class, feed, metric, timeframe, + price, volume, as_of, raw, now() + FROM doomed + RETURNING id + ) + DELETE FROM market_data_snapshots m + USING doomed d + WHERE m.id = d.id + RETURNING m.id + '''; + int removed = 0; + await _connection.runTx((TxSession tx) async { + _onExecute?.call(sql); + final Result archived = await tx.execute( + Sql.named(sql), + parameters: { + 'cutoff': cutoff, + 'batch_size': batchSize, + }, + ); + removed = archived.length; + }); + return removed; + } +} diff --git a/server/lib/trading/market_history_admin_actions.dart b/server/lib/trading/market_history_admin_actions.dart new file mode 100644 index 0000000..c4874cb --- /dev/null +++ b/server/lib/trading/market_history_admin_actions.dart @@ -0,0 +1,107 @@ +import 'package:postgres/postgres.dart'; + +import 'sync_run_recorder.dart'; + +typedef AdminRunStage = Future Function(DateTime now); + +class AdminTriggerResult { + AdminTriggerResult({required this.runIds}); + + final List runIds; + + Map toJson() => {'runIds': runIds}; +} + +/// On-demand market-history operations for the admin portal. +class MarketHistoryAdminActions { + MarketHistoryAdminActions({ + required Connection connection, + required AdminRunStage runUniverse, + required AdminRunStage runBackfill, + required Future Function(DateTime now, bool archive, int windowDays) + runCleanup, + this.defaultArchiveEnabled = false, + this.defaultWindowDays = 7, + }) : _connection = connection, + _recorder = SyncRunRecorder(connection), + _runUniverse = runUniverse, + _runBackfill = runBackfill, + _runCleanup = runCleanup; + + final Connection _connection; + final SyncRunRecorder _recorder; + final AdminRunStage _runUniverse; + final AdminRunStage _runBackfill; + final Future Function(DateTime now, bool archive, int windowDays) + _runCleanup; + final bool defaultArchiveEnabled; + final int defaultWindowDays; + + Future hasInProgressRun() async { + final Result rows = await _connection.execute( + ''' + SELECT 1 + FROM market_data_sync_runs + WHERE finished_at IS NULL + LIMIT 1 + ''', + ); + return rows.isNotEmpty; + } + + Future _abortPreviousSyncRuns(DateTime tick) async { + await _recorder.abortAllInProgressRuns( + now: tick, + message: 'aborted: superseded by admin trigger', + ); + } + + Future resync({DateTime? now}) async { + final DateTime tick = (now ?? DateTime.now()).toUtc(); + await _abortPreviousSyncRuns(tick); + final int beforeMaxId = await _maxRunId(); + await _runUniverse(tick); + await _runBackfill(tick); + final List runIds = await _runIdsAfter(beforeMaxId); + return AdminTriggerResult(runIds: runIds); + } + + Future cleanup({ + DateTime? now, + bool? archive, + int? windowDays, + }) async { + final DateTime tick = (now ?? DateTime.now()).toUtc(); + await _abortPreviousSyncRuns(tick); + final int beforeMaxId = await _maxRunId(); + await _runCleanup( + tick, + archive ?? defaultArchiveEnabled, + windowDays ?? defaultWindowDays, + ); + final List runIds = await _runIdsAfter(beforeMaxId); + return AdminTriggerResult(runIds: runIds); + } + + Future _maxRunId() async { + final Result rows = await _connection.execute( + 'SELECT COALESCE(MAX(id), 0) FROM market_data_sync_runs', + ); + return (rows.first[0]! as num).toInt(); + } + + Future> _runIdsAfter(int afterId) async { + final Result rows = await _connection.execute( + Sql.named( + ''' + SELECT id + FROM market_data_sync_runs + WHERE id > @after_id + ORDER BY id ASC + ''', + ), + parameters: {'after_id': afterId}, + ); + return rows.map((ResultRow row) => (row[0]! as num).toInt()).toList(); + } +} diff --git a/server/lib/trading/market_history_admin_logic.dart b/server/lib/trading/market_history_admin_logic.dart new file mode 100644 index 0000000..4fff08b --- /dev/null +++ b/server/lib/trading/market_history_admin_logic.dart @@ -0,0 +1,180 @@ +import 'backfill_sync_item.dart'; + +/// Errors that only mean Alpaca had no bars but placeholders were stored. +bool isBenignEmptyMarketError(String? error, {required int rowsWritten}) { + if (error == null || error.trim().isEmpty || rowsWritten <= 0) { + return false; + } + final String normalized = error.toLowerCase(); + if (normalized.contains('rate limited') || normalized.contains('429')) { + return false; + } + return normalized.contains('alpaca returned no persistable'); +} + +String? effectiveRunError(AdminSyncRunRecord run) { + if (isBenignEmptyMarketError(run.error, rowsWritten: run.rowsWritten)) { + return null; + } + return run.error; +} + +enum AdminRunSeverity { + ok('ok'), + warning('warning'), + error('error'), + rateLimit('rate_limit'); + + const AdminRunSeverity(this.wireValue); + final String wireValue; +} + +enum AdminRunStatus { + success('success'), + failed('failed'), + partial('partial'), + inProgress('in_progress'); + + const AdminRunStatus(this.wireValue); + final String wireValue; +} + +class AdminSyncRunRecord { + const AdminSyncRunRecord({ + required this.id, + required this.kind, + required this.startedAt, + required this.finishedAt, + required this.rowsWritten, + required this.rowsRemoved, + required this.slotsSynced, + required this.backfillItems, + required this.error, + }); + + final int id; + final String kind; + final DateTime startedAt; + final DateTime? finishedAt; + final int rowsWritten; + final int rowsRemoved; + final int slotsSynced; + final List backfillItems; + final String? error; +} + +AdminRunSeverity deriveSeverity({ + required String? error, + required DateTime startedAt, + required DateTime? finishedAt, + required DateTime now, + Duration staleThreshold = const Duration(minutes: 30), + int rowsWritten = 0, +}) { + final String? effectiveError = error; + if (effectiveError != null && effectiveError.trim().isNotEmpty) { + if (isBenignEmptyMarketError(effectiveError, rowsWritten: rowsWritten)) { + return AdminRunSeverity.ok; + } + final String normalized = effectiveError.toLowerCase(); + final bool looksRateLimited = + normalized.contains('429') || + normalized.contains('rate limit') || + normalized.contains('rate limited'); + final bool looksAlpacaError = + normalized.contains('alpacamarketdataexception') || + normalized.contains('alpacaassetsexception') || + normalized.contains('alpacatradingexception'); + final bool looks5xx = RegExp(r'\b5\d\d\b').hasMatch(normalized); + if (looksRateLimited || looksAlpacaError || looks5xx) { + return AdminRunSeverity.rateLimit; + } + return AdminRunSeverity.error; + } + + if (finishedAt == null) { + final bool stale = now.toUtc().difference(startedAt.toUtc()) >= staleThreshold; + return stale ? AdminRunSeverity.warning : AdminRunSeverity.warning; + } + + return AdminRunSeverity.ok; +} + +AdminRunStatus deriveStatus({ + required String? error, + required DateTime? finishedAt, + required int rowsWritten, + required int rowsRemoved, +}) { + if (finishedAt == null) { + return AdminRunStatus.inProgress; + } + if (isBenignEmptyMarketError(error, rowsWritten: rowsWritten)) { + return AdminRunStatus.success; + } + if (error != null && error.trim().isNotEmpty) { + if (rowsWritten > 0) { + return AdminRunStatus.partial; + } + return AdminRunStatus.failed; + } + return AdminRunStatus.success; +} + +List sortNewestFirst(Iterable runs) { + final List sorted = List.from(runs); + sorted.sort( + (AdminSyncRunRecord a, AdminSyncRunRecord b) => + b.startedAt.compareTo(a.startedAt), + ); + return sorted; +} + +List computePinned(Iterable runs) { + final List sorted = sortNewestFirst(runs); + final Set resolvedKinds = {}; + final List pinned = []; + + for (final AdminSyncRunRecord run in sorted) { + final String? error = effectiveRunError(run); + final bool unresolved = + (error != null && error.trim().isNotEmpty) || run.finishedAt == null; + if (!unresolved) { + resolvedKinds.add(run.kind); + continue; + } + if (!resolvedKinds.contains(run.kind)) { + pinned.add(run); + } + } + + return pinned; +} + +String toSummary(AdminSyncRunRecord run) { + final String? error = effectiveRunError(run); + switch (run.kind) { + case 'universe': + return '${run.rowsWritten} assets refreshed'; + case 'backfill': + final String slotNote = + run.slotsSynced > 0 ? '${run.slotsSynced} slots, ' : ''; + if (error != null && run.rowsWritten > 0) { + return 'Backfill partial: $slotNote${run.rowsWritten} rows written'; + } + if (error != null) { + return 'Backfill failed'; + } + if (isBenignEmptyMarketError(run.error, rowsWritten: run.rowsWritten)) { + return '$slotNote${run.rowsWritten} no-data placeholders stored'; + } + if (run.slotsSynced > 0) { + return '$slotNote${run.rowsWritten} bar rows written'; + } + return '${run.rowsWritten} bar rows written'; + case 'cleanup': + return '${run.rowsRemoved} rows removed'; + default: + return error == null ? 'Run completed' : 'Run failed'; + } +} diff --git a/server/lib/trading/market_history_api_rate_limiter.dart b/server/lib/trading/market_history_api_rate_limiter.dart new file mode 100644 index 0000000..12d6102 --- /dev/null +++ b/server/lib/trading/market_history_api_rate_limiter.dart @@ -0,0 +1,51 @@ +/// Throttles Alpaca market-data HTTP calls to a max count per rolling minute. +class MarketHistoryApiRateLimiter { + MarketHistoryApiRateLimiter({ + required int requestsPerMinute, + DateTime Function()? clock, + Future Function(Duration delay)? sleep, + }) : _requestsPerMinute = requestsPerMinute, + _clock = clock ?? DateTime.now, + _sleep = sleep ?? Future.delayed; + + final int _requestsPerMinute; + final DateTime Function() _clock; + final Future Function(Duration delay) _sleep; + + final List _requestTimes = []; + + /// Blocks until another request is allowed under [requestsPerMinute]. + Future acquire() async { + if (_requestsPerMinute <= 0) { + throw ArgumentError.value( + _requestsPerMinute, + 'requestsPerMinute', + 'must be positive', + ); + } + + while (true) { + final DateTime now = _clock().toUtc(); + _dropOlderThan(now.subtract(const Duration(minutes: 1))); + + if (_requestTimes.length < _requestsPerMinute) { + _requestTimes.add(now); + return; + } + + final DateTime oldest = _requestTimes.first; + final Duration wait = oldest + .add(const Duration(minutes: 1)) + .difference(now); + if (wait > Duration.zero) { + await _sleep(wait); + } + } + } + + void _dropOlderThan(DateTime cutoff) { + while (_requestTimes.isNotEmpty && !_requestTimes.first.isAfter(cutoff)) { + _requestTimes.removeAt(0); + } + } +} diff --git a/server/lib/trading/market_history_bar_placeholder.dart b/server/lib/trading/market_history_bar_placeholder.dart new file mode 100644 index 0000000..caffe01 --- /dev/null +++ b/server/lib/trading/market_history_bar_placeholder.dart @@ -0,0 +1,13 @@ +/// Marker rows written when Alpaca has no 4Hour bar for a symbol × slot. +abstract final class MarketHistoryBarPlaceholder { + static const String rawKey = 'no_data'; + static const String sourceAlpacaEmpty = 'alpaca_empty'; + static const String sourceMarketClosed = 'market_closed'; + + static bool isPlaceholder(Map? raw) => + raw?[rawKey] == true; + + /// SQL predicate: row is a real bar, not a no-data tombstone. + static const String sqlExcludePlaceholders = + "(raw->>'$rawKey' IS NULL OR raw->>'$rawKey' <> 'true')"; +} diff --git a/server/lib/trading/market_history_config.dart b/server/lib/trading/market_history_config.dart new file mode 100644 index 0000000..21a0c3f --- /dev/null +++ b/server/lib/trading/market_history_config.dart @@ -0,0 +1,34 @@ +/// Defaults for 4-hour slot market history ([MarketHistoryFourHourSlot]). +/// Env overrides via [MarketHistoryEnv] in [ServerEnv.load]. +abstract final class MarketHistoryConfig { + /// Rolling window length in calendar days (UTC). + static const int windowDays = 7; + + /// Alpaca bar aggregation for market-history backfill (six slots per UTC day). + static const String barTimeframe = '4Hour'; + + /// Width of each history slot in hours (`24 / slotsPerDay`). + static const int slotHours = 4; + + /// Symbols per Alpaca `GET /v2/stocks/bars` request (max ~100). + static const int historySyncBatchSize = 100; + + /// Hard cap on symbols synced per run (Alpaca Basic rate-limit safety). + static const int historySyncMaxSymbols = 2000; + + /// Minimum 4-hour bars required before a symbol is eligible for the + /// guess-the-move question rule. + static const int minBarsForGuess = 5; + + /// Hours before the same symbol can fire another guess question. + static const int guessCooldownHours = 24; + + /// Rows deleted per cleanup loop iteration (Postgres batched DELETE). + static const int retentionBatchSize = 5000; + + /// Max Alpaca HTTP calls per rolling minute during history backfill. + static const int apiRequestsPerMinute = 200; + + /// Wait after HTTP 429 before retrying the same bars request. + static const Duration rateLimitCooldown = Duration(minutes: 1); +} diff --git a/server/lib/trading/market_history_four_hour_slot.dart b/server/lib/trading/market_history_four_hour_slot.dart new file mode 100644 index 0000000..9bab14a --- /dev/null +++ b/server/lib/trading/market_history_four_hour_slot.dart @@ -0,0 +1,89 @@ +/// Six UTC 4-hour slots per day. Sync only [completedSlotStartsInWindow]. +abstract final class MarketHistoryFourHourSlot { + static const int slotHours = 4; + static const int slotsPerDay = 24 ~/ slotHours; + static const String alpacaTimeframe = '4Hour'; + + /// Left edge of the 4-hour bucket containing [instant] (UTC). + static DateTime slotStartContaining(DateTime instant) { + final DateTime u = instant.toUtc(); + final int slotHour = (u.hour ~/ slotHours) * slotHours; + return DateTime.utc(u.year, u.month, u.day, slotHour); + } + + /// Inclusive end of the slot for Alpaca `start`/`end` (Option A: 00:00–03:59:59). + static DateTime endInclusive(DateTime slotStart) { + return slotStart + .add(const Duration(hours: slotHours)) + .subtract(const Duration(seconds: 1)); + } + + /// Exclusive end (current slot begins here). + static DateTime endExclusive(DateTime slotStart) { + return slotStart.add(const Duration(hours: slotHours)); + } + + /// `true` when [now] is at or after the end of the slot that began at [slotStart]. + static bool hasEnded(DateTime slotStart, DateTime now) { + return !now.toUtc().isBefore(endExclusive(slotStart)); + } + + /// Start of the most recently completed slot (never the in-progress slot). + static DateTime lastCompletedSlotStart(DateTime now) { + final DateTime current = slotStartContaining(now); + return current.subtract(const Duration(hours: slotHours)); + } + + /// Earliest slot start included in a [windowDays] rolling window ending at [now]. + static DateTime windowFirstSlotStart(DateTime now, int windowDays) { + final DateTime windowStart = + now.toUtc().subtract(Duration(days: windowDays)); + return slotStartContaining(windowStart); + } + + /// Completed slot starts from the rolling window through [lastCompletedSlotStart]. + static List completedSlotStartsInWindow( + DateTime now, + int windowDays, + ) { + final DateTime last = lastCompletedSlotStart(now); + final DateTime first = windowFirstSlotStart(now, windowDays); + if (last.isBefore(first)) { + return []; + } + + final List slots = []; + DateTime cursor = first; + while (!cursor.isAfter(last)) { + if (hasEnded(cursor, now)) { + slots.add(cursor); + } + cursor = cursor.add(const Duration(hours: slotHours)); + } + return slots; + } + + /// Canonical UTC wire form: `YYYY-MM-DDTHH:MM:SSZ` (no fractional seconds). + /// + /// Used for Alpaca bar-range query params and [raw.slot_start] / [raw.t] in + /// [market_data_snapshots] so fetch, persist, and gap checks always agree. + static String wireUtc(DateTime value) { + final DateTime u = value.toUtc(); + String two(int n) => n.toString().padLeft(2, '0'); + return '${u.year.toString().padLeft(4, '0')}-' + '${two(u.month)}-${two(u.day)}T' + '${two(u.hour)}:${two(u.minute)}:${two(u.second)}Z'; + } + + /// Wire form for the left edge of the 4-hour slot containing [slotStart]. + static String slotStartWire(DateTime slotStart) => + wireUtc(slotStartContaining(slotStart)); + + /// Parses a [wireUtc] / Alpaca RFC3339 timestamp, or `null` when invalid. + static DateTime? parseWire(String? wire) { + if (wire == null || wire.isEmpty) { + return null; + } + return DateTime.tryParse(wire)?.toUtc(); + } +} diff --git a/server/lib/trading/market_history_query.dart b/server/lib/trading/market_history_query.dart new file mode 100644 index 0000000..660b005 --- /dev/null +++ b/server/lib/trading/market_history_query.dart @@ -0,0 +1,138 @@ +import 'dart:math'; + +import 'package:postgres/postgres.dart'; + +import 'market_data_db.dart' show MarketDataDb; +import 'market_history_bar_placeholder.dart'; +import 'market_history_config.dart'; +import 'tradable_assets_db.dart'; + +/// One symbol's rolling-window 4-hour bar summary for the guessing game. +class WeeklyMover { + WeeklyMover({ + required this.symbol, + required this.openClose, + required this.currentClose, + required this.days, + }); + + final String symbol; + final num openClose; + final num currentClose; + + /// Number of 4-hour bars in the window (≥ [MarketHistoryConfig.minBarsForGuess]). + final int days; +} + +/// Read-side queries over 4-hour history bars for question rules. +class MarketHistoryQuery { + MarketHistoryQuery({ + required Connection connection, + TradableAssetsDb? tradableAssetsDb, + }) : _connection = connection, + _tradableAssetsDb = tradableAssetsDb ?? TradableAssetsDb(connection); + + final Connection _connection; + final TradableAssetsDb _tradableAssetsDb; + + /// Symbols with ≥ [minBars] 4-hour closes in [`asOf` − window, `asOf`), whose + /// newest bar is not older than [maxStalenessDays] before [asOf]. + /// + /// When [random] is set, eligible rows are sorted by symbol then shuffled for + /// a stable pick order across runs with the same seed. + Future> weeklyMovers({ + required DateTime asOf, + int minBars = MarketHistoryConfig.minBarsForGuess, + int windowDays = MarketHistoryConfig.windowDays, + int maxStalenessDays = 2, + Random? random, + }) async { + final DateTime until = asOf.toUtc(); + final DateTime since = + until.subtract(Duration(days: windowDays)); + final DateTime freshSince = + until.subtract(Duration(days: maxStalenessDays)); + + final List active = + await _tradableAssetsDb.listActiveTradableSymbols(); + if (active.isEmpty) { + return []; + } + + final Result rows = await _connection.execute( + Sql.named( + ''' + WITH bars AS ( + SELECT symbol, as_of, price + FROM market_data_snapshots + WHERE metric = 'bar' + AND timeframe = @timeframe + AND as_of >= @since + AND as_of < @until + AND symbol = ANY(@symbols) + AND ${MarketHistoryBarPlaceholder.sqlExcludePlaceholders} + AND price IS NOT NULL + ), + agg AS ( + SELECT + symbol, + COUNT(*)::int AS bar_count, + MIN(as_of) AS oldest_as_of, + MAX(as_of) AS newest_as_of + FROM bars + GROUP BY symbol + HAVING COUNT(*) >= @min_bars + AND MAX(as_of) >= @fresh_since + ) + SELECT + a.symbol, + a.bar_count, + ( + SELECT b.price + FROM bars b + WHERE b.symbol = a.symbol AND b.as_of = a.oldest_as_of + LIMIT 1 + ) AS open_close, + ( + SELECT b.price + FROM bars b + WHERE b.symbol = a.symbol AND b.as_of = a.newest_as_of + LIMIT 1 + ) AS current_close + FROM agg a + ORDER BY a.symbol ASC + ''', + ), + parameters: { + 'since': since, + 'until': until, + 'symbols': active, + 'timeframe': MarketHistoryConfig.barTimeframe, + 'min_bars': minBars, + 'fresh_since': freshSince, + }, + ); + + final List movers = []; + for (final ResultRow row in rows) { + final num? openClose = MarketDataDb.readOptionalNumeric(row[2]); + final num? currentClose = MarketDataDb.readOptionalNumeric(row[3]); + if (openClose == null || currentClose == null) { + continue; + } + movers.add( + WeeklyMover( + symbol: row[0]! as String, + openClose: openClose, + currentClose: currentClose, + days: (row[1]! as num).toInt(), + ), + ); + } + + if (random != null && movers.length > 1) { + movers.shuffle(random); + } + return movers; + } +} diff --git a/server/lib/trading/market_history_question_audit.dart b/server/lib/trading/market_history_question_audit.dart new file mode 100644 index 0000000..068cc7f --- /dev/null +++ b/server/lib/trading/market_history_question_audit.dart @@ -0,0 +1,457 @@ +import 'dart:convert'; + +import 'package:postgres/postgres.dart'; + +import 'market_data_db.dart' show MarketDataDb; +import 'market_history_bar_placeholder.dart'; +import 'market_history_config.dart'; +import 'market_history_four_hour_slot.dart'; +import 'tradable_assets_db.dart'; + +/// One 4-hour bar snapshot used in question audit comparisons. +class QuestionAuditBarSlot { + QuestionAuditBarSlot({ + required this.asOf, + required this.avgPrice, + required this.volume, + this.open, + this.high, + this.low, + this.close, + }); + + final DateTime asOf; + final num? open; + final num? high; + final num? low; + final num? close; + final num avgPrice; + final num volume; + + Map toJson() => { + 'asOf': asOf.toIso8601String(), + if (open != null) 'open': open, + if (high != null) 'high': high, + if (low != null) 'low': low, + if (close != null) 'close': close, + 'avgPrice': avgPrice, + 'volume': volume, + }; +} + +/// One tradable symbol's last-two 4-hour bar deltas for question auditing. +class QuestionAuditAsset { + QuestionAuditAsset({ + required this.symbol, + required this.priceDelta, + required this.volumeDelta, + required this.olderSlot, + required this.newerSlot, + }); + + final String symbol; + final num priceDelta; + final num volumeDelta; + final QuestionAuditBarSlot olderSlot; + final QuestionAuditBarSlot newerSlot; + + Map toJson() => { + 'symbol': symbol, + 'priceDelta': priceDelta, + 'volumeDelta': volumeDelta, + 'olderSlot': olderSlot.toJson(), + 'newerSlot': newerSlot.toJson(), + }; +} + +/// OHLC average for a bar; falls back to [closePrice] when [raw] lacks legs. +num averageBarPrice({required num? closePrice, Map? raw}) { + if (raw != null) { + final num? open = MarketDataDb.readOptionalNumeric(raw['o']); + final num? high = MarketDataDb.readOptionalNumeric(raw['h']); + final num? low = MarketDataDb.readOptionalNumeric(raw['l']); + final num? close = MarketDataDb.readOptionalNumeric(raw['c']); + if (open != null && high != null && low != null && close != null) { + return (open + high + low + close) / 4; + } + } + if (closePrice != null) { + return closePrice; + } + throw ArgumentError('Bar has no OHLC or close price'); +} + +QuestionAuditBarSlot barRowToSlot(_BarRow row) { + final Map? raw = row.raw; + final num? open = raw == null ? null : MarketDataDb.readOptionalNumeric(raw['o']); + final num? high = raw == null ? null : MarketDataDb.readOptionalNumeric(raw['h']); + final num? low = raw == null ? null : MarketDataDb.readOptionalNumeric(raw['l']); + final num? close = + raw == null ? row.closePrice : MarketDataDb.readOptionalNumeric(raw['c']); + return QuestionAuditBarSlot( + asOf: row.asOf, + open: open, + high: high, + low: low, + close: close ?? row.closePrice, + avgPrice: averageBarPrice(closePrice: row.closePrice, raw: raw), + volume: row.volume!, + ); +} + +/// Paginated question-audit payload for one [compareUntil] exclusive bound. +class QuestionAuditPage { + QuestionAuditPage({ + required this.compareUntil, + required this.newerSlotStart, + required this.olderSlotStart, + required this.windowDays, + required this.assets, + required this.canStepOlder, + required this.canStepNewer, + this.stepOlderCompareUntil, + this.stepNewerCompareUntil, + }); + + /// Exclusive upper bound for this page (`endExclusive(newerSlotStart)`). + final DateTime compareUntil; + final DateTime newerSlotStart; + final DateTime olderSlotStart; + final int windowDays; + final List assets; + final bool canStepOlder; + final bool canStepNewer; + final DateTime? stepOlderCompareUntil; + final DateTime? stepNewerCompareUntil; + + Map toJson() => { + 'compareUntil': compareUntil.toIso8601String(), + 'newerSlotStart': newerSlotStart.toIso8601String(), + 'olderSlotStart': olderSlotStart.toIso8601String(), + 'windowDays': windowDays, + 'canStepOlder': canStepOlder, + 'canStepNewer': canStepNewer, + if (stepOlderCompareUntil != null) + 'stepOlderCompareUntil': stepOlderCompareUntil!.toIso8601String(), + if (stepNewerCompareUntil != null) + 'stepNewerCompareUntil': stepNewerCompareUntil!.toIso8601String(), + 'assets': assets.map((QuestionAuditAsset a) => a.toJson()).toList(), + }; +} + +/// Default view: last two **completed** 4-hour slots (newer = last completed). +DateTime questionAuditDefaultCompareUntil(DateTime now) { + final DateTime last = + MarketHistoryFourHourSlot.lastCompletedSlotStart(now.toUtc()); + return MarketHistoryFourHourSlot.endExclusive(last); +} + +/// Earliest [compareUntil] that still pairs two slots in the rolling window. +DateTime questionAuditMinCompareUntil(DateTime now, int windowDays) { + final DateTime first = MarketHistoryFourHourSlot.windowFirstSlotStart( + now.toUtc(), + windowDays, + ); + final DateTime minNewerSlot = first.add( + const Duration(hours: MarketHistoryFourHourSlot.slotHours), + ); + return MarketHistoryFourHourSlot.endExclusive(minNewerSlot); +} + +/// The newer bar's slot start for a page keyed by [compareUntil]. +/// +/// [compareUntil] is always `endExclusive(newerSlotStart)`. +DateTime questionAuditNewerSlotStart(DateTime compareUntil) { + return MarketHistoryFourHourSlot.slotStartContaining( + compareUntil.toUtc().subtract( + const Duration(hours: MarketHistoryFourHourSlot.slotHours), + ), + ); +} + +/// Snaps [requested] to a slot-aligned bound within [minUntil, maxUntil]. +DateTime snapQuestionAuditCompareUntil({ + required DateTime requested, + required DateTime minUntil, + required DateTime maxUntil, +}) { + final DateTime r = requested.toUtc(); + if (r.isAfter(maxUntil)) { + return maxUntil; + } + final DateTime snapped = MarketHistoryFourHourSlot.endExclusive( + questionAuditNewerSlotStart(r), + ); + if (snapped.isBefore(minUntil)) { + return minUntil; + } + if (snapped.isAfter(maxUntil)) { + return maxUntil; + } + return snapped; +} + +/// Pair of slot starts for the page keyed by [compareUntil]. +(DateTime newer, DateTime older) questionAuditSlotPair(DateTime compareUntil) { + final DateTime newer = questionAuditNewerSlotStart(compareUntil); + final DateTime older = newer.subtract( + const Duration(hours: MarketHistoryFourHourSlot.slotHours), + ); + return (newer, older); +} + +/// Steps back: newer becomes previous older (e.g. #1 vs #2 → #2 vs #3). +DateTime questionAuditStepOlderCompareUntil({ + required DateTime compareUntil, + required DateTime now, +}) { + final DateTime newerSlot = questionAuditNewerSlotStart(compareUntil); + final DateTime priorNewerSlot = newerSlot.subtract( + const Duration(hours: MarketHistoryFourHourSlot.slotHours), + ); + return MarketHistoryFourHourSlot.endExclusive(priorNewerSlot); +} + +/// Steps forward one completed slot, capped at [maxUntil]. +DateTime questionAuditStepNewerCompareUntil({ + required DateTime compareUntil, + required DateTime maxUntil, +}) { + final DateTime newerSlot = questionAuditNewerSlotStart(compareUntil); + final DateTime nextNewerSlot = newerSlot.add( + const Duration(hours: MarketHistoryFourHourSlot.slotHours), + ); + final DateTime candidate = + MarketHistoryFourHourSlot.endExclusive(nextNewerSlot); + return candidate.isAfter(maxUntil) ? maxUntil : candidate; +} + +/// Read-side audit of the two newest 4-hour bars per active tradable symbol. +class MarketHistoryQuestionAudit { + MarketHistoryQuestionAudit({ + required Connection connection, + TradableAssetsDb? tradableAssetsDb, + }) : _connection = connection, + _tradableAssetsDb = tradableAssetsDb ?? TradableAssetsDb(connection); + + final Connection _connection; + final TradableAssetsDb _tradableAssetsDb; + + Future page({ + required DateTime now, + DateTime? compareUntil, + int windowDays = MarketHistoryConfig.windowDays, + String timeframe = MarketHistoryConfig.barTimeframe, + }) async { + final DateTime n = now.toUtc(); + final DateTime calendarMax = questionAuditDefaultCompareUntil(n); + final DateTime minUntil = questionAuditMinCompareUntil(n, windowDays); + final DateTime? latestBarSlot = await _latestBarSlotStart(timeframe: timeframe); + final DateTime maxUntil = _effectiveMaxCompareUntil( + calendarMax: calendarMax, + latestBarSlot: latestBarSlot, + ); + + final DateTime until = compareUntil == null + ? maxUntil + : snapQuestionAuditCompareUntil( + requested: compareUntil, + minUntil: minUntil, + maxUntil: maxUntil, + ); + + final (DateTime newerSlotStart, DateTime olderSlotStart) = + questionAuditSlotPair(until); + + final bool canStepOlder = until.isAfter(minUntil); + final bool canStepNewer = until.isBefore(maxUntil); + + final List assets = await assetsForSlotPair( + newerSlotStart: newerSlotStart, + olderSlotStart: olderSlotStart, + timeframe: timeframe, + ); + + return QuestionAuditPage( + compareUntil: until, + newerSlotStart: newerSlotStart, + olderSlotStart: olderSlotStart, + windowDays: windowDays, + assets: assets, + canStepOlder: canStepOlder, + canStepNewer: canStepNewer, + stepOlderCompareUntil: canStepOlder + ? questionAuditStepOlderCompareUntil( + compareUntil: until, + now: n, + ) + : null, + stepNewerCompareUntil: canStepNewer + ? questionAuditStepNewerCompareUntil( + compareUntil: until, + maxUntil: maxUntil, + ) + : null, + ); + } + + DateTime _effectiveMaxCompareUntil({ + required DateTime calendarMax, + required DateTime? latestBarSlot, + }) { + if (latestBarSlot == null) { + return calendarMax; + } + final DateTime dataMax = + MarketHistoryFourHourSlot.endExclusive(latestBarSlot); + return dataMax.isBefore(calendarMax) ? dataMax : calendarMax; + } + + /// Latest synced 4-hour bar slot, or `null` when history is empty. + Future _latestBarSlotStart({required String timeframe}) async { + final Result result = await _connection.execute( + Sql.named( + ''' + SELECT MAX(as_of) AS latest + FROM market_data_snapshots + WHERE metric = 'bar' + AND timeframe = @timeframe + AND price IS NOT NULL + AND ${MarketHistoryBarPlaceholder.sqlExcludePlaceholders} + ''', + ), + parameters: {'timeframe': timeframe}, + ); + if (result.isEmpty || result.first[0] == null) { + return null; + } + return MarketHistoryFourHourSlot.slotStartContaining( + (result.first[0]! as DateTime).toUtc(), + ); + } + + /// Loads the bar row for each symbol at [newerSlotStart] and [olderSlotStart]. + Future> assetsForSlotPair({ + required DateTime newerSlotStart, + required DateTime olderSlotStart, + String timeframe = MarketHistoryConfig.barTimeframe, + }) async { + final DateTime newer = MarketHistoryFourHourSlot.slotStartContaining( + newerSlotStart.toUtc(), + ); + final DateTime older = MarketHistoryFourHourSlot.slotStartContaining( + olderSlotStart.toUtc(), + ); + final String newerWire = MarketHistoryFourHourSlot.slotStartWire(newer); + final String olderWire = MarketHistoryFourHourSlot.slotStartWire(older); + + final List active = + await _tradableAssetsDb.listActiveTradableSymbols(); + if (active.isEmpty) { + return []; + } + + final Result rows = await _connection.execute( + Sql.named( + ''' + SELECT symbol, as_of, price, volume, raw + FROM market_data_snapshots + WHERE metric = 'bar' + AND timeframe = @timeframe + AND symbol = ANY(@symbols) + AND price IS NOT NULL + AND ${MarketHistoryBarPlaceholder.sqlExcludePlaceholders} + AND ( + as_of = @newer_slot + OR as_of = @older_slot + OR raw->>'slot_start' = @newer_wire + OR raw->>'slot_start' = @older_wire + ) + ORDER BY symbol ASC, as_of DESC + ''', + ), + parameters: { + 'symbols': active, + 'timeframe': timeframe, + 'newer_slot': newer, + 'older_slot': older, + 'newer_wire': newerWire, + 'older_wire': olderWire, + }, + ); + + final Map> bySymbol = + >{}; + for (final ResultRow row in rows) { + final String symbol = row[0]! as String; + final DateTime asOf = MarketHistoryFourHourSlot.slotStartContaining( + (row[1]! as DateTime).toUtc(), + ); + final Map? raw = _decodeRaw(row[4]); + bySymbol.putIfAbsent(symbol, () => {})[asOf] = _BarRow( + asOf: asOf, + closePrice: MarketDataDb.readOptionalNumeric(row[2]), + volume: MarketDataDb.readOptionalNumeric(row[3]), + raw: raw, + ); + } + + final List assets = []; + for (final MapEntry> entry + in bySymbol.entries) { + final _BarRow? newerRow = entry.value[newer]; + final _BarRow? olderRow = entry.value[older]; + if (newerRow == null || + olderRow == null || + newerRow.volume == null || + olderRow.volume == null) { + continue; + } + try { + final QuestionAuditBarSlot newerBar = barRowToSlot(newerRow); + final QuestionAuditBarSlot olderBar = barRowToSlot(olderRow); + assets.add( + QuestionAuditAsset( + symbol: entry.key, + priceDelta: newerBar.avgPrice - olderBar.avgPrice, + volumeDelta: newerBar.volume - olderBar.volume, + olderSlot: olderBar, + newerSlot: newerBar, + ), + ); + } on ArgumentError { + continue; + } + } + + assets.sort( + (QuestionAuditAsset a, QuestionAuditAsset b) => + a.symbol.compareTo(b.symbol), + ); + return assets; + } + + Map? _decodeRaw(Object? rawValue) { + if (rawValue == null) { + return null; + } + if (rawValue is Map) { + return rawValue; + } + return jsonDecode(rawValue.toString()) as Map; + } +} + +class _BarRow { + const _BarRow({ + required this.asOf, + required this.closePrice, + required this.volume, + required this.raw, + }); + + final DateTime asOf; + final num? closePrice; + final num? volume; + final Map? raw; +} diff --git a/server/lib/trading/market_history_trading_calendar.dart b/server/lib/trading/market_history_trading_calendar.dart new file mode 100644 index 0000000..bd8688e --- /dev/null +++ b/server/lib/trading/market_history_trading_calendar.dart @@ -0,0 +1,50 @@ +/// US equity regular-session calendar checks for 4-hour history slots (UTC). +abstract final class MarketHistoryTradingCalendar { + /// Saturday or Sunday (UTC calendar date of [instant]). + static bool isWeekendDayUtc(DateTime instant) { + final DateTime day = DateTime.utc(instant.year, instant.month, instant.day); + return day.weekday == DateTime.saturday || day.weekday == DateTime.sunday; + } + + /// NYSE full-day closures (UTC date). Extend as needed. + static bool isNyseHolidayUtc(DateTime instant) { + return _nyseHolidays.contains(_dateKey(instant)); + } + + /// No regular US cash session on this UTC calendar day (weekend or NYSE holiday). + static bool isLikelyNoRegularSession(DateTime slotStart) { + final DateTime utc = slotStart.toUtc(); + return isWeekendDayUtc(utc) || isNyseHolidayUtc(utc); + } + + static String _dateKey(DateTime instant) { + final DateTime d = instant.toUtc(); + String two(int n) => n.toString().padLeft(2, '0'); + return '${d.year}-${two(d.month)}-${two(d.day)}'; + } + + /// NYSE observed full closures (UTC dates, YYYY-MM-DD). + static const Set _nyseHolidays = { + '2025-01-01', + '2025-01-20', + '2025-02-17', + '2025-04-18', + '2025-05-26', + '2025-06-19', + '2025-07-04', + '2025-09-01', + '2025-11-27', + '2025-12-25', + '2026-01-01', + '2026-01-19', + '2026-02-16', + '2026-04-03', + '2026-05-25', + '2026-06-19', + '2026-07-03', + '2026-09-07', + '2026-11-26', + '2026-12-25', + '2027-01-01', + }; +} diff --git a/server/lib/trading/market_history_week_coverage.dart b/server/lib/trading/market_history_week_coverage.dart new file mode 100644 index 0000000..a9e3682 --- /dev/null +++ b/server/lib/trading/market_history_week_coverage.dart @@ -0,0 +1,282 @@ +import 'dart:convert'; + +import 'package:postgres/postgres.dart'; + +import 'market_history_config.dart'; +import 'market_history_four_hour_slot.dart'; +import 'tradable_assets_db.dart'; + +/// One UTC 4-hour slot within the rolling window. +class MarketHistorySlotCoverage { + const MarketHistorySlotCoverage({ + required this.slotStart, + required this.completed, + required this.fullySynced, + required this.syncedSymbolCount, + required this.expectedSymbolCount, + }); + + final DateTime slotStart; + final bool completed; + final bool fullySynced; + final int syncedSymbolCount; + final int expectedSymbolCount; + + Map toJson() => { + 'slotStart': slotStart.toIso8601String(), + 'completed': completed, + 'fullySynced': fullySynced, + 'syncedSymbolCount': syncedSymbolCount, + 'expectedSymbolCount': expectedSymbolCount, + }; +} + +/// Slot rollup for one UTC calendar day. +class MarketHistoryDayCoverage { + const MarketHistoryDayCoverage({ + required this.date, + required this.slots, + required this.completedSlots, + required this.fullySyncedSlots, + }); + + final DateTime date; + final List slots; + final int completedSlots; + final int fullySyncedSlots; + + Map toJson() => { + 'date': dateWire(date), + 'slotsPerDay': MarketHistoryFourHourSlot.slotsPerDay, + 'completedSlots': completedSlots, + 'fullySyncedSlots': fullySyncedSlots, + 'slots': slots.map((MarketHistorySlotCoverage s) => s.toJson()).toList(), + }; +} + +/// Rolling-window slot consistency for the admin calendar. +class MarketHistoryWeekCoverageReport { + const MarketHistoryWeekCoverageReport({ + required this.asOf, + required this.windowDays, + required this.slotsPerDay, + required this.symbolCount, + required this.days, + required this.isConsistent, + }); + + final DateTime asOf; + final int windowDays; + final int slotsPerDay; + final int symbolCount; + final List days; + final bool isConsistent; + + Map toJson() => { + 'asOf': asOf.toIso8601String(), + 'windowDays': windowDays, + 'slotsPerDay': slotsPerDay, + 'symbolCount': symbolCount, + 'isConsistent': isConsistent, + 'days': days.map((MarketHistoryDayCoverage d) => d.toJson()).toList(), + }; +} + +/// Validates 4-hour bar coverage per UTC day for the admin week view. +class MarketHistoryWeekCoverage { + MarketHistoryWeekCoverage({ + required Connection connection, + TradableAssetsDb? tradableAssetsDb, + this.windowDays = MarketHistoryConfig.windowDays, + this.timeframe = MarketHistoryConfig.barTimeframe, + this.maxSymbols = MarketHistoryConfig.historySyncMaxSymbols, + }) : _tradableAssetsDb = tradableAssetsDb ?? TradableAssetsDb(connection), + _connection = connection; + + final Connection _connection; + final TradableAssetsDb _tradableAssetsDb; + final int windowDays; + final String timeframe; + final int maxSymbols; + + Future compute({DateTime? now}) async { + final DateTime tick = (now ?? DateTime.now()).toUtc(); + final List symbols = await _activeSymbols(); + final int symbolCount = symbols.length; + + final List calendarDays = _calendarDaysEndingToday(tick, windowDays); + final Map> symbolsBySlot = + await _loadSyncedSymbolsBySlot(tick, symbols); + + final List days = []; + var isConsistent = symbolCount > 0; + + for (final DateTime day in calendarDays) { + final List slots = []; + var completedSlots = 0; + var fullySyncedSlots = 0; + + for (int hour = 0; hour < 24; hour += MarketHistoryFourHourSlot.slotHours) { + final DateTime slotStart = DateTime.utc(day.year, day.month, day.day, hour); + final bool completed = MarketHistoryFourHourSlot.hasEnded(slotStart, tick); + final Set synced = + symbolsBySlot[_slotKey(slotStart)] ?? {}; + final int syncedCount = _countSyncedSymbols(synced, symbols); + final bool fullySynced = + symbolCount > 0 && completed && syncedCount >= symbolCount; + + if (completed) { + completedSlots++; + if (fullySynced) { + fullySyncedSlots++; + } else { + isConsistent = false; + } + } + + slots.add( + MarketHistorySlotCoverage( + slotStart: slotStart, + completed: completed, + fullySynced: fullySynced, + syncedSymbolCount: syncedCount, + expectedSymbolCount: symbolCount, + ), + ); + } + + days.add( + MarketHistoryDayCoverage( + date: day, + slots: slots, + completedSlots: completedSlots, + fullySyncedSlots: fullySyncedSlots, + ), + ); + } + + if (symbolCount == 0) { + isConsistent = false; + } + + return MarketHistoryWeekCoverageReport( + asOf: tick, + windowDays: windowDays, + slotsPerDay: MarketHistoryFourHourSlot.slotsPerDay, + symbolCount: symbolCount, + days: days, + isConsistent: isConsistent, + ); + } + + Future> _activeSymbols() async { + List symbols = await _tradableAssetsDb.listActiveTradableSymbols(); + if (symbols.length > maxSymbols) { + symbols = symbols.sublist(0, maxSymbols); + } + return symbols; + } + + Future>> _loadSyncedSymbolsBySlot( + DateTime now, + List symbols, + ) async { + if (symbols.isEmpty) { + return >{}; + } + + final DateTime firstDay = + _calendarDaysEndingToday(now, windowDays).first; + final DateTime since = DateTime.utc(firstDay.year, firstDay.month, firstDay.day); + final DateTime until = + MarketHistoryFourHourSlot.endExclusive(MarketHistoryFourHourSlot.slotStartContaining(now)); + + final Result rows = await _connection.execute( + Sql.named( + ''' + SELECT symbol, as_of, raw + FROM market_data_snapshots + WHERE metric = 'bar' + AND timeframe = @timeframe + AND as_of >= @since + AND as_of < @until + AND symbol = ANY(@symbols) + ''', + ), + parameters: { + 'timeframe': timeframe, + 'since': since, + 'until': until, + 'symbols': symbols, + }, + ); + + final Map> bySlot = >{}; + for (final ResultRow row in rows) { + final String symbol = row[0]! as String; + final DateTime slotStart = _slotStartFromRow((row[1]! as DateTime).toUtc(), row[2]); + final String key = _slotKey(slotStart); + bySlot.putIfAbsent(key, () => {}).add(symbol); + } + return bySlot; + } + + static DateTime _slotStartFromRow(DateTime asOf, Object? rawValue) { + if (rawValue is Map) { + final String? slotStartRaw = rawValue['slot_start'] as String?; + if (slotStartRaw != null) { + final DateTime? parsed = DateTime.tryParse(slotStartRaw); + if (parsed != null) { + return parsed.toUtc(); + } + } + } else if (rawValue != null) { + try { + final Map raw = + jsonDecode(rawValue.toString()) as Map; + final String? slotStartRaw = raw['slot_start'] as String?; + if (slotStartRaw != null) { + final DateTime? parsed = DateTime.tryParse(slotStartRaw); + if (parsed != null) { + return parsed.toUtc(); + } + } + } on Object { + // Fall back to as_of bucketing below. + } + } + return MarketHistoryFourHourSlot.slotStartContaining(asOf); + } + + static List calendarDaysEndingToday(DateTime now, int windowDays) { + final DateTime today = DateTime.utc(now.year, now.month, now.day); + return List.generate( + windowDays, + (int index) => today.subtract(Duration(days: windowDays - 1 - index)), + ); + } + + static List _calendarDaysEndingToday(DateTime now, int windowDays) => + calendarDaysEndingToday(now, windowDays); + + static String _slotKey(DateTime slotStart) => + MarketHistoryFourHourSlot.slotStartWire(slotStart); + + static int _countSyncedSymbols(Set synced, List expected) { + if (expected.isEmpty) { + return 0; + } + var count = 0; + for (final String symbol in expected) { + if (synced.contains(symbol)) { + count++; + } + } + return count; + } +} + +String dateWire(DateTime date) => + '${date.year.toString().padLeft(4, '0')}-' + '${date.month.toString().padLeft(2, '0')}-' + '${date.day.toString().padLeft(2, '0')}'; diff --git a/server/lib/trading/rule_engine.dart b/server/lib/trading/rule_engine.dart index 556c413..d209651 100644 --- a/server/lib/trading/rule_engine.dart +++ b/server/lib/trading/rule_engine.dart @@ -1,4 +1,5 @@ import 'market_data_db.dart'; +import 'market_history_query.dart'; import 'trading_config.dart'; /// Why a rule did not fire. `null` means the rule fired. @@ -9,6 +10,7 @@ enum RuleSkipReason { aboveThreshold, cooldown, zeroReferencePrice, + insufficientBars, } /// Result of evaluating a single [TradingRuleConfig] against snapshots. @@ -22,6 +24,10 @@ class RuleEvaluation { this.observedPrice, this.questionText, this.asOf, + this.symbolToken, + this.guessSymbol, + this.correctAnswer, + this.refDaysAgo, }); final TradingRuleConfig rule; @@ -38,6 +44,18 @@ class RuleEvaluation { /// Most recent `as_of` across the snapshots used. final DateTime? asOf; + + /// Obfuscated ticker token for `guess_weekly_move` (e.g. `ASSET_A`). + final String? symbolToken; + + /// Real symbol stored server-side only (not in [questionText]). + final String? guessSymbol; + + /// Expected swipe answer: `10` (up) or `-10` (down) for guess rules. + final num? correctAnswer; + + /// Days ago label for the reference close in guess templates. + final int? refDaysAgo; } /// Pure evaluation of trading rules over [MarketDataSnapshot] inputs. @@ -59,9 +77,21 @@ class RuleEngine { required Map snapshots, DateTime? lastFiredAt, DateTime? now, + WeeklyMover? weeklyMover, + String? symbolToken, }) { final DateTime evaluatedAt = (now ?? _clock()).toUtc(); + if (rule.type == 'guess_weekly_move') { + return evaluateGuessWeeklyMove( + rule: rule, + mover: weeklyMover, + symbolToken: symbolToken, + lastFiredAt: lastFiredAt, + now: now, + ); + } + if (rule.type != 'price_below_pct_of_ref') { return RuleEvaluation( rule: rule, @@ -145,6 +175,66 @@ class RuleEngine { ); } + /// Evaluates [guess_weekly_move] using a pre-selected [mover]. + RuleEvaluation evaluateGuessWeeklyMove({ + required TradingRuleConfig rule, + required WeeklyMover? mover, + required String? symbolToken, + DateTime? lastFiredAt, + DateTime? now, + }) { + final DateTime evaluatedAt = (now ?? _clock()).toUtc(); + + if (rule.type != 'guess_weekly_move') { + return RuleEvaluation( + rule: rule, + fired: false, + skipReason: RuleSkipReason.unknownType, + ); + } + + if (mover == null || symbolToken == null) { + return RuleEvaluation( + rule: rule, + fired: false, + skipReason: RuleSkipReason.insufficientBars, + ); + } + + final num refPrice = mover.openClose; + final num currentClose = mover.currentClose; + if (refPrice == 0) { + return RuleEvaluation( + rule: rule, + fired: false, + skipReason: RuleSkipReason.zeroReferencePrice, + ); + } + + final num correctAnswer = + currentClose > refPrice ? 10 : (currentClose < refPrice ? -10 : 10); + final int refDaysAgo = mover.days; + + final String questionText = _renderGuessTemplate( + rule.questionTemplate, + token: symbolToken, + refPrice: refPrice, + refDaysAgo: refDaysAgo, + ); + + return RuleEvaluation( + rule: rule, + fired: true, + refPrice: refPrice, + observedPrice: currentClose, + questionText: questionText, + symbolToken: symbolToken, + guessSymbol: mover.symbol, + correctAnswer: correctAnswer, + refDaysAgo: refDaysAgo, + ); + } + bool _isCooldown(DateTime? lastFiredAt, DateTime now) { if (lastFiredAt == null) { return false; @@ -175,4 +265,16 @@ class RuleEngine { final num abs = value.abs(); return abs.toStringAsFixed(abs < 10 ? 2 : 1); } + + String _renderGuessTemplate( + String template, { + required String token, + required num refPrice, + required int refDaysAgo, + }) { + return template + .replaceAll('{{token}}', token) + .replaceAll('{{ref_price}}', _formatPrice(refPrice)) + .replaceAll('{{ref_days_ago}}', refDaysAgo.toString()); + } } diff --git a/server/lib/trading/symbol_obfuscator.dart b/server/lib/trading/symbol_obfuscator.dart new file mode 100644 index 0000000..57b2bda --- /dev/null +++ b/server/lib/trading/symbol_obfuscator.dart @@ -0,0 +1,23 @@ +/// Maps real ticker symbols to opaque tokens shown in question text. +class SymbolObfuscator { + SymbolObfuscator({this.prefix = 'ASSET_'}); + + final String prefix; + + /// Stable token for [symbol] among [universe] sorted lexicographically. + String tokenFor(String symbol, List universe) { + final List sorted = List.from(universe)..sort(); + final int index = sorted.indexOf(symbol); + if (index < 0) { + throw ArgumentError.value(symbol, 'symbol', 'not in universe'); + } + return '$prefix${_letterForIndex(index)}'; + } + + static String _letterForIndex(int index) { + if (index < 26) { + return String.fromCharCode(65 + index); + } + return '${_letterForIndex(index ~/ 26 - 1)}${_letterForIndex(index % 26)}'; + } +} diff --git a/server/lib/trading/sync_run_recorder.dart b/server/lib/trading/sync_run_recorder.dart new file mode 100644 index 0000000..8db77d2 --- /dev/null +++ b/server/lib/trading/sync_run_recorder.dart @@ -0,0 +1,229 @@ +import 'dart:convert'; + +import 'package:postgres/postgres.dart'; + +import 'backfill_sync_item.dart'; + +/// Outcome of one [SyncRunRecorder.record] body. +/// +/// The recorder ALWAYS emits one of these regardless of whether the body +/// threw, so callers can convert to a domain-specific result type without +/// re-implementing the start/finish bookkeeping. +class SyncRunOutcome { + SyncRunOutcome({ + required this.id, + required this.kind, + required this.startedAt, + required this.finishedAt, + required this.rowsWritten, + required this.rowsRemoved, + this.slotsSynced, + this.error, + }); + + final int id; + final String kind; + final DateTime startedAt; + final DateTime finishedAt; + final int rowsWritten; + final int rowsRemoved; + final int? slotsSynced; + final String? error; + + bool get succeeded => error == null; +} + +/// Counts the body of a sync run reports back to the recorder. +class SyncRunCounts { + const SyncRunCounts({ + this.rowsWritten = 0, + this.rowsRemoved = 0, + this.slotsSynced, + this.backfillItems, + this.error, + }); + + final int rowsWritten; + final int rowsRemoved; + + /// Completed 4-hour slots written (market-history backfill only). + final int? slotsSynced; + + /// Slot starts and symbol lists requested from Alpaca (backfill only). + final List? backfillItems; + + /// Non-fatal partial failure (e.g. one Alpaca batch 500 while others + /// succeeded). Recorded on the sync run without discarding [rowsWritten]. + final String? error; +} + +/// Wraps a closure with a `market_data_sync_runs` audit row. +/// +/// On entry, INSERTs a row with `kind`, `started_at` and returns its id. +/// After the body completes (success or thrown exception), UPDATEs the +/// row with `finished_at`, `rows_written`, `rows_removed`, and `error`. +/// +/// The body's exception is **swallowed** — the recorder records it and +/// returns a [SyncRunOutcome] with `error` set instead. This is the +/// "scheduler-friendly" contract every sync stage in §2/§3/§4 needs. +/// +/// **Do not change that contract for §3/§4:** `MarketHistoryScheduler` +/// (§5) runs universe → backfill → cleanup in sequence and expects a +/// thrown Alpaca 500 in backfill to be recorded here without aborting +/// cleanup. Partial batch failures in `MarketDataHistorySync` rely on +/// the same behaviour. +class SyncRunRecorder { + SyncRunRecorder(this._connection); + + final Connection _connection; + + static const String abortSupersededMessage = + 'aborted: superseded by new worker sync'; + + /// Closes every run still marked in-progress (crashed or superseded worker). + Future abortAllInProgressRuns({ + DateTime? now, + String? message, + }) async { + return _abortInProgress( + now: now, + olderThan: null, + message: message ?? abortSupersededMessage, + ); + } + + /// Closes in-progress runs older than [olderThan] (hung without finishing). + Future abortStaleInProgressRuns({ + DateTime? now, + Duration olderThan = const Duration(minutes: 30), + String? message, + }) async { + return _abortInProgress( + now: now, + olderThan: olderThan, + message: message ?? 'aborted: stale in-progress sync run', + ); + } + + Future _abortInProgress({ + required DateTime? now, + required Duration? olderThan, + required String message, + }) async { + final DateTime tick = (now ?? DateTime.now()).toUtc(); + final Result rows; + if (olderThan == null) { + rows = await _connection.execute( + Sql.named( + ''' + UPDATE market_data_sync_runs + SET finished_at = @finished_at, + error = COALESCE(error, @message) + WHERE finished_at IS NULL + ''', + ), + parameters: { + 'finished_at': tick, + 'message': message, + }, + ); + } else { + rows = await _connection.execute( + Sql.named( + ''' + UPDATE market_data_sync_runs + SET finished_at = @finished_at, + error = COALESCE(error, @message) + WHERE finished_at IS NULL + AND started_at < @cutoff + ''', + ), + parameters: { + 'finished_at': tick, + 'message': message, + 'cutoff': tick.subtract(olderThan), + }, + ); + } + return rows.affectedRows; + } + + Future record( + String kind, + Future Function() body, { + DateTime? now, + }) async { + final DateTime startedAt = (now ?? DateTime.now()).toUtc(); + + final Result inserted = await _connection.execute( + Sql.named( + ''' + INSERT INTO market_data_sync_runs (kind, started_at) + VALUES (@kind, @started_at) + RETURNING id + ''', + ), + parameters: { + 'kind': kind, + 'started_at': startedAt, + }, + ); + final int id = (inserted.first[0]! as num).toInt(); + + int rowsWritten = 0; + int rowsRemoved = 0; + int? slotsSynced; + List? backfillItems; + String? error; + try { + final SyncRunCounts counts = await body(); + rowsWritten = counts.rowsWritten; + rowsRemoved = counts.rowsRemoved; + slotsSynced = counts.slotsSynced; + backfillItems = counts.backfillItems; + error = counts.error; + } on Object catch (e) { + // Always recorded, never rethrown — the scheduler in §5 expects + // each stage to fail in isolation, not to bubble up. + error = e.toString(); + } + + final DateTime finishedAt = (now ?? DateTime.now()).toUtc(); + await _connection.execute( + Sql.named( + ''' + UPDATE market_data_sync_runs + SET finished_at = @finished_at, + rows_written = @rows_written, + rows_removed = @rows_removed, + slots_synced = @slots_synced, + backfill_items = @backfill_items::jsonb, + error = @error + WHERE id = @id + ''', + ), + parameters: { + 'id': id, + 'finished_at': finishedAt, + 'rows_written': rowsWritten, + 'rows_removed': rowsRemoved, + 'slots_synced': slotsSynced ?? 0, + 'backfill_items': backfillItems == null || backfillItems.isEmpty + ? null + : jsonEncode(BackfillSyncItem.encodeList(backfillItems)), + 'error': error, + }, + ); + + return SyncRunOutcome( + id: id, + kind: kind, + startedAt: startedAt, + finishedAt: finishedAt, + rowsWritten: rowsWritten, + rowsRemoved: rowsRemoved, + slotsSynced: slotsSynced, + error: error, + ); + } +} diff --git a/server/lib/trading/tradable_assets_db.dart b/server/lib/trading/tradable_assets_db.dart new file mode 100644 index 0000000..fc58162 --- /dev/null +++ b/server/lib/trading/tradable_assets_db.dart @@ -0,0 +1,163 @@ +import 'dart:convert'; + +import 'package:postgres/postgres.dart'; + +import '../alpaca/alpaca_models.dart'; + +/// Read-side row for [tradable_assets]. +class TradableAssetRow { + TradableAssetRow({ + required this.symbol, + required this.assetClass, + required this.tradable, + required this.fractionable, + required this.status, + required this.refreshedAt, + this.exchange, + this.name, + this.raw, + }); + + final String symbol; + final String assetClass; + final String? exchange; + final String? name; + final bool tradable; + final bool fractionable; + final String status; + final Map? raw; + final DateTime refreshedAt; +} + +/// Postgres access for [tradable_assets] (the daily Alpaca asset universe). +/// +/// Writers: only [TradableAssetsSync] (§2.2). Readers: the historical +/// backfill in §3 plus operator/query tooling. +class TradableAssetsDb { + TradableAssetsDb(this._connection); + + final Connection _connection; + + /// Upserts every entry in [assets] (PK on `symbol`) using [now] as the + /// shared `refreshed_at`, then marks any rows whose `refreshed_at` is + /// older than [now] as `status='inactive', tradable=false` — preserving + /// the historical row but flagging it as no longer in the live universe. + /// + /// The two phases run inside a single transaction so a partial failure + /// can't leave the universe half-updated. + Future upsertAll( + List assets, { + DateTime? now, + }) async { + final DateTime ts = (now ?? DateTime.now()).toUtc(); + + await _connection.runTx((TxSession tx) async { + for (final AlpacaAsset asset in assets) { + await tx.execute( + Sql.named( + ''' + INSERT INTO tradable_assets ( + symbol, asset_class, exchange, name, tradable, + fractionable, status, raw, refreshed_at + ) VALUES ( + @symbol, @asset_class, @exchange, @name, @tradable, + @fractionable, @status, @raw::jsonb, @refreshed_at + ) + ON CONFLICT (symbol) DO UPDATE SET + asset_class = EXCLUDED.asset_class, + exchange = EXCLUDED.exchange, + name = EXCLUDED.name, + tradable = EXCLUDED.tradable, + fractionable = EXCLUDED.fractionable, + status = EXCLUDED.status, + raw = EXCLUDED.raw, + refreshed_at = EXCLUDED.refreshed_at + ''', + ), + parameters: { + 'symbol': asset.symbol, + 'asset_class': asset.assetClass, + 'exchange': asset.exchange, + 'name': asset.name, + 'tradable': asset.tradable, + 'fractionable': asset.fractionable, + 'status': asset.status, + 'raw': asset.raw == null ? null : jsonEncode(asset.raw), + 'refreshed_at': ts, + }, + ); + } + + // Anything not seen in this batch hasn't been refreshed at [ts] yet. + // Flip it to inactive without bumping refreshed_at, so the audit + // trail still records "last seen as part of the live universe." + await tx.execute( + Sql.named( + ''' + UPDATE tradable_assets + SET status = 'inactive', tradable = false + WHERE refreshed_at < @now + AND (status <> 'inactive' OR tradable = true) + ''', + ), + parameters: {'now': ts}, + ); + }); + } + + /// Symbols currently tradable on the active universe. + Future> listActiveTradableSymbols() async { + final Result result = await _connection.execute( + ''' + SELECT symbol + FROM tradable_assets + WHERE status = 'active' AND tradable = true + ORDER BY symbol ASC + ''', + ); + return result.map((ResultRow r) => r[0]! as String).toList(growable: false); + } + + /// Single-row lookup, primarily for tests and admin tooling. + Future getBySymbol(String symbol) async { + final Result result = await _connection.execute( + Sql.named( + ''' + SELECT symbol, asset_class, exchange, name, tradable, + fractionable, status, raw, refreshed_at + FROM tradable_assets + WHERE symbol = @symbol + ''', + ), + parameters: {'symbol': symbol}, + ); + if (result.isEmpty) { + return null; + } + return _rowToModel(result.first); + } + + TradableAssetRow _rowToModel(ResultRow row) { + final Object? rawValue = row[7]; + Map? raw; + if (rawValue is Map) { + raw = rawValue; + } else if (rawValue is Map) { + raw = Map.from(rawValue); + } else if (rawValue != null) { + raw = jsonDecode(rawValue.toString()) as Map; + } + + return TradableAssetRow( + symbol: row[0]! as String, + assetClass: row[1]! as String, + exchange: row[2] as String?, + name: row[3] as String?, + tradable: row[4]! as bool, + fractionable: row[5]! as bool, + status: row[6]! as String, + raw: raw, + refreshedAt: (row[8]! as DateTime).toUtc(), + ); + } +} diff --git a/server/lib/trading/tradable_assets_sync.dart b/server/lib/trading/tradable_assets_sync.dart new file mode 100644 index 0000000..6fe661a --- /dev/null +++ b/server/lib/trading/tradable_assets_sync.dart @@ -0,0 +1,77 @@ +import 'package:postgres/postgres.dart'; + +import '../alpaca/alpaca_assets_client.dart'; +import '../alpaca/alpaca_models.dart'; +import 'sync_run_recorder.dart'; +import 'tradable_assets_db.dart'; + +/// Outcome of a single [TradableAssetsSync.runOnce] invocation. +/// +/// On the happy path [error] is `null` and [rowsWritten] is the number of +/// asset rows upserted. On a failure path [error] holds the upstream +/// message and [rowsWritten] is `0` — the exception itself is never +/// rethrown to callers (the scheduler in §5 must keep going). +class TradableAssetsSyncResult { + TradableAssetsSyncResult({ + required this.rowsWritten, + required this.startedAt, + required this.finishedAt, + this.error, + }); + + final int rowsWritten; + final DateTime startedAt; + final DateTime finishedAt; + final String? error; + + bool get succeeded => error == null; +} + +/// Daily refresh of the `tradable_assets` cache from Alpaca. +/// +/// One pass: +/// 1. [SyncRunRecorder] inserts a `market_data_sync_runs` row with +/// `kind='universe'`. +/// 2. Pull `/v2/assets?status=active&asset_class=us_equity`. +/// 3. Upsert into `tradable_assets`, marking missing symbols inactive. +/// 4. Recorder closes the row with `finished_at` + `rows_written` (or +/// `error` on failure). +/// +/// The scheduler in §5 runs this once per `MARKET_UNIVERSE_REFRESH_HOURS`. +class TradableAssetsSync { + TradableAssetsSync({ + required AlpacaAssetsClient assetsClient, + required TradableAssetsDb assetsDb, + required Connection connection, + }) : _assetsClient = assetsClient, + _assetsDb = assetsDb, + _recorder = SyncRunRecorder(connection); + + final AlpacaAssetsClient _assetsClient; + final TradableAssetsDb _assetsDb; + final SyncRunRecorder _recorder; + + static const String kind = 'universe'; + + Future runOnce({DateTime? now}) async { + final DateTime started = (now ?? DateTime.now()).toUtc(); + + final SyncRunOutcome outcome = await _recorder.record( + kind, + () async { + final List assets = + await _assetsClient.listActiveTradable(); + await _assetsDb.upsertAll(assets, now: started); + return SyncRunCounts(rowsWritten: assets.length); + }, + now: started, + ); + + return TradableAssetsSyncResult( + rowsWritten: outcome.rowsWritten, + startedAt: outcome.startedAt, + finishedAt: outcome.finishedAt, + error: outcome.error, + ); + } +} diff --git a/server/lib/trading/trading_config.dart b/server/lib/trading/trading_config.dart index e2208ff..fc9be1b 100644 --- a/server/lib/trading/trading_config.dart +++ b/server/lib/trading/trading_config.dart @@ -170,7 +170,7 @@ class TradingRuleConfig { return TradingRuleConfig( id: json['id']! as String, type: json['type']! as String, - symbol: json['symbol']! as String, + symbol: json['symbol'] as String? ?? '*', refMetric: json['ref_metric'] as String? ?? 'prev_close', thresholdPct: json['threshold_pct'] as num? ?? 0, questionTemplate: json['question_template'] as String? ?? '', diff --git a/server/lib/trading/trading_pipeline.dart b/server/lib/trading/trading_pipeline.dart index 7565cac..a5f3bc8 100644 --- a/server/lib/trading/trading_pipeline.dart +++ b/server/lib/trading/trading_pipeline.dart @@ -7,7 +7,10 @@ import '../question_service.dart'; import '../questions_db.dart'; import 'guardrails.dart'; import 'market_data_db.dart'; +import '../market_history_env.dart'; +import 'market_history_query.dart'; import 'rule_engine.dart'; +import 'symbol_obfuscator.dart'; import 'trading_config.dart'; import 'trading_config_db.dart'; import 'user_trading_state_db.dart'; @@ -36,6 +39,9 @@ class TradingPipeline { required TradingConfigDb tradingConfigDb, required UserTradingStateDb tradingStateDb, RuleEngine? ruleEngine, + MarketHistoryQuery? marketHistoryQuery, + MarketHistoryEnv? marketHistoryEnv, + SymbolObfuscator? symbolObfuscator, Guardrails? guardrails, int maxQueuedQuestions = 3, DateTime Function()? clock, @@ -44,7 +50,10 @@ class TradingPipeline { _marketDataDb = marketDataDb, _tradingConfigDb = tradingConfigDb, _tradingStateDb = tradingStateDb, - _ruleEngine = ruleEngine ?? RuleEngine(), + _ruleEngine = ruleEngine ?? RuleEngine(clock: clock), + _marketHistoryQuery = marketHistoryQuery, + _marketHistoryEnv = marketHistoryEnv ?? MarketHistoryEnv.fromMap({}), + _symbolObfuscator = symbolObfuscator ?? SymbolObfuscator(), _guardrails = guardrails ?? Guardrails(), _maxQueuedQuestions = maxQueuedQuestions, _clock = clock ?? DateTime.now; @@ -55,6 +64,9 @@ class TradingPipeline { final TradingConfigDb _tradingConfigDb; final UserTradingStateDb _tradingStateDb; final RuleEngine _ruleEngine; + final MarketHistoryQuery? _marketHistoryQuery; + final MarketHistoryEnv _marketHistoryEnv; + final SymbolObfuscator _symbolObfuscator; final Guardrails _guardrails; final int _maxQueuedQuestions; final DateTime Function() _clock; @@ -99,46 +111,74 @@ class TradingPipeline { continue; } - final Map snapshots = - await _loadSnapshotsForRule(rule); final DateTime? lastFiredAt = await _tradingStateDb.getRuleLastFiredAt(firebaseUid, rule.id); - final RuleEvaluation result = _ruleEngine.evaluate( - rule: rule, - snapshots: snapshots, - lastFiredAt: lastFiredAt, - now: now, - ); + final RuleEvaluation result; + if (rule.type == 'guess_weekly_move') { + result = await _evaluateGuessRule( + firebaseUid: firebaseUid, + rule: rule, + lastFiredAt: lastFiredAt, + now: now, + ); + } else { + final Map snapshots = + await _loadSnapshotsForRule(rule); + result = _ruleEngine.evaluate( + rule: rule, + snapshots: snapshots, + lastFiredAt: lastFiredAt, + now: now, + ); + } if (!result.fired) { skipped.add('${rule.id}(${result.skipReason?.name ?? 'no_fire'})'); continue; } + final String phase = rule.type == 'guess_weekly_move' + ? TradingPhases.awaitAnswer + : TradingPhases.awaitConfirm; + final num correctAnswer = + result.correctAnswer ?? (rule.type == 'guess_weekly_move' ? 10 : 10); + final Map question = await _questionService.createAndDeliverQuestion( assignedUserId: firebaseUid, questionText: result.questionText!, - correctAnswer: 10, + correctAnswer: correctAnswer, sourceTag: 'trading:rule:${rule.id}', pipelineKey: PipelineKeys.trading, - pipelineStep: '${rule.id}:${TradingPhases.awaitConfirm}', + pipelineStep: '${rule.id}:$phase', + metadata: result.guessSymbol == null + ? null + : {'guess_symbol': result.guessSymbol}, ); questionsCreated++; fired.add(rule.id); + if (rule.type == 'guess_weekly_move' && result.guessSymbol != null) { + await _tradingStateDb.recordGuessSymbolPicked( + firebaseUid: firebaseUid, + symbol: result.guessSymbol!, + at: now, + ); + } + await _tradingStateDb.setRuleState( firebaseUid: firebaseUid, ruleId: rule.id, state: { - 'phase': TradingPhases.awaitConfirm, + 'phase': phase, 'last_fired_at': now.toIso8601String(), 'question_id': question['id'], - 'symbol': rule.symbol, + 'symbol': result.guessSymbol ?? rule.symbol, 'observed_price': result.observedPrice, 'ref_price': result.refPrice, 'pct': result.pricePct, + if (result.symbolToken != null) 'symbol_token': result.symbolToken, }, ); } catch (e, st) { @@ -171,7 +211,12 @@ class TradingPipeline { return; } final List parts = pipelineStep.split(':'); - if (parts.length < 2 || parts[1] != TradingPhases.awaitConfirm) { + if (parts.length < 2) { + return; + } + final String phase = parts[1]; + if (phase != TradingPhases.awaitConfirm && + phase != TradingPhases.awaitAnswer) { return; } final String ruleId = parts.first; @@ -195,13 +240,26 @@ class TradingPipeline { return; } + final Map? priorState = + await _tradingStateDb.getRuleState(firebaseUid, ruleId); + + if (rule.type == 'guess_weekly_move') { + await _handleGuessAnswer( + firebaseUid: firebaseUid, + rule: rule, + questionId: questionId, + userResponse: userResponse, + correctAnswer: correctAnswer, + priorState: priorState, + now: now, + ); + return; + } + final BranchOutcome outcome = BranchDecision.yesNo( userResponse: userResponse, correctAnswer: correctAnswer, ); - - final Map? priorState = - await _tradingStateDb.getRuleState(firebaseUid, ruleId); final Map baseState = { ...?priorState, }; @@ -259,6 +317,96 @@ class TradingPipeline { } } + Future _evaluateGuessRule({ + required String firebaseUid, + required TradingRuleConfig rule, + required DateTime? lastFiredAt, + required DateTime now, + }) async { + if (_marketHistoryQuery == null) { + return RuleEvaluation( + rule: rule, + fired: false, + skipReason: RuleSkipReason.insufficientBars, + ); + } + + final List movers = await _marketHistoryQuery.weeklyMovers( + asOf: now, + minBars: _marketHistoryEnv.minBarsForGuess, + windowDays: _marketHistoryEnv.windowDays, + ); + if (movers.isEmpty) { + return RuleEvaluation( + rule: rule, + fired: false, + skipReason: RuleSkipReason.insufficientBars, + ); + } + + final List universe = + movers.map((WeeklyMover m) => m.symbol).toList(); + WeeklyMover? picked; + String? token; + for (final WeeklyMover mover in movers) { + final bool onCooldown = await _tradingStateDb.isGuessSymbolOnCooldown( + firebaseUid: firebaseUid, + symbol: mover.symbol, + now: now, + cooldownHours: _marketHistoryEnv.guessCooldownHours, + ); + if (onCooldown) { + continue; + } + picked = mover; + token = _symbolObfuscator.tokenFor(mover.symbol, universe); + break; + } + + return _ruleEngine.evaluateGuessWeeklyMove( + rule: rule, + mover: picked, + symbolToken: token, + lastFiredAt: lastFiredAt, + now: now, + ); + } + + Future _handleGuessAnswer({ + required String firebaseUid, + required TradingRuleConfig rule, + required String questionId, + required num userResponse, + required num correctAnswer, + required Map? priorState, + required DateTime now, + }) async { + final int scoreDelta = userResponse == correctAnswer ? 1 : -1; + final String symbol = + (priorState?['symbol'] as String?) ?? rule.symbol; + + await _tradingStateDb.recordGuessScore( + firebaseUid: firebaseUid, + scoreDelta: scoreDelta, + symbol: symbol, + at: now, + ); + + final Map baseState = { + ...?priorState, + 'phase': TradingPhases.done, + 'question_id': questionId, + 'answer': userResponse == correctAnswer ? 'match' : 'miss', + 'score_delta': scoreDelta, + 'answered_at': now.toIso8601String(), + }; + await _tradingStateDb.setRuleState( + firebaseUid: firebaseUid, + ruleId: rule.id, + state: baseState, + ); + } + Future> _loadSnapshotsForRule( TradingRuleConfig rule, ) async { diff --git a/server/lib/trading/user_trading_state_db.dart b/server/lib/trading/user_trading_state_db.dart index 28fe617..c1f0467 100644 --- a/server/lib/trading/user_trading_state_db.dart +++ b/server/lib/trading/user_trading_state_db.dart @@ -12,6 +12,8 @@ class UserTradingStateDb { static const String rulesContextKey = 'rules'; static const String pendingOrdersContextKey = 'pending_orders'; static const String skippedContextKey = 'skipped'; + static const String guessScoreContextKey = 'guess_score'; + static const String guessSymbolCooldownContextKey = 'guess_symbol_cooldown'; Future ensureExists(String firebaseUid) async { await _connection.execute( @@ -177,6 +179,82 @@ class UserTradingStateDb { await _writeContext(firebaseUid, context, touchEvalAt: false); } + Future?> getGuessScore(String firebaseUid) async { + final Map context = await getContext(firebaseUid); + final Object? raw = context[guessScoreContextKey]; + if (raw is Map) { + return Map.from(raw); + } + return null; + } + + Future recordGuessScore({ + required String firebaseUid, + required int scoreDelta, + required String symbol, + required DateTime at, + }) async { + await ensureExists(firebaseUid); + final Map context = await getContext(firebaseUid); + final Map prior = Map.from( + context[guessScoreContextKey] as Map? ?? {}, + ); + final int total = ((prior['total'] as num?)?.toInt() ?? 0) + scoreDelta; + context[guessScoreContextKey] = { + 'total': total, + 'last': { + 'score_delta': scoreDelta, + 'symbol': symbol, + 'at': at.toUtc().toIso8601String(), + }, + }; + await _writeContext(firebaseUid, context, touchEvalAt: true); + } + + Future getGuessSymbolLastPickedAt( + String firebaseUid, + String symbol, + ) async { + final Map context = await getContext(firebaseUid); + final Map cooldown = Map.from( + context[guessSymbolCooldownContextKey] as Map? ?? {}, + ); + final String? raw = cooldown[symbol] as String?; + if (raw == null || raw.isEmpty) { + return null; + } + return DateTime.parse(raw).toUtc(); + } + + Future isGuessSymbolOnCooldown({ + required String firebaseUid, + required String symbol, + required DateTime now, + int cooldownHours = 24, + }) async { + final DateTime? last = + await getGuessSymbolLastPickedAt(firebaseUid, symbol); + if (last == null) { + return false; + } + return now.difference(last) < Duration(hours: cooldownHours); + } + + Future recordGuessSymbolPicked({ + required String firebaseUid, + required String symbol, + required DateTime at, + }) async { + await ensureExists(firebaseUid); + final Map context = await getContext(firebaseUid); + final Map cooldown = Map.from( + context[guessSymbolCooldownContextKey] as Map? ?? {}, + ); + cooldown[symbol] = at.toUtc().toIso8601String(); + context[guessSymbolCooldownContextKey] = cooldown; + await _writeContext(firebaseUid, context, touchEvalAt: true); + } + Future recordSkip({ required String firebaseUid, required String ruleId, diff --git a/server/lib/workers/market_history_scheduler.dart b/server/lib/workers/market_history_scheduler.dart new file mode 100644 index 0000000..5b9761d --- /dev/null +++ b/server/lib/workers/market_history_scheduler.dart @@ -0,0 +1,193 @@ +import 'dart:io'; + +import 'package:postgres/postgres.dart'; + +import '../trading/market_data_history.dart'; +import '../trading/market_data_retention.dart'; +import '../trading/sync_run_recorder.dart'; +import '../trading/tradable_assets_sync.dart'; +import 'market_history_scheduler_config.dart'; + +/// Which scheduler stages ran during [MarketHistoryScheduler.runIfDue]. +class MarketHistorySchedulerReport { + MarketHistorySchedulerReport({required this.ranStages}); + + final List ranStages; +} + +/// Market-history pipeline: universe → backfill → cleanup. +/// +/// Before each run, stale or orphaned in-progress `market_data_sync_runs` rows +/// are aborted so a hung prior sync cannot block the worker. +class MarketHistoryScheduler { + MarketHistoryScheduler({ + required Connection connection, + this.config = const MarketHistorySchedulerConfig(), + Future Function(DateTime now)? runUniverse, + Future Function(DateTime now)? runBackfill, + Future Function(DateTime now)? runCleanup, + Future Function(DateTime now)? backfillIsDue, + }) : _connection = connection, + _recorder = SyncRunRecorder(connection), + _runUniverse = runUniverse, + _runBackfill = runBackfill, + _runCleanup = runCleanup, + _backfillIsDue = backfillIsDue; + + final Connection _connection; + final SyncRunRecorder _recorder; + final MarketHistorySchedulerConfig config; + final Future Function(DateTime now)? _runUniverse; + final Future Function(DateTime now)? _runBackfill; + final Future Function(DateTime now)? _runCleanup; + final Future Function(DateTime now)? _backfillIsDue; + + bool _pipelineActive = false; + + Future runIfDue(DateTime now) async { + final DateTime tick = now.toUtc(); + + await _prepareForPipeline(tick); + + if (_pipelineActive) { + return MarketHistorySchedulerReport(ranStages: []); + } + + _pipelineActive = true; + try { + final List ran = []; + + await _maybeRunStage( + tick: tick, + kind: TradableAssetsSync.kind, + cadenceHours: config.universeRefreshHours, + runner: _runUniverse, + ran: ran, + ); + await _maybeRunStage( + tick: tick, + kind: MarketDataHistorySync.kind, + cadenceHours: config.historySyncHours, + runner: _runBackfill, + isDue: _backfillIsDue == null + ? null + : (DateTime tick) => _isDue( + tick, + MarketDataHistorySync.kind, + config.historySyncHours, + slotGate: _backfillIsDue, + ), + ran: ran, + ); + await _maybeRunStage( + tick: tick, + kind: MarketDataRetention.kind, + cadenceHours: config.cleanupHours, + runner: _runCleanup, + ran: ran, + ); + + return MarketHistorySchedulerReport(ranStages: ran); + } finally { + _pipelineActive = false; + } + } + + Future _prepareForPipeline(DateTime tick) async { + final int staleAborted = await _recorder.abortStaleInProgressRuns( + now: tick, + olderThan: Duration(minutes: config.staleSyncRunMinutes), + ); + if (_pipelineActive) { + if (staleAborted == 0) { + stderr.writeln( + 'Market history sync still running; skipped overlapping scheduler tick', + ); + return; + } + _pipelineActive = false; + stderr.writeln( + 'Aborted $staleAborted stale market history sync run(s); starting new pipeline', + ); + } + + final int aborted = await _recorder.abortAllInProgressRuns(now: tick); + if (aborted > 0) { + stderr.writeln( + 'Aborted $aborted orphaned market history sync run(s) before new pipeline', + ); + } + } + + Future _maybeRunStage({ + required DateTime tick, + required String kind, + required int cadenceHours, + required Future Function(DateTime now)? runner, + Future Function(DateTime now)? isDue, + required List ran, + }) async { + if (runner == null) { + return; + } + if (!await _isDue(tick, kind, cadenceHours, slotGate: isDue)) { + return; + } + await runner(tick); + ran.add(kind); + } + + Future _isDue( + DateTime now, + String kind, + int cadenceHours, { + Future Function(DateTime now)? slotGate, + }) async { + final DateTime? last = await _lastFinishedAt(kind); + + if (config.syncHourUtc != null) { + if (now.hour < config.syncHourUtc!) { + return false; + } + if (slotGate == null && last != null && _sameUtcDate(last, now)) { + return false; + } + } + + if (slotGate != null) { + return slotGate(now); + } + + if (last == null) { + return true; + } + return now.difference(last) >= Duration(hours: cadenceHours); + } + + Future _lastFinishedAt(String kind) async { + final Result result = await _connection.execute( + Sql.named( + ''' + SELECT finished_at + FROM market_data_sync_runs + WHERE kind = @kind + AND finished_at IS NOT NULL + AND (error IS NULL OR error NOT LIKE 'aborted:%') + ORDER BY finished_at DESC + LIMIT 1 + ''', + ), + parameters: {'kind': kind}, + ); + if (result.isEmpty) { + return null; + } + return (result.first[0]! as DateTime).toUtc(); + } + + static bool _sameUtcDate(DateTime a, DateTime b) { + final DateTime au = a.toUtc(); + final DateTime bu = b.toUtc(); + return au.year == bu.year && au.month == bu.month && au.day == bu.day; + } +} diff --git a/server/lib/workers/market_history_scheduler_config.dart b/server/lib/workers/market_history_scheduler_config.dart new file mode 100644 index 0000000..55d55c3 --- /dev/null +++ b/server/lib/workers/market_history_scheduler_config.dart @@ -0,0 +1,21 @@ +/// Cadence for [MarketHistoryScheduler]. Backfill uses slot pending, not [historySyncHours]. +class MarketHistorySchedulerConfig { + const MarketHistorySchedulerConfig({ + this.universeRefreshHours = 24, + this.historySyncHours = 24, + this.cleanupHours = 24, + this.syncHourUtc, + this.staleSyncRunMinutes = 30, + }); + + final int universeRefreshHours; + final int historySyncHours; + final int cleanupHours; + + /// In-progress sync rows older than this are aborted before a new pipeline run. + final int staleSyncRunMinutes; + + /// When set, stages only run when `now.hour >= syncHourUtc` (UTC) and + /// no successful run has already finished on the same UTC calendar day. + final int? syncHourUtc; +} diff --git a/server/lib/workers/question_background_worker.dart b/server/lib/workers/question_background_worker.dart index d1cdcda..9b08910 100644 --- a/server/lib/workers/question_background_worker.dart +++ b/server/lib/workers/question_background_worker.dart @@ -1,23 +1,38 @@ import 'dart:async'; import 'dart:io'; +import 'package:meta/meta.dart'; + import '../pipeline/question_pipeline.dart'; import '../trading/trading_orchestrator.dart'; +import 'market_history_scheduler.dart'; /// Runs [QuestionPipeline.runMaintenanceCycle] on a fixed interval, and /// optionally [TradingOrchestrator.runMaintenanceCycle] right after when /// trading is enabled. +/// +/// When [marketHistoryScheduler] is set, [MarketHistoryScheduler.runIfDue] +/// runs at the start of each tick (before the question pipeline). class QuestionBackgroundWorker { QuestionBackgroundWorker({ required QuestionPipeline pipeline, required Duration interval, TradingOrchestrator? tradingOrchestrator, + MarketHistoryScheduler? marketHistoryScheduler, + Future Function()? tradingMaintenanceRunner, + DateTime Function()? clock, }) : _pipeline = pipeline, _interval = interval, - _tradingOrchestrator = tradingOrchestrator; + _tradingOrchestrator = tradingOrchestrator, + _marketHistoryScheduler = marketHistoryScheduler, + _tradingMaintenanceRunner = tradingMaintenanceRunner, + _clock = clock ?? DateTime.now; final QuestionPipeline _pipeline; final TradingOrchestrator? _tradingOrchestrator; + final MarketHistoryScheduler? _marketHistoryScheduler; + final Future Function()? _tradingMaintenanceRunner; + final DateTime Function() _clock; final Duration _interval; Timer? _timer; bool _running = false; @@ -28,7 +43,8 @@ class QuestionBackgroundWorker { } stdout.writeln( 'Question background worker started (interval ${_interval.inSeconds}s, ' - 'trading=${_tradingOrchestrator != null})', + 'trading=${_tradingOrchestrator != null || _tradingMaintenanceRunner != null}, ' + 'marketHistory=${_marketHistoryScheduler != null})', ); _timer = Timer.periodic(_interval, (_) => _tick()); unawaited(_tick()); @@ -40,17 +56,33 @@ class QuestionBackgroundWorker { _pipeline.close(); } + @visibleForTesting + Future runTickForTest() => _tick(); + Future _tick() async { if (_running) { return; } _running = true; + if (_marketHistoryScheduler != null) { + try { + await _marketHistoryScheduler.runIfDue(_clock()); + } catch (e, st) { + stderr.writeln('Market history scheduler tick failed: $e\n$st'); + } + } try { await _pipeline.runMaintenanceCycle(); } catch (e, st) { stderr.writeln('Question background worker tick failed: $e\n$st'); } - if (_tradingOrchestrator != null) { + if (_tradingMaintenanceRunner != null) { + try { + await _tradingMaintenanceRunner(); + } catch (e, st) { + stderr.writeln('Trading orchestrator tick failed: $e\n$st'); + } + } else if (_tradingOrchestrator != null) { try { await _tradingOrchestrator.runMaintenanceCycle(); } catch (e, st) { diff --git a/server/migrations/005_market_history.sql b/server/migrations/005_market_history.sql new file mode 100644 index 0000000..f4fc392 --- /dev/null +++ b/server/migrations/005_market_history.sql @@ -0,0 +1,53 @@ +-- 005_market_history.sql +-- +-- Adds the rolling 7-day market-history substrate: +-- * timeframe column + idempotent unique observation key on +-- market_data_snapshots +-- * tradable_assets cache for the daily Alpaca asset universe +-- * market_data_sync_runs audit table +-- See TODO.md §1. + +ALTER TABLE market_data_snapshots + ADD COLUMN IF NOT EXISTS timeframe TEXT NOT NULL DEFAULT 'tick'; + +ALTER TABLE market_data_snapshots + DROP CONSTRAINT IF EXISTS market_data_snapshots_unique_obs; + +ALTER TABLE market_data_snapshots + ADD CONSTRAINT market_data_snapshots_unique_obs + UNIQUE (symbol, metric, timeframe, as_of); + +CREATE INDEX IF NOT EXISTS market_data_snapshots_asof_idx + ON market_data_snapshots (as_of DESC); + +CREATE TABLE IF NOT EXISTS tradable_assets ( + symbol TEXT PRIMARY KEY, + asset_class TEXT NOT NULL DEFAULT 'us_equity', + exchange TEXT, + name TEXT, + tradable BOOLEAN NOT NULL DEFAULT true, + fractionable BOOLEAN NOT NULL DEFAULT false, + status TEXT NOT NULL DEFAULT 'active', + raw JSONB, + refreshed_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS tradable_assets_status_idx + ON tradable_assets (status, tradable); + +CREATE TABLE IF NOT EXISTS market_data_sync_runs ( + id BIGSERIAL PRIMARY KEY, + kind TEXT NOT NULL, + started_at TIMESTAMPTZ NOT NULL DEFAULT now(), + finished_at TIMESTAMPTZ, + rows_written INTEGER NOT NULL DEFAULT 0, + rows_removed INTEGER NOT NULL DEFAULT 0, + error TEXT +); + +CREATE INDEX IF NOT EXISTS market_data_sync_runs_kind_started_idx + ON market_data_sync_runs (kind, started_at DESC); + +-- The market_data_archive table is intentionally deferred to TODO.md +-- section 4.2 (Phase 2 archive mode) and will be added when +-- MarketDataRetention.runArchiveAndCleanup is implemented. diff --git a/server/migrations/006_market_data_archive.sql b/server/migrations/006_market_data_archive.sql new file mode 100644 index 0000000..c8277da --- /dev/null +++ b/server/migrations/006_market_data_archive.sql @@ -0,0 +1,21 @@ +-- 006_market_data_archive.sql — Phase 2 archive-before-delete (TODO §4.2). + +CREATE TABLE IF NOT EXISTS market_data_archive ( + id BIGSERIAL PRIMARY KEY, + symbol TEXT NOT NULL, + asset_class TEXT NOT NULL DEFAULT 'us_equity', + feed TEXT NOT NULL DEFAULT 'iex', + metric TEXT NOT NULL, + timeframe TEXT NOT NULL DEFAULT 'tick', + price NUMERIC, + volume NUMERIC, + as_of TIMESTAMPTZ NOT NULL, + raw JSONB, + archived_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS market_data_archive_asof_idx + ON market_data_archive (as_of DESC); + +CREATE INDEX IF NOT EXISTS market_data_archive_symbol_asof_idx + ON market_data_archive (symbol, as_of DESC); diff --git a/server/migrations/007_questions_metadata.sql b/server/migrations/007_questions_metadata.sql new file mode 100644 index 0000000..aeb1eb3 --- /dev/null +++ b/server/migrations/007_questions_metadata.sql @@ -0,0 +1,3 @@ +-- Server-side question metadata (e.g. guess_symbol for obfuscated trading questions). +ALTER TABLE questions + ADD COLUMN IF NOT EXISTS metadata JSONB NOT NULL DEFAULT '{}'; diff --git a/server/migrations/008_market_history_four_hour.sql b/server/migrations/008_market_history_four_hour.sql new file mode 100644 index 0000000..ee71068 --- /dev/null +++ b/server/migrations/008_market_history_four_hour.sql @@ -0,0 +1,25 @@ +-- 008_market_history_four_hour.sql +-- +-- Market history uses Alpaca 4Hour bars (six UTC slots per day). +-- Drops legacy 1Day history rows, constrains timeframes, adds query index, +-- and records slots_synced on backfill audit rows. + +DELETE FROM market_data_snapshots +WHERE metric = 'bar' AND timeframe = '1Day'; + +DELETE FROM market_data_archive +WHERE metric = 'bar' AND timeframe = '1Day'; + +ALTER TABLE market_data_snapshots + DROP CONSTRAINT IF EXISTS market_data_snapshots_timeframe_check; + +ALTER TABLE market_data_snapshots + ADD CONSTRAINT market_data_snapshots_timeframe_check + CHECK (timeframe IN ('tick', '1Min', '1Hour', '4Hour', '1Day')); + +CREATE INDEX IF NOT EXISTS market_data_snapshots_bar_4h_idx + ON market_data_snapshots (symbol, as_of DESC) + WHERE metric = 'bar' AND timeframe = '4Hour'; + +ALTER TABLE market_data_sync_runs + ADD COLUMN IF NOT EXISTS slots_synced INTEGER NOT NULL DEFAULT 0; diff --git a/server/migrations/009_market_history_backfill_items.sql b/server/migrations/009_market_history_backfill_items.sql new file mode 100644 index 0000000..beea237 --- /dev/null +++ b/server/migrations/009_market_history_backfill_items.sql @@ -0,0 +1,4 @@ +-- Per-slot backfill audit: UTC slot start + symbols requested from Alpaca. + +ALTER TABLE market_data_sync_runs + ADD COLUMN IF NOT EXISTS backfill_items JSONB; diff --git a/server/test/alpaca/alpaca_assets_client_test.dart b/server/test/alpaca/alpaca_assets_client_test.dart new file mode 100644 index 0000000..e2916ab --- /dev/null +++ b/server/test/alpaca/alpaca_assets_client_test.dart @@ -0,0 +1,154 @@ +import 'dart:convert'; + +import 'package:cyberhybridhub_server/alpaca/alpaca_assets_client.dart'; +import 'package:cyberhybridhub_server/alpaca/alpaca_env.dart'; +import 'package:cyberhybridhub_server/alpaca/alpaca_models.dart'; +import 'package:http/http.dart' as http; +import 'package:test/test.dart'; + +import '../helpers/fixture_loader.dart'; +import '../helpers/mock_http_client.dart'; + +void main() { + late FixtureLoader fixtures; + late AlpacaEnv env; + + setUp(() { + fixtures = FixtureLoader(); + env = AlpacaEnv.fromMap({ + 'ALPACA_API_KEY_ID': 'test-key', + 'ALPACA_API_SECRET_KEY': 'test-secret', + 'ALPACA_TRADING_BASE_URL': 'https://paper-api.alpaca.markets', + 'ALPACA_DATA_BASE_URL': 'https://data.alpaca.markets', + }); + }); + + test('listActiveTradable issues GET with auth headers and query', () async { + final String body = await fixtures.loadString('alpaca_assets_active.json'); + final MockHttpClient mock = MockHttpClient() + ..whenGet( + '/v2/assets', + http.Response(body, 200, headers: { + 'content-type': 'application/json', + }), + ); + + final AlpacaAssetsClient client = + AlpacaAssetsClient(env: env, httpClient: mock); + await client.listActiveTradable(); + + expect(mock.requests, hasLength(1)); + final http.BaseRequest req = mock.requests.single; + expect(req.method, 'GET'); + expect( + req.url.toString(), + startsWith('https://paper-api.alpaca.markets/v2/assets'), + ); + expect(req.url.queryParameters['status'], 'active'); + expect(req.url.queryParameters['asset_class'], 'us_equity'); + expect(req.headers['APCA-API-KEY-ID'], 'test-key'); + expect(req.headers['APCA-API-SECRET-KEY'], 'test-secret'); + }); + + test('listActiveTradable parses the fixture into AlpacaAsset rows', + () async { + final String body = await fixtures.loadString('alpaca_assets_active.json'); + final MockHttpClient mock = MockHttpClient() + ..whenGet( + '/v2/assets', + http.Response(body, 200, headers: { + 'content-type': 'application/json', + }), + ); + + final AlpacaAssetsClient client = + AlpacaAssetsClient(env: env, httpClient: mock); + final List assets = await client.listActiveTradable(); + + expect(assets, hasLength(5)); + + final AlpacaAsset aapl = + assets.firstWhere((AlpacaAsset a) => a.symbol == 'AAPL'); + expect(aapl.assetClass, 'us_equity'); + expect(aapl.exchange, 'NASDAQ'); + expect(aapl.name, 'Apple Inc. Common Stock'); + expect(aapl.status, 'active'); + expect(aapl.tradable, isTrue); + expect(aapl.fractionable, isTrue); + + final AlpacaAsset pinkSheet = + assets.firstWhere((AlpacaAsset a) => a.symbol == 'PNKZZ'); + expect(pinkSheet.tradable, isFalse); + expect(pinkSheet.fractionable, isFalse); + + final AlpacaAsset brk = + assets.firstWhere((AlpacaAsset a) => a.symbol == 'BRK.B'); + expect(brk.tradable, isTrue); + expect(brk.fractionable, isFalse); + }); + + test('401 unauthorized throws AlpacaAssetsException with code + body', + () async { + final MockHttpClient mock = MockHttpClient() + ..whenGet( + '/v2/assets', + http.Response( + jsonEncode({'message': 'forbidden.'}), + 401, + headers: {'content-type': 'application/json'}, + ), + ); + + final AlpacaAssetsClient client = + AlpacaAssetsClient(env: env, httpClient: mock); + + await expectLater( + client.listActiveTradable(), + throwsA( + isA() + .having((AlpacaAssetsException e) => e.message, 'message', + contains('401')) + .having((AlpacaAssetsException e) => e.message, 'body', + contains('forbidden')), + ), + ); + }); + + test('500 server error throws AlpacaAssetsException with code + body', + () async { + final MockHttpClient mock = MockHttpClient() + ..whenGet( + '/v2/assets', + http.Response('upstream exploded', 500), + ); + + final AlpacaAssetsClient client = + AlpacaAssetsClient(env: env, httpClient: mock); + + await expectLater( + client.listActiveTradable(), + throwsA( + isA() + .having((AlpacaAssetsException e) => e.message, 'message', + contains('500')) + .having((AlpacaAssetsException e) => e.message, 'body', + contains('upstream exploded')), + ), + ); + }); + + test('empty array response returns [] and does not throw', () async { + final MockHttpClient mock = MockHttpClient() + ..whenGet( + '/v2/assets', + http.Response('[]', 200, headers: { + 'content-type': 'application/json', + }), + ); + + final AlpacaAssetsClient client = + AlpacaAssetsClient(env: env, httpClient: mock); + final List assets = await client.listActiveTradable(); + expect(assets, isEmpty); + }); +} diff --git a/server/test/alpaca/alpaca_assets_live_test.dart b/server/test/alpaca/alpaca_assets_live_test.dart new file mode 100644 index 0000000..4031958 --- /dev/null +++ b/server/test/alpaca/alpaca_assets_live_test.dart @@ -0,0 +1,64 @@ +@Tags(['alpaca']) +library; + +import 'package:cyberhybridhub_server/alpaca/alpaca_assets_client.dart'; +import 'package:cyberhybridhub_server/alpaca/alpaca_env.dart'; +import 'package:cyberhybridhub_server/alpaca/alpaca_models.dart'; +import 'package:dotenv/dotenv.dart'; +import 'package:test/test.dart'; + +void main() { + late AlpacaEnv env; + AlpacaAssetsClient? client; + + setUpAll(() { + final DotEnv dotenv = DotEnv(includePlatformEnvironment: true) + ..load(['.env']); + const List alpacaKeys = [ + 'ALPACA_API_KEY_ID', + 'ALPACA_API_SECRET_KEY', + 'ALPACA_TRADING_BASE_URL', + 'ALPACA_DATA_BASE_URL', + 'ALPACA_DATA_FEED', + 'ALPACA_ALLOW_LIVE', + ]; + final Map envMap = { + for (final String key in alpacaKeys) + if (dotenv[key] != null && dotenv[key]!.isNotEmpty) key: dotenv[key]!, + }; + env = AlpacaEnv.fromMap(envMap); + if (env.hasCredentials) { + client = AlpacaAssetsClient(env: env); + } + }); + + tearDownAll(() { + client?.close(); + }); + + test('live listActiveTradable returns >100 us_equity assets', () async { + if (!env.hasCredentials) { + markTestSkipped( + 'Set ALPACA_API_KEY_ID and ALPACA_API_SECRET_KEY in server/.env', + ); + return; + } + + final List assets = await client!.listActiveTradable(); + + // Alpaca's live US-equity catalog has thousands of entries; >100 is a + // safe lower bound that catches "we got an empty/auth-error payload" + // without being brittle. + expect(assets.length, greaterThan(100)); + expect( + assets.every((AlpacaAsset a) => a.assetClass == 'us_equity'), + isTrue, + reason: 'asset_class=us_equity filter should be enforced server-side', + ); + expect( + assets.every((AlpacaAsset a) => a.status == 'active'), + isTrue, + reason: 'status=active filter should be enforced server-side', + ); + }); +} diff --git a/server/test/alpaca/alpaca_market_data_client_test.dart b/server/test/alpaca/alpaca_market_data_client_test.dart index 32bbd7f..f960b5d 100644 --- a/server/test/alpaca/alpaca_market_data_client_test.dart +++ b/server/test/alpaca/alpaca_market_data_client_test.dart @@ -1,5 +1,7 @@ import 'package:cyberhybridhub_server/alpaca/alpaca_env.dart'; import 'package:cyberhybridhub_server/alpaca/alpaca_market_data_client.dart'; +import 'package:cyberhybridhub_server/alpaca/alpaca_models.dart'; +import 'package:http/http.dart' as http; import 'package:test/test.dart'; import '../helpers/fixture_loader.dart'; @@ -53,4 +55,105 @@ void main() { expect(mock.requests.single.url.queryParameters['symbols'], 'SPY'); expect(mock.requests.single.url.queryParameters['timeframe'], '1Day'); }); + + group('getBarsRange', () { + final DateTime start = DateTime.utc(2026, 5, 20); + final DateTime end = DateTime.utc(2026, 5, 27); + + test('builds query string with start, end, timeframe, feed, symbols, limit', + () async { + final Map barsJson = + await fixtures.loadJson('alpaca_daily_bars.json'); + final MockHttpClient mock = MockHttpClient() + ..whenGetJson('/bars', barsJson); + + final AlpacaMarketDataClient client = + AlpacaMarketDataClient(env: env, httpClient: mock); + await client.getBarsRange( + symbols: ['SPY', 'AAPL'], + timeframe: '1Day', + start: start, + end: end, + ); + + expect(mock.requests, hasLength(1)); + final Uri url = mock.requests.single.url; + expect(url.queryParameters['symbols'], 'SPY,AAPL'); + expect(url.queryParameters['timeframe'], '1Day'); + expect(url.queryParameters['feed'], 'iex'); + expect(url.queryParameters['limit'], '10000'); + expect(url.queryParameters['start'], '2026-05-20T00:00:00Z'); + expect(url.queryParameters['end'], '2026-05-27T00:00:00Z'); + }); + + test('follows pagination and merges bars per symbol', () async { + final Map page1 = + await fixtures.loadJson('alpaca_bars_7d_multi_page1.json'); + final Map page2 = + await fixtures.loadJson('alpaca_bars_7d_multi_page2.json'); + final MockHttpClient mock = MockHttpClient() + ..whenGetQueuedJson('/bars', page1) + ..whenGetQueuedJson('/bars', page2); + + final AlpacaMarketDataClient client = + AlpacaMarketDataClient(env: env, httpClient: mock); + final AlpacaBarsResponse merged = await client.getBarsRange( + symbols: ['SPY', 'AAPL'], + timeframe: '1Day', + start: start, + end: end, + ); + + expect(mock.requests, hasLength(2)); + expect(mock.requests[1].url.queryParameters['page_token'], 'abc'); + expect(merged.barsBySymbol['SPY'], hasLength(3)); + expect(merged.barsBySymbol['AAPL'], hasLength(3)); + }); + + test('stops after maxPages even when next_page_token is present', () async { + final Map page1 = + await fixtures.loadJson('alpaca_bars_7d_multi_page1.json'); + final MockHttpClient mock = MockHttpClient() + ..whenGetQueuedJson('/bars', page1); + + final AlpacaMarketDataClient client = + AlpacaMarketDataClient(env: env, httpClient: mock); + await client.getBarsRange( + symbols: ['SPY'], + timeframe: '1Day', + start: start, + end: end, + maxPages: 1, + ); + + expect(mock.requests, hasLength(1)); + }); + + test('429 throws AlpacaMarketDataException containing rate', () async { + final MockHttpClient mock = MockHttpClient() + ..whenGet( + '/bars', + http.Response('rate limit exceeded', 429), + ); + + final AlpacaMarketDataClient client = + AlpacaMarketDataClient(env: env, httpClient: mock); + + await expectLater( + client.getBarsRange( + symbols: ['SPY'], + timeframe: '1Day', + start: start, + end: end, + ), + throwsA( + isA().having( + (AlpacaMarketDataException e) => e.message, + 'message', + contains('rate'), + ), + ), + ); + }); + }); } diff --git a/server/test/alpaca/alpaca_market_data_history_live_test.dart b/server/test/alpaca/alpaca_market_data_history_live_test.dart new file mode 100644 index 0000000..aaa284b --- /dev/null +++ b/server/test/alpaca/alpaca_market_data_history_live_test.dart @@ -0,0 +1,62 @@ +@Tags(['alpaca']) +library; + +import 'package:cyberhybridhub_server/alpaca/alpaca_env.dart'; +import 'package:cyberhybridhub_server/alpaca/alpaca_market_data_client.dart'; +import 'package:cyberhybridhub_server/alpaca/alpaca_models.dart'; +import 'package:dotenv/dotenv.dart'; +import 'package:test/test.dart'; + +void main() { + late AlpacaEnv env; + AlpacaMarketDataClient? client; + + setUpAll(() { + final DotEnv dotenv = DotEnv(includePlatformEnvironment: true) + ..load(['.env']); + const List alpacaKeys = [ + 'ALPACA_API_KEY_ID', + 'ALPACA_API_SECRET_KEY', + 'ALPACA_TRADING_BASE_URL', + 'ALPACA_DATA_BASE_URL', + 'ALPACA_DATA_FEED', + 'ALPACA_ALLOW_LIVE', + ]; + final Map envMap = { + for (final String key in alpacaKeys) + if (dotenv[key] != null && dotenv[key]!.isNotEmpty) key: dotenv[key]!, + }; + env = AlpacaEnv.fromMap(envMap); + if (env.hasCredentials) { + client = AlpacaMarketDataClient(env: env); + } + }); + + tearDownAll(() { + client?.close(); + }); + + test('live getBarsRange for SPY returns at least 3 daily bars in 7d window', + () async { + if (!env.hasCredentials) { + markTestSkipped( + 'Set ALPACA_API_KEY_ID and ALPACA_API_SECRET_KEY in server/.env', + ); + return; + } + + final DateTime end = DateTime.now().toUtc(); + final DateTime start = end.subtract(const Duration(days: 7)); + + final AlpacaBarsResponse response = await client!.getBarsRange( + symbols: ['SPY'], + timeframe: '1Day', + start: start, + end: end, + ); + + final List? bars = response.barsBySymbol['SPY']; + expect(bars, isNotNull); + expect(bars!.length, greaterThanOrEqualTo(3)); + }); +} diff --git a/server/test/env/market_history_env_test.dart b/server/test/env/market_history_env_test.dart new file mode 100644 index 0000000..23f1d7a --- /dev/null +++ b/server/test/env/market_history_env_test.dart @@ -0,0 +1,77 @@ +import 'package:cyberhybridhub_server/market_history_env.dart'; +import 'package:test/test.dart'; + +void main() { + group('MarketHistoryEnv.fromMap', () { + test('defaults when env keys are empty', () { + final MarketHistoryEnv env = MarketHistoryEnv.fromMap({}); + + expect(env.syncEnabled, isFalse); + expect(env.windowDays, 7); + expect(env.retentionDays, 7); + expect(env.archiveEnabled, isFalse); + expect(env.universeRefreshHours, 24); + expect(env.historySyncHours, 24); + expect(env.cleanupHours, 24); + expect(env.syncHourUtc, isNull); + expect(env.historySyncBatchSize, 50); + expect(env.historySyncMaxSymbols, 2000); + expect(env.minBarsForGuess, 5); + expect(env.guessCooldownHours, 24); + expect(env.apiRequestsPerMinute, 200); + expect(env.staleSyncRunMinutes, 30); + }); + + test('MARKET_HISTORY_API_REQUESTS_PER_MINUTE overrides default', () { + final MarketHistoryEnv env = MarketHistoryEnv.fromMap({ + 'MARKET_HISTORY_API_REQUESTS_PER_MINUTE': '120', + }); + expect(env.apiRequestsPerMinute, 120); + }); + + test('MARKET_HISTORY_SYNC_ENABLED=true without trading throws', () { + final MarketHistoryEnv env = MarketHistoryEnv.fromMap({ + 'MARKET_HISTORY_SYNC_ENABLED': 'true', + }); + + expect( + () => env.assertConsistent(tradingEnabled: false), + throwsStateError, + ); + }); + + test('MARKET_HISTORY_WINDOW_DAYS=0 throws', () { + expect( + () => MarketHistoryEnv.fromMap({ + 'MARKET_HISTORY_WINDOW_DAYS': '0', + }), + throwsArgumentError, + ); + }); + + test('MARKET_HISTORY_WINDOW_DAYS negative throws', () { + expect( + () => MarketHistoryEnv.fromMap({ + 'MARKET_HISTORY_WINDOW_DAYS': '-3', + }), + throwsArgumentError, + ); + }); + + test('MARKET_HISTORY_SYNC_HOUR_UTC=24 throws', () { + expect( + () => MarketHistoryEnv.fromMap({ + 'MARKET_HISTORY_SYNC_HOUR_UTC': '24', + }), + throwsArgumentError, + ); + }); + + test('MARKET_HISTORY_SYNC_HOUR_UTC=10 is accepted', () { + final MarketHistoryEnv env = MarketHistoryEnv.fromMap({ + 'MARKET_HISTORY_SYNC_HOUR_UTC': '10', + }); + expect(env.syncHourUtc, 10); + }); + }); +} diff --git a/server/test/fixtures/alpaca_assets_active.json b/server/test/fixtures/alpaca_assets_active.json new file mode 100644 index 0000000..b0bc1a6 --- /dev/null +++ b/server/test/fixtures/alpaca_assets_active.json @@ -0,0 +1,67 @@ +[ + { + "id": "b0b6dd9d-8b9b-48a9-ba46-b9d54906e415", + "class": "us_equity", + "exchange": "NASDAQ", + "symbol": "AAPL", + "name": "Apple Inc. Common Stock", + "status": "active", + "tradable": true, + "marginable": true, + "shortable": true, + "easy_to_borrow": true, + "fractionable": true + }, + { + "id": "8ccae427-5dd0-45b3-b5fe-7ba5e422c766", + "class": "us_equity", + "exchange": "NASDAQ", + "symbol": "MSFT", + "name": "Microsoft Corporation Common Stock", + "status": "active", + "tradable": true, + "marginable": true, + "shortable": true, + "easy_to_borrow": true, + "fractionable": true + }, + { + "id": "a8ab8ab8-7777-4444-aaaa-cccccccccccc", + "class": "us_equity", + "exchange": "ARCA", + "symbol": "SPY", + "name": "SPDR S&P 500 ETF Trust", + "status": "active", + "tradable": true, + "marginable": true, + "shortable": true, + "easy_to_borrow": true, + "fractionable": true + }, + { + "id": "11111111-2222-3333-4444-555555555555", + "class": "us_equity", + "exchange": "OTC", + "symbol": "PNKZZ", + "name": "Pink Sheet Test Co.", + "status": "active", + "tradable": false, + "marginable": false, + "shortable": false, + "easy_to_borrow": false, + "fractionable": false + }, + { + "id": "66666666-7777-8888-9999-000000000000", + "class": "us_equity", + "exchange": "NYSE", + "symbol": "BRK.B", + "name": "Berkshire Hathaway Inc. Class B", + "status": "active", + "tradable": true, + "marginable": true, + "shortable": true, + "easy_to_borrow": true, + "fractionable": false + } +] diff --git a/server/test/fixtures/alpaca_bars_4h_window.json b/server/test/fixtures/alpaca_bars_4h_window.json new file mode 100644 index 0000000..6e2ecd6 --- /dev/null +++ b/server/test/fixtures/alpaca_bars_4h_window.json @@ -0,0 +1,29 @@ +{ + "bars": { + "SPY": [ + { "t": "2026-05-25T12:00:00Z", "o": 490, "h": 492, "l": 488, "c": 491, "v": 4000000 }, + { "t": "2026-05-25T16:00:00Z", "o": 491, "h": 493, "l": 489, "c": 492, "v": 4100000 }, + { "t": "2026-05-25T20:00:00Z", "o": 492, "h": 494, "l": 490, "c": 493, "v": 4200000 }, + { "t": "2026-05-26T00:00:00Z", "o": 493, "h": 495, "l": 491, "c": 494, "v": 4300000 }, + { "t": "2026-05-26T04:00:00Z", "o": 494, "h": 496, "l": 492, "c": 495, "v": 4400000 }, + { "t": "2026-05-26T08:00:00Z", "o": 495, "h": 497, "l": 493, "c": 496, "v": 4500000 } + ], + "AAPL": [ + { "t": "2026-05-25T12:00:00Z", "o": 180, "h": 182, "l": 179, "c": 181.5, "v": 5000000 }, + { "t": "2026-05-25T16:00:00Z", "o": 181.5, "h": 183, "l": 180, "c": 182, "v": 5100000 }, + { "t": "2026-05-25T20:00:00Z", "o": 182, "h": 184, "l": 181, "c": 183, "v": 5200000 }, + { "t": "2026-05-26T00:00:00Z", "o": 183, "h": 185, "l": 182, "c": 184, "v": 5300000 }, + { "t": "2026-05-26T04:00:00Z", "o": 184, "h": 186, "l": 183, "c": 185, "v": 5400000 }, + { "t": "2026-05-26T08:00:00Z", "o": 185, "h": 187, "l": 184, "c": 186, "v": 5500000 } + ], + "MSFT": [ + { "t": "2026-05-25T12:00:00Z", "o": 410, "h": 412, "l": 408, "c": 411, "v": 3000000 }, + { "t": "2026-05-25T16:00:00Z", "o": 411, "h": 413, "l": 409, "c": 412, "v": 3100000 }, + { "t": "2026-05-25T20:00:00Z", "o": 412, "h": 414, "l": 410, "c": 413, "v": 3200000 }, + { "t": "2026-05-26T00:00:00Z", "o": 413, "h": 415, "l": 411, "c": 414, "v": 3300000 }, + { "t": "2026-05-26T04:00:00Z", "o": 414, "h": 416, "l": 412, "c": 415, "v": 3400000 }, + { "t": "2026-05-26T08:00:00Z", "o": 415, "h": 417, "l": 413, "c": 416, "v": 3500000 } + ] + }, + "next_page_token": null +} diff --git a/server/test/fixtures/alpaca_bars_7d_3symbols.json b/server/test/fixtures/alpaca_bars_7d_3symbols.json new file mode 100644 index 0000000..7ba75d7 --- /dev/null +++ b/server/test/fixtures/alpaca_bars_7d_3symbols.json @@ -0,0 +1,32 @@ +{ + "bars": { + "SPY": [ + { "t": "2026-05-20T04:00:00Z", "o": 490, "h": 492, "l": 488, "c": 491, "v": 40000000 }, + { "t": "2026-05-21T04:00:00Z", "o": 491, "h": 495, "l": 489, "c": 494, "v": 41000000 }, + { "t": "2026-05-22T04:00:00Z", "o": 494, "h": 498, "l": 492, "c": 497, "v": 42000000 }, + { "t": "2026-05-23T04:00:00Z", "o": 497, "h": 500, "l": 495, "c": 499, "v": 43000000 }, + { "t": "2026-05-24T04:00:00Z", "o": 499, "h": 501, "l": 496, "c": 500, "v": 44000000 }, + { "t": "2026-05-25T04:00:00Z", "o": 500, "h": 502, "l": 497, "c": 501, "v": 45000000 }, + { "t": "2026-05-26T04:00:00Z", "o": 501, "h": 504, "l": 499, "c": 503, "v": 46000000 } + ], + "AAPL": [ + { "t": "2026-05-20T04:00:00Z", "o": 180, "h": 182, "l": 179, "c": 181.5, "v": 50000000 }, + { "t": "2026-05-21T04:00:00Z", "o": 181.5, "h": 184, "l": 180.5, "c": 183, "v": 48000000 }, + { "t": "2026-05-22T04:00:00Z", "o": 183, "h": 185, "l": 182, "c": 184.5, "v": 46000000 }, + { "t": "2026-05-23T04:00:00Z", "o": 184.5, "h": 186, "l": 183, "c": 185, "v": 47000000 }, + { "t": "2026-05-24T04:00:00Z", "o": 185, "h": 187, "l": 184, "c": 186, "v": 48000000 }, + { "t": "2026-05-25T04:00:00Z", "o": 186, "h": 188, "l": 185, "c": 187, "v": 49000000 }, + { "t": "2026-05-26T04:00:00Z", "o": 187, "h": 189, "l": 186, "c": 188, "v": 50000000 } + ], + "MSFT": [ + { "t": "2026-05-20T04:00:00Z", "o": 410, "h": 412, "l": 408, "c": 411, "v": 30000000 }, + { "t": "2026-05-21T04:00:00Z", "o": 411, "h": 414, "l": 409, "c": 413, "v": 31000000 }, + { "t": "2026-05-22T04:00:00Z", "o": 413, "h": 416, "l": 411, "c": 415, "v": 32000000 }, + { "t": "2026-05-23T04:00:00Z", "o": 415, "h": 417, "l": 412, "c": 416, "v": 33000000 }, + { "t": "2026-05-24T04:00:00Z", "o": 416, "h": 418, "l": 413, "c": 417, "v": 34000000 }, + { "t": "2026-05-25T04:00:00Z", "o": 417, "h": 419, "l": 414, "c": 418, "v": 35000000 }, + { "t": "2026-05-26T04:00:00Z", "o": 418, "h": 421, "l": 416, "c": 420, "v": 36000000 } + ] + }, + "next_page_token": null +} diff --git a/server/test/fixtures/alpaca_bars_7d_multi_page1.json b/server/test/fixtures/alpaca_bars_7d_multi_page1.json new file mode 100644 index 0000000..ae83924 --- /dev/null +++ b/server/test/fixtures/alpaca_bars_7d_multi_page1.json @@ -0,0 +1,33 @@ +{ + "bars": { + "SPY": [ + { + "t": "2026-05-20T04:00:00Z", + "o": 490.0, + "h": 492.0, + "l": 488.0, + "c": 491.0, + "v": 40000000 + }, + { + "t": "2026-05-21T04:00:00Z", + "o": 491.0, + "h": 495.0, + "l": 489.0, + "c": 494.0, + "v": 41000000 + } + ], + "AAPL": [ + { + "t": "2026-05-20T04:00:00Z", + "o": 180.0, + "h": 182.0, + "l": 179.0, + "c": 181.5, + "v": 50000000 + } + ] + }, + "next_page_token": "abc" +} diff --git a/server/test/fixtures/alpaca_bars_7d_multi_page2.json b/server/test/fixtures/alpaca_bars_7d_multi_page2.json new file mode 100644 index 0000000..7e3cb7b --- /dev/null +++ b/server/test/fixtures/alpaca_bars_7d_multi_page2.json @@ -0,0 +1,33 @@ +{ + "bars": { + "SPY": [ + { + "t": "2026-05-22T04:00:00Z", + "o": 494.0, + "h": 498.0, + "l": 492.0, + "c": 497.0, + "v": 42000000 + } + ], + "AAPL": [ + { + "t": "2026-05-21T04:00:00Z", + "o": 181.5, + "h": 184.0, + "l": 180.5, + "c": 183.0, + "v": 48000000 + }, + { + "t": "2026-05-22T04:00:00Z", + "o": 183.0, + "h": 185.0, + "l": 182.0, + "c": 184.5, + "v": 46000000 + } + ] + }, + "next_page_token": null +} diff --git a/server/test/helpers/admin_sync_run_fixtures.dart b/server/test/helpers/admin_sync_run_fixtures.dart new file mode 100644 index 0000000..1a91c3a --- /dev/null +++ b/server/test/helpers/admin_sync_run_fixtures.dart @@ -0,0 +1,142 @@ +import 'package:cyberhybridhub_server/trading/backfill_sync_item.dart'; +import 'package:cyberhybridhub_server/trading/market_history_admin_logic.dart'; + +AdminSyncRunRecord adminSyncRun({ + required int id, + required String kind, + required DateTime startedAt, + DateTime? finishedAt, + int rowsWritten = 0, + int rowsRemoved = 0, + int slotsSynced = 0, + List? backfillItems, + String? error, +}) { + return AdminSyncRunRecord( + id: id, + kind: kind, + startedAt: startedAt, + finishedAt: finishedAt, + rowsWritten: rowsWritten, + rowsRemoved: rowsRemoved, + slotsSynced: slotsSynced, + backfillItems: backfillItems ?? const [], + error: error, + ); +} + +List fixtureAllSuccessRecentFirst(DateTime base) { + return [ + adminSyncRun( + id: 3, + kind: 'cleanup', + startedAt: base, + finishedAt: base.add(const Duration(minutes: 1)), + rowsRemoved: 100, + ), + adminSyncRun( + id: 2, + kind: 'backfill', + startedAt: base.subtract(const Duration(hours: 1)), + finishedAt: base.subtract(const Duration(minutes: 59)), + rowsWritten: 500, + ), + adminSyncRun( + id: 1, + kind: 'universe', + startedAt: base.subtract(const Duration(hours: 2)), + finishedAt: base.subtract(const Duration(hours: 1, minutes: 59)), + rowsWritten: 8000, + ), + ]; +} + +List fixtureRateLimitUnresolved(DateTime base) { + return [ + adminSyncRun( + id: 10, + kind: 'backfill', + startedAt: base, + finishedAt: base.add(const Duration(minutes: 2)), + rowsWritten: 0, + error: 'AlpacaMarketDataException: rate limited: 429', + ), + ]; +} + +List fixtureFailedThenSuccessSameKind(DateTime base) { + return [ + adminSyncRun( + id: 21, + kind: 'backfill', + startedAt: base.subtract(const Duration(hours: 2)), + finishedAt: base.subtract(const Duration(hours: 1, minutes: 58)), + error: '429', + ), + adminSyncRun( + id: 22, + kind: 'backfill', + startedAt: base, + finishedAt: base.add(const Duration(minutes: 1)), + rowsWritten: 1200, + ), + ]; +} + +List fixturePartialBackfillError(DateTime base) { + return [ + adminSyncRun( + id: 30, + kind: 'backfill', + startedAt: base, + finishedAt: base.add(const Duration(minutes: 5)), + rowsWritten: 12000, + error: 'batch MSFT,AAPL: server 500', + ), + ]; +} + +List fixtureInProgressStale(DateTime base) { + return [ + adminSyncRun( + id: 40, + kind: 'cleanup', + startedAt: base.subtract(const Duration(hours: 2)), + finishedAt: null, + ), + ]; +} + +List fixtureMixedKindsMixedOutcomes(DateTime base) { + return [ + adminSyncRun( + id: 50, + kind: 'cleanup', + startedAt: base, + finishedAt: base.add(const Duration(minutes: 1)), + rowsRemoved: 4200, + ), + adminSyncRun( + id: 51, + kind: 'backfill', + startedAt: base.subtract(const Duration(hours: 1)), + finishedAt: base.subtract(const Duration(minutes: 55)), + rowsWritten: 8000, + error: 'partial batch failure', + ), + adminSyncRun( + id: 52, + kind: 'universe', + startedAt: base.subtract(const Duration(hours: 3)), + finishedAt: base.subtract(const Duration(hours: 2, minutes: 58)), + rowsWritten: 8200, + ), + adminSyncRun( + id: 53, + kind: 'backfill', + startedAt: base.subtract(const Duration(hours: 5)), + finishedAt: base.subtract(const Duration(hours: 4, minutes: 58)), + error: '429', + ), + ]; +} diff --git a/server/test/helpers/mock_http_client.dart b/server/test/helpers/mock_http_client.dart index cd60e88..5a0baff 100644 --- a/server/test/helpers/mock_http_client.dart +++ b/server/test/helpers/mock_http_client.dart @@ -8,6 +8,7 @@ class MockHttpClient extends http.BaseClient { : _responses = responses ?? {}; final Map _responses; + final List<_QueuedGet> _getQueue = <_QueuedGet>[]; final List requests = []; /// Captured request bodies indexed by request order in [requests]. @@ -27,6 +28,53 @@ class MockHttpClient extends http.BaseClient { ); } + /// Returns [response] for the next GET whose path ends with [pathSuffix]. + /// + /// Useful for paginated endpoints where the same path is hit multiple + /// times with different `page_token` query values. + void whenGetQueued(String pathSuffix, http.Response response) { + _getQueue.add(_QueuedGet(pathSuffix, response)); + } + + void whenGetQueuedJson(String pathSuffix, Map body, + {int statusCode = 200}) { + whenGetQueued( + pathSuffix, + http.Response(jsonEncode(body), statusCode, headers: { + 'content-type': 'application/json', + }), + ); + } + + /// Returns [response] when [predicate] matches the request URI. + void whenGetWhere( + String pathSuffix, + bool Function(Uri uri) predicate, + http.Response response, + ) { + _responses['GET:$pathSuffix:${_responses.length}'] = _MatchedResponse( + method: 'GET', + response: response, + pathSuffix: pathSuffix, + uriPredicate: predicate, + ); + } + + void whenGetWhereJson( + String pathSuffix, + bool Function(Uri uri) predicate, + Map body, { + int statusCode = 200, + }) { + whenGetWhere( + pathSuffix, + predicate, + http.Response(jsonEncode(body), statusCode, headers: { + 'content-type': 'application/json', + }), + ); + } + void whenPost(String pathSuffix, http.Response response) { _responses['POST:$pathSuffix'] = _MatchedResponse(method: 'POST', response: response); @@ -50,21 +98,42 @@ class MockHttpClient extends http.BaseClient { final String path = request.url.path; final String method = request.method.toUpperCase(); - for (final MapEntry entry in _responses.entries) { - final _MatchedResponse match = entry.value; - if (match.method != method) continue; - final String suffix = entry.key.startsWith('$method:') - ? entry.key.substring(method.length + 1) - : entry.key; - if (path.endsWith(suffix) || request.url.toString().contains(suffix)) { + + if (method == 'GET' && _getQueue.isNotEmpty) { + final int idx = _getQueue.indexWhere( + (_QueuedGet q) => path.endsWith(q.pathSuffix), + ); + if (idx >= 0) { + final _QueuedGet queued = _getQueue.removeAt(idx); return http.StreamedResponse( - Stream>.value(utf8.encode(match.response.body)), - match.response.statusCode, - headers: match.response.headers, + Stream>.value(utf8.encode(queued.response.body)), + queued.response.statusCode, + headers: queued.response.headers, request: request, ); } } + + for (final MapEntry entry in _responses.entries) { + final _MatchedResponse match = entry.value; + if (match.method != method) continue; + final String suffix = match.pathSuffix ?? + (entry.key.startsWith('$method:') + ? entry.key.substring(method.length + 1).split(':').first + : entry.key); + final bool pathOk = + path.endsWith(suffix) || request.url.toString().contains(suffix); + if (!pathOk) continue; + if (match.uriPredicate != null && !match.uriPredicate!(request.url)) { + continue; + } + return http.StreamedResponse( + Stream>.value(utf8.encode(match.response.body)), + match.response.statusCode, + headers: match.response.headers, + request: request, + ); + } return http.StreamedResponse( Stream>.value(utf8.encode('{}')), 404, @@ -84,9 +153,23 @@ class MockHttpClient extends http.BaseClient { void close() {} } +class _QueuedGet { + _QueuedGet(this.pathSuffix, this.response); + + final String pathSuffix; + final http.Response response; +} + class _MatchedResponse { - _MatchedResponse({required this.method, required this.response}); + _MatchedResponse({ + required this.method, + required this.response, + this.pathSuffix, + this.uriPredicate, + }); final String method; final http.Response response; + final String? pathSuffix; + final bool Function(Uri uri)? uriPredicate; } diff --git a/server/test/helpers/test_db.dart b/server/test/helpers/test_db.dart index 375923a..7dad35f 100644 --- a/server/test/helpers/test_db.dart +++ b/server/test/helpers/test_db.dart @@ -11,7 +11,7 @@ import 'package:cyberhybridhub_server/trading/user_trading_state_db.dart'; import 'package:dotenv/dotenv.dart'; import 'package:postgres/postgres.dart'; -/// Integration test Postgres: [cyberhybridhub_test] with migrations 001–004. +/// Integration test Postgres: [cyberhybridhub_test] with migrations 001–009. class TestDb { TestDb._(this.db, this._connection, this.databaseUrl); @@ -125,6 +125,8 @@ class TestDb { TRUNCATE TABLE trade_orders, market_data_snapshots, + market_data_sync_runs, + tradable_assets, user_trading_state, user_trading_config, questions, diff --git a/server/test/integration/market_data_db_test.dart b/server/test/integration/market_data_db_test.dart index 52f2c9d..9c540f0 100644 --- a/server/test/integration/market_data_db_test.dart +++ b/server/test/integration/market_data_db_test.dart @@ -2,6 +2,7 @@ library; import 'package:cyberhybridhub_server/trading/market_data_db.dart'; +import 'package:cyberhybridhub_server/trading/market_history_four_hour_slot.dart'; import 'package:test/test.dart'; import '../helpers/test_db.dart'; @@ -56,4 +57,295 @@ void main() { greaterThan(older.toUtc().millisecondsSinceEpoch), ); }); + + group('upsertSnapshot', () { + test('re-upsert updates price and raw without duplicating row', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final MarketDataDb db = testDb!.marketDataDb; + final DateTime asOf = DateTime.utc(2026, 5, 23, 13); + + await db.upsertSnapshot( + symbol: 'SPY', + metric: 'bar', + timeframe: '1Day', + asOf: asOf, + price: 500, + volume: 1000, + raw: {'c': 500, 'v': 1000}, + ); + await db.upsertSnapshot( + symbol: 'SPY', + metric: 'bar', + timeframe: '1Day', + asOf: asOf, + price: 505, + volume: 1100, + raw: {'c': 505, 'v': 1100}, + ); + + final List rows = await db.barsForSymbol( + symbol: 'SPY', + timeframe: '1Day', + since: asOf.subtract(const Duration(seconds: 1)), + until: asOf.add(const Duration(seconds: 1)), + ); + + expect(rows, hasLength(1)); + expect(rows.single.price, 505); + expect(rows.single.volume, 1100); + expect(rows.single.raw?['c'], 505); + expect(rows.single.raw?['v'], 1100); + }); + }); + + group('barsForSymbol', () { + test('returns rows ordered by as_of ASC within [since, until)', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final MarketDataDb db = testDb!.marketDataDb; + final DateTime t1 = DateTime.utc(2026, 5, 20); + final DateTime t2 = DateTime.utc(2026, 5, 21); + final DateTime t3 = DateTime.utc(2026, 5, 22); + + for (final DateTime t in [t1, t2, t3]) { + await db.upsertSnapshot( + symbol: 'SPY', + metric: 'bar', + timeframe: '1Day', + asOf: t, + price: t.day.toDouble(), + ); + } + + final List rows = await db.barsForSymbol( + symbol: 'SPY', + timeframe: '1Day', + since: t1, + until: t3, + ); + + expect(rows, hasLength(2)); + expect(rows.map((MarketDataSnapshot r) => r.asOf), [t1, t2]); + }); + + test('returns empty list when no rows match', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final List rows = + await testDb!.marketDataDb.barsForSymbol( + symbol: 'NOPE', + timeframe: '1Day', + since: DateTime.utc(2026, 1, 1), + until: DateTime.utc(2026, 1, 2), + ); + + expect(rows, isEmpty); + }); + }); + + test('latestSyncedAsOf returns newest as_of or null', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final MarketDataDb db = testDb!.marketDataDb; + final DateTime older = DateTime.utc(2026, 5, 20); + final DateTime newer = DateTime.utc(2026, 5, 22); + + expect(await db.latestSyncedAsOf('SPY', '1Day'), isNull); + + await db.upsertSnapshot( + symbol: 'SPY', + metric: 'bar', + timeframe: '1Day', + asOf: older, + price: 490, + ); + await db.upsertSnapshot( + symbol: 'SPY', + metric: 'bar', + timeframe: '1Day', + asOf: newer, + price: 500, + ); + + expect(await db.latestSyncedAsOf('SPY', '1Day'), newer); + }); + + test('symbolsWithBarForSlot matches canonical slot_start wire string', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final MarketDataDb db = testDb!.marketDataDb; + const String timeframe = '4Hour'; + final DateTime slotStart = DateTime.utc(2026, 5, 26, 8); + final String slotWire = MarketHistoryFourHourSlot.slotStartWire(slotStart); + + await db.upsertSnapshot( + symbol: 'AAPL', + metric: 'bar', + timeframe: timeframe, + asOf: slotStart, + price: 186, + raw: { + 'slot_start': slotWire, + 't': slotWire, + }, + ); + + final Set synced = await db.symbolsWithBarForSlot( + symbols: ['AAPL', 'MSFT'], + slotStart: slotStart, + timeframe: timeframe, + ); + + expect(synced, {'AAPL'}); + }); + + test('symbolsWithBarForSlot falls back to as_of slot bucket for legacy rows', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final MarketDataDb db = testDb!.marketDataDb; + const String timeframe = '4Hour'; + final DateTime slotStart = DateTime.utc(2026, 5, 26, 8); + final DateTime barAt = slotStart.add(const Duration(hours: 1)); + + await db.upsertSnapshot( + symbol: 'AAPL', + metric: 'bar', + timeframe: timeframe, + asOf: barAt, + price: 186, + raw: { + // Different wire format than Dart's toIso8601String() — must still count. + 'slot_start': '2026-05-26T08:00:00Z', + }, + ); + + final Set synced = await db.symbolsWithBarForSlot( + symbols: ['AAPL', 'MSFT'], + slotStart: slotStart, + timeframe: timeframe, + ); + + expect(synced, {'AAPL'}); + }); + + test('symbolsWithBarForSlot matches via slot_start bucket when wire differs', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final MarketDataDb db = testDb!.marketDataDb; + const String timeframe = '4Hour'; + final DateTime slotStart = DateTime.utc(2026, 5, 26, 8); + + await db.upsertSnapshot( + symbol: 'AAPL', + metric: 'bar', + timeframe: timeframe, + asOf: slotStart.add(const Duration(hours: 4)), + price: 186, + raw: { + 'slot_start': '2026-05-26T08:00:00.000Z', + }, + ); + + final Set synced = await db.symbolsWithBarForSlot( + symbols: ['AAPL'], + slotStart: slotStart, + timeframe: timeframe, + ); + + expect(synced, {'AAPL'}); + }); + + test('symbolsWithBarForSlot does not match the next slot boundary as prior slot', + () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final MarketDataDb db = testDb!.marketDataDb; + const String timeframe = '4Hour'; + final DateTime slotStart = DateTime.utc(2026, 5, 26, 8); + + await db.upsertSnapshot( + symbol: 'AAPL', + metric: 'bar', + timeframe: timeframe, + asOf: DateTime.utc(2026, 5, 26, 12), + price: 186, + ); + + final Set synced = await db.symbolsWithBarForSlot( + symbols: ['AAPL'], + slotStart: slotStart, + timeframe: timeframe, + ); + + expect(synced, isEmpty); + }); + + test('symbolsWithBarForSlot counts no_data placeholder as synced', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final MarketDataDb db = testDb!.marketDataDb; + const String timeframe = '4Hour'; + final DateTime slotStart = DateTime.utc(2026, 5, 30, 20); + + await db.upsertNoDataBarPlaceholder( + symbol: 'A', + slotStart: slotStart, + timeframe: timeframe, + checkedAt: DateTime.utc(2026, 5, 31), + ); + + final Set synced = await db.symbolsWithBarForSlot( + symbols: ['A', 'B'], + slotStart: slotStart, + timeframe: timeframe, + ); + + expect(synced, {'A'}); + }); } diff --git a/server/test/integration/market_data_history_sync_test.dart b/server/test/integration/market_data_history_sync_test.dart new file mode 100644 index 0000000..fecc405 --- /dev/null +++ b/server/test/integration/market_data_history_sync_test.dart @@ -0,0 +1,775 @@ +@Tags(['integration', 'postgres']) +library; + +import 'package:cyberhybridhub_server/alpaca/alpaca_env.dart'; +import 'package:cyberhybridhub_server/alpaca/alpaca_market_data_client.dart'; +import 'package:cyberhybridhub_server/alpaca/alpaca_models.dart'; +import 'package:cyberhybridhub_server/trading/market_data_history.dart'; +import 'package:cyberhybridhub_server/trading/market_history_config.dart'; +import 'package:cyberhybridhub_server/trading/market_history_api_rate_limiter.dart'; +import 'package:cyberhybridhub_server/trading/market_history_four_hour_slot.dart'; +import 'package:cyberhybridhub_server/trading/tradable_assets_db.dart'; +import 'package:http/http.dart' as http; +import 'package:postgres/postgres.dart'; +import 'package:test/test.dart'; + +import '../helpers/fixture_loader.dart'; +import '../helpers/mock_http_client.dart'; +import '../helpers/test_db.dart'; + +Future _seedTradables( + Connection connection, + List symbols, +) async { + final TradableAssetsDb db = TradableAssetsDb(connection); + await db.upsertAll( + symbols + .map( + (String symbol) => AlpacaAsset( + symbol: symbol, + assetClass: 'us_equity', + status: 'active', + tradable: true, + fractionable: true, + ), + ) + .toList(), + now: DateTime.utc(2026, 5, 26, 10), + ); +} + +void main() { + TestDb? testDb; + late FixtureLoader fixtures; + late AlpacaEnv env; + + setUpAll(() async { + testDb = await TestDb.open(); + fixtures = FixtureLoader(); + env = AlpacaEnv.fromMap({ + 'ALPACA_API_KEY_ID': 'test-key', + 'ALPACA_API_SECRET_KEY': 'test-secret', + 'ALPACA_DATA_BASE_URL': 'https://data.alpaca.markets', + 'ALPACA_DATA_FEED': 'iex', + }); + }); + + tearDown(() async { + if (testDb != null) { + await testDb!.truncateTradingTables(); + } + }); + + tearDownAll(() async { + await testDb?.close(); + }); + + MarketDataHistorySync makeSync({ + required MockHttpClient mock, + int batchSize = 100, + int maxSymbols = 2000, + int windowDays = 1, + int apiRequestsPerMinute = 10000, + MarketHistoryApiRateLimiter? rateLimiter, + Duration rateLimitCooldown = Duration.zero, + List? sleepLog, + }) { + return MarketDataHistorySync( + marketDataClient: AlpacaMarketDataClient(env: env, httpClient: mock), + tradableAssetsDb: TradableAssetsDb(testDb!.connection), + marketDataDb: testDb!.marketDataDb, + connection: testDb!.connection, + batchSize: batchSize, + maxSymbols: maxSymbols, + windowDays: windowDays, + apiRequestsPerMinute: apiRequestsPerMinute, + rateLimiter: rateLimiter, + rateLimitCooldown: rateLimitCooldown, + sleep: sleepLog == null + ? null + : (Duration d) async { + sleepLog.add(d); + }, + ); + } + + group('runOnce — 4-hour slots', () { + final DateTime now = DateTime.utc(2026, 5, 26, 12); + + test('cold start upserts completed slots in window and uses 4Hour', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + await _seedTradables( + testDb!.connection, + ['SPY', 'AAPL', 'MSFT'], + ); + + final Map barsJson = + await fixtures.loadJson('alpaca_bars_4h_window.json'); + final MockHttpClient mock = MockHttpClient() + ..whenGetJson('/bars', barsJson); + + final MarketDataHistorySyncResult result = + await makeSync(mock: mock, windowDays: 1).runOnce(now: now); + + expect(result.error, isNull); + expect(result.rowsWritten, 18); + expect(result.slotsSynced, 6); + + final Result rows = await testDb!.connection.execute( + ''' + SELECT metric, timeframe, COUNT(*)::int + FROM market_data_snapshots + GROUP BY metric, timeframe + ''', + ); + expect(rows.first[0], 'bar'); + expect(rows.first[1], MarketHistoryConfig.barTimeframe); + expect((rows.first[2]! as num).toInt(), 18); + + final Uri firstBarRequest = mock.requests + .firstWhere((http.BaseRequest r) => r.url.path.endsWith('/bars')) + .url; + expect( + firstBarRequest.queryParameters['timeframe'], + MarketHistoryFourHourSlot.alpacaTimeframe, + ); + expect( + DateTime.parse(firstBarRequest.queryParameters['start']!).toUtc(), + DateTime.utc(2026, 5, 26, 8), + ); + expect( + DateTime.parse(firstBarRequest.queryParameters['end']!).toUtc(), + MarketHistoryFourHourSlot.endInclusive( + DateTime.utc(2026, 5, 26, 8), + ), + ); + + final Result runs = await testDb!.connection.execute( + ''' + SELECT kind, rows_written, slots_synced, backfill_items, error + FROM market_data_sync_runs + ''', + ); + expect(runs.single[0], 'backfill'); + expect((runs.single[1]! as num).toInt(), 18); + expect((runs.single[2]! as num).toInt(), 6); + final List items = + runs.single[3]! as List; + expect(items, isNotEmpty); + final Map firstItem = + items.first as Map; + expect(firstItem['slotStart'], isNotNull); + expect(firstItem['symbols'], isA>()); + expect(runs.single[4], isNull); + }); + + test('stored slot_start wire matches Alpaca start query param', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + await _seedTradables( + testDb!.connection, + ['SPY'], + ); + + final Map barsJson = + await fixtures.loadJson('alpaca_bars_4h_window.json'); + final MockHttpClient mock = MockHttpClient() + ..whenGetJson('/bars', barsJson); + + await makeSync(mock: mock, windowDays: 1).runOnce(now: now); + + final String alpacaStart = mock.requests + .firstWhere((http.BaseRequest r) => r.url.path.endsWith('/bars')) + .url + .queryParameters['start']!; + + final Result rows = await testDb!.connection.execute( + Sql.named( + ''' + SELECT raw->>'slot_start' AS slot_start + FROM market_data_snapshots + WHERE symbol = 'SPY' + AND metric = 'bar' + AND timeframe = @timeframe + AND raw->>'slot_start' = @slot_start + LIMIT 1 + ''', + ), + parameters: { + 'timeframe': MarketHistoryConfig.barTimeframe, + 'slot_start': alpacaStart, + }, + ); + expect(rows.single[0], alpacaStart); + expect(alpacaStart, '2026-05-26T08:00:00Z'); + }); + + test('re-run is idempotent with zero rows when fully synced', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + await _seedTradables(testDb!.connection, ['SPY', 'AAPL', 'MSFT']); + final Map barsJson = + await fixtures.loadJson('alpaca_bars_4h_window.json'); + final MockHttpClient mock = MockHttpClient() + ..whenGetJson('/bars', barsJson); + + final MarketDataHistorySync sync = + makeSync(mock: mock, windowDays: 1); + final MarketDataHistorySyncResult r1 = await sync.runOnce(now: now); + mock.requests.clear(); + final MarketDataHistorySyncResult r2 = await sync.runOnce(now: now); + + expect(r1.rowsWritten, 18); + expect(r2.rowsWritten, 0); + expect( + mock.requests.where((http.BaseRequest r) => r.url.path.endsWith('/bars')), + isEmpty, + ); + }); + + test('re-run skips symbols when bar is stored with alternate slot_start wire format', + () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + await _seedTradables( + testDb!.connection, + ['AAPL', 'MSFT', 'SPY'], + ); + + final List completed = + MarketHistoryFourHourSlot.completedSlotStartsInWindow(now, 1); + for (final DateTime slotStart in completed) { + for (final String symbol in ['AAPL', 'MSFT', 'SPY']) { + await testDb!.marketDataDb.upsertSnapshot( + symbol: symbol, + metric: 'bar', + timeframe: MarketHistoryConfig.barTimeframe, + asOf: slotStart.add(const Duration(minutes: 30)), + price: 100, + raw: { + 'slot_start': slotStart.toUtc().toIso8601String().replaceAll( + '.000', + '', + ), + }, + ); + } + } + + final MockHttpClient mock = MockHttpClient(); + final MarketDataHistorySyncResult result = await makeSync( + mock: mock, + windowDays: 1, + ).runOnce(now: now); + + expect(result.rowsWritten, 0); + expect( + mock.requests.where((http.BaseRequest r) => r.url.path.endsWith('/bars')), + isEmpty, + ); + }); + + test('does not fetch while current slot is still open', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + await _seedTradables(testDb!.connection, ['SPY']); + final Map barsJson = + await fixtures.loadJson('alpaca_bars_4h_window.json'); + final MockHttpClient mock = MockHttpClient() + ..whenGetJson('/bars', barsJson); + + final MarketDataHistorySync sync = + makeSync(mock: mock, windowDays: 1); + final DateTime midSlot = DateTime.utc(2026, 5, 26, 10, 30); + await sync.runOnce(now: midSlot); + mock.requests.clear(); + + final MarketDataHistorySyncResult second = + await sync.runOnce(now: midSlot); + + expect(second.rowsWritten, 0); + final Iterable barRequests = mock.requests.where( + (http.BaseRequest r) => r.url.path.endsWith('/bars'), + ); + for (final http.BaseRequest request in barRequests) { + final String? start = request.url.queryParameters['start']; + expect(start, isNotNull); + expect( + DateTime.parse(start!).toUtc(), + isNot(DateTime.utc(2026, 5, 26, 12)), + reason: 'must not fetch the still-open slot', + ); + } + }); + + test('fetches only the newly completed slot after prior sync', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + await _seedTradables(testDb!.connection, ['SPY']); + final List completed = + MarketHistoryFourHourSlot.completedSlotStartsInWindow(now, 1); + final DateTime targetSlot = DateTime.utc(2026, 5, 26, 8); + for (final DateTime slotStart in completed) { + if (slotStart == targetSlot) { + continue; + } + await testDb!.marketDataDb.upsertSnapshot( + symbol: 'SPY', + metric: 'bar', + timeframe: MarketHistoryConfig.barTimeframe, + asOf: slotStart.add(const Duration(hours: 1)), + price: 495, + raw: { + 'slot_start': slotStart.toIso8601String(), + }, + ); + } + + final MockHttpClient mock = MockHttpClient() + ..whenGetJson('/bars', { + 'bars': { + 'SPY': >[ + { + 't': '2026-05-26T08:00:00Z', + 'o': 495, + 'h': 497, + 'l': 493, + 'c': 496, + 'v': 4500000, + }, + ], + }, + 'next_page_token': null, + }); + + await makeSync(mock: mock, windowDays: 1).runOnce(now: now); + + final String start = mock.requests.single.url.queryParameters['start']!; + expect(DateTime.parse(start).toUtc(), DateTime.utc(2026, 5, 26, 8)); + expect( + mock.requests.single.url.queryParameters['timeframe'], + '4Hour', + ); + }); + + test('partial outage persists successful batch and records error', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + await _seedTradables( + testDb!.connection, + ['SPY', 'AAPL', 'MSFT'], + ); + + final Map okJson = + await fixtures.loadJson('alpaca_bars_4h_window.json'); + final MockHttpClient mock = MockHttpClient() + ..whenGetWhereJson( + '/bars', + (Uri uri) => uri.queryParameters['symbols'] == 'AAPL,MSFT', + okJson, + ) + ..whenGetWhere( + '/bars', + (Uri uri) => uri.queryParameters['symbols'] == 'SPY', + http.Response('upstream exploded', 500), + ); + + final MarketDataHistorySyncResult result = await makeSync( + mock: mock, + batchSize: 2, + windowDays: 1, + ).runOnce(now: now); + + expect(result.error, isNotNull); + expect(result.error, contains('SPY')); + expect(result.rowsWritten, greaterThan(0)); + }); + + test('stores no_data placeholder when Alpaca returns empty bars for requested symbols', + () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + await _seedTradables( + testDb!.connection, + ['A', 'AA'], + ); + + final MockHttpClient mock = MockHttpClient() + ..whenGetJson('/bars', { + 'bars': {}, + 'next_page_token': null, + }); + + final MarketDataHistorySyncResult result = await makeSync( + mock: mock, + batchSize: 10, + windowDays: 1, + ).runOnce(now: now); + + expect(result.error, isNull); + expect(result.rowsWritten, greaterThan(0)); + + final Result rows = await testDb!.connection.execute( + Sql.named( + ''' + SELECT symbol, raw->>'no_data' AS no_data, raw->>'slot_start' AS slot_start + FROM market_data_snapshots + WHERE symbol IN ('A', 'AA') + AND metric = 'bar' + AND timeframe = @timeframe + ''', + ), + parameters: { + 'timeframe': MarketHistoryConfig.barTimeframe, + }, + ); + expect(rows.length, greaterThanOrEqualTo(2)); + for (final ResultRow row in rows) { + expect(row[1], 'true'); + expect(row[2], isNotNull); + } + + mock.requests.clear(); + final MarketDataHistorySyncResult second = await makeSync( + mock: mock, + batchSize: 10, + windowDays: 1, + ).runOnce(now: now); + + expect( + mock.requests.where((http.BaseRequest r) => r.url.path.endsWith('/bars')), + isEmpty, + reason: 'placeholders should satisfy gap check for synced symbols', + ); + expect(second.error, isNull); + }); + + test('stores no_data placeholder when Alpaca bars are outside the requested slot', + () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + await _seedTradables(testDb!.connection, ['SPY']); + + final MockHttpClient mock = MockHttpClient() + ..whenGetJson('/bars', { + 'bars': { + 'SPY': >[ + { + 't': '2026-05-26T12:00:00Z', + 'o': 495, + 'h': 497, + 'l': 493, + 'c': 496, + 'v': 4500000, + }, + ], + }, + 'next_page_token': null, + }); + + final MarketDataHistorySyncResult result = await makeSync( + mock: mock, + windowDays: 1, + ).runOnce(now: now); + + expect(result.error, isNull); + expect(result.rowsWritten, greaterThan(0)); + + final Result rows = await testDb!.connection.execute( + Sql.named( + ''' + SELECT raw->>'no_data' AS no_data, raw->>'slot_start' AS slot_start + FROM market_data_snapshots + WHERE symbol = 'SPY' AND metric = 'bar' AND timeframe = @timeframe + AND raw->>'slot_start' = '2026-05-26T08:00:00Z' + ''', + ), + parameters: { + 'timeframe': MarketHistoryConfig.barTimeframe, + }, + ); + expect(rows.single[0], 'true'); + expect(rows.single[1], '2026-05-26T08:00:00Z'); + }); + + test('stores market_closed placeholder on weekend with no error', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + await _seedTradables(testDb!.connection, ['A']); + final DateTime sundayAfterWeekend = DateTime.utc(2026, 5, 31, 0, 30); + final MockHttpClient mock = MockHttpClient() + ..whenGetJson('/bars', { + 'bars': {}, + 'next_page_token': null, + }); + + final MarketDataHistorySyncResult result = await makeSync( + mock: mock, + batchSize: 10, + windowDays: 1, + ).runOnce(now: sundayAfterWeekend); + + expect(result.error, isNull); + expect(result.rowsWritten, greaterThan(0)); + + final Result rows = await testDb!.connection.execute( + Sql.named( + ''' + SELECT raw->>'source' AS source + FROM market_data_snapshots + WHERE symbol = 'A' + AND metric = 'bar' + AND timeframe = @timeframe + AND raw->>'slot_start' = '2026-05-30T20:00:00Z' + LIMIT 1 + ''', + ), + parameters: { + 'timeframe': MarketHistoryConfig.barTimeframe, + }, + ); + expect(rows, isNotEmpty); + expect(rows.first[0], 'market_closed'); + }); + + test('batching issues one Alpaca call per slot per batch', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + await _seedTradables( + testDb!.connection, + ['S1', 'S2', 'S3', 'S4', 'S5'], + ); + + final Map barsJson = + await fixtures.loadJson('alpaca_bars_4h_window.json'); + final MockHttpClient mock = MockHttpClient() + ..whenGetJson('/bars', barsJson); + + await makeSync(mock: mock, batchSize: 2, windowDays: 1).runOnce( + now: now, + ); + + final int barRequests = mock.requests + .where((http.BaseRequest r) => r.url.path.endsWith('/bars')) + .length; + expect(barRequests, 6 * 3); + }); + + test('new symbol is fetched without re-requesting fully synced symbols', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + await _seedTradables( + testDb!.connection, + ['SPY', 'AAPL', 'MSFT'], + ); + final Map barsJson = + await fixtures.loadJson('alpaca_bars_4h_window.json'); + final MockHttpClient mock = MockHttpClient() + ..whenGetJson('/bars', barsJson); + + final MarketDataHistorySync sync = + makeSync(mock: mock, windowDays: 1); + await sync.runOnce(now: now); + + await _seedTradables( + testDb!.connection, + ['SPY', 'AAPL', 'MSFT', 'NVDA'], + ); + mock.requests.clear(); + + await sync.runOnce(now: now); + + final Iterable barRequests = mock.requests.where( + (http.BaseRequest r) => r.url.path.endsWith('/bars'), + ); + expect(barRequests, isNotEmpty); + for (final http.BaseRequest request in barRequests) { + final String? symbols = request.url.queryParameters['symbols']; + expect(symbols, isNotNull); + expect(symbols!.split(','), everyElement('NVDA')); + } + }); + }); + + group('rate limit', () { + final DateTime now = DateTime.utc(2026, 5, 26, 12); + + test('429 waits one minute, retries, and saves partial progress', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + await _seedTradables(testDb!.connection, ['SPY']); + final Map barsJson = + await fixtures.loadJson('alpaca_bars_4h_window.json'); + + final MockHttpClient mock = MockHttpClient() + ..whenGetQueued('/bars', http.Response('rate limited', 429)) + ..whenGetQueuedJson('/bars', barsJson) + ..whenGetJson('/bars', barsJson); + + final List sleeps = []; + final MarketDataHistorySyncResult result = await makeSync( + mock: mock, + batchSize: 1, + windowDays: 1, + rateLimitCooldown: const Duration(minutes: 1), + sleepLog: sleeps, + ).runOnce(now: now); + + expect(sleeps, [const Duration(minutes: 1)]); + expect(result.rowsWritten, greaterThan(0)); + expect(result.error, isNull); + }); + + test('429 after cooldown stops run and keeps partial rows', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + await _seedTradables( + testDb!.connection, + ['SPY', 'AAPL', 'MSFT'], + ); + final Map okJson = + await fixtures.loadJson('alpaca_bars_4h_window.json'); + + final MockHttpClient mock = MockHttpClient() + ..whenGetWhereJson( + '/bars', + (Uri uri) => uri.queryParameters['symbols'] == 'AAPL,MSFT', + okJson, + ) + ..whenGetWhere( + '/bars', + (Uri uri) => uri.queryParameters['symbols'] == 'SPY', + http.Response('rate limited', 429), + ); + + final List sleeps = []; + final MarketDataHistorySyncResult result = await makeSync( + mock: mock, + batchSize: 2, + windowDays: 1, + rateLimitCooldown: const Duration(minutes: 1), + sleepLog: sleeps, + ).runOnce(now: now); + + expect(sleeps, [const Duration(minutes: 1)]); + expect(result.error, isNotNull); + expect(result.error, contains('rate limited')); + expect(result.error, contains('partial sync saved')); + expect(result.rowsWritten, greaterThan(0)); + + final Result okBatchCount = await testDb!.connection.execute( + Sql.named( + ''' + SELECT COUNT(*)::int FROM market_data_snapshots + WHERE symbol IN ('AAPL', 'MSFT') AND metric = 'bar' + ''', + ), + ); + expect((okBatchCount.first[0]! as num).toInt(), greaterThan(0)); + + final Result spyCount = await testDb!.connection.execute( + "SELECT COUNT(*)::int FROM market_data_snapshots WHERE symbol = 'SPY'", + ); + expect((spyCount.first[0]! as num).toInt(), 0); + }); + }); + + group('hasPendingSlots', () { + test('true on cold start and false when window is caught up', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + await _seedTradables(testDb!.connection, ['SPY']); + final MarketDataHistorySync sync = makeSync( + mock: MockHttpClient(), + windowDays: 1, + ); + final DateTime now = DateTime.utc(2026, 5, 26, 12); + + expect(await sync.hasPendingSlots(now), isTrue); + + final Map barsJson = + await fixtures.loadJson('alpaca_bars_4h_window.json'); + final MockHttpClient mock = MockHttpClient() + ..whenGetJson('/bars', barsJson); + await makeSync(mock: mock, windowDays: 1).runOnce(now: now); + + expect(await sync.hasPendingSlots(now), isFalse); + }); + }); +} diff --git a/server/test/integration/market_data_retention_test.dart b/server/test/integration/market_data_retention_test.dart new file mode 100644 index 0000000..1714be6 --- /dev/null +++ b/server/test/integration/market_data_retention_test.dart @@ -0,0 +1,319 @@ +@Tags(['integration', 'postgres']) +library; + +import 'package:cyberhybridhub_server/trading/market_data_db.dart'; +import 'package:cyberhybridhub_server/trading/market_data_retention.dart'; +import 'package:postgres/postgres.dart'; +import 'package:test/test.dart'; + +import '../helpers/test_db.dart'; + +void main() { + TestDb? testDb; + + setUpAll(() async { + testDb = await TestDb.open(); + }); + + tearDown(() async { + if (testDb != null) { + await testDb!.connection.execute('TRUNCATE TABLE market_data_archive'); + await testDb!.truncateTradingTables(); + } + }); + + tearDownAll(() async { + await testDb?.close(); + }); + + MarketDataRetention retention({void Function(String sql)? onExecute}) { + return MarketDataRetention( + connection: testDb!.connection, + onExecute: onExecute, + ); + } + + group('runCleanup (hard delete)', () { + test('deletes rows older than window and keeps rows within window', + () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final MarketDataDb db = testDb!.marketDataDb; + final DateTime now = DateTime.utc(2026, 5, 26, 12); + final DateTime cutoff = now.subtract(const Duration(days: 7)); + + int removedCount = 0; + int keptCount = 0; + + for (int day = 0; day < 10; day++) { + final DateTime asOf = now.subtract(Duration(days: 14 - day)); + await db.upsertSnapshot( + symbol: 'SPY', + metric: 'bar', + timeframe: '1Day', + asOf: asOf, + price: 490 + day, + ); + if (asOf.isBefore(cutoff)) { + removedCount++; + } else { + keptCount++; + } + } + + final MarketDataRetentionResult result = await retention().runCleanup( + now: now, + ); + + expect(result.error, isNull); + expect(result.rowsRemoved, removedCount); + expect(keptCount, greaterThan(0)); + + final Result remaining = await testDb!.connection.execute( + 'SELECT COUNT(*)::int FROM market_data_snapshots', + ); + expect((remaining.first[0]! as num).toInt(), keptCount); + }); + + test('empty table returns rowsRemoved 0 without throwing', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final MarketDataRetentionResult result = await retention().runCleanup( + now: DateTime.utc(2026, 5, 26, 12), + ); + + expect(result.rowsRemoved, 0); + expect(result.error, isNull); + }); + + test('batchSize issues multiple DELETE statements for large backlog', + () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + const int rowCount = 5000; + const int batchSize = 1000; + final DateTime now = DateTime.utc(2026, 5, 26, 12); + final DateTime oldAsOf = now.subtract(const Duration(days: 14)); + + await testDb!.connection.execute( + Sql.named( + ''' + INSERT INTO market_data_snapshots (symbol, metric, timeframe, as_of, price) + SELECT 'SYM' || g::text, 'bar', '1Day', @as_of, 100 + FROM generate_series(1, @count) AS g + ''', + ), + parameters: { + 'as_of': oldAsOf, + 'count': rowCount, + }, + ); + + int deleteStatements = 0; + final MarketDataRetentionResult result = await MarketDataRetention( + connection: testDb!.connection, + batchSize: batchSize, + onExecute: (String sql) { + if (sql.toUpperCase().contains('DELETE FROM MARKET_DATA_SNAPSHOTS')) { + deleteStatements++; + } + }, + ).runCleanup(now: now); + + expect(result.rowsRemoved, rowCount); + expect(deleteStatements, greaterThanOrEqualTo(5)); + }); + + test('writes market_data_sync_runs row with kind cleanup and rows_removed', + () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final DateTime now = DateTime.utc(2026, 5, 26, 12); + await testDb!.marketDataDb.upsertSnapshot( + symbol: 'SPY', + metric: 'bar', + timeframe: '1Day', + asOf: now.subtract(const Duration(days: 10)), + price: 480, + ); + + await retention().runCleanup(now: now); + + final Result runs = await testDb!.connection.execute( + ''' + SELECT kind, rows_removed, finished_at, error + FROM market_data_sync_runs + ORDER BY id DESC + LIMIT 1 + ''', + ); + expect(runs, hasLength(1)); + expect(runs.first[0], 'cleanup'); + expect((runs.first[1]! as num).toInt(), 1); + expect(runs.first[2], isNotNull); + expect(runs.first[3], isNull); + }); + + test('rows within window are never deleted (by id)', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final MarketDataDb db = testDb!.marketDataDb; + final DateTime now = DateTime.utc(2026, 5, 26, 12); + + final MarketDataSnapshot kept = await db.upsertSnapshot( + symbol: 'SPY', + metric: 'bar', + timeframe: '1Day', + asOf: now.subtract(const Duration(days: 2)), + price: 500, + ); + await db.upsertSnapshot( + symbol: 'SPY', + metric: 'bar', + timeframe: '1Day', + asOf: now.subtract(const Duration(days: 10)), + price: 480, + ); + + await retention().runCleanup(now: now); + + final Result row = await testDb!.connection.execute( + Sql.named('SELECT id FROM market_data_snapshots WHERE id = @id'), + parameters: {'id': kept.id}, + ); + expect(row, hasLength(1)); + }); + }); + + group('runArchiveAndCleanup', () { + test('archiveEnabled copies expired rows then deletes them', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final DateTime now = DateTime.utc(2026, 5, 26, 12); + await testDb!.marketDataDb.upsertSnapshot( + symbol: 'SPY', + metric: 'bar', + timeframe: '1Day', + asOf: now.subtract(const Duration(days: 10)), + price: 480, + volume: 1000, + raw: {'c': 480}, + ); + await testDb!.marketDataDb.upsertSnapshot( + symbol: 'SPY', + metric: 'bar', + timeframe: '1Day', + asOf: now.subtract(const Duration(days: 2)), + price: 500, + ); + + final MarketDataRetentionResult result = + await retention().runArchiveAndCleanup(now: now); + + expect(result.error, isNull); + expect(result.rowsRemoved, 1); + + final Result archiveCount = await testDb!.connection.execute( + 'SELECT COUNT(*)::int FROM market_data_archive', + ); + expect((archiveCount.first[0]! as num).toInt(), 1); + + final Result liveCount = await testDb!.connection.execute( + 'SELECT COUNT(*)::int FROM market_data_snapshots', + ); + expect((liveCount.first[0]! as num).toInt(), 1); + }); + + test('archive failure rolls back delete and records error', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final DateTime now = DateTime.utc(2026, 5, 26, 12); + await testDb!.marketDataDb.upsertSnapshot( + symbol: 'SPY', + metric: 'bar', + timeframe: '1Day', + asOf: now.subtract(const Duration(days: 10)), + price: 480, + ); + + await testDb!.connection.execute( + 'ALTER TABLE market_data_archive RENAME TO market_data_archive_hidden', + ); + + final MarketDataRetentionResult result = + await retention().runArchiveAndCleanup(now: now); + + expect(result.error, isNotNull); + expect(result.rowsRemoved, 0); + + final Result liveCount = await testDb!.connection.execute( + 'SELECT COUNT(*)::int FROM market_data_snapshots', + ); + expect((liveCount.first[0]! as num).toInt(), 1); + + await testDb!.connection.execute( + 'ALTER TABLE market_data_archive_hidden RENAME TO market_data_archive', + ); + }); + + test('runCleanup does not write to archive table', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final DateTime now = DateTime.utc(2026, 5, 26, 12); + await testDb!.marketDataDb.upsertSnapshot( + symbol: 'SPY', + metric: 'bar', + timeframe: '1Day', + asOf: now.subtract(const Duration(days: 10)), + price: 480, + ); + + await retention().runCleanup(now: now); + + final Result archiveCount = await testDb!.connection.execute( + 'SELECT COUNT(*)::int FROM market_data_archive', + ); + expect((archiveCount.first[0]! as num).toInt(), 0); + }); + }); +} diff --git a/server/test/integration/market_history_admin_handler_test.dart b/server/test/integration/market_history_admin_handler_test.dart new file mode 100644 index 0000000..f265db7 --- /dev/null +++ b/server/test/integration/market_history_admin_handler_test.dart @@ -0,0 +1,821 @@ +@Tags(['integration', 'postgres']) +library; + +import 'dart:convert'; + +import 'package:cyberhybridhub_server/firebase_auth.dart'; +import 'package:cyberhybridhub_server/handlers/market_history_admin_handler.dart'; +import 'package:cyberhybridhub_server/trading/market_data_history.dart'; +import 'package:cyberhybridhub_server/trading/market_history_four_hour_slot.dart'; +import 'package:cyberhybridhub_server/trading/market_data_retention.dart'; +import 'package:cyberhybridhub_server/trading/market_history_admin_actions.dart'; +import 'package:cyberhybridhub_server/trading/sync_run_recorder.dart'; +import 'package:cyberhybridhub_server/trading/tradable_assets_sync.dart'; +import 'package:postgres/postgres.dart'; +import 'package:shelf/shelf.dart'; +import 'package:test/test.dart'; + +import '../helpers/test_db.dart'; + +class _FakeAuthVerifier extends FirebaseAuthVerifier { + _FakeAuthVerifier() : super('test-key'); + + @override + Future verifyBearerToken(String? authorization) async { + if (authorization == null || !authorization.startsWith('Bearer ')) { + return null; + } + final String token = authorization.substring('Bearer '.length).trim(); + return token.isEmpty ? null : token; + } +} + +Future _get( + Handler handler, { + required String path, + String? bearer, +}) async { + return await Future.value( + handler( + Request( + 'GET', + Uri.parse('http://localhost$path'), + headers: { + if (bearer != null) 'Authorization': 'Bearer $bearer', + }, + ), + ), + ); +} + +Future _post( + Handler handler, { + required String path, + String? bearer, +}) async { + return await Future.value( + handler( + Request( + 'POST', + Uri.parse('http://localhost$path'), + headers: { + if (bearer != null) 'Authorization': 'Bearer $bearer', + }, + ), + ), + ); +} + +Future _insertRun( + Connection connection, { + required String kind, + required DateTime startedAt, + DateTime? finishedAt, + int rowsWritten = 0, + int rowsRemoved = 0, + String? error, +}) async { + await connection.execute( + Sql.named( + ''' + INSERT INTO market_data_sync_runs + (kind, started_at, finished_at, rows_written, rows_removed, error) + VALUES + (@kind, @started_at, @finished_at, @rows_written, @rows_removed, @error) + ''', + ), + parameters: { + 'kind': kind, + 'started_at': startedAt.toUtc(), + 'finished_at': finishedAt?.toUtc(), + 'rows_written': rowsWritten, + 'rows_removed': rowsRemoved, + 'error': error, + }, + ); +} + +void main() { + TestDb? testDb; + + setUpAll(() async { + testDb = await TestDb.open(); + }); + + tearDown(() async { + if (testDb != null) { + await testDb!.truncateTradingTables(); + } + }); + + tearDownAll(() async { + await testDb?.close(); + }); + + test('returns 403 for authenticated non-admin user', () async { + if (testDb == null) { + markTestSkipped('Set DATABASE_URL or TEST_DATABASE_URL for integration tests'); + return; + } + + final Handler handler = marketHistoryAdminHandler( + auth: _FakeAuthVerifier(), + connection: testDb!.connection, + adminFirebaseUids: {'admin-uid'}, + ); + + final Response response = await _get( + handler, + path: '/v1/admin/market-history/sync-runs', + bearer: 'non-admin', + ); + expect(response.statusCode, 403); + }); + + test('returns newest-first runs and pinned unresolved failures', () async { + if (testDb == null) { + markTestSkipped('Set DATABASE_URL or TEST_DATABASE_URL for integration tests'); + return; + } + final DateTime t0 = DateTime.utc(2026, 5, 26, 10); + await _insertRun( + testDb!.connection, + kind: 'backfill', + startedAt: t0, + finishedAt: t0.add(const Duration(minutes: 1)), + rowsWritten: 10, + error: 'rate limited: 429', + ); + await _insertRun( + testDb!.connection, + kind: 'cleanup', + startedAt: t0.add(const Duration(hours: 1)), + finishedAt: t0.add(const Duration(hours: 1, minutes: 1)), + rowsRemoved: 100, + ); + + final Handler handler = marketHistoryAdminHandler( + auth: _FakeAuthVerifier(), + connection: testDb!.connection, + adminFirebaseUids: {'admin-uid'}, + ); + final Response response = await _get( + handler, + path: '/v1/admin/market-history/sync-runs?limit=50', + bearer: 'admin-uid', + ); + expect(response.statusCode, 200); + + final Map body = + jsonDecode(await response.readAsString()) as Map; + final List pinned = body['pinned'] as List; + final List runs = body['runs'] as List; + expect((pinned.first as Map)['kind'], 'backfill'); + expect((pinned.first as Map)['severity'], 'rate_limit'); + expect((runs.first as Map)['kind'], 'cleanup'); + }); + + test('pinned clears after later success for same kind', () async { + if (testDb == null) { + markTestSkipped('Set DATABASE_URL or TEST_DATABASE_URL for integration tests'); + return; + } + final DateTime t0 = DateTime.utc(2026, 5, 26, 10); + await _insertRun( + testDb!.connection, + kind: 'backfill', + startedAt: t0, + finishedAt: t0.add(const Duration(minutes: 1)), + rowsWritten: 10, + error: '429', + ); + await _insertRun( + testDb!.connection, + kind: 'backfill', + startedAt: t0.add(const Duration(hours: 2)), + finishedAt: t0.add(const Duration(hours: 2, minutes: 1)), + rowsWritten: 40, + ); + + final Handler handler = marketHistoryAdminHandler( + auth: _FakeAuthVerifier(), + connection: testDb!.connection, + adminFirebaseUids: {'admin-uid'}, + ); + final Response response = await _get( + handler, + path: '/v1/admin/market-history/sync-runs', + bearer: 'admin-uid', + ); + final Map body = + jsonDecode(await response.readAsString()) as Map; + expect((body['pinned'] as List), isEmpty); + }); + + test('supports kind filter, limit, and before pagination', () async { + if (testDb == null) { + markTestSkipped('Set DATABASE_URL or TEST_DATABASE_URL for integration tests'); + return; + } + final DateTime t0 = DateTime.utc(2026, 5, 26, 10); + await _insertRun( + testDb!.connection, + kind: 'cleanup', + startedAt: t0, + finishedAt: t0.add(const Duration(minutes: 1)), + rowsRemoved: 10, + ); + await _insertRun( + testDb!.connection, + kind: 'cleanup', + startedAt: t0.add(const Duration(hours: 1)), + finishedAt: t0.add(const Duration(hours: 1, minutes: 1)), + rowsRemoved: 20, + ); + await _insertRun( + testDb!.connection, + kind: 'backfill', + startedAt: t0.add(const Duration(hours: 2)), + finishedAt: t0.add(const Duration(hours: 2, minutes: 1)), + rowsWritten: 99, + ); + + final Handler handler = marketHistoryAdminHandler( + auth: _FakeAuthVerifier(), + connection: testDb!.connection, + adminFirebaseUids: {'admin-uid'}, + ); + + final Response filtered = await _get( + handler, + path: '/v1/admin/market-history/sync-runs?kind=cleanup&limit=1', + bearer: 'admin-uid', + ); + final Map filteredBody = + jsonDecode(await filtered.readAsString()) as Map; + final List filteredRuns = filteredBody['runs'] as List; + expect(filteredRuns, hasLength(1)); + expect((filteredRuns.first as Map)['kind'], 'cleanup'); + + final String before = filteredBody['nextBefore'] as String; + final Response nextPage = await _get( + handler, + path: '/v1/admin/market-history/sync-runs?kind=cleanup&before=$before&limit=5', + bearer: 'admin-uid', + ); + final Map nextBody = + jsonDecode(await nextPage.readAsString()) as Map; + final List nextRuns = nextBody['runs'] as List; + expect(nextRuns, hasLength(1)); + }); + + test('POST resync returns 202 and creates run rows', () async { + if (testDb == null) { + markTestSkipped('Set DATABASE_URL or TEST_DATABASE_URL for integration tests'); + return; + } + + Future record(String kind, DateTime now) async { + final SyncRunRecorder recorder = SyncRunRecorder(testDb!.connection); + await recorder.record( + kind, + () async => const SyncRunCounts(rowsWritten: 1), + now: now, + ); + } + + final MarketHistoryAdminActions actions = MarketHistoryAdminActions( + connection: testDb!.connection, + runUniverse: (DateTime now) => record(TradableAssetsSync.kind, now), + runBackfill: (DateTime now) => record(MarketDataHistorySync.kind, now), + runCleanup: (DateTime now, bool archive, int windowDays) => + record(MarketDataRetention.kind, now), + ); + final Handler handler = marketHistoryAdminHandler( + auth: _FakeAuthVerifier(), + connection: testDb!.connection, + adminFirebaseUids: {'admin-uid'}, + actions: actions, + ); + + final Response response = await _post( + handler, + path: '/v1/admin/market-data/resync', + bearer: 'admin-uid', + ); + expect(response.statusCode, 202); + final Map body = + jsonDecode(await response.readAsString()) as Map; + expect((body['runIds'] as List), hasLength(2)); + }); + + test('POST cleanup routes archive flag for admin', () async { + if (testDb == null) { + markTestSkipped('Set DATABASE_URL or TEST_DATABASE_URL for integration tests'); + return; + } + + bool? seenArchive; + final MarketHistoryAdminActions actions = MarketHistoryAdminActions( + connection: testDb!.connection, + runUniverse: (_) async {}, + runBackfill: (_) async {}, + runCleanup: (DateTime now, bool archive, int windowDays) async { + seenArchive = archive; + final SyncRunRecorder recorder = SyncRunRecorder(testDb!.connection); + await recorder.record( + MarketDataRetention.kind, + () async => const SyncRunCounts(rowsRemoved: 5), + now: now, + ); + }, + ); + final Handler handler = marketHistoryAdminHandler( + auth: _FakeAuthVerifier(), + connection: testDb!.connection, + adminFirebaseUids: {'admin-uid'}, + actions: actions, + ); + + final Response response = await _post( + handler, + path: '/v1/admin/market-data/cleanup?archive=true', + bearer: 'admin-uid', + ); + expect(response.statusCode, 202); + expect(seenArchive, isTrue); + }); + + test('POST trigger returns 403 for non-admin', () async { + if (testDb == null) { + markTestSkipped('Set DATABASE_URL or TEST_DATABASE_URL for integration tests'); + return; + } + + final Handler handler = marketHistoryAdminHandler( + auth: _FakeAuthVerifier(), + connection: testDb!.connection, + adminFirebaseUids: {'admin-uid'}, + actions: MarketHistoryAdminActions( + connection: testDb!.connection, + runUniverse: (_) async {}, + runBackfill: (_) async {}, + runCleanup: (_, __, ___) async {}, + ), + ); + + final Response response = await _post( + handler, + path: '/v1/admin/market-data/resync', + bearer: 'not-admin', + ); + expect(response.statusCode, 403); + }); + + test('POST resync aborts orphaned in-progress run and returns 202', () async { + if (testDb == null) { + markTestSkipped('Set DATABASE_URL or TEST_DATABASE_URL for integration tests'); + return; + } + + await testDb!.connection.execute( + Sql.named( + ''' + INSERT INTO market_data_sync_runs (kind, started_at, finished_at) + VALUES ('backfill', @started_at, NULL) + ''', + ), + parameters: { + 'started_at': DateTime.utc(2026, 5, 27, 12), + }, + ); + + final MarketHistoryAdminActions actions = MarketHistoryAdminActions( + connection: testDb!.connection, + runUniverse: (_) async {}, + runBackfill: (_) async {}, + runCleanup: (_, __, ___) async {}, + ); + final Handler handler = marketHistoryAdminHandler( + auth: _FakeAuthVerifier(), + connection: testDb!.connection, + adminFirebaseUids: {'admin-uid'}, + actions: actions, + ); + + final Response response = await _post( + handler, + path: '/v1/admin/market-data/resync', + bearer: 'admin-uid', + ); + expect(response.statusCode, 202); + }); + + test('returns portal config for admin sync-runs', () async { + if (testDb == null) { + markTestSkipped('Set DATABASE_URL or TEST_DATABASE_URL for integration tests'); + return; + } + + final Handler handler = marketHistoryAdminHandler( + auth: _FakeAuthVerifier(), + connection: testDb!.connection, + adminFirebaseUids: {'admin-uid'}, + portalConfig: const MarketHistoryAdminPortalConfig( + archiveEnabled: true, + windowDays: 7, + retentionDays: 14, + syncEnabled: true, + ), + ); + final Response response = await _get( + handler, + path: '/v1/admin/market-history/sync-runs', + bearer: 'admin-uid', + ); + expect(response.statusCode, 200); + + final Map body = + jsonDecode(await response.readAsString()) as Map; + final Map config = + body['config'] as Map; + expect(config['archiveEnabled'], isTrue); + expect(config['windowDays'], 7); + expect(config['retentionDays'], 14); + expect(config['syncEnabled'], isTrue); + }); + + test('sync-runs response excludes raw snapshot fields', () async { + if (testDb == null) { + markTestSkipped('Set DATABASE_URL or TEST_DATABASE_URL for integration tests'); + return; + } + + final DateTime t0 = DateTime.utc(2026, 5, 26, 10); + await _insertRun( + testDb!.connection, + kind: 'backfill', + startedAt: t0, + finishedAt: t0.add(const Duration(minutes: 1)), + rowsWritten: 100, + error: 'batch failed: upstream 500', + ); + + final Handler handler = marketHistoryAdminHandler( + auth: _FakeAuthVerifier(), + connection: testDb!.connection, + adminFirebaseUids: {'admin-uid'}, + ); + final Response response = await _get( + handler, + path: '/v1/admin/market-history/sync-runs', + bearer: 'admin-uid', + ); + final Map body = + jsonDecode(await response.readAsString()) as Map; + final List all = [ + ...(body['runs'] as List? ?? []), + ...(body['pinned'] as List? ?? []), + ]; + for (final dynamic item in all) { + final Map run = item as Map; + expect(run.containsKey('raw'), isFalse); + expect(run.containsKey('bars'), isFalse); + expect(run.containsKey('snapshots'), isFalse); + } + }); + + test('question-audit returns last-two bar deltas for admin', () async { + if (testDb == null) { + markTestSkipped('Set DATABASE_URL or TEST_DATABASE_URL for integration tests'); + return; + } + + final DateTime now = DateTime.now().toUtc(); + final DateTime newerSlot = MarketHistoryFourHourSlot.lastCompletedSlotStart(now); + final DateTime olderSlot = newerSlot.subtract(const Duration(hours: 4)); + final DateTime oldestSlot = olderSlot.subtract(const Duration(hours: 4)); + + await testDb!.connection.execute( + Sql.named( + ''' + INSERT INTO tradable_assets (symbol, asset_class, status, tradable, refreshed_at) + VALUES ('AAA', 'us_equity', 'active', true, @refreshed_at) + ''', + ), + parameters: {'refreshed_at': now}, + ); + + Future insertBar({ + required DateTime asOf, + required num open, + required num high, + required num low, + required num close, + required num volume, + }) async { + await testDb!.connection.execute( + Sql.named( + ''' + INSERT INTO market_data_snapshots ( + symbol, asset_class, feed, metric, timeframe, price, volume, as_of, raw + ) VALUES ( + 'AAA', 'us_equity', 'iex', 'bar', '4Hour', @close, @volume, @as_of, @raw::jsonb + ) + ''', + ), + parameters: { + 'as_of': asOf, + 'close': close, + 'volume': volume, + 'raw': jsonEncode({ + 'o': open, + 'h': high, + 'l': low, + 'c': close, + 'v': volume, + 'slot_start': asOf.toIso8601String(), + }), + }, + ); + } + + await insertBar( + asOf: oldestSlot, + open: 8, + high: 9, + low: 7, + close: 8, + volume: 80, + ); + await insertBar( + asOf: olderSlot, + open: 10, + high: 12, + low: 8, + close: 10, + volume: 100, + ); + await insertBar( + asOf: newerSlot, + open: 12, + high: 14, + low: 10, + close: 12, + volume: 150, + ); + + final Handler handler = marketHistoryAdminHandler( + auth: _FakeAuthVerifier(), + connection: testDb!.connection, + adminFirebaseUids: {'admin-uid'}, + portalConfig: const MarketHistoryAdminPortalConfig( + archiveEnabled: false, + windowDays: 7, + retentionDays: 7, + syncEnabled: false, + ), + ); + + final Response response = await _get( + handler, + path: '/v1/admin/market-history/question-audit', + bearer: 'admin-uid', + ); + expect(response.statusCode, 200); + + final Map body = + jsonDecode(await response.readAsString()) as Map; + expect(body['windowDays'], 7); + expect(body['canStepNewer'], isFalse); + expect(body['canStepOlder'], isTrue); + expect( + DateTime.parse(body['compareUntil'] as String).toUtc(), + MarketHistoryFourHourSlot.endExclusive(newerSlot), + ); + final List assets = body['assets'] as List; + expect(assets, hasLength(1)); + + final Map aaa = assets.first as Map; + expect(aaa['symbol'], 'AAA'); + expect(aaa['priceDelta'], 2); + expect(aaa['volumeDelta'], 50); + expect(aaa.containsKey('raw'), isFalse); + + final Map older = + aaa['olderSlot'] as Map; + final Map newer = + aaa['newerSlot'] as Map; + expect(older['avgPrice'], 10); + expect(older['volume'], 100); + expect(older['open'], 10); + expect(newer['avgPrice'], 12); + expect(newer['volume'], 150); + expect(newer['close'], 12); + expect( + DateTime.parse(body['newerSlotStart'] as String).toUtc(), + newerSlot, + ); + expect( + DateTime.parse(body['olderSlotStart'] as String).toUtc(), + olderSlot, + ); + + final String olderBound = body['stepOlderCompareUntil'] as String; + final Response stepped = await _get( + handler, + path: + '/v1/admin/market-history/question-audit?asOf=${Uri.encodeQueryComponent(olderBound)}', + bearer: 'admin-uid', + ); + expect(stepped.statusCode, 200); + final Map steppedBody = + jsonDecode(await stepped.readAsString()) as Map; + expect(steppedBody['canStepNewer'], isTrue); + final Map steppedAsset = + (steppedBody['assets'] as List).first as Map; + expect((steppedAsset['newerSlot'] as Map)['close'], 10); + expect( + DateTime.parse(steppedBody['newerSlotStart'] as String).toUtc(), + olderSlot, + ); + }); + + test('question-audit omits symbols missing either slot', () async { + if (testDb == null) { + markTestSkipped('Set DATABASE_URL or TEST_DATABASE_URL for integration tests'); + return; + } + + final DateTime now = DateTime.now().toUtc(); + final DateTime newerSlot = MarketHistoryFourHourSlot.lastCompletedSlotStart(now); + final DateTime olderSlot = newerSlot.subtract(const Duration(hours: 4)); + + await testDb!.connection.execute( + Sql.named( + ''' + INSERT INTO tradable_assets (symbol, asset_class, status, tradable, refreshed_at) + VALUES + ('AAA', 'us_equity', 'active', true, @refreshed_at), + ('BBB', 'us_equity', 'active', true, @refreshed_at) + ''', + ), + parameters: {'refreshed_at': now}, + ); + + Future insertBar(String symbol, DateTime asOf, num close) async { + await testDb!.connection.execute( + Sql.named( + ''' + INSERT INTO market_data_snapshots ( + symbol, asset_class, feed, metric, timeframe, price, volume, as_of, raw + ) VALUES ( + @symbol, 'us_equity', 'iex', 'bar', '4Hour', @close, 100, @as_of, + @raw::jsonb + ) + ''', + ), + parameters: { + 'symbol': symbol, + 'as_of': asOf, + 'close': close, + 'raw': jsonEncode({ + 'o': close, + 'h': close, + 'l': close, + 'c': close, + 'v': 100, + 'slot_start': asOf.toIso8601String(), + }), + }, + ); + } + + await insertBar('AAA', olderSlot, 10); + await insertBar('AAA', newerSlot, 12); + await insertBar('BBB', newerSlot, 20); + + final Handler handler = marketHistoryAdminHandler( + auth: _FakeAuthVerifier(), + connection: testDb!.connection, + adminFirebaseUids: {'admin-uid'}, + ); + + final Response response = await _get( + handler, + path: '/v1/admin/market-history/question-audit', + bearer: 'admin-uid', + ); + expect(response.statusCode, 200); + + final Map body = + jsonDecode(await response.readAsString()) as Map; + final List assets = body['assets'] as List; + expect(assets, hasLength(1)); + expect((assets.first as Map)['symbol'], 'AAA'); + }); + + test('week-coverage validates slot consistency for admin', () async { + if (testDb == null) { + markTestSkipped('Set DATABASE_URL or TEST_DATABASE_URL for integration tests'); + return; + } + + final DateTime now = DateTime.now().toUtc(); + final DateTime slotStart = + MarketHistoryFourHourSlot.lastCompletedSlotStart(now); + + await testDb!.connection.execute( + Sql.named( + ''' + INSERT INTO tradable_assets (symbol, asset_class, status, tradable, refreshed_at) + VALUES ('AAA', 'us_equity', 'active', true, @refreshed_at) + ''', + ), + parameters: {'refreshed_at': now}, + ); + await testDb!.connection.execute( + Sql.named( + ''' + INSERT INTO market_data_snapshots ( + symbol, asset_class, feed, metric, timeframe, price, as_of, raw + ) VALUES ( + 'AAA', 'us_equity', 'iex', 'bar', '4Hour', 100, + @as_of, + @raw::jsonb + ) + ''', + ), + parameters: { + 'as_of': slotStart.add(const Duration(hours: 1)), + 'raw': jsonEncode({ + 'slot_start': slotStart.toIso8601String(), + }), + }, + ); + + final Handler handler = marketHistoryAdminHandler( + auth: _FakeAuthVerifier(), + connection: testDb!.connection, + adminFirebaseUids: {'admin-uid'}, + portalConfig: const MarketHistoryAdminPortalConfig( + archiveEnabled: false, + windowDays: 7, + retentionDays: 7, + syncEnabled: false, + ), + ); + + final Response response = await _get( + handler, + path: '/v1/admin/market-history/week-coverage', + bearer: 'admin-uid', + ); + expect(response.statusCode, 200); + + final Map body = + jsonDecode(await response.readAsString()) as Map; + expect(body['windowDays'], 7); + expect(body['slotsPerDay'], 6); + expect(body['symbolCount'], 1); + expect(body['isConsistent'], isFalse); + + final List days = body['days'] as List; + expect(days, hasLength(7)); + final Map today = + days.last as Map; + expect(today['fullySyncedSlots'], 1); + expect(today['completedSlots'], greaterThan(0)); + }); + + test('resync returns 503 when sync is disabled in portal config', () async { + if (testDb == null) { + markTestSkipped('Set DATABASE_URL or TEST_DATABASE_URL for integration tests'); + return; + } + + final Handler handler = marketHistoryAdminHandler( + auth: _FakeAuthVerifier(), + connection: testDb!.connection, + adminFirebaseUids: {'admin-uid'}, + actions: null, + portalConfig: const MarketHistoryAdminPortalConfig( + archiveEnabled: false, + windowDays: 7, + retentionDays: 7, + syncEnabled: false, + ), + ); + + final Response response = await _post( + handler, + path: '/v1/admin/market-data/resync', + bearer: 'admin-uid', + ); + expect(response.statusCode, 503); + + final Map body = + jsonDecode(await response.readAsString()) as Map; + expect(body['error'], contains('MARKET_HISTORY_SYNC_ENABLED=false')); + }); +} diff --git a/server/test/integration/market_history_admin_risk_test.dart b/server/test/integration/market_history_admin_risk_test.dart new file mode 100644 index 0000000..12b130e --- /dev/null +++ b/server/test/integration/market_history_admin_risk_test.dart @@ -0,0 +1,184 @@ +@Tags(['integration', 'postgres', 'risk']) +library; + +import 'dart:convert'; + +import 'package:cyberhybridhub_server/firebase_auth.dart'; +import 'package:cyberhybridhub_server/handlers/market_history_admin_handler.dart'; +import 'package:cyberhybridhub_server/trading/market_history_admin_logic.dart'; +import 'package:postgres/postgres.dart'; +import 'package:shelf/shelf.dart'; +import 'package:test/test.dart'; + +import '../helpers/admin_sync_run_fixtures.dart'; +import '../helpers/test_db.dart'; + +class _FakeAuthVerifier extends FirebaseAuthVerifier { + _FakeAuthVerifier() : super('test-key'); + + @override + Future verifyBearerToken(String? authorization) async { + if (authorization == null || !authorization.startsWith('Bearer ')) { + return null; + } + final String token = authorization.substring('Bearer '.length).trim(); + return token.isEmpty ? null : token; + } +} + +Future _get(Handler handler, {required String bearer}) async { + return await Future.value( + handler( + Request( + 'GET', + Uri.parse('http://localhost/v1/admin/market-history/sync-runs'), + headers: {'Authorization': 'Bearer $bearer'}, + ), + ), + ); +} + +Future _insertRecord( + Connection connection, + AdminSyncRunRecord run, +) async { + await connection.execute( + Sql.named( + ''' + INSERT INTO market_data_sync_runs + (kind, started_at, finished_at, rows_written, rows_removed, error) + VALUES + (@kind, @started_at, @finished_at, @rows_written, @rows_removed, @error) + ''', + ), + parameters: { + 'kind': run.kind, + 'started_at': run.startedAt.toUtc(), + 'finished_at': run.finishedAt?.toUtc(), + 'rows_written': run.rowsWritten, + 'rows_removed': run.rowsRemoved, + 'error': run.error, + }, + ); +} + +void main() { + TestDb? testDb; + + setUpAll(() async { + testDb = await TestDb.open(); + }); + + tearDown(() async { + if (testDb != null) { + await testDb!.truncateTradingTables(); + } + }); + + tearDownAll(() async { + await testDb?.close(); + }); + + group('Section 8 risk checklist — server', () { + test('mixed kinds interleaving pins only unresolved backfill failures', () async { + if (testDb == null) { + markTestSkipped('Set DATABASE_URL or TEST_DATABASE_URL for integration tests'); + return; + } + + final DateTime base = DateTime.utc(2026, 5, 27, 12); + for (final AdminSyncRunRecord run in fixtureMixedKindsMixedOutcomes(base)) { + await _insertRecord(testDb!.connection, run); + } + + final Handler handler = marketHistoryAdminHandler( + auth: _FakeAuthVerifier(), + connection: testDb!.connection, + adminFirebaseUids: {'admin-uid'}, + ); + final Response response = await _get(handler, bearer: 'admin-uid'); + expect(response.statusCode, 200); + + final Map body = + jsonDecode(await response.readAsString()) as Map; + final List pinned = body['pinned'] as List; + expect(pinned, isNotEmpty); + expect( + pinned.every( + (dynamic item) => (item as Map)['kind'] == 'backfill', + ), + isTrue, + ); + expect( + pinned.any( + (dynamic item) => + (item as Map)['severity'] == 'rate_limit', + ), + isTrue, + ); + }); + + test('empty database returns safe empty payload', () async { + if (testDb == null) { + markTestSkipped('Set DATABASE_URL or TEST_DATABASE_URL for integration tests'); + return; + } + + final Handler handler = marketHistoryAdminHandler( + auth: _FakeAuthVerifier(), + connection: testDb!.connection, + adminFirebaseUids: {'admin-uid'}, + ); + final Response response = await _get(handler, bearer: 'admin-uid'); + expect(response.statusCode, 200); + + final Map body = + jsonDecode(await response.readAsString()) as Map; + expect(body['runs'], isEmpty); + expect(body['pinned'], isEmpty); + expect(body['nextBefore'], isNull); + }); + + test('rate-limit capitalization variants surface as rate_limit severity', () async { + if (testDb == null) { + markTestSkipped('Set DATABASE_URL or TEST_DATABASE_URL for integration tests'); + return; + } + + final DateTime t0 = DateTime.utc(2026, 5, 27, 10); + for (final String message in [ + 'HTTP 429 Too Many Requests', + 'Rate Limit exceeded', + 'RATE LIMITED by upstream', + ]) { + await _insertRecord( + testDb!.connection, + adminSyncRun( + id: 0, + kind: 'backfill', + startedAt: t0, + finishedAt: t0.add(const Duration(minutes: 1)), + error: message, + ), + ); + + final Handler handler = marketHistoryAdminHandler( + auth: _FakeAuthVerifier(), + connection: testDb!.connection, + adminFirebaseUids: {'admin-uid'}, + ); + final Response response = await _get(handler, bearer: 'admin-uid'); + final Map body = + jsonDecode(await response.readAsString()) as Map; + final List pinned = body['pinned'] as List; + expect( + (pinned.first as Map)['severity'], + 'rate_limit', + reason: message, + ); + + await testDb!.truncateTradingTables(); + } + }); + }); +} diff --git a/server/test/integration/market_history_query_test.dart b/server/test/integration/market_history_query_test.dart new file mode 100644 index 0000000..8bedd57 --- /dev/null +++ b/server/test/integration/market_history_query_test.dart @@ -0,0 +1,169 @@ +@Tags(['integration', 'postgres']) +library; + +import 'dart:math'; + +import 'package:cyberhybridhub_server/alpaca/alpaca_models.dart'; +import 'package:cyberhybridhub_server/trading/market_history_config.dart'; +import 'package:cyberhybridhub_server/trading/market_history_query.dart'; +import 'package:cyberhybridhub_server/trading/tradable_assets_db.dart'; +import 'package:test/test.dart'; + +import '../helpers/test_db.dart'; + +void main() { + TestDb? testDb; + + setUpAll(() async { + testDb = await TestDb.open(); + }); + + tearDown(() async { + if (testDb != null) { + await testDb!.truncateTradingTables(); + } + }); + + tearDownAll(() async { + await testDb?.close(); + }); + + Future seedBars({ + required String symbol, + required List closes, + required DateTime asOf, + }) async { + for (int i = 0; i < closes.length; i++) { + await testDb!.marketDataDb.upsertSnapshot( + symbol: symbol, + metric: 'bar', + timeframe: MarketHistoryConfig.barTimeframe, + asOf: asOf.subtract(Duration(hours: 4 * (closes.length - 1 - i))), + price: closes[i], + volume: 1000, + ); + } + } + + group('weeklyMovers', () { + test('returns symbols with minBars and open/current closes', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final DateTime asOf = DateTime.utc(2026, 5, 26, 12); + final DateTime windowStart = asOf.subtract(const Duration(days: 7)); + + await TradableAssetsDb(testDb!.connection).upsertAll( + [ + AlpacaAsset( + symbol: 'SPY', + assetClass: 'us_equity', + tradable: true, + fractionable: true, + status: 'active', + ), + AlpacaAsset( + symbol: 'STALE', + assetClass: 'us_equity', + tradable: true, + fractionable: true, + status: 'active', + ), + ], + now: asOf, + ); + + await seedBars( + symbol: 'SPY', + closes: [500, 501, 502, 503, 504, 505, 510], + asOf: asOf, + ); + await seedBars( + symbol: 'STALE', + closes: [100, 101, 102, 103, 104], + asOf: asOf, + ); + // Newest STALE bar is 9 days before asOf (> 2d stale). + await testDb!.marketDataDb.upsertSnapshot( + symbol: 'STALE', + metric: 'bar', + timeframe: MarketHistoryConfig.barTimeframe, + asOf: asOf.subtract(const Duration(days: 9)), + price: 104, + ); + + final MarketHistoryQuery query = + MarketHistoryQuery(connection: testDb!.connection); + final List movers = await query.weeklyMovers( + asOf: asOf, + minBars: 5, + ); + + expect(movers, hasLength(1)); + expect(movers.single.symbol, 'SPY'); + expect(movers.single.openClose, 500); + expect(movers.single.currentClose, 505); + expect(movers.single.days, 6); + }); + + test('deterministic selection order with seeded random', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final DateTime asOf = DateTime.utc(2026, 5, 26, 12); + + await TradableAssetsDb(testDb!.connection).upsertAll( + [ + AlpacaAsset( + symbol: 'AAA', + assetClass: 'us_equity', + tradable: true, + fractionable: true, + status: 'active', + ), + AlpacaAsset( + symbol: 'ZZZ', + assetClass: 'us_equity', + tradable: true, + fractionable: true, + status: 'active', + ), + ], + now: asOf, + ); + + for (final String symbol in ['AAA', 'ZZZ']) { + await seedBars( + symbol: symbol, + closes: [10, 11, 12, 13, 14, 15, 16], + asOf: asOf, + ); + } + + final MarketHistoryQuery query = + MarketHistoryQuery(connection: testDb!.connection); + final List a = await query.weeklyMovers( + asOf: asOf, + minBars: 5, + random: Random(42), + ); + final List b = await query.weeklyMovers( + asOf: asOf, + minBars: 5, + random: Random(42), + ); + + expect(a.map((WeeklyMover m) => m.symbol).toList(), + b.map((WeeklyMover m) => m.symbol).toList()); + expect(a.map((WeeklyMover m) => m.symbol).toList(), isNotEmpty); + }); + }); +} diff --git a/server/test/integration/market_history_scheduler_test.dart b/server/test/integration/market_history_scheduler_test.dart new file mode 100644 index 0000000..54337e0 --- /dev/null +++ b/server/test/integration/market_history_scheduler_test.dart @@ -0,0 +1,361 @@ +@Tags(['integration', 'postgres']) +library; + +import 'package:cyberhybridhub_server/trading/market_data_history.dart'; +import 'package:cyberhybridhub_server/trading/market_data_retention.dart'; +import 'package:cyberhybridhub_server/trading/sync_run_recorder.dart'; +import 'package:cyberhybridhub_server/trading/tradable_assets_sync.dart'; +import 'package:cyberhybridhub_server/workers/market_history_scheduler.dart'; +import 'package:cyberhybridhub_server/workers/market_history_scheduler_config.dart'; +import 'package:postgres/postgres.dart'; +import 'package:test/test.dart'; + +import '../helpers/test_db.dart'; + +void main() { + TestDb? testDb; + + setUpAll(() async { + testDb = await TestDb.open(); + }); + + tearDown(() async { + if (testDb != null) { + await testDb!.truncateTradingTables(); + } + }); + + tearDownAll(() async { + await testDb?.close(); + }); + + MarketHistoryScheduler scheduler({ + MarketHistorySchedulerConfig? config, + Future Function(DateTime now)? runUniverse, + Future Function(DateTime now)? runBackfill, + Future Function(DateTime now)? runCleanup, + Future Function(DateTime now)? backfillIsDue, + }) { + return MarketHistoryScheduler( + connection: testDb!.connection, + config: config ?? const MarketHistorySchedulerConfig(), + runUniverse: runUniverse, + runBackfill: runBackfill, + runCleanup: runCleanup, + backfillIsDue: backfillIsDue, + ); + } + + Future recordStage(String kind, DateTime now) async { + final SyncRunRecorder recorder = SyncRunRecorder(testDb!.connection); + await recorder.record( + kind, + () async => const SyncRunCounts(rowsWritten: 1), + now: now, + ); + } + + group('runIfDue', () { + test('cold start runs universe, backfill, cleanup in order', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final List order = []; + final DateTime t0 = DateTime.utc(2026, 5, 26, 12); + + await scheduler( + runUniverse: (DateTime now) async { + order.add('universe'); + await recordStage(TradableAssetsSync.kind, now); + }, + runBackfill: (DateTime now) async { + order.add('backfill'); + await recordStage(MarketDataHistorySync.kind, now); + }, + runCleanup: (DateTime now) async { + order.add('cleanup'); + await recordStage(MarketDataRetention.kind, now); + }, + ).runIfDue(t0); + + expect(order, ['universe', 'backfill', 'cleanup']); + + final Result runs = await testDb!.connection.execute( + ''' + SELECT kind FROM market_data_sync_runs + ORDER BY id ASC + ''', + ); + expect( + runs.map((ResultRow r) => r[0]).toList(), + ['universe', 'backfill', 'cleanup'], + ); + }); + + test('same-day re-run does not execute stages again', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final DateTime t0 = DateTime.utc(2026, 5, 26, 12); + int calls = 0; + + final MarketHistoryScheduler s = scheduler( + runUniverse: (DateTime now) async { + calls++; + await recordStage(TradableAssetsSync.kind, now); + }, + runBackfill: (DateTime now) async { + calls++; + await recordStage(MarketDataHistorySync.kind, now); + }, + runCleanup: (DateTime now) async { + calls++; + await recordStage(MarketDataRetention.kind, now); + }, + ); + + await s.runIfDue(t0); + + final int runsBefore = + (await testDb!.connection.execute('SELECT COUNT(*) FROM market_data_sync_runs')) + .first[0]! as int; + + await s.runIfDue(t0.add(const Duration(hours: 1))); + + expect(calls, 3); + + final int runsAfter = + (await testDb!.connection.execute('SELECT COUNT(*) FROM market_data_sync_runs')) + .first[0]! as int; + expect(runsAfter, runsBefore); + }); + + test('next day runs all stages again', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final DateTime t0 = DateTime.utc(2026, 5, 26, 12); + int calls = 0; + + final MarketHistoryScheduler s = scheduler( + runUniverse: (DateTime now) async { + calls++; + await recordStage(TradableAssetsSync.kind, now); + }, + runBackfill: (DateTime now) async { + calls++; + await recordStage(MarketDataHistorySync.kind, now); + }, + runCleanup: (DateTime now) async { + calls++; + await recordStage(MarketDataRetention.kind, now); + }, + ); + + await s.runIfDue(t0); + + await s.runIfDue(t0.add(const Duration(hours: 24))); + + expect(calls, 6); + }); + + test('per-stage cadence: at T0+24h only backfill and cleanup run', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final DateTime t0 = DateTime.utc(2026, 5, 26, 12); + final List ran = []; + + final MarketHistoryScheduler s = scheduler( + config: const MarketHistorySchedulerConfig( + universeRefreshHours: 48, + historySyncHours: 24, + cleanupHours: 24, + ), + runUniverse: (DateTime now) async { + ran.add('universe'); + await recordStage(TradableAssetsSync.kind, now); + }, + runBackfill: (DateTime now) async { + ran.add('backfill'); + await recordStage(MarketDataHistorySync.kind, now); + }, + runCleanup: (DateTime now) async { + ran.add('cleanup'); + await recordStage(MarketDataRetention.kind, now); + }, + ); + + await s.runIfDue(t0); + + await s.runIfDue(t0.add(const Duration(hours: 24))); + + expect(ran, ['universe', 'backfill', 'cleanup', 'backfill', 'cleanup']); + }); + + test('failure isolation: backfill error still runs cleanup', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final DateTime t0 = DateTime.utc(2026, 5, 26, 12); + final Connection connection = testDb!.connection; + + await scheduler( + runUniverse: (DateTime now) => recordStage(TradableAssetsSync.kind, now), + runBackfill: (DateTime now) async { + final SyncRunRecorder recorder = SyncRunRecorder(connection); + await recorder.record( + MarketDataHistorySync.kind, + () async { + throw Exception('backfill boom'); + }, + now: now, + ); + }, + runCleanup: (DateTime now) => recordStage(MarketDataRetention.kind, now), + ).runIfDue(t0); + + final Result runs = await connection.execute( + ''' + SELECT kind, error IS NOT NULL AS has_error + FROM market_data_sync_runs + ORDER BY id ASC + ''', + ); + expect(runs, hasLength(3)); + expect(runs[0][0], 'universe'); + expect(runs[0][1], isFalse); + expect(runs[1][0], 'backfill'); + expect(runs[1][1], isTrue); + expect(runs[2][0], 'cleanup'); + expect(runs[2][1], isFalse); + }); + + test('slot-based backfill runs again same day when new slot is pending', + () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + bool pending = true; + int backfillCalls = 0; + + final MarketHistoryScheduler s = scheduler( + backfillIsDue: (DateTime now) async => pending, + runUniverse: (DateTime now) => recordStage(TradableAssetsSync.kind, now), + runBackfill: (DateTime now) async { + backfillCalls++; + await recordStage(MarketDataHistorySync.kind, now); + pending = false; + }, + runCleanup: (DateTime now) => recordStage(MarketDataRetention.kind, now), + ); + + final DateTime t0 = DateTime.utc(2026, 5, 26, 12); + await s.runIfDue(t0); + expect(backfillCalls, 1); + + pending = true; + await s.runIfDue(t0.add(const Duration(hours: 1))); + expect(backfillCalls, 2); + }); + + test('aborts orphaned in-progress run before starting pipeline', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + await testDb!.connection.execute( + Sql.named( + ''' + INSERT INTO market_data_sync_runs (kind, started_at, finished_at) + VALUES ('backfill', @started_at, NULL) + ''', + ), + parameters: { + 'started_at': DateTime.utc(2026, 5, 26, 10), + }, + ); + + final DateTime t0 = DateTime.utc(2026, 5, 26, 12); + await scheduler( + runUniverse: (DateTime now) => recordStage(TradableAssetsSync.kind, now), + runBackfill: (DateTime now) => recordStage(MarketDataHistorySync.kind, now), + runCleanup: (DateTime now) => recordStage(MarketDataRetention.kind, now), + ).runIfDue(t0); + + final Result rows = await testDb!.connection.execute( + ''' + SELECT kind, finished_at IS NULL AS open, error + FROM market_data_sync_runs + ORDER BY id ASC + ''', + ); + expect(rows.first[0], 'backfill'); + expect(rows.first[1], false); + expect(rows.first[2], contains('aborted')); + expect(rows, hasLength(4)); + }); + + test('syncHourUtc blocks before hour and same UTC day', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final MarketHistoryScheduler s = scheduler( + config: const MarketHistorySchedulerConfig(syncHourUtc: 10), + runUniverse: (DateTime now) async { + await recordStage(TradableAssetsSync.kind, now); + }, + runBackfill: (DateTime now) async { + await recordStage(MarketDataHistorySync.kind, now); + }, + runCleanup: (DateTime now) async { + await recordStage(MarketDataRetention.kind, now); + }, + ); + + final MarketHistorySchedulerReport beforeHour = + await s.runIfDue(DateTime.utc(2026, 5, 26, 9)); + expect(beforeHour.ranStages, isEmpty); + + final DateTime tRun = DateTime.utc(2026, 5, 26, 11); + final MarketHistorySchedulerReport first = + await s.runIfDue(tRun); + expect(first.ranStages, hasLength(3)); + + final MarketHistorySchedulerReport sameDay = + await s.runIfDue(DateTime.utc(2026, 5, 26, 12)); + expect(sameDay.ranStages, isEmpty); + }); + }); +} diff --git a/server/test/integration/market_history_schema_test.dart b/server/test/integration/market_history_schema_test.dart new file mode 100644 index 0000000..fcd09a6 --- /dev/null +++ b/server/test/integration/market_history_schema_test.dart @@ -0,0 +1,411 @@ +@Tags(['integration', 'postgres']) +library; + +import 'package:cyberhybridhub_server/trading/market_data_history.dart'; +import 'package:cyberhybridhub_server/trading/sync_run_recorder.dart'; +import 'package:postgres/postgres.dart'; +import 'package:test/test.dart'; + +import '../helpers/test_db.dart'; + +void main() { + TestDb? testDb; + + setUpAll(() async { + testDb = await TestDb.open(); + }); + + tearDown(() async { + if (testDb != null) { + await testDb!.truncateTradingTables(); + } + }); + + tearDownAll(() async { + await testDb?.close(); + }); + + group('market_data_snapshots — timeframe + unique observation', () { + test( + 'upsert on (symbol, metric, timeframe, as_of) keeps a single row ' + 'and updates price', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final Connection connection = testDb!.connection; + final DateTime asOf = DateTime.utc(2026, 5, 23, 13); + + // First insert. + await connection.execute( + Sql.named( + ''' + INSERT INTO market_data_snapshots ( + symbol, metric, timeframe, price, as_of + ) VALUES ( + 'SPY', 'bar', '1Day', 500, @as_of + ) + ON CONFLICT (symbol, metric, timeframe, as_of) DO UPDATE + SET price = EXCLUDED.price + ''', + ), + parameters: {'as_of': asOf}, + ); + + // Re-upsert the same observation with a new price. + await connection.execute( + Sql.named( + ''' + INSERT INTO market_data_snapshots ( + symbol, metric, timeframe, price, as_of + ) VALUES ( + 'SPY', 'bar', '1Day', 505, @as_of + ) + ON CONFLICT (symbol, metric, timeframe, as_of) DO UPDATE + SET price = EXCLUDED.price + ''', + ), + parameters: {'as_of': asOf}, + ); + + final Result rows = await connection.execute( + Sql.named( + ''' + SELECT price::float + FROM market_data_snapshots + WHERE symbol = 'SPY' + AND metric = 'bar' + AND timeframe = '1Day' + AND as_of = @as_of + ''', + ), + parameters: {'as_of': asOf}, + ); + + expect(rows, hasLength(1)); + expect(rows.first[0], 505.0); + }); + + test('timeframe defaults to tick and accepts 1Min/1Hour/1Day', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final Connection connection = testDb!.connection; + final DateTime asOfBase = DateTime.utc(2026, 5, 23, 14); + + // Insert without naming timeframe — must default to 'tick'. + await connection.execute( + Sql.named( + ''' + INSERT INTO market_data_snapshots (symbol, metric, price, as_of) + VALUES ('SPY', 'last_trade', 491, @as_of) + ''', + ), + parameters: {'as_of': asOfBase}, + ); + + final Result defaulted = await connection.execute( + Sql.named( + ''' + SELECT timeframe + FROM market_data_snapshots + WHERE symbol = 'SPY' AND as_of = @as_of + ''', + ), + parameters: {'as_of': asOfBase}, + ); + expect(defaulted, hasLength(1)); + expect(defaulted.first[0], 'tick'); + + // Each accepted timeframe value can be inserted. + const List timeframes = ['1Min', '1Hour', '4Hour', '1Day']; + for (int i = 0; i < timeframes.length; i++) { + final String tf = timeframes[i]; + final DateTime asOf = asOfBase.add(Duration(minutes: i + 1)); + await connection.execute( + Sql.named( + ''' + INSERT INTO market_data_snapshots ( + symbol, metric, timeframe, price, as_of + ) VALUES ( + 'SPY', 'bar', @timeframe, 500, @as_of + ) + ''', + ), + parameters: {'timeframe': tf, 'as_of': asOf}, + ); + } + + final Result accepted = await connection.execute( + Sql.named( + ''' + SELECT timeframe + FROM market_data_snapshots + WHERE symbol = 'SPY' AND metric = 'bar' + ORDER BY timeframe + ''', + ), + ); + expect( + accepted.map((ResultRow r) => r[0]).toList(), + ['1Day', '1Hour', '1Min', '4Hour'], + ); + }); + }); + + group('tradable_assets', () { + test('rejects duplicate symbol via primary key', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final Connection connection = testDb!.connection; + + await connection.execute( + ''' + INSERT INTO tradable_assets (symbol, asset_class, status, tradable) + VALUES ('SPY', 'us_equity', 'active', true) + ''', + ); + + await expectLater( + connection.execute( + ''' + INSERT INTO tradable_assets (symbol, asset_class, status, tradable) + VALUES ('SPY', 'us_equity', 'active', true) + ''', + ), + throwsA(isA()), + ); + }); + + test( + 'status/tradable lookup uses the tradable_assets_status_idx index', + () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final Connection connection = testDb!.connection; + + // Seed enough rows that the planner has a real choice. + for (int i = 0; i < 60; i++) { + await connection.execute( + Sql.named( + ''' + INSERT INTO tradable_assets (symbol, asset_class, status, tradable) + VALUES (@symbol, 'us_equity', @status, @tradable) + ''', + ), + parameters: { + 'symbol': 'SYM$i', + 'status': i.isEven ? 'active' : 'inactive', + 'tradable': i.isEven, + }, + ); + } + await connection.execute('ANALYZE tradable_assets'); + + // Force the planner to consider the index even on a small dataset. + // SET LOCAL only persists inside an explicit transaction; use plain + // SET on this short-lived test session and restore afterwards. + await connection.execute('SET enable_seqscan = off'); + final Result plan; + try { + plan = await connection.execute( + ''' + EXPLAIN (FORMAT TEXT) + SELECT symbol FROM tradable_assets + WHERE status = 'active' AND tradable = true + ''', + ); + } finally { + await connection.execute('SET enable_seqscan = on'); + } + final String planText = + plan.map((ResultRow r) => r[0]?.toString() ?? '').join('\n'); + + // Either an Index Scan or a Bitmap Index Scan referencing the index + // is acceptable; both prove the index is in use. + expect( + planText.contains('tradable_assets_status_idx'), + isTrue, + reason: 'expected plan to use tradable_assets_status_idx, ' + 'got:\n$planText', + ); + }); + }); + + group('market_data_sync_runs', () { + test( + 'records kind/started_at/finished_at/rows_written/rows_removed/error', + () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final Connection connection = testDb!.connection; + final DateTime started = DateTime.utc(2026, 5, 26, 10); + final DateTime finished = DateTime.utc(2026, 5, 26, 10, 0, 5); + + await connection.execute( + Sql.named( + ''' + INSERT INTO market_data_sync_runs ( + kind, started_at, finished_at, rows_written, rows_removed, error + ) VALUES ( + 'universe', @started, @finished, 12, 0, NULL + ) + ''', + ), + parameters: { + 'started': started, + 'finished': finished, + }, + ); + + await connection.execute( + Sql.named( + ''' + INSERT INTO market_data_sync_runs ( + kind, started_at, finished_at, rows_written, rows_removed, error + ) VALUES ( + 'cleanup', @started, @finished, 0, 42, 'partial outage: MSFT' + ) + ''', + ), + parameters: { + 'started': started, + 'finished': finished, + }, + ); + + // kind must reject NULL. + await expectLater( + connection.execute( + ''' + INSERT INTO market_data_sync_runs (started_at) + VALUES (now()) + ''', + ), + throwsA(isA()), + ); + + final Result runs = await connection.execute( + ''' + SELECT kind, rows_written, rows_removed, error + FROM market_data_sync_runs + ORDER BY kind + ''', + ); + expect( + runs.map((ResultRow r) => r[0]).toList(), + ['cleanup', 'universe'], + ); + expect((runs.elementAt(0)[1]! as num).toInt(), 0); + expect((runs.elementAt(0)[2]! as num).toInt(), 42); + expect(runs.elementAt(0)[3], 'partial outage: MSFT'); + expect((runs.elementAt(1)[1]! as num).toInt(), 12); + expect((runs.elementAt(1)[2]! as num).toInt(), 0); + expect(runs.elementAt(1)[3], isNull); + }); + + test('slots_synced defaults to 0 and is stored on backfill runs', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final SyncRunRecorder recorder = SyncRunRecorder(testDb!.connection); + await recorder.record( + MarketDataHistorySync.kind, + () async => const SyncRunCounts(rowsWritten: 9, slotsSynced: 3), + now: DateTime.utc(2026, 5, 26, 12), + ); + + final Result rows = await testDb!.connection.execute( + ''' + SELECT slots_synced, rows_written + FROM market_data_sync_runs + WHERE kind = 'backfill' + ''', + ); + expect((rows.single[0]! as num).toInt(), 3); + expect((rows.single[1]! as num).toInt(), 9); + }); + }); + + group('008 four-hour history constraints', () { + test('rejects invalid bar timeframe', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + await expectLater( + testDb!.connection.execute( + Sql.named( + ''' + INSERT INTO market_data_snapshots ( + symbol, metric, timeframe, price, as_of + ) VALUES ( + 'SPY', 'bar', '2Hour', 100, @as_of + ) + ''', + ), + parameters: { + 'as_of': DateTime.utc(2026, 5, 26, 8), + }, + ), + throwsA(isA()), + ); + }); + + test('accepts 4Hour bar rows and partial index exists', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + await testDb!.marketDataDb.upsertSnapshot( + symbol: 'SPY', + metric: 'bar', + timeframe: '4Hour', + asOf: DateTime.utc(2026, 5, 26, 8), + price: 500, + ); + + final Result indexes = await testDb!.connection.execute( + ''' + SELECT indexname + FROM pg_indexes + WHERE tablename = 'market_data_snapshots' + AND indexname = 'market_data_snapshots_bar_4h_idx' + ''', + ); + expect(indexes, hasLength(1)); + }); + }); +} diff --git a/server/test/integration/market_history_week_coverage_test.dart b/server/test/integration/market_history_week_coverage_test.dart new file mode 100644 index 0000000..c9f761b --- /dev/null +++ b/server/test/integration/market_history_week_coverage_test.dart @@ -0,0 +1,83 @@ +@Tags(['integration', 'postgres']) +library; + +import 'package:cyberhybridhub_server/trading/market_history_config.dart'; +import 'package:cyberhybridhub_server/trading/market_history_four_hour_slot.dart'; +import 'package:cyberhybridhub_server/trading/market_history_week_coverage.dart'; +import 'package:cyberhybridhub_server/trading/tradable_assets_db.dart'; +import 'package:cyberhybridhub_server/alpaca/alpaca_models.dart'; +import 'package:test/test.dart'; + +import '../helpers/test_db.dart'; + +void main() { + TestDb? testDb; + + setUpAll(() async { + testDb = await TestDb.open(); + }); + + tearDown(() async { + if (testDb != null) { + await testDb!.truncateTradingTables(); + } + }); + + tearDownAll(() async { + await testDb?.close(); + }); + + test('no_data placeholder counts toward fullySynced slot', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final DateTime now = DateTime.utc(2026, 5, 31, 0, 30); + final DateTime slotStart = DateTime.utc(2026, 5, 30, 20); + + await TradableAssetsDb(testDb!.connection).upsertAll( + [ + AlpacaAsset( + symbol: 'AAA', + assetClass: 'us_equity', + status: 'active', + tradable: true, + fractionable: true, + ), + ], + now: now, + ); + + await testDb!.marketDataDb.upsertNoDataBarPlaceholder( + symbol: 'AAA', + slotStart: slotStart, + timeframe: MarketHistoryConfig.barTimeframe, + checkedAt: now, + source: 'market_closed', + ); + + final MarketHistoryWeekCoverageReport report = + await MarketHistoryWeekCoverage( + connection: testDb!.connection, + windowDays: 2, + maxSymbols: 2000, + ).compute(now: now); + + expect(report.symbolCount, 1); + + final MarketHistoryDayCoverage saturday = report.days.singleWhere( + (MarketHistoryDayCoverage day) => day.date == DateTime.utc(2026, 5, 30), + ); + final MarketHistorySlotCoverage slot = saturday.slots.singleWhere( + (MarketHistorySlotCoverage s) => + s.slotStart == DateTime.utc(2026, 5, 30, 20), + ); + + expect(slot.completed, isTrue); + expect(slot.fullySynced, isTrue); + expect(slot.syncedSymbolCount, 1); + }); +} diff --git a/server/test/integration/market_history_worker_wireup_test.dart b/server/test/integration/market_history_worker_wireup_test.dart new file mode 100644 index 0000000..37a9ccc --- /dev/null +++ b/server/test/integration/market_history_worker_wireup_test.dart @@ -0,0 +1,112 @@ +@Tags(['integration', 'postgres']) +library; + +import 'package:cyberhybridhub_server/pipeline/question_pipeline.dart'; +import 'package:cyberhybridhub_server/question_service.dart'; +import 'package:cyberhybridhub_server/questions_db.dart'; +import 'package:cyberhybridhub_server/signalr/questions_hub_connections.dart'; +import 'package:cyberhybridhub_server/workers/market_history_scheduler.dart'; +import 'package:cyberhybridhub_server/workers/question_background_worker.dart'; +import 'package:test/test.dart'; + +import '../helpers/test_db.dart'; + +void main() { + TestDb? testDb; + + setUpAll(() async { + testDb = await TestDb.open(); + }); + + tearDownAll(() async { + await testDb?.close(); + }); + + group('QuestionBackgroundWorker wireup', () { + test('scheduler runs before trading maintenance', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final List callOrder = []; + final QuestionsDb questionsDb = QuestionsDb(testDb!.connection); + final QuestionPipeline pipeline = QuestionPipeline( + questionsDb: questionsDb, + questionService: QuestionService( + questionsDb: questionsDb, + hubConnections: QuestionsHubConnections(), + ), + testMode: true, + ); + + final MarketHistoryScheduler scheduler = MarketHistoryScheduler( + connection: testDb!.connection, + runUniverse: (_) async { + callOrder.add('scheduler'); + }, + runBackfill: (_) async {}, + runCleanup: (_) async {}, + ); + + final QuestionBackgroundWorker worker = QuestionBackgroundWorker( + pipeline: pipeline, + interval: const Duration(hours: 1), + marketHistoryScheduler: scheduler, + tradingMaintenanceRunner: () async { + callOrder.add('trading'); + }, + ); + + await worker.runTickForTest(); + + expect(callOrder.indexOf('scheduler'), lessThan(callOrder.indexOf('trading'))); + expect(callOrder, contains('scheduler')); + expect(callOrder, contains('trading')); + }); + + test('scheduler exception does not block trading', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final QuestionsDb questionsDb = QuestionsDb(testDb!.connection); + final QuestionPipeline pipeline = QuestionPipeline( + questionsDb: questionsDb, + questionService: QuestionService( + questionsDb: questionsDb, + hubConnections: QuestionsHubConnections(), + ), + testMode: true, + ); + + final MarketHistoryScheduler scheduler = MarketHistoryScheduler( + connection: testDb!.connection, + runUniverse: (_) async { + throw Exception('scheduler boom'); + }, + runBackfill: (_) async {}, + runCleanup: (_) async {}, + ); + + var tradingRan = false; + final QuestionBackgroundWorker worker = QuestionBackgroundWorker( + pipeline: pipeline, + interval: const Duration(hours: 1), + marketHistoryScheduler: scheduler, + tradingMaintenanceRunner: () async { + tradingRan = true; + }, + ); + + await worker.runTickForTest(); + + expect(tradingRan, isTrue); + }); + }); +} diff --git a/server/test/integration/tradable_assets_db_test.dart b/server/test/integration/tradable_assets_db_test.dart new file mode 100644 index 0000000..4d64fed --- /dev/null +++ b/server/test/integration/tradable_assets_db_test.dart @@ -0,0 +1,174 @@ +@Tags(['integration', 'postgres']) +library; + +import 'package:cyberhybridhub_server/alpaca/alpaca_models.dart'; +import 'package:cyberhybridhub_server/trading/tradable_assets_db.dart'; +import 'package:test/test.dart'; + +import '../helpers/test_db.dart'; + +AlpacaAsset _asset( + String symbol, { + bool tradable = true, + bool fractionable = true, + String status = 'active', + String? exchange = 'NASDAQ', + String? name, +}) { + return AlpacaAsset( + symbol: symbol, + assetClass: 'us_equity', + exchange: exchange, + name: name ?? '$symbol Inc.', + status: status, + tradable: tradable, + fractionable: fractionable, + raw: { + 'symbol': symbol, + 'class': 'us_equity', + 'exchange': exchange, + 'name': name ?? '$symbol Inc.', + 'status': status, + 'tradable': tradable, + 'fractionable': fractionable, + }, + ); +} + +void main() { + TestDb? testDb; + + setUpAll(() async { + testDb = await TestDb.open(); + }); + + tearDown(() async { + if (testDb != null) { + await testDb!.truncateTradingTables(); + } + }); + + tearDownAll(() async { + await testDb?.close(); + }); + + test('upsertAll inserts new symbols with the supplied refreshed_at', + () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final TradableAssetsDb db = TradableAssetsDb(testDb!.connection); + final DateTime t0 = DateTime.utc(2026, 5, 26, 10); + + await db.upsertAll([_asset('A'), _asset('B'), _asset('C')], + now: t0); + + final TradableAssetRow? a = await db.getBySymbol('A'); + final TradableAssetRow? b = await db.getBySymbol('B'); + final TradableAssetRow? c = await db.getBySymbol('C'); + + expect(a, isNotNull); + expect(b, isNotNull); + expect(c, isNotNull); + expect(a!.tradable, isTrue); + expect(a.status, 'active'); + expect(a.refreshedAt, t0); + expect(b!.refreshedAt, t0); + expect(c!.refreshedAt, t0); + }); + + test( + 'second upsertAll updates B*, leaves C content unchanged but bumps ' + 'refreshed_at, inserts D, and marks A inactive without deleting it', + () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final TradableAssetsDb db = TradableAssetsDb(testDb!.connection); + + final DateTime t0 = DateTime.utc(2026, 5, 26, 10); + await db.upsertAll( + [ + _asset('A'), + _asset('B', name: 'B Original Inc.', exchange: 'NYSE'), + _asset('C', exchange: 'NASDAQ'), + ], + now: t0, + ); + + final DateTime t1 = DateTime.utc(2026, 5, 27, 10); + await db.upsertAll( + [ + // B with new content (name + exchange). + _asset('B', name: 'B Renamed Corp.', exchange: 'NASDAQ'), + // C with identical content as t0 — should bump refreshed_at only. + _asset('C', exchange: 'NASDAQ'), + // D is new. + _asset('D'), + ], + now: t1, + ); + + final TradableAssetRow? a = await db.getBySymbol('A'); + final TradableAssetRow? b = await db.getBySymbol('B'); + final TradableAssetRow? c = await db.getBySymbol('C'); + final TradableAssetRow? d = await db.getBySymbol('D'); + + // A is preserved as a historical record but flipped to inactive. + expect(a, isNotNull, reason: 'A must NOT be deleted — history preserved'); + expect(a!.status, 'inactive'); + expect(a.tradable, isFalse); + // refreshed_at on the inactivated row stays at the original t0 so + // operators can see "last seen as active" in the audit trail. + expect(a.refreshedAt, t0); + + // B was updated in place: same row, new content, new refreshed_at. + expect(b, isNotNull); + expect(b!.name, 'B Renamed Corp.'); + expect(b.exchange, 'NASDAQ'); + expect(b.refreshedAt, t1); + + // C content unchanged but refreshed_at bumped to t1. + expect(c, isNotNull); + expect(c!.exchange, 'NASDAQ'); + expect(c.refreshedAt, t1); + + // D inserted. + expect(d, isNotNull); + expect(d!.refreshedAt, t1); + }); + + test('listActiveTradableSymbols filters to active AND tradable', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final TradableAssetsDb db = TradableAssetsDb(testDb!.connection); + final DateTime t0 = DateTime.utc(2026, 5, 26, 10); + + await db.upsertAll( + [ + _asset('AAA'), // active + tradable + _asset('BBB'), // active + tradable + _asset('CCC', tradable: false), // active but not tradable + _asset('DDD', status: 'inactive'), // inactive + ], + now: t0, + ); + + final List symbols = await db.listActiveTradableSymbols(); + + expect(symbols.toSet(), {'AAA', 'BBB'}); + }); +} diff --git a/server/test/integration/tradable_assets_sync_test.dart b/server/test/integration/tradable_assets_sync_test.dart new file mode 100644 index 0000000..fe78683 --- /dev/null +++ b/server/test/integration/tradable_assets_sync_test.dart @@ -0,0 +1,177 @@ +@Tags(['integration', 'postgres']) +library; + +import 'package:cyberhybridhub_server/alpaca/alpaca_assets_client.dart'; +import 'package:cyberhybridhub_server/alpaca/alpaca_env.dart'; +import 'package:cyberhybridhub_server/trading/tradable_assets_db.dart'; +import 'package:cyberhybridhub_server/trading/tradable_assets_sync.dart'; +import 'package:http/http.dart' as http; +import 'package:postgres/postgres.dart'; +import 'package:test/test.dart'; + +import '../helpers/fixture_loader.dart'; +import '../helpers/mock_http_client.dart'; +import '../helpers/test_db.dart'; + +void main() { + TestDb? testDb; + late FixtureLoader fixtures; + late AlpacaEnv env; + + setUpAll(() async { + testDb = await TestDb.open(); + fixtures = FixtureLoader(); + env = AlpacaEnv.fromMap({ + 'ALPACA_API_KEY_ID': 'test-key', + 'ALPACA_API_SECRET_KEY': 'test-secret', + 'ALPACA_TRADING_BASE_URL': 'https://paper-api.alpaca.markets', + }); + }); + + tearDown(() async { + if (testDb != null) { + await testDb!.truncateTradingTables(); + } + }); + + tearDownAll(() async { + await testDb?.close(); + }); + + TradableAssetsSync makeSync({required MockHttpClient mock}) { + return TradableAssetsSync( + assetsClient: AlpacaAssetsClient(env: env, httpClient: mock), + assetsDb: TradableAssetsDb(testDb!.connection), + connection: testDb!.connection, + ); + } + + test( + 'runOnce with the fixture upserts 5 assets and writes a universe ' + 'sync_runs row with finished_at', + () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final String body = await fixtures.loadString('alpaca_assets_active.json'); + final MockHttpClient mock = MockHttpClient() + ..whenGet( + '/v2/assets', + http.Response(body, 200, headers: { + 'content-type': 'application/json', + }), + ); + + final TradableAssetsSync sync = makeSync(mock: mock); + final TradableAssetsSyncResult result = await sync.runOnce(); + + expect(result.error, isNull); + expect(result.rowsWritten, 5); + + final List active = await TradableAssetsDb(testDb!.connection) + .listActiveTradableSymbols(); + expect( + active.toSet(), + {'AAPL', 'MSFT', 'SPY', 'BRK.B'}, + reason: 'PNKZZ has tradable=false and must be excluded', + ); + + final Result runs = await testDb!.connection.execute( + ''' + SELECT kind, rows_written, rows_removed, error, + started_at, finished_at + FROM market_data_sync_runs + ORDER BY id ASC + ''', + ); + expect(runs, hasLength(1)); + final ResultRow row = runs.first; + expect(row[0], 'universe'); + expect((row[1]! as num).toInt(), 5); + expect((row[2]! as num).toInt(), 0); + expect(row[3], isNull, reason: 'no error on happy path'); + expect(row[4], isNotNull); + expect(row[5], isNotNull); + }); + + test( + 'runOnce records the error when the Alpaca client throws and does ' + 'not propagate the exception', + () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final MockHttpClient mock = MockHttpClient() + ..whenGet( + '/v2/assets', + http.Response('upstream exploded', 500), + ); + + final TradableAssetsSync sync = makeSync(mock: mock); + + // Must not throw — failures get recorded, not raised. + final TradableAssetsSyncResult result = await sync.runOnce(); + + expect(result.rowsWritten, 0); + expect(result.error, isNotNull); + expect(result.error, contains('500')); + + final Result runs = await testDb!.connection.execute( + ''' + SELECT kind, rows_written, error, finished_at + FROM market_data_sync_runs + ORDER BY id ASC + ''', + ); + expect(runs, hasLength(1)); + expect(runs.first[0], 'universe'); + expect((runs.first[1]! as num).toInt(), 0); + expect(runs.first[2]?.toString(), contains('500')); + expect(runs.first[3], isNotNull, + reason: 'finished_at must always be set, even on failure'); + }); + + test('two consecutive runs produce identical row counts (idempotent)', + () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + final String body = await fixtures.loadString('alpaca_assets_active.json'); + final MockHttpClient mock = MockHttpClient() + ..whenGet( + '/v2/assets', + http.Response(body, 200, headers: { + 'content-type': 'application/json', + }), + ); + + final TradableAssetsSync sync = makeSync(mock: mock); + final TradableAssetsSyncResult r1 = await sync.runOnce(); + final TradableAssetsSyncResult r2 = await sync.runOnce(); + + expect(r1.rowsWritten, 5); + expect(r2.rowsWritten, 5); + expect(r1.error, isNull); + expect(r2.error, isNull); + + final Result count = await testDb!.connection + .execute("SELECT COUNT(*) FROM tradable_assets WHERE status = 'active'"); + expect((count.first[0]! as num).toInt(), 5); + + final Result runs = await testDb!.connection + .execute('SELECT COUNT(*) FROM market_data_sync_runs'); + expect((runs.first[0]! as num).toInt(), 2); + }); +} diff --git a/server/test/integration/trading_pipeline_guess_weekly_move_test.dart b/server/test/integration/trading_pipeline_guess_weekly_move_test.dart new file mode 100644 index 0000000..e6f3c11 --- /dev/null +++ b/server/test/integration/trading_pipeline_guess_weekly_move_test.dart @@ -0,0 +1,292 @@ +@Tags(['integration', 'postgres']) +library; + +import 'package:cyberhybridhub_server/alpaca/alpaca_models.dart'; +import 'package:cyberhybridhub_server/pipeline/question_pipeline.dart'; +import 'package:cyberhybridhub_server/trading/market_history_config.dart'; +import 'package:cyberhybridhub_server/trading/market_history_query.dart'; +import 'package:cyberhybridhub_server/trading/tradable_assets_db.dart'; +import 'package:cyberhybridhub_server/trading/trading_pipeline.dart'; +import 'package:cyberhybridhub_server/trading/user_trading_state_db.dart'; +import 'package:postgres/postgres.dart'; +import 'package:test/test.dart'; + +import '../helpers/test_db.dart'; + + +void main() { + TestDb? testDb; + + setUpAll(() async { + testDb = await TestDb.open(); + }); + + tearDown(() async { + if (testDb != null) { + await testDb!.truncateTradingTables(); + } + }); + + tearDownAll(() async { + await testDb?.close(); + }); + + final DateTime _testNow = DateTime.utc(2026, 5, 26, 12); + + Future _seedGuessUniverse() async { + await TradableAssetsDb(testDb!.connection).upsertAll( + [ + AlpacaAsset( + symbol: 'SPY', + assetClass: 'us_equity', + tradable: true, + fractionable: true, + status: 'active', + ), + ], + now: _testNow, + ); + final List closes = [500, 501, 502, 503, 504, 505, 510]; + for (int i = 0; i < closes.length; i++) { + await testDb!.marketDataDb.upsertSnapshot( + symbol: 'SPY', + metric: 'bar', + timeframe: MarketHistoryConfig.barTimeframe, + asOf: _testNow.subtract(Duration(hours: 4 * (closes.length - 1 - i))), + price: closes[i], + ); + } + } + + Future _guessPipeline() async { + return TradingPipeline( + questionsDb: testDb!.questionsDb, + questionService: testDb!.questionService(), + marketDataDb: testDb!.marketDataDb, + tradingConfigDb: testDb!.tradingConfigDb, + tradingStateDb: testDb!.userTradingStateDb, + marketHistoryQuery: MarketHistoryQuery(connection: testDb!.connection), + clock: () => _testNow, + ); + } + + Future _enableGuessRule(String uid) async { + await testDb!.seedUser(uid); + await testDb!.tradingConfigDb.upsertUserConfig( + firebaseUid: uid, + templateName: 'default_paper_watchlist', + enabled: true, + config: { + 'rules': >[ + { + 'id': 'guess_weekly_move', + 'type': 'guess_weekly_move', + 'question_template': + '{{token}} was {{ref_price}} {{ref_days_ago}} days ago. Swipe +10 if up, -10 if down.', + }, + { + 'id': 'dip_confirm', + 'threshold_pct': 0, + }, + ], + }, + ); + } + + group('guess_weekly_move pipeline', () { + test('evaluate creates obfuscated question with metadata.guess_symbol', + () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + const String uid = 'guess-eval-uid'; + await _enableGuessRule(uid); + await _seedGuessUniverse(); + + final TradingPipeline pipeline = await _guessPipeline(); + final TradingEvaluationResult result = await pipeline.evaluate(uid); + + expect(result.questionsCreated, 1); + expect(result.rulesFired, ['guess_weekly_move']); + + final Result rows = await testDb!.connection.execute( + Sql.named( + ''' + SELECT pipeline_key, pipeline_step, question_text, metadata + FROM questions + WHERE assigned_user_id = @uid + ''', + ), + parameters: {'uid': uid}, + ); + expect(rows, hasLength(1)); + expect(rows.first[0], PipelineKeys.trading); + expect(rows.first[1], 'guess_weekly_move:${TradingPhases.awaitAnswer}'); + final String text = rows.first[2]! as String; + expect(text, contains('ASSET_A')); + expect(text, isNot(contains('SPY'))); + final Map metadata = + rows.first[3]! as Map; + expect(metadata['guess_symbol'], 'SPY'); + }); + + test('matching direction records score_delta +1', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + const String uid = 'guess-score-match-uid'; + await _enableGuessRule(uid); + await _seedGuessUniverse(); + + final TradingPipeline pipeline = await _guessPipeline(); + await pipeline.evaluate(uid); + + final List> open = + await testDb!.questionsDb.listUnansweredQuestions(uid); + final Map? updated = + await testDb!.questionsDb.submitAnswer( + questionId: open.single['id'] as String, + assignedUserId: uid, + userResponse: 10, + ); + + await pipeline.handleAnswer( + firebaseUid: uid, + answeredQuestion: updated!, + userResponse: 10, + ); + + final Map? score = + await testDb!.userTradingStateDb.getGuessScore(uid); + expect(score, isNotNull); + expect(score!['total'], 1); + expect((score['last'] as Map)['score_delta'], 1); + }); + + test('non-matching direction records score_delta -1', () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + const String uid = 'guess-score-miss-uid'; + await _enableGuessRule(uid); + await _seedGuessUniverse(); + + final TradingPipeline pipeline = await _guessPipeline(); + await pipeline.evaluate(uid); + + final List> open = + await testDb!.questionsDb.listUnansweredQuestions(uid); + final Map? updated = + await testDb!.questionsDb.submitAnswer( + questionId: open.single['id'] as String, + assignedUserId: uid, + userResponse: -10, + ); + + await pipeline.handleAnswer( + firebaseUid: uid, + answeredQuestion: updated!, + userResponse: -10, + ); + + final Map? score = + await testDb!.userTradingStateDb.getGuessScore(uid); + expect(score!['total'], -1); + expect((score['last'] as Map)['score_delta'], -1); + }); + + test('handleAnswer never stages pending orders; actuator not invoked', + () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + const String uid = 'guess-no-actuator-uid'; + await _enableGuessRule(uid); + await _seedGuessUniverse(); + + final TradingPipeline pipeline = await _guessPipeline(); + await pipeline.evaluate(uid); + + final List> open = + await testDb!.questionsDb.listUnansweredQuestions(uid); + final Map? updated = + await testDb!.questionsDb.submitAnswer( + questionId: open.single['id'] as String, + assignedUserId: uid, + userResponse: 10, + ); + + await pipeline.handleAnswer( + firebaseUid: uid, + answeredQuestion: updated!, + userResponse: 10, + ); + + final List> pending = + await testDb!.userTradingStateDb.listPendingOrders(uid); + expect(pending, isEmpty); + + final Result orders = await testDb!.connection.execute( + Sql.named( + 'SELECT COUNT(*)::int FROM trade_orders WHERE firebase_uid = @uid', + ), + parameters: {'uid': uid}, + ); + expect(orders.first[0], 0); + }); + + test('symbol cooldown prevents re-pick within GUESS_COOLDOWN_HOURS', + () async { + if (testDb == null) { + markTestSkipped( + 'Set DATABASE_URL or TEST_DATABASE_URL for integration tests', + ); + return; + } + + const String uid = 'guess-cooldown-uid'; + await _enableGuessRule(uid); + await _seedGuessUniverse(); + + final TradingPipeline pipeline = await _guessPipeline(); + await pipeline.evaluate(uid); + + final List> open = + await testDb!.questionsDb.listUnansweredQuestions(uid); + final Map? updated = + await testDb!.questionsDb.submitAnswer( + questionId: open.single['id'] as String, + assignedUserId: uid, + userResponse: 10, + ); + await pipeline.handleAnswer( + firebaseUid: uid, + answeredQuestion: updated!, + userResponse: 10, + ); + + final TradingEvaluationResult again = await pipeline.evaluate(uid); + expect(again.questionsCreated, 0); + expect( + again.rulesSkipped, + contains('guess_weekly_move(insufficientBars)'), + ); + }); + }); +} diff --git a/server/test/trading/market_history_admin_actions_test.dart b/server/test/trading/market_history_admin_actions_test.dart new file mode 100644 index 0000000..ba4f72d --- /dev/null +++ b/server/test/trading/market_history_admin_actions_test.dart @@ -0,0 +1,137 @@ +import 'package:cyberhybridhub_server/trading/market_data_history.dart'; +import 'package:cyberhybridhub_server/trading/market_data_retention.dart'; +import 'package:cyberhybridhub_server/trading/market_history_admin_actions.dart'; +import 'package:cyberhybridhub_server/trading/sync_run_recorder.dart'; +import 'package:cyberhybridhub_server/trading/tradable_assets_sync.dart'; +import 'package:postgres/postgres.dart'; +import 'package:test/test.dart'; + +import '../helpers/test_db.dart'; + +void main() { + TestDb? testDb; + + setUpAll(() async { + testDb = await TestDb.open(); + }); + + tearDown(() async { + if (testDb != null) { + await testDb!.truncateTradingTables(); + } + }); + + tearDownAll(() async { + await testDb?.close(); + }); + + Future recordStage(String kind, [DateTime? now]) async { + final SyncRunRecorder recorder = SyncRunRecorder(testDb!.connection); + await recorder.record( + kind, + () async => const SyncRunCounts(rowsWritten: 1), + now: now, + ); + } + + MarketHistoryAdminActions actions({ + Future Function(DateTime now)? runUniverse, + Future Function(DateTime now)? runBackfill, + Future Function(DateTime now, bool archive, int windowDays)? + runCleanup, + }) { + return MarketHistoryAdminActions( + connection: testDb!.connection, + runUniverse: runUniverse ?? + (DateTime now) => recordStage(TradableAssetsSync.kind, now), + runBackfill: runBackfill ?? + (DateTime now) => recordStage(MarketDataHistorySync.kind, now), + runCleanup: runCleanup ?? + (DateTime now, bool archive, int windowDays) => + recordStage(MarketDataRetention.kind, now), + defaultArchiveEnabled: false, + defaultWindowDays: 7, + ); + } + + test('resync records universe and backfill run ids', () async { + if (testDb == null) { + markTestSkipped('Set DATABASE_URL or TEST_DATABASE_URL for integration tests'); + return; + } + + final AdminTriggerResult result = await actions().resync( + now: DateTime.utc(2026, 5, 27, 12), + ); + expect(result.runIds, hasLength(2)); + + final Result kinds = await testDb!.connection.execute( + ''' + SELECT kind + FROM market_data_sync_runs + WHERE id IN (${result.runIds.join(',')}) + ORDER BY id ASC + ''', + ); + expect( + kinds.map((ResultRow row) => row[0]).toList(), + ['universe', 'backfill'], + ); + }); + + test('cleanup records cleanup run id', () async { + if (testDb == null) { + markTestSkipped('Set DATABASE_URL or TEST_DATABASE_URL for integration tests'); + return; + } + + final AdminTriggerResult result = await actions().cleanup( + now: DateTime.utc(2026, 5, 27, 12), + archive: true, + ); + expect(result.runIds, hasLength(1)); + + final Result row = await testDb!.connection.execute( + Sql.named('SELECT kind FROM market_data_sync_runs WHERE id = @id'), + parameters: {'id': result.runIds.single}, + ); + expect(row.first[0], 'cleanup'); + }); + + test('aborts orphaned in-progress run then triggers resync', () async { + if (testDb == null) { + markTestSkipped('Set DATABASE_URL or TEST_DATABASE_URL for integration tests'); + return; + } + + await testDb!.connection.execute( + Sql.named( + ''' + INSERT INTO market_data_sync_runs (kind, started_at, finished_at) + VALUES ('backfill', @started_at, NULL) + ''', + ), + parameters: { + 'started_at': DateTime.utc(2026, 5, 27, 12), + }, + ); + + final AdminTriggerResult result = await actions().resync( + now: DateTime.utc(2026, 5, 27, 12, 1), + ); + + expect(result.runIds, isNotEmpty); + + final Result rows = await testDb!.connection.execute( + ''' + SELECT finished_at, error + FROM market_data_sync_runs + WHERE kind = 'backfill' + ORDER BY id ASC + LIMIT 1 + ''', + ); + expect(rows.first[0], isNotNull); + expect(rows.first[1] as String, contains('aborted')); + }); +} diff --git a/server/test/trading/market_history_admin_logic_test.dart b/server/test/trading/market_history_admin_logic_test.dart new file mode 100644 index 0000000..f4778c5 --- /dev/null +++ b/server/test/trading/market_history_admin_logic_test.dart @@ -0,0 +1,236 @@ +import 'package:cyberhybridhub_server/trading/backfill_sync_item.dart'; +import 'package:cyberhybridhub_server/trading/market_history_admin_logic.dart'; +import 'package:test/test.dart'; + +import '../helpers/admin_sync_run_fixtures.dart'; + +AdminSyncRunRecord _run({ + required int id, + required String kind, + required DateTime startedAt, + DateTime? finishedAt, + int rowsWritten = 0, + int rowsRemoved = 0, + int slotsSynced = 0, + List? backfillItems, + String? error, +}) { + return AdminSyncRunRecord( + id: id, + kind: kind, + startedAt: startedAt, + finishedAt: finishedAt, + rowsWritten: rowsWritten, + rowsRemoved: rowsRemoved, + slotsSynced: slotsSynced, + backfillItems: backfillItems ?? const [], + error: error, + ); +} + +void main() { + group('computePinned', () { + test('pins unresolved failure when no later success exists', () { + final DateTime t0 = DateTime.utc(2026, 5, 26, 12); + final List runs = [ + _run( + id: 1, + kind: 'backfill', + startedAt: t0, + finishedAt: t0.add(const Duration(minutes: 1)), + error: 'getBarsRange rate limited: 429', + ), + ]; + + final List pinned = computePinned(runs); + expect(pinned.map((AdminSyncRunRecord r) => r.id), [1]); + }); + + test('unpins failure after later success of same kind', () { + final DateTime t0 = DateTime.utc(2026, 5, 26, 12); + final List runs = [ + _run( + id: 1, + kind: 'backfill', + startedAt: t0, + finishedAt: t0.add(const Duration(minutes: 1)), + error: '429', + ), + _run( + id: 2, + kind: 'backfill', + startedAt: t0.add(const Duration(hours: 1)), + finishedAt: t0.add(const Duration(hours: 1, minutes: 1)), + rowsWritten: 20, + ), + ]; + + final List pinned = computePinned(runs); + expect(pinned, isEmpty); + }); + + test('pin clearance is per kind only', () { + final DateTime t0 = DateTime.utc(2026, 5, 26, 12); + final List runs = [ + _run( + id: 1, + kind: 'backfill', + startedAt: t0, + finishedAt: t0.add(const Duration(minutes: 1)), + error: '429', + ), + _run( + id: 2, + kind: 'cleanup', + startedAt: t0.add(const Duration(hours: 1)), + finishedAt: t0.add(const Duration(hours: 1, minutes: 1)), + rowsRemoved: 100, + ), + ]; + + final List pinned = computePinned(runs); + expect(pinned.map((AdminSyncRunRecord r) => r.id), [1]); + }); + }); + + group('deriveSeverity', () { + test('classifies rate-limit variants as rate_limit', () { + final DateTime now = DateTime.utc(2026, 5, 27, 12); + final AdminRunSeverity severity = deriveSeverity( + error: 'AlpacaMarketDataException: rate limited: 429', + startedAt: now.subtract(const Duration(minutes: 2)), + finishedAt: now.subtract(const Duration(minutes: 1)), + now: now, + ); + expect(severity, AdminRunSeverity.rateLimit); + }); + + test('classifies stale in-progress runs as warning', () { + final DateTime now = DateTime.utc(2026, 5, 27, 12); + final AdminRunSeverity severity = deriveSeverity( + error: null, + startedAt: now.subtract(const Duration(hours: 1)), + finishedAt: null, + now: now, + ); + expect(severity, AdminRunSeverity.warning); + }); + + test('treats empty Alpaca response with placeholders as ok', () { + final DateTime now = DateTime.utc(2026, 5, 30, 22); + final AdminRunSeverity severity = deriveSeverity( + error: + 'Alpaca returned no persistable 4Hour bars; slot=2026-05-30T20:00:00Z; rows_written=0', + startedAt: now.subtract(const Duration(minutes: 5)), + finishedAt: now, + now: now, + rowsWritten: 200, + ); + expect(severity, AdminRunSeverity.ok); + }); + }); + + group('deriveStatus', () { + test('classifies partial success when rows written and error exists', () { + final AdminRunStatus status = deriveStatus( + error: 'batch AAPL failed', + finishedAt: DateTime.utc(2026, 5, 27, 12, 1), + rowsWritten: 50, + rowsRemoved: 0, + ); + expect(status, AdminRunStatus.partial); + }); + + test('classifies in-progress when finishedAt is null', () { + final AdminRunStatus status = deriveStatus( + error: null, + finishedAt: null, + rowsWritten: 0, + rowsRemoved: 0, + ); + expect(status, AdminRunStatus.inProgress); + }); + + test('classifies placeholder-only empty market runs as success', () { + final AdminRunStatus status = deriveStatus( + error: + 'Alpaca returned no persistable 4Hour bars; slot=2026-05-30T20:00:00Z', + finishedAt: DateTime.utc(2026, 5, 30, 22), + rowsWritten: 100, + rowsRemoved: 0, + ); + expect(status, AdminRunStatus.success); + }); + }); + + test('sortNewestFirst orders by started_at descending', () { + final DateTime t0 = DateTime.utc(2026, 5, 26, 12); + final List sorted = sortNewestFirst([ + _run(id: 1, kind: 'cleanup', startedAt: t0), + _run(id: 2, kind: 'cleanup', startedAt: t0.add(const Duration(hours: 1))), + _run(id: 3, kind: 'cleanup', startedAt: t0.subtract(const Duration(hours: 1))), + ]); + expect(sorted.map((AdminSyncRunRecord r) => r.id), [2, 1, 3]); + }); + + test('toSummary returns partial backfill summary', () { + final String summary = toSummary( + _run( + id: 1, + kind: 'backfill', + startedAt: DateTime.utc(2026, 5, 26, 12), + finishedAt: DateTime.utc(2026, 5, 26, 12, 1), + rowsWritten: 12000, + error: 'batch MSFT failed', + ), + ); + expect(summary, contains('partial')); + expect(summary, contains('12000')); + }); + + group('fixture-driven risk cases', () { + final DateTime base = DateTime.utc(2026, 5, 27, 12); + + test('mixed kinds interleaving keeps only unresolved kinds pinned', () { + final List pinned = + computePinned(fixtureMixedKindsMixedOutcomes(base)); + expect(pinned.every((AdminSyncRunRecord r) => r.kind == 'backfill'), isTrue); + expect(pinned, isNotEmpty); + }); + + test('failed then success fixture unpins older failure', () { + expect(computePinned(fixtureFailedThenSuccessSameKind(base)), isEmpty); + }); + + test('rate-limit capitalization variants classify as rate_limit', () { + for (final String message in [ + 'HTTP 429 Too Many Requests', + 'Rate Limit exceeded', + 'RATE LIMITED by upstream', + ]) { + expect( + deriveSeverity( + error: message, + startedAt: base, + finishedAt: base.add(const Duration(minutes: 1)), + now: base.add(const Duration(minutes: 2)), + ), + AdminRunSeverity.rateLimit, + ); + } + }); + + test('partial backfill fixture maps to partial status', () { + final AdminSyncRunRecord run = fixturePartialBackfillError(base).single; + expect( + deriveStatus( + error: run.error, + finishedAt: run.finishedAt, + rowsWritten: run.rowsWritten, + rowsRemoved: run.rowsRemoved, + ), + AdminRunStatus.partial, + ); + }); + }); +} diff --git a/server/test/trading/market_history_api_rate_limiter_test.dart b/server/test/trading/market_history_api_rate_limiter_test.dart new file mode 100644 index 0000000..2c1bfbd --- /dev/null +++ b/server/test/trading/market_history_api_rate_limiter_test.dart @@ -0,0 +1,25 @@ +import 'package:cyberhybridhub_server/trading/market_history_api_rate_limiter.dart'; +import 'package:test/test.dart'; + +void main() { + test('blocks when requestsPerMinute is exhausted', () async { + DateTime now = DateTime.utc(2026, 5, 26, 12); + final List sleeps = []; + + final MarketHistoryApiRateLimiter limiter = MarketHistoryApiRateLimiter( + requestsPerMinute: 2, + clock: () => now, + sleep: (Duration d) async { + sleeps.add(d); + now = now.add(d); + }, + ); + + await limiter.acquire(); + await limiter.acquire(); + await limiter.acquire(); + + expect(sleeps, hasLength(1)); + expect(sleeps.single, const Duration(minutes: 1)); + }); +} diff --git a/server/test/trading/market_history_four_hour_slot_test.dart b/server/test/trading/market_history_four_hour_slot_test.dart new file mode 100644 index 0000000..1d63e32 --- /dev/null +++ b/server/test/trading/market_history_four_hour_slot_test.dart @@ -0,0 +1,88 @@ +import 'package:cyberhybridhub_server/trading/market_history_four_hour_slot.dart'; +import 'package:test/test.dart'; + +void main() { + group('MarketHistoryFourHourSlot', () { + test('slotStartContaining floors to UTC 4-hour boundary', () { + expect( + MarketHistoryFourHourSlot.slotStartContaining( + DateTime.utc(2026, 5, 26, 10, 30), + ), + DateTime.utc(2026, 5, 26, 8), + ); + expect( + MarketHistoryFourHourSlot.slotStartContaining( + DateTime.utc(2026, 5, 26, 0), + ), + DateTime.utc(2026, 5, 26, 0), + ); + }); + + test('endInclusive is three hours fifty-nine minutes after start', () { + final DateTime start = DateTime.utc(2026, 5, 30, 0); + expect( + MarketHistoryFourHourSlot.endInclusive(start), + DateTime.utc(2026, 5, 30, 3, 59, 59), + ); + }); + + test('lastCompletedSlotStart at slot boundary is previous slot', () { + expect( + MarketHistoryFourHourSlot.lastCompletedSlotStart( + DateTime.utc(2026, 5, 26, 12), + ), + DateTime.utc(2026, 5, 26, 8), + ); + }); + + test('lastCompletedSlotStart mid-slot is previous slot', () { + expect( + MarketHistoryFourHourSlot.lastCompletedSlotStart( + DateTime.utc(2026, 5, 26, 10, 30), + ), + DateTime.utc(2026, 5, 26, 4), + ); + }); + + test('completedSlotStartsInWindow excludes in-progress slot', () { + final DateTime now = DateTime.utc(2026, 5, 26, 10, 30); + final List slots = + MarketHistoryFourHourSlot.completedSlotStartsInWindow(now, 1); + expect(slots, isNot(contains(DateTime.utc(2026, 5, 26, 8)))); + expect(slots, contains(DateTime.utc(2026, 5, 26, 4))); + }); + + test('five completed slots on a UTC day before the 20:00 block ends', () { + final List slots = + MarketHistoryFourHourSlot.completedSlotStartsInWindow( + DateTime.utc(2026, 5, 26, 23, 59), + 1, + ); + final Set hours = slots + .where((DateTime s) => s.day == 26) + .map((DateTime s) => s.hour) + .toSet(); + expect(hours, {0, 4, 8, 12, 16}); + }); + + test('wireUtc uses canonical YYYY-MM-DDTHH:MM:SSZ without fractional seconds', + () { + expect( + MarketHistoryFourHourSlot.wireUtc(DateTime.utc(2026, 5, 26, 8)), + '2026-05-26T08:00:00Z', + ); + expect( + MarketHistoryFourHourSlot.wireUtc( + DateTime.utc(2026, 5, 26, 8, 0, 0, 500), + ), + '2026-05-26T08:00:00Z', + ); + expect( + MarketHistoryFourHourSlot.slotStartWire( + DateTime.utc(2026, 5, 26, 10, 30), + ), + '2026-05-26T08:00:00Z', + ); + }); + }); +} diff --git a/server/test/trading/market_history_question_audit_test.dart b/server/test/trading/market_history_question_audit_test.dart new file mode 100644 index 0000000..1fba551 --- /dev/null +++ b/server/test/trading/market_history_question_audit_test.dart @@ -0,0 +1,71 @@ +import 'package:cyberhybridhub_server/trading/market_history_four_hour_slot.dart'; +import 'package:cyberhybridhub_server/trading/market_history_question_audit.dart'; +import 'package:test/test.dart'; + +void main() { + group('averageBarPrice', () { + test('uses OHLC average when raw legs are present', () { + expect( + averageBarPrice( + closePrice: 99, + raw: { + 'o': 10, + 'h': 14, + 'l': 8, + 'c': 12, + }, + ), + 11, + ); + }); + + test('falls back to close price when raw is incomplete', () { + expect( + averageBarPrice(closePrice: 42.5, raw: {'c': 12}), + 42.5, + ); + }); + + test('throws when no price data', () { + expect( + () => averageBarPrice(closePrice: null, raw: null), + throwsArgumentError, + ); + }); + }); + + group('compareUntil navigation', () { + final DateTime now = DateTime.utc(2026, 5, 30, 15, 30); + late DateTime defaultUntil; + + setUp(() { + defaultUntil = questionAuditDefaultCompareUntil(now); + }); + + test('step older shifts slot pair back one slot', () { + final DateTime stepped = questionAuditStepOlderCompareUntil( + compareUntil: defaultUntil, + now: now, + ); + final (DateTime newer, DateTime older) = questionAuditSlotPair(stepped); + final DateTime last = + MarketHistoryFourHourSlot.lastCompletedSlotStart(now); + expect(newer, last.subtract(const Duration(hours: 4))); + expect(older, last.subtract(const Duration(hours: 8))); + }); + + test('step newer from stepped returns to default pair', () { + final DateTime stepped = questionAuditStepOlderCompareUntil( + compareUntil: defaultUntil, + now: now, + ); + expect( + questionAuditStepNewerCompareUntil( + compareUntil: stepped, + maxUntil: defaultUntil, + ), + defaultUntil, + ); + }); + }); +} diff --git a/server/test/trading/market_history_trading_calendar_test.dart b/server/test/trading/market_history_trading_calendar_test.dart new file mode 100644 index 0000000..0a75325 --- /dev/null +++ b/server/test/trading/market_history_trading_calendar_test.dart @@ -0,0 +1,33 @@ +import 'package:cyberhybridhub_server/trading/market_history_trading_calendar.dart'; +import 'package:test/test.dart'; + +void main() { + group('MarketHistoryTradingCalendar', () { + test('Saturday UTC is no regular session', () { + expect( + MarketHistoryTradingCalendar.isLikelyNoRegularSession( + DateTime.utc(2026, 5, 30, 20), + ), + isTrue, + ); + }); + + test('weekday UTC is regular session day', () { + expect( + MarketHistoryTradingCalendar.isLikelyNoRegularSession( + DateTime.utc(2026, 5, 28, 20), + ), + isFalse, + ); + }); + + test('NYSE holiday is no regular session', () { + expect( + MarketHistoryTradingCalendar.isLikelyNoRegularSession( + DateTime.utc(2026, 12, 25, 16), + ), + isTrue, + ); + }); + }); +} diff --git a/server/test/trading/market_history_week_coverage_test.dart b/server/test/trading/market_history_week_coverage_test.dart new file mode 100644 index 0000000..b783496 --- /dev/null +++ b/server/test/trading/market_history_week_coverage_test.dart @@ -0,0 +1,35 @@ +import 'package:cyberhybridhub_server/trading/market_history_four_hour_slot.dart'; +import 'package:cyberhybridhub_server/trading/market_history_week_coverage.dart'; +import 'package:test/test.dart'; + +void main() { + group('MarketHistoryWeekCoverage calendar days', () { + test('returns windowDays UTC days ending on today', () { + final DateTime now = DateTime.utc(2026, 5, 30, 15, 30); + final List days = + MarketHistoryWeekCoverage.calendarDaysEndingToday(now, 7); + + expect(days, hasLength(7)); + expect(days.first, DateTime.utc(2026, 5, 24)); + expect(days.last, DateTime.utc(2026, 5, 30)); + }); + }); + + group('slot completion for today', () { + test('marks only ended slots completed at 15:30 UTC', () { + final DateTime now = DateTime.utc(2026, 5, 30, 15, 30); + final DateTime day = DateTime.utc(2026, 5, 30); + var completed = 0; + + for (int hour = 0; hour < 24; hour += MarketHistoryFourHourSlot.slotHours) { + final DateTime slotStart = DateTime.utc(day.year, day.month, day.day, hour); + if (MarketHistoryFourHourSlot.hasEnded(slotStart, now)) { + completed++; + } + } + + // Slots 00, 04, 08 end before 15:30; 12:00 slot ends at 16:00 UTC. + expect(completed, 3); + }); + }); +} diff --git a/server/test/trading/rule_engine_guess_weekly_move_test.dart b/server/test/trading/rule_engine_guess_weekly_move_test.dart new file mode 100644 index 0000000..f47dd24 --- /dev/null +++ b/server/test/trading/rule_engine_guess_weekly_move_test.dart @@ -0,0 +1,97 @@ +import 'package:cyberhybridhub_server/trading/market_history_query.dart'; +import 'package:cyberhybridhub_server/trading/rule_engine.dart'; +import 'package:cyberhybridhub_server/trading/trading_config.dart'; +import 'package:test/test.dart'; + +void main() { + late TradingRuleConfig guessRule; + late DateTime now; + + setUp(() { + guessRule = TradingRuleConfig.fromJson({ + 'id': 'guess_weekly_move', + 'type': 'guess_weekly_move', + 'symbol': '*', + 'question_template': + '{{token}} was {{ref_price}} {{ref_days_ago}} days ago. +10 up, -10 down.', + }); + now = DateTime.utc(2026, 5, 26, 12); + }); + + WeeklyMover mover({ + required num openClose, + required num currentClose, + int days = 5, + }) { + return WeeklyMover( + symbol: 'SPY', + openClose: openClose, + currentClose: currentClose, + days: days, + ); + } + + group('RuleEngine.evaluateGuessWeeklyMove', () { + test('up move produces ASSET_A token and correct_answer 10', () { + final RuleEngine engine = RuleEngine(); + final RuleEvaluation result = engine.evaluateGuessWeeklyMove( + rule: guessRule, + mover: mover(openClose: 500, currentClose: 510), + symbolToken: 'ASSET_A', + now: now, + ); + + expect(result.fired, isTrue); + expect(result.symbolToken, 'ASSET_A'); + expect(result.guessSymbol, 'SPY'); + expect(result.correctAnswer, 10); + expect(result.refPrice, 500); + expect(result.observedPrice, 510); + expect(result.refDaysAgo, 5); + expect( + result.questionText, + 'ASSET_A was 500.00 5 days ago. +10 up, -10 down.', + ); + expect(result.questionText, isNot(contains('SPY'))); + expect(result.questionText, isNot(contains('{{symbol}}'))); + }); + + test('down move produces correct_answer -10', () { + final RuleEngine engine = RuleEngine(); + final RuleEvaluation result = engine.evaluateGuessWeeklyMove( + rule: guessRule, + mover: mover(openClose: 510, currentClose: 500), + symbolToken: 'ASSET_A', + now: now, + ); + + expect(result.fired, isTrue); + expect(result.correctAnswer, -10); + }); + + test('insufficient bars (no mover) does not fire', () { + final RuleEngine engine = RuleEngine(); + final RuleEvaluation result = engine.evaluateGuessWeeklyMove( + rule: guessRule, + mover: null, + symbolToken: null, + now: now, + ); + + expect(result.fired, isFalse); + expect(result.skipReason, RuleSkipReason.insufficientBars); + }); + + test('guessSymbol is exposed for pipeline metadata', () { + final RuleEngine engine = RuleEngine(); + final RuleEvaluation result = engine.evaluateGuessWeeklyMove( + rule: guessRule, + mover: mover(openClose: 500, currentClose: 510), + symbolToken: 'ASSET_A', + now: now, + ); + + expect(result.guessSymbol, 'SPY'); + }); + }); +} diff --git a/server/test/trading/symbol_obfuscator_test.dart b/server/test/trading/symbol_obfuscator_test.dart new file mode 100644 index 0000000..33027b6 --- /dev/null +++ b/server/test/trading/symbol_obfuscator_test.dart @@ -0,0 +1,13 @@ +import 'package:cyberhybridhub_server/trading/symbol_obfuscator.dart'; +import 'package:test/test.dart'; + +void main() { + test('tokenFor assigns ASSET_A, ASSET_B in sorted universe order', () { + final SymbolObfuscator obfuscator = SymbolObfuscator(); + final List universe = ['MSFT', 'SPY', 'AAPL']; + + expect(obfuscator.tokenFor('AAPL', universe), 'ASSET_A'); + expect(obfuscator.tokenFor('MSFT', universe), 'ASSET_B'); + expect(obfuscator.tokenFor('SPY', universe), 'ASSET_C'); + }); +} diff --git a/server/test/trading/sync_run_recorder_test.dart b/server/test/trading/sync_run_recorder_test.dart new file mode 100644 index 0000000..9e02f7b --- /dev/null +++ b/server/test/trading/sync_run_recorder_test.dart @@ -0,0 +1,105 @@ +@Tags(['integration', 'postgres']) +library; + +import 'package:cyberhybridhub_server/trading/sync_run_recorder.dart'; +import 'package:postgres/postgres.dart'; +import 'package:test/test.dart'; + +import '../helpers/test_db.dart'; + +void main() { + TestDb? testDb; + + setUpAll(() async { + testDb = await TestDb.open(); + }); + + tearDown(() async { + if (testDb != null) { + await testDb!.truncateTradingTables(); + } + }); + + tearDownAll(() async { + await testDb?.close(); + }); + + group('abort in-progress runs', () { + test('abortAllInProgressRuns closes orphaned rows', () async { + if (testDb == null) { + markTestSkipped('Set DATABASE_URL or TEST_DATABASE_URL for integration tests'); + return; + } + + final DateTime started = DateTime.utc(2026, 5, 27, 10); + await testDb!.connection.execute( + Sql.named( + ''' + INSERT INTO market_data_sync_runs (kind, started_at, finished_at) + VALUES ('backfill', @started_at, NULL) + ''', + ), + parameters: {'started_at': started}, + ); + + final SyncRunRecorder recorder = SyncRunRecorder(testDb!.connection); + final int aborted = await recorder.abortAllInProgressRuns( + now: DateTime.utc(2026, 5, 27, 11), + ); + + expect(aborted, 1); + + final Result rows = await testDb!.connection.execute( + ''' + SELECT finished_at, error + FROM market_data_sync_runs + ''', + ); + expect(rows.first[0], isNotNull); + expect(rows.first[1], SyncRunRecorder.abortSupersededMessage); + }); + + test('abortStaleInProgressRuns leaves recent in-progress rows', () async { + if (testDb == null) { + markTestSkipped('Set DATABASE_URL or TEST_DATABASE_URL for integration tests'); + return; + } + + final DateTime now = DateTime.utc(2026, 5, 27, 12); + await testDb!.connection.execute( + Sql.named( + ''' + INSERT INTO market_data_sync_runs (kind, started_at, finished_at) + VALUES + ('backfill', @recent, NULL), + ('universe', @stale, NULL) + ''', + ), + parameters: { + 'recent': now.subtract(const Duration(minutes: 5)), + 'stale': now.subtract(const Duration(hours: 2)), + }, + ); + + final SyncRunRecorder recorder = SyncRunRecorder(testDb!.connection); + final int aborted = await recorder.abortStaleInProgressRuns( + now: now, + olderThan: const Duration(minutes: 30), + ); + + expect(aborted, 1); + + final Result rows = await testDb!.connection.execute( + ''' + SELECT kind, finished_at IS NULL AS open + FROM market_data_sync_runs + ORDER BY kind + ''', + ); + expect(rows.first[0], 'backfill'); + expect(rows.first[1], true); + expect(rows.last[0], 'universe'); + expect(rows.last[1], false); + }); + }); +} diff --git a/startup.sh b/startup.sh index 10b257c..8a1a29d 100755 --- a/startup.sh +++ b/startup.sh @@ -92,6 +92,10 @@ if [ "${SKIP_WEB_BUILD:-0}" != "1" ]; then --dart-define=API_BASE_URL="http://localhost:${API_PORT}" \ 2>&1 | prefix_lines web-build ) + if [ -x "$ROOT/scripts/copy_flutter_js_source_map.sh" ]; then + "$ROOT/scripts/copy_flutter_js_source_map.sh" "$ROOT/build/web" \ + 2>&1 | prefix_lines web-build + fi else log "SKIP_WEB_BUILD=1 — reusing existing build/web bundle." fi diff --git a/test/admin/acceptance/admin_portal_acceptance_test.dart b/test/admin/acceptance/admin_portal_acceptance_test.dart new file mode 100644 index 0000000..9ed5370 --- /dev/null +++ b/test/admin/acceptance/admin_portal_acceptance_test.dart @@ -0,0 +1,225 @@ +import 'package:cyberhybridhub/admin/models/market_history_admin_config.dart'; +import 'package:cyberhybridhub/admin/models/sync_run_event.dart'; +import 'package:cyberhybridhub/admin/repositories/sync_run_log_repository.dart'; +import 'package:cyberhybridhub/admin/screens/market_history_log_screen.dart'; +import 'package:cyberhybridhub/admin/widgets/sync_run_expansion_tile.dart'; +import 'package:cyberhybridhub/theme/app_theme.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; + +/// FLUTTER-TDD-PLAN.md Section 9 — automated definition-of-done checks. +class _AcceptanceController implements SyncRunLogController { + _AcceptanceController(this._states); + + final List _states; + int _index = 0; + bool? lastCleanupArchive; + + SyncRunLogRepositoryState _current = const SyncRunLogRepositoryState( + pinned: [], + history: [], + nextBefore: null, + isLoading: true, + errorMessage: null, + ); + + @override + SyncRunLogRepositoryState get state => _current; + + @override + Future loadInitial({int limit = 50}) async { + _current = _states[_index]; + return _current; + } + + @override + Future refresh({int limit = 50}) async { + _index = 0; + _current = _states[_index]; + return _current; + } + + @override + Future loadMore({int limit = 50}) async { + if (_index < _states.length - 1) { + _index++; + } + _current = _states[_index]; + return _current; + } + + @override + Future triggerResync() => refresh(); + + @override + Future triggerCleanup({bool archive = false}) async { + lastCleanupArchive = archive; + return refresh(); + } +} + +SyncRunEvent _run({ + required int id, + required String kind, + required DateTime startedAt, + String? error, + SyncRunSeverity severity = SyncRunSeverity.ok, + SyncRunStatus status = SyncRunStatus.success, +}) { + return SyncRunEvent( + id: id, + kind: kind, + startedAt: startedAt, + finishedAt: startedAt.add(const Duration(minutes: 1)), + rowsWritten: kind == 'cleanup' ? 0 : 100, + rowsRemoved: kind == 'cleanup' ? 50 : 0, + error: error, + severity: severity, + status: status, + durationMs: 60000, + summary: error ?? 'summary $id', + ); +} + +Future _pump( + WidgetTester tester, + _AcceptanceController controller, +) async { + await tester.pumpWidget( + MaterialApp( + theme: buildAppTheme(), + home: MarketHistoryLogScreen( + controller: controller, + now: DateTime.utc(2026, 5, 27, 12), + autoRefreshInterval: null, + ), + ), + ); + await tester.pumpAndSettle(); +} + +void main() { + testWidgets('newest history row appears first', (WidgetTester tester) async { + final DateTime t0 = DateTime.utc(2026, 5, 27, 10); + final _AcceptanceController controller = _AcceptanceController( + [ + SyncRunLogRepositoryState( + pinned: const [], + history: [ + _run(id: 2, kind: 'cleanup', startedAt: t0, error: null), + _run( + id: 1, + kind: 'backfill', + startedAt: t0.subtract(const Duration(hours: 2)), + ), + ], + nextBefore: null, + isLoading: false, + errorMessage: null, + ), + ], + ); + + await _pump(tester, controller); + expect(find.textContaining('summary 2'), findsOneWidget); + }); + + testWidgets('pinned unresolved failures appear in needs attention', ( + WidgetTester tester, + ) async { + final DateTime t0 = DateTime.utc(2026, 5, 27, 10); + final _AcceptanceController controller = _AcceptanceController( + [ + SyncRunLogRepositoryState( + pinned: [ + _run( + id: 9, + kind: 'backfill', + startedAt: t0, + error: '429 rate limited', + severity: SyncRunSeverity.rateLimit, + status: SyncRunStatus.failed, + ), + ], + history: const [], + nextBefore: null, + isLoading: false, + errorMessage: null, + ), + ], + ); + + await _pump(tester, controller); + expect(find.byKey(const Key('pinned-section')), findsOneWidget); + expect(find.byKey(const Key('sync-run-9')), findsOneWidget); + }); + + testWidgets('expanded row shows full error without raw payload labels', ( + WidgetTester tester, + ) async { + const String errorText = 'AlpacaMarketDataException: batch failed 500'; + final _AcceptanceController controller = _AcceptanceController( + [ + SyncRunLogRepositoryState( + pinned: [ + _run( + id: 11, + kind: 'backfill', + startedAt: DateTime.utc(2026, 5, 27, 10), + error: errorText, + severity: SyncRunSeverity.error, + status: SyncRunStatus.failed, + ), + ], + history: const [], + nextBefore: null, + isLoading: false, + errorMessage: null, + ), + ], + ); + + await _pump(tester, controller); + await tester.tap(find.byType(ExpansionTile)); + await tester.pumpAndSettle(); + + expect(find.text(errorText), findsWidgets); + expect(find.textContaining('"raw"'), findsNothing); + expect(find.textContaining('bars'), findsNothing); + }); + + testWidgets('cleanup archive toggle passes archive flag when enabled', ( + WidgetTester tester, + ) async { + final DateTime t0 = DateTime.utc(2026, 5, 27, 10); + final _AcceptanceController controller = _AcceptanceController( + [ + SyncRunLogRepositoryState( + pinned: const [], + history: [_run(id: 3, kind: 'cleanup', startedAt: t0)], + nextBefore: null, + isLoading: false, + errorMessage: null, + config: const MarketHistoryAdminConfig( + archiveEnabled: true, + windowDays: 7, + retentionDays: 7, + syncEnabled: true, + ), + ), + ], + ); + + await _pump(tester, controller); + await tester.tap(find.byKey(const Key('actions-menu'))); + await tester.pumpAndSettle(); + await tester.tap(find.byKey(const Key('action-cleanup'))); + await tester.pumpAndSettle(); + await tester.tap(find.byKey(const Key('cleanup-archive-toggle'))); + await tester.pumpAndSettle(); + await tester.tap(find.byKey(const Key('cleanup-confirm'))); + await tester.pumpAndSettle(); + + expect(controller.lastCleanupArchive, isTrue); + }); +} diff --git a/test/admin/helpers/sync_run_fixtures.dart b/test/admin/helpers/sync_run_fixtures.dart new file mode 100644 index 0000000..95e85e8 --- /dev/null +++ b/test/admin/helpers/sync_run_fixtures.dart @@ -0,0 +1,44 @@ +import 'package:cyberhybridhub/admin/models/sync_run_event.dart'; + +SyncRunEvent syncRunFixture({ + required int id, + required String kind, + required DateTime startedAt, + DateTime? finishedAt, + int rowsWritten = 0, + int rowsRemoved = 0, + String? error, + SyncRunSeverity severity = SyncRunSeverity.ok, + SyncRunStatus status = SyncRunStatus.success, + String? summary, +}) { + return SyncRunEvent( + id: id, + kind: kind, + startedAt: startedAt, + finishedAt: finishedAt, + rowsWritten: rowsWritten, + rowsRemoved: rowsRemoved, + error: error, + severity: severity, + status: status, + durationMs: finishedAt == null + ? null + : finishedAt.difference(startedAt).inMilliseconds, + summary: summary ?? 'summary $id', + ); +} + +List fixtureRateLimitUnresolved(DateTime base) { + return [ + syncRunFixture( + id: 10, + kind: 'backfill', + startedAt: base, + finishedAt: base.add(const Duration(minutes: 2)), + error: 'rate limited: 429', + severity: SyncRunSeverity.rateLimit, + status: SyncRunStatus.failed, + ), + ]; +} diff --git a/test/admin/models/sync_run_event_test.dart b/test/admin/models/sync_run_event_test.dart new file mode 100644 index 0000000..5e7abb7 --- /dev/null +++ b/test/admin/models/sync_run_event_test.dart @@ -0,0 +1,54 @@ +import 'package:cyberhybridhub/admin/models/sync_run_event.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + test('parses wire severity/status and title mapping', () { + final SyncRunEvent event = SyncRunEvent.fromJson({ + 'id': 10, + 'kind': 'backfill', + 'startedAt': '2026-05-27T01:00:00Z', + 'finishedAt': '2026-05-27T01:00:05Z', + 'rowsWritten': 100, + 'rowsRemoved': 0, + 'error': null, + 'severity': 'ok', + 'status': 'success', + 'durationMs': 5000, + 'summary': '100 bar rows written', + }); + + expect(event.severity, SyncRunSeverity.ok); + expect(event.status, SyncRunStatus.success); + expect(event.displayTitle, 'Market history backfill'); + }); + + test('falls back to rate-limit severity parsing', () { + final SyncRunEvent event = SyncRunEvent.fromJson({ + 'id': 11, + 'kind': 'backfill', + 'startedAt': '2026-05-27T01:00:00Z', + 'finishedAt': '2026-05-27T01:00:05Z', + 'rowsWritten': 0, + 'rowsRemoved': 0, + 'error': 'AlpacaMarketDataException rate limited: 429', + }); + + expect(event.severity, SyncRunSeverity.rateLimit); + expect(event.status, SyncRunStatus.failed); + }); + + test('falls back to partial status when rows written and error', () { + final SyncRunEvent event = SyncRunEvent.fromJson({ + 'id': 12, + 'kind': 'backfill', + 'startedAt': '2026-05-27T01:00:00Z', + 'finishedAt': '2026-05-27T01:00:05Z', + 'rowsWritten': 33, + 'rowsRemoved': 0, + 'error': 'batch failed', + }); + + expect(event.status, SyncRunStatus.partial); + expect(event.summary, contains('Partial success')); + }); +} diff --git a/test/admin/repositories/sync_run_log_repository_test.dart b/test/admin/repositories/sync_run_log_repository_test.dart new file mode 100644 index 0000000..12967c5 --- /dev/null +++ b/test/admin/repositories/sync_run_log_repository_test.dart @@ -0,0 +1,441 @@ +import 'package:cyberhybridhub/admin/models/sync_run_event.dart'; +import 'package:cyberhybridhub/admin/repositories/sync_run_log_repository.dart'; +import 'package:cyberhybridhub/admin/services/market_history_admin_api.dart'; +import 'package:flutter_test/flutter_test.dart'; + +class _FakeApi extends MarketHistoryAdminApi { + _FakeApi(this._responses) : super(tokenProvider: () async => 'token'); + + final List _responses; + int _cursor = 0; + int triggerResyncCalls = 0; + int triggerCleanupCalls = 0; + + @override + Future fetchSyncRuns({ + int limit = 50, + DateTime? before, + String? kind, + }) async { + final SyncRunLogPage page = _responses[_cursor]; + if (_cursor < _responses.length - 1) { + _cursor++; + } + return page; + } + + @override + Future triggerResync() async { + triggerResyncCalls++; + return const AdminTriggerResponse(runIds: [1, 2]); + } + + @override + Future triggerCleanup({bool archive = false}) async { + triggerCleanupCalls++; + return const AdminTriggerResponse(runIds: [3]); + } +} + +SyncRunEvent _event({ + required int id, + required String kind, + required DateTime startedAt, + String? error, +}) { + return SyncRunEvent( + id: id, + kind: kind, + startedAt: startedAt, + finishedAt: startedAt.add(const Duration(minutes: 1)), + rowsWritten: 0, + rowsRemoved: 0, + error: error, + severity: error == null ? SyncRunSeverity.ok : SyncRunSeverity.error, + status: error == null ? SyncRunStatus.success : SyncRunStatus.failed, + durationMs: 60000, + summary: 'summary', + ); +} + +void main() { + test('loadInitial keeps pinned above newest-first history', () async { + final DateTime t0 = DateTime.utc(2026, 5, 27, 10); + final SyncRunLogRepository repository = SyncRunLogRepository( + api: _FakeApi([ + SyncRunLogPage( + pinned: [ + _event(id: 1, kind: 'backfill', startedAt: t0, error: '429'), + ], + runs: [ + _event(id: 2, kind: 'cleanup', startedAt: t0.subtract(const Duration(hours: 2))), + _event(id: 3, kind: 'universe', startedAt: t0.subtract(const Duration(hours: 1))), + ], + nextBefore: DateTime.utc(2026, 5, 26), + ), + ]), + ); + + final SyncRunLogRepositoryState state = await repository.loadInitial(); + expect(state.pinned.map((SyncRunEvent e) => e.id), [1]); + expect(state.history.map((SyncRunEvent e) => e.id), [3, 2]); + }); + + test('loadMore appends and deduplicates by id', () async { + final DateTime t0 = DateTime.utc(2026, 5, 27, 10); + final SyncRunLogRepository repository = SyncRunLogRepository( + api: _FakeApi([ + SyncRunLogPage( + pinned: const [], + runs: [ + _event(id: 10, kind: 'cleanup', startedAt: t0), + ], + nextBefore: DateTime.utc(2026, 5, 26), + ), + SyncRunLogPage( + pinned: const [], + runs: [ + _event(id: 10, kind: 'cleanup', startedAt: t0), + _event(id: 9, kind: 'backfill', startedAt: t0.subtract(const Duration(hours: 1))), + ], + nextBefore: null, + ), + ]), + ); + + await repository.loadInitial(); + final SyncRunLogRepositoryState state = await repository.loadMore(); + expect(state.history.map((SyncRunEvent e) => e.id), [10, 9]); + expect(state.hasMore, isFalse); + }); + + test('loadInitial stores 403 error message', () async { + final _ErrorApi api = _ErrorApi(); + final SyncRunLogRepository repository = SyncRunLogRepository(api: api); + final SyncRunLogRepositoryState state = await repository.loadInitial(); + expect(state.errorMessage, contains('Forbidden')); + expect(state.history, isEmpty); + }); + + test('refresh after loadMore replaces history without duplicates', () async { + final DateTime t0 = DateTime.utc(2026, 5, 27, 10); + final SyncRunLogRepository repository = SyncRunLogRepository( + api: _FakeApi([ + SyncRunLogPage( + pinned: const [], + runs: [ + _event(id: 10, kind: 'cleanup', startedAt: t0), + ], + nextBefore: DateTime.utc(2026, 5, 26), + ), + SyncRunLogPage( + pinned: const [], + runs: [ + _event(id: 10, kind: 'cleanup', startedAt: t0), + _event(id: 9, kind: 'backfill', startedAt: t0.subtract(const Duration(hours: 1))), + ], + nextBefore: null, + ), + SyncRunLogPage( + pinned: const [], + runs: [ + _event(id: 20, kind: 'universe', startedAt: t0.add(const Duration(hours: 1))), + ], + nextBefore: null, + ), + ]), + ); + + await repository.loadInitial(); + await repository.loadMore(); + final SyncRunLogRepositoryState refreshed = await repository.refresh(); + expect(refreshed.history.map((SyncRunEvent e) => e.id), [20]); + expect(refreshed.history, hasLength(1)); + }); + + test('triggerResync refreshes log after API call', () async { + final DateTime t0 = DateTime.utc(2026, 5, 27, 10); + final _FakeApi api = _FakeApi([ + SyncRunLogPage( + pinned: const [], + runs: [_event(id: 1, kind: 'backfill', startedAt: t0)], + nextBefore: null, + ), + SyncRunLogPage( + pinned: const [], + runs: [ + _event(id: 2, kind: 'backfill', startedAt: t0.add(const Duration(hours: 1))), + ], + nextBefore: null, + ), + ]); + final SyncRunLogRepository repository = SyncRunLogRepository(api: api); + await repository.loadInitial(); + final SyncRunLogRepositoryState state = await repository.triggerResync(); + expect(api.triggerResyncCalls, 1); + expect(state.history.map((SyncRunEvent e) => e.id), [2]); + }); + + test('triggerCleanup stores API errors', () async { + final _TriggerErrorApi api = _TriggerErrorApi(); + final SyncRunLogRepository repository = SyncRunLogRepository(api: api); + final SyncRunLogRepositoryState state = + await repository.triggerCleanup(archive: true); + expect(state.errorMessage, contains('Conflict')); + expect(state.isLoading, isFalse); + }); + + test('loadMore no-ops when there is no next page', () async { + final DateTime t0 = DateTime.utc(2026, 5, 27, 10); + final _FakeApi api = _FakeApi([ + SyncRunLogPage( + pinned: const [], + runs: [_event(id: 1, kind: 'cleanup', startedAt: t0)], + nextBefore: null, + ), + ]); + final SyncRunLogRepository repository = SyncRunLogRepository(api: api); + await repository.loadInitial(); + final SyncRunLogRepositoryState state = await repository.loadMore(); + expect(state.history, hasLength(1)); + }); + + test('loadMore stores API errors without clearing history', () async { + final DateTime t0 = DateTime.utc(2026, 5, 27, 10); + final _LoadMoreErrorApi api = _LoadMoreErrorApi( + first: SyncRunLogPage( + pinned: const [], + runs: [_event(id: 5, kind: 'cleanup', startedAt: t0)], + nextBefore: DateTime.utc(2026, 5, 26), + ), + ); + final SyncRunLogRepository repository = SyncRunLogRepository(api: api); + await repository.loadInitial(); + final SyncRunLogRepositoryState state = await repository.loadMore(); + expect(state.errorMessage, contains('Server error')); + expect(state.history.map((SyncRunEvent e) => e.id), [5]); + }); + + test('triggerResync stores generic errors', () async { + final _ResyncErrorApi api = _ResyncErrorApi(); + final SyncRunLogRepository repository = SyncRunLogRepository(api: api); + final SyncRunLogRepositoryState state = await repository.triggerResync(); + expect(state.errorMessage, contains('boom')); + }); + + test('triggerCleanup refreshes log after successful API call', () async { + final DateTime t0 = DateTime.utc(2026, 5, 27, 10); + final _FakeApi api = _FakeApi([ + SyncRunLogPage( + pinned: const [], + runs: [_event(id: 1, kind: 'cleanup', startedAt: t0)], + nextBefore: null, + ), + SyncRunLogPage( + pinned: const [], + runs: [ + _event(id: 2, kind: 'cleanup', startedAt: t0.add(const Duration(hours: 1))), + ], + nextBefore: null, + ), + ]); + final SyncRunLogRepository repository = SyncRunLogRepository(api: api); + await repository.loadInitial(); + final SyncRunLogRepositoryState state = + await repository.triggerCleanup(archive: true); + expect(api.triggerCleanupCalls, 1); + expect(state.history.map((SyncRunEvent e) => e.id), [2]); + }); + + test('triggerResync stores API errors', () async { + final _ResyncApiErrorApi api = _ResyncApiErrorApi(); + final SyncRunLogRepository repository = SyncRunLogRepository(api: api); + final SyncRunLogRepositoryState state = await repository.triggerResync(); + expect(state.errorMessage, contains('Conflict')); + expect(state.isLoading, isFalse); + }); + + test('triggerCleanup stores generic errors', () async { + final _CleanupGenericErrorApi api = _CleanupGenericErrorApi(); + final SyncRunLogRepository repository = SyncRunLogRepository(api: api); + final SyncRunLogRepositoryState state = await repository.triggerCleanup(); + expect(state.errorMessage, contains('cleanup failed')); + }); + + test('loadMore stores generic errors without clearing history', () async { + final DateTime t0 = DateTime.utc(2026, 5, 27, 10); + final _LoadMoreGenericErrorApi api = _LoadMoreGenericErrorApi( + first: SyncRunLogPage( + pinned: const [], + runs: [_event(id: 5, kind: 'cleanup', startedAt: t0)], + nextBefore: DateTime.utc(2026, 5, 26), + ), + ); + final SyncRunLogRepository repository = SyncRunLogRepository(api: api); + await repository.loadInitial(); + final SyncRunLogRepositoryState state = await repository.loadMore(); + expect(state.errorMessage, contains('timeout')); + expect(state.history.map((SyncRunEvent e) => e.id), [5]); + }); + + test('loadInitial stores generic errors', () async { + final SyncRunLogRepository repository = + SyncRunLogRepository(api: _GenericErrorApi()); + final SyncRunLogRepositoryState state = await repository.loadInitial(); + expect(state.errorMessage, contains('network down')); + }); +} + +class _ErrorApi extends MarketHistoryAdminApi { + _ErrorApi() : super(tokenProvider: () async => 'token'); + + @override + Future fetchSyncRuns({ + int limit = 50, + DateTime? before, + String? kind, + }) { + throw MarketHistoryAdminApiException('Forbidden', statusCode: 403); + } +} + +class _TriggerErrorApi extends MarketHistoryAdminApi { + _TriggerErrorApi() : super(tokenProvider: () async => 'token'); + + @override + Future fetchSyncRuns({ + int limit = 50, + DateTime? before, + String? kind, + }) async { + return const SyncRunLogPage( + runs: [], + pinned: [], + nextBefore: null, + ); + } + + @override + Future triggerCleanup({bool archive = false}) { + throw MarketHistoryAdminApiException('Conflict', statusCode: 409); + } +} + +class _LoadMoreErrorApi extends MarketHistoryAdminApi { + _LoadMoreErrorApi({required this.first}) : super(tokenProvider: () async => 'token'); + + final SyncRunLogPage first; + int _calls = 0; + + @override + Future fetchSyncRuns({ + int limit = 50, + DateTime? before, + String? kind, + }) async { + _calls++; + if (_calls == 1) { + return first; + } + throw MarketHistoryAdminApiException('Server error', statusCode: 500); + } +} + +class _ResyncErrorApi extends MarketHistoryAdminApi { + _ResyncErrorApi() : super(tokenProvider: () async => 'token'); + + @override + Future fetchSyncRuns({ + int limit = 50, + DateTime? before, + String? kind, + }) async { + return const SyncRunLogPage( + runs: [], + pinned: [], + nextBefore: null, + ); + } + + @override + Future triggerResync() { + throw Exception('boom'); + } +} + +class _ResyncApiErrorApi extends MarketHistoryAdminApi { + _ResyncApiErrorApi() : super(tokenProvider: () async => 'token'); + + @override + Future fetchSyncRuns({ + int limit = 50, + DateTime? before, + String? kind, + }) async { + return const SyncRunLogPage( + runs: [], + pinned: [], + nextBefore: null, + ); + } + + @override + Future triggerResync() { + throw MarketHistoryAdminApiException('Conflict', statusCode: 409); + } +} + +class _CleanupGenericErrorApi extends MarketHistoryAdminApi { + _CleanupGenericErrorApi() : super(tokenProvider: () async => 'token'); + + @override + Future fetchSyncRuns({ + int limit = 50, + DateTime? before, + String? kind, + }) async { + return const SyncRunLogPage( + runs: [], + pinned: [], + nextBefore: null, + ); + } + + @override + Future triggerCleanup({bool archive = false}) { + throw Exception('cleanup failed'); + } +} + +class _LoadMoreGenericErrorApi extends MarketHistoryAdminApi { + _LoadMoreGenericErrorApi({required this.first}) + : super(tokenProvider: () async => 'token'); + + final SyncRunLogPage first; + int _calls = 0; + + @override + Future fetchSyncRuns({ + int limit = 50, + DateTime? before, + String? kind, + }) async { + _calls++; + if (_calls == 1) { + return first; + } + throw Exception('timeout'); + } +} +class _GenericErrorApi extends MarketHistoryAdminApi { + _GenericErrorApi() : super(tokenProvider: () async => 'token'); + + @override + Future fetchSyncRuns({ + int limit = 50, + DateTime? before, + String? kind, + }) { + throw Exception('network down'); + } +} diff --git a/test/admin/risk/admin_portal_risk_test.dart b/test/admin/risk/admin_portal_risk_test.dart new file mode 100644 index 0000000..31a47c4 --- /dev/null +++ b/test/admin/risk/admin_portal_risk_test.dart @@ -0,0 +1,314 @@ +import 'dart:async'; +import 'dart:convert'; + +import 'package:cyberhybridhub/admin/models/sync_run_event.dart'; +import 'package:cyberhybridhub/admin/repositories/sync_run_log_repository.dart'; +import 'package:cyberhybridhub/admin/services/market_history_admin_api.dart'; +import 'package:cyberhybridhub/admin/utils/sync_run_formatters.dart'; +import 'package:cyberhybridhub/admin/widgets/sync_run_expansion_tile.dart'; +import 'package:cyberhybridhub/theme/app_theme.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:http/http.dart' as http; +import 'package:http/testing.dart'; + +/// FLUTTER-TDD-PLAN.md Section 8 — risk-driven checklist (Flutter layer). +void main() { + group('rate-limit text variants', () { + for (final String message in [ + 'HTTP 429 Too Many Requests', + 'Rate Limit exceeded', + 'RATE LIMITED by upstream', + '429', + ]) { + test('SyncRunEvent infers rate_limit from "$message"', () { + final SyncRunEvent event = SyncRunEvent.fromJson({ + 'id': 1, + 'kind': 'backfill', + 'startedAt': '2026-05-27T01:00:00Z', + 'finishedAt': '2026-05-27T01:00:05Z', + 'rowsWritten': 0, + 'rowsRemoved': 0, + 'error': message, + }); + expect(event.severity, SyncRunSeverity.rateLimit); + if (message.contains('429')) { + expect(parseHttpStatus(message), '429'); + } + }); + } + }); + + group('timezone display', () { + test('formatLocalTimestamp matches DateTime.toLocal()', () { + final DateTime utc = DateTime.utc(2026, 5, 27, 14, 30); + final DateTime local = utc.toLocal(); + final String formatted = formatLocalTimestamp(utc); + expect( + formatted, + '${local.year}-${local.month.toString().padLeft(2, '0')}-' + '${local.day.toString().padLeft(2, '0')} ' + '${local.hour.toString().padLeft(2, '0')}:' + '${local.minute.toString().padLeft(2, '0')}', + ); + expect(formatted, isNot(contains('Z'))); + }); + + testWidgets('expansion tile shows Started in local timezone', ( + WidgetTester tester, + ) async { + final DateTime utc = DateTime.utc(2026, 5, 27, 14, 30); + final SyncRunEvent event = SyncRunEvent( + id: 42, + kind: 'cleanup', + startedAt: utc, + finishedAt: utc.add(const Duration(minutes: 1)), + rowsWritten: 0, + rowsRemoved: 10, + error: null, + severity: SyncRunSeverity.ok, + status: SyncRunStatus.success, + durationMs: 60000, + summary: '10 rows removed', + ); + + await tester.pumpWidget( + MaterialApp( + theme: buildAppTheme(), + home: Scaffold(body: SyncRunExpansionTile(event: event)), + ), + ); + await tester.tap(find.byType(ExpansionTile)); + await tester.pumpAndSettle(); + + expect(find.text(formatLocalTimestamp(utc)), findsOneWidget); + }); + }); + + group('pagination refresh + load-more race', () { + test('loadMore ignores overlapping calls while first page fetch is in flight', () async { + final _DelayedLoadMoreApi api = _DelayedLoadMoreApi(); + final SyncRunLogRepository repository = SyncRunLogRepository(api: api); + await repository.loadInitial(); + + final Future first = repository.loadMore(); + final SyncRunLogRepositoryState skipped = await repository.loadMore(); + expect(api.inFlightLoadMoreCalls, 1); + expect(skipped.history.map((SyncRunEvent e) => e.id), [10]); + + api.releaseLoadMore(); + final SyncRunLogRepositoryState loaded = await first; + expect(loaded.history.map((SyncRunEvent e) => e.id), [10, 9]); + expect(api.completedLoadMoreCalls, 1); + }); + + test('refresh after loadMore replaces history without holes or duplicates', () async { + final DateTime t0 = DateTime.utc(2026, 5, 27, 10); + final SyncRunLogRepository repository = SyncRunLogRepository( + api: _FakePagedApi(t0), + ); + + await repository.loadInitial(); + await repository.loadMore(); + final SyncRunLogRepositoryState beforeRefresh = repository.state; + expect(beforeRefresh.history.map((SyncRunEvent e) => e.id), [10, 9]); + + final SyncRunLogRepositoryState refreshed = await repository.refresh(); + expect(refreshed.history.map((SyncRunEvent e) => e.id), [20]); + expect(refreshed.history.map((SyncRunEvent e) => e.id).toSet(), hasLength(1)); + }); + }); + + group('empty payload safety', () { + test('fetchSyncRuns accepts empty runs and pinned arrays', () async { + final MarketHistoryAdminApi api = MarketHistoryAdminApi( + tokenProvider: () async => 'token', + client: MockClient( + (http.Request request) async => http.Response( + jsonEncode({ + 'runs': [], + 'pinned': [], + }), + 200, + ), + ), + ); + + final SyncRunLogPage page = await api.fetchSyncRuns(); + expect(page.runs, isEmpty); + expect(page.pinned, isEmpty); + expect(page.nextBefore, isNull); + }); + + test('fetchSyncRuns accepts minimal empty object', () async { + final MarketHistoryAdminApi api = MarketHistoryAdminApi( + tokenProvider: () async => 'token', + client: MockClient( + (http.Request request) async => http.Response('{}', 200), + ), + ); + + final SyncRunLogPage page = await api.fetchSyncRuns(); + expect(page.runs, isEmpty); + expect(page.pinned, isEmpty); + expect(page.nextBefore, isNull); + }); + + test('loadInitial with empty API response does not crash repository', () async { + final SyncRunLogRepository repository = SyncRunLogRepository( + api: _EmptyApi(), + ); + final SyncRunLogRepositoryState state = await repository.loadInitial(); + expect(state.pinned, isEmpty); + expect(state.history, isEmpty); + expect(state.errorMessage, isNull); + }); + }); + + testWidgets('very long error text renders when expansion tile is opened', ( + WidgetTester tester, + ) async { + tester.view.physicalSize = const Size(800, 1600); + tester.view.devicePixelRatio = 1.0; + addTearDown(tester.view.reset); + + final String longError = 'X' * 600; + final SyncRunEvent event = SyncRunEvent( + id: 99, + kind: 'backfill', + startedAt: DateTime.utc(2026, 5, 27, 10), + finishedAt: DateTime.utc(2026, 5, 27, 10, 5), + rowsWritten: 0, + rowsRemoved: 0, + error: longError, + severity: SyncRunSeverity.error, + status: SyncRunStatus.failed, + durationMs: 300000, + summary: 'Run failed', + ); + + await tester.pumpWidget( + MaterialApp( + theme: buildAppTheme(), + home: Scaffold(body: SyncRunExpansionTile(event: event)), + ), + ); + await tester.tap(find.byType(ExpansionTile)); + await tester.pumpAndSettle(); + + expect(find.byKey(const Key('sync-run-error-99')), findsOneWidget); + expect(find.text(longError), findsWidgets); + }); +} + +SyncRunEvent _event({ + required int id, + required DateTime startedAt, +}) { + return SyncRunEvent( + id: id, + kind: 'cleanup', + startedAt: startedAt, + finishedAt: startedAt.add(const Duration(minutes: 1)), + rowsWritten: 0, + rowsRemoved: 1, + error: null, + severity: SyncRunSeverity.ok, + status: SyncRunStatus.success, + durationMs: 60000, + summary: 'summary', + ); +} + +class _DelayedLoadMoreApi extends MarketHistoryAdminApi { + _DelayedLoadMoreApi() : super(tokenProvider: () async => 'token'); + + final Completer _gate = Completer(); + int inFlightLoadMoreCalls = 0; + int completedLoadMoreCalls = 0; + + void releaseLoadMore() { + if (!_gate.isCompleted) { + _gate.complete(); + } + } + + @override + Future fetchSyncRuns({ + int limit = 50, + DateTime? before, + String? kind, + }) async { + if (before == null) { + return SyncRunLogPage( + runs: [_event(id: 10, startedAt: DateTime.utc(2026, 5, 27, 10))], + pinned: const [], + nextBefore: DateTime.utc(2026, 5, 26), + ); + } + + inFlightLoadMoreCalls++; + await _gate.future; + completedLoadMoreCalls++; + return SyncRunLogPage( + runs: [_event(id: 9, startedAt: DateTime.utc(2026, 5, 26, 10))], + pinned: const [], + nextBefore: null, + ); + } +} + +class _FakePagedApi extends MarketHistoryAdminApi { + _FakePagedApi(this._t0) : super(tokenProvider: () async => 'token'); + + final DateTime _t0; + int _cursor = 0; + + @override + Future fetchSyncRuns({ + int limit = 50, + DateTime? before, + String? kind, + }) async { + if (_cursor == 0) { + _cursor++; + return SyncRunLogPage( + runs: [_event(id: 10, startedAt: _t0)], + pinned: const [], + nextBefore: DateTime.utc(2026, 5, 26), + ); + } + if (_cursor == 1 && before != null) { + _cursor++; + return SyncRunLogPage( + runs: [ + _event(id: 9, startedAt: _t0.subtract(const Duration(hours: 1))), + ], + pinned: const [], + nextBefore: null, + ); + } + return SyncRunLogPage( + runs: [_event(id: 20, startedAt: _t0.add(const Duration(hours: 2)))], + pinned: const [], + nextBefore: null, + ); + } +} + +class _EmptyApi extends MarketHistoryAdminApi { + _EmptyApi() : super(tokenProvider: () async => 'token'); + + @override + Future fetchSyncRuns({ + int limit = 50, + DateTime? before, + String? kind, + }) async { + return const SyncRunLogPage( + runs: [], + pinned: [], + nextBefore: null, + ); + } +} diff --git a/test/admin/screens/market_history_log_screen_test.dart b/test/admin/screens/market_history_log_screen_test.dart new file mode 100644 index 0000000..72821e9 --- /dev/null +++ b/test/admin/screens/market_history_log_screen_test.dart @@ -0,0 +1,583 @@ +import 'dart:async'; + +import 'package:cyberhybridhub/admin/models/market_history_admin_config.dart'; +import 'package:cyberhybridhub/admin/models/market_history_week_coverage.dart'; +import 'package:cyberhybridhub/admin/models/question_audit_asset.dart'; +import 'package:cyberhybridhub/admin/models/sync_run_event.dart'; +import 'package:cyberhybridhub/admin/repositories/sync_run_log_repository.dart'; +import 'package:cyberhybridhub/admin/screens/market_history_log_screen.dart'; +import 'package:cyberhybridhub/admin/services/market_history_admin_api.dart'; +import 'package:cyberhybridhub/theme/app_theme.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:http/http.dart' as http; +import 'package:http/testing.dart'; + +class _FakeController implements SyncRunLogController { + _FakeController(this._pages); + + final List _pages; + int _index = 0; + int loadInitialCalls = 0; + int refreshCalls = 0; + int loadMoreCalls = 0; + bool? lastCleanupArchive; + + SyncRunLogRepositoryState _current = const SyncRunLogRepositoryState( + pinned: [], + history: [], + nextBefore: null, + isLoading: true, + errorMessage: null, + ); + + @override + SyncRunLogRepositoryState get state => _current; + + @override + Future loadInitial({int limit = 50}) async { + loadInitialCalls++; + _current = _pages[_index]; + return _current; + } + + @override + Future refresh({int limit = 50}) async { + refreshCalls++; + _index = 0; + _current = _pages[_index]; + return _current; + } + + @override + Future loadMore({int limit = 50}) async { + loadMoreCalls++; + if (_index < _pages.length - 1) { + _index++; + } + _current = _pages[_index]; + return _current; + } + + @override + Future triggerResync() async { + return refresh(); + } + + @override + Future triggerCleanup({bool archive = false}) async { + lastCleanupArchive = archive; + return refresh(); + } +} + +SyncRunEvent _event({ + required int id, + required String kind, + required DateTime startedAt, + String? error, + SyncRunSeverity severity = SyncRunSeverity.ok, + SyncRunStatus status = SyncRunStatus.success, + String? summary, +}) { + return SyncRunEvent( + id: id, + kind: kind, + startedAt: startedAt, + finishedAt: startedAt.add(const Duration(minutes: 1)), + rowsWritten: 100, + rowsRemoved: kind == 'cleanup' ? 4200 : 0, + error: error, + severity: severity, + status: status, + durationMs: 60000, + summary: summary ?? 'summary $id', + ); +} + +Future _pumpScreen( + WidgetTester tester, + _FakeController controller, { + MarketHistoryAdminApi? coverageApi, +}) async { + await tester.pumpWidget( + MaterialApp( + theme: buildAppTheme(), + home: MarketHistoryLogScreen( + controller: controller, + coverageApi: coverageApi, + now: DateTime.utc(2026, 5, 27, 12), + autoRefreshInterval: null, + ), + ), + ); + await tester.pumpAndSettle(); +} + +class _FakeCoverageApi extends MarketHistoryAdminApi { + _FakeCoverageApi(this.report) + : super( + baseUrl: 'http://localhost:3000', + tokenProvider: () async => 'token', + client: MockClient((http.Request request) async => http.Response('', 500)), + ); + + final MarketHistoryWeekCoverageReport report; + + @override + Future fetchWeekCoverage() async => report; + + @override + Future fetchQuestionAudit({DateTime? asOf}) async => + QuestionAuditReport( + compareUntil: DateTime.utc(2026, 5, 30, 16), + newerSlotStart: DateTime.utc(2026, 5, 30, 12), + olderSlotStart: DateTime.utc(2026, 5, 30, 8), + windowDays: 7, + canStepOlder: false, + canStepNewer: false, + assets: const [], + ); +} + +MarketHistoryWeekCoverageReport _sampleWeekReport() { + final DateTime day = DateTime.utc(2026, 5, 30); + return MarketHistoryWeekCoverageReport( + asOf: DateTime.utc(2026, 5, 30, 15), + windowDays: 7, + slotsPerDay: 6, + symbolCount: 10, + isConsistent: true, + days: List.generate( + 7, + (int index) { + final DateTime date = day.subtract(Duration(days: 6 - index)); + return MarketHistoryDayCoverage( + date: date, + slotsPerDay: 6, + completedSlots: 6, + fullySyncedSlots: 6, + slots: List.generate( + 6, + (int slotIndex) => MarketHistorySlotCoverage( + slotStart: DateTime.utc(date.year, date.month, date.day, slotIndex * 4), + completed: true, + fullySynced: true, + syncedSymbolCount: 10, + expectedSymbolCount: 10, + ), + ), + ); + }, + ), + ); +} + +void main() { + testWidgets('shows pinned section when pinned rows exist', ( + WidgetTester tester, + ) async { + final DateTime t0 = DateTime.utc(2026, 5, 27, 10); + final _FakeController controller = _FakeController([ + SyncRunLogRepositoryState( + pinned: [ + _event( + id: 1, + kind: 'backfill', + startedAt: t0, + error: '429', + severity: SyncRunSeverity.rateLimit, + status: SyncRunStatus.failed, + ), + ], + history: [ + _event(id: 2, kind: 'cleanup', startedAt: t0.subtract(const Duration(hours: 1))), + ], + nextBefore: null, + isLoading: false, + errorMessage: null, + ), + ]); + + await _pumpScreen(tester, controller); + expect(find.byKey(const Key('pinned-section')), findsOneWidget); + expect(find.byKey(const Key('sync-run-1')), findsOneWidget); + }); + + testWidgets('hides pinned section when none exist', ( + WidgetTester tester, + ) async { + final _FakeController controller = _FakeController([ + SyncRunLogRepositoryState( + pinned: const [], + history: [ + _event( + id: 3, + kind: 'universe', + startedAt: DateTime.utc(2026, 5, 27, 9), + ), + ], + nextBefore: null, + isLoading: false, + errorMessage: null, + ), + ]); + + await _pumpScreen(tester, controller); + expect(find.byKey(const Key('pinned-section')), findsNothing); + expect(find.byKey(const Key('history-section')), findsOneWidget); + }); + + testWidgets('renders newest history row first', ( + WidgetTester tester, + ) async { + final DateTime t0 = DateTime.utc(2026, 5, 27, 10); + final _FakeController controller = _FakeController([ + SyncRunLogRepositoryState( + pinned: const [], + history: [ + _event(id: 10, kind: 'cleanup', startedAt: t0, summary: 'newest'), + _event( + id: 9, + kind: 'backfill', + startedAt: t0.subtract(const Duration(hours: 1)), + summary: 'older', + ), + ], + nextBefore: null, + isLoading: false, + errorMessage: null, + ), + ]); + + await _pumpScreen(tester, controller); + + final Finder tiles = find.byType(ExpansionTile); + expect(tiles, findsNWidgets(2)); + expect(find.textContaining('newest'), findsOneWidget); + }); + + testWidgets('pull-to-refresh triggers repository refresh', ( + WidgetTester tester, + ) async { + final _FakeController controller = _FakeController([ + SyncRunLogRepositoryState( + pinned: const [], + history: [ + _event(id: 4, kind: 'cleanup', startedAt: DateTime.utc(2026, 5, 27, 8)), + ], + nextBefore: null, + isLoading: false, + errorMessage: null, + ), + ]); + + await _pumpScreen(tester, controller); + expect(controller.refreshCalls, 0); + + await tester.fling(find.byType(CustomScrollView), const Offset(0, 300), 1200); + await tester.pump(); + await tester.pump(const Duration(seconds: 1)); + await tester.pumpAndSettle(); + + expect(controller.refreshCalls, 1); + }); + + testWidgets('refresh button triggers repository refresh', ( + WidgetTester tester, + ) async { + final _FakeController controller = _FakeController([ + SyncRunLogRepositoryState( + pinned: const [], + history: [ + _event(id: 4, kind: 'cleanup', startedAt: DateTime.utc(2026, 5, 27, 8)), + ], + nextBefore: null, + isLoading: false, + errorMessage: null, + ), + ]); + + await _pumpScreen(tester, controller); + expect(controller.refreshCalls, 0); + + await tester.tap(find.byKey(const Key('refresh-button'))); + await tester.pumpAndSettle(); + + expect(controller.refreshCalls, 1); + }); + + testWidgets('scroll near end triggers loadMore', ( + WidgetTester tester, + ) async { + tester.view.physicalSize = const Size(400, 600); + tester.view.devicePixelRatio = 1.0; + addTearDown(tester.view.reset); + + final DateTime t0 = DateTime.utc(2026, 5, 27, 10); + final List firstPage = List.generate( + 12, + (int i) => _event( + id: 100 + i, + kind: 'cleanup', + startedAt: t0.subtract(Duration(hours: i)), + ), + ); + final _FakeController controller = _FakeController([ + SyncRunLogRepositoryState( + pinned: const [], + history: firstPage, + nextBefore: DateTime.utc(2026, 5, 25), + isLoading: false, + errorMessage: null, + ), + SyncRunLogRepositoryState( + pinned: const [], + history: [ + ...firstPage, + _event(id: 50, kind: 'universe', startedAt: DateTime.utc(2026, 5, 24)), + ], + nextBefore: null, + isLoading: false, + errorMessage: null, + ), + ]); + + await _pumpScreen(tester, controller); + expect(controller.loadMoreCalls, 0); + + await tester.drag(find.byType(CustomScrollView), const Offset(0, -2400)); + await tester.pumpAndSettle(); + + expect(controller.loadMoreCalls, greaterThan(0)); + }); + + testWidgets('renders empty state', (WidgetTester tester) async { + final _FakeController empty = _FakeController([ + const SyncRunLogRepositoryState( + pinned: [], + history: [], + nextBefore: null, + isLoading: false, + errorMessage: null, + ), + ]); + await _pumpScreen(tester, empty); + expect(find.byKey(const Key('empty-state')), findsOneWidget); + }); + + testWidgets('renders error state', (WidgetTester tester) async { + final _FakeController error = _FakeController([ + const SyncRunLogRepositoryState( + pinned: [], + history: [], + nextBefore: null, + isLoading: false, + errorMessage: 'Forbidden', + ), + ]); + await _pumpScreen(tester, error); + expect(find.byKey(const Key('error-state')), findsOneWidget); + expect(find.text('Forbidden'), findsOneWidget); + }); + + testWidgets('renders loading indicator before initial load completes', ( + WidgetTester tester, + ) async { + final Completer pending = + Completer(); + final _PendingController loading = _PendingController(pending); + + await tester.pumpWidget( + MaterialApp( + theme: buildAppTheme(), + home: MarketHistoryLogScreen(controller: loading, autoRefreshInterval: null), + ), + ); + await tester.pump(); + expect(find.byKey(const Key('loading-indicator')), findsOneWidget); + + pending.complete( + const SyncRunLogRepositoryState( + pinned: [], + history: [], + nextBefore: null, + isLoading: false, + errorMessage: null, + ), + ); + await tester.pumpAndSettle(); + expect(find.byKey(const Key('empty-state')), findsOneWidget); + }); + + testWidgets('cleanup confirm dialog triggers controller cleanup', ( + WidgetTester tester, + ) async { + final DateTime t0 = DateTime.utc(2026, 5, 27, 10); + final _FakeController controller = _FakeController([ + SyncRunLogRepositoryState( + pinned: const [], + history: [ + _event(id: 1, kind: 'cleanup', startedAt: t0), + ], + nextBefore: null, + isLoading: false, + errorMessage: null, + config: const MarketHistoryAdminConfig( + archiveEnabled: false, + windowDays: 7, + retentionDays: 7, + syncEnabled: true, + ), + ), + ]); + + await _pumpScreen(tester, controller); + await tester.tap(find.byKey(const Key('actions-menu'))); + await tester.pumpAndSettle(); + await tester.tap(find.byKey(const Key('action-cleanup'))); + await tester.pumpAndSettle(); + expect(find.byKey(const Key('cleanup-confirm-dialog')), findsOneWidget); + + await tester.tap(find.byKey(const Key('cleanup-confirm'))); + await tester.pumpAndSettle(); + expect(controller.refreshCalls, greaterThan(0)); + }); + + testWidgets('pinned failure shows retry button that triggers resync', ( + WidgetTester tester, + ) async { + final DateTime t0 = DateTime.utc(2026, 5, 27, 10); + final _FakeController controller = _FakeController([ + SyncRunLogRepositoryState( + pinned: [ + _event( + id: 99, + kind: 'backfill', + startedAt: t0, + error: '429', + severity: SyncRunSeverity.rateLimit, + status: SyncRunStatus.failed, + ), + ], + history: const [], + nextBefore: null, + isLoading: false, + errorMessage: null, + config: const MarketHistoryAdminConfig( + archiveEnabled: false, + windowDays: 7, + retentionDays: 7, + syncEnabled: true, + ), + ), + ]); + + await _pumpScreen(tester, controller); + await tester.tap(find.byType(ExpansionTile)); + await tester.pumpAndSettle(); + await tester.tap(find.byKey(const Key('retry-run-99'))); + await tester.pumpAndSettle(); + expect(controller.refreshCalls, greaterThan(0)); + }); + + testWidgets('help button opens question audit dialog', ( + WidgetTester tester, + ) async { + final _FakeController controller = _FakeController([ + const SyncRunLogRepositoryState( + pinned: [], + history: [], + nextBefore: null, + isLoading: false, + errorMessage: null, + ), + ]); + + await _pumpScreen( + tester, + controller, + coverageApi: _FakeCoverageApi(_sampleWeekReport()), + ); + + await tester.tap(find.byKey(const Key('question-audit-button'))); + await tester.pumpAndSettle(); + + expect(find.byKey(const Key('question-audit-dialog')), findsOneWidget); + expect(find.text('Prospective question data'), findsOneWidget); + expect(find.byKey(const Key('question-audit-symbol-count')), findsOneWidget); + + await tester.tap(find.byKey(const Key('question-audit-close'))); + await tester.pumpAndSettle(); + expect(find.byKey(const Key('question-audit-dialog')), findsNothing); + }); + + testWidgets('calendar button opens week coverage dialog', ( + WidgetTester tester, + ) async { + final _FakeController controller = _FakeController([ + const SyncRunLogRepositoryState( + pinned: [], + history: [], + nextBefore: null, + isLoading: false, + errorMessage: null, + ), + ]); + + await _pumpScreen( + tester, + controller, + coverageApi: _FakeCoverageApi(_sampleWeekReport()), + ); + + await tester.tap(find.byKey(const Key('week-coverage-button'))); + await tester.pumpAndSettle(); + + expect(find.byKey(const Key('week-coverage-dialog')), findsOneWidget); + expect(find.text('7-day sync health'), findsOneWidget); + expect(find.textContaining('6/6'), findsWidgets); + + await tester.tap(find.byKey(const Key('week-coverage-close'))); + await tester.pumpAndSettle(); + expect(find.byKey(const Key('week-coverage-dialog')), findsNothing); + }); +} + +class _PendingController implements SyncRunLogController { + _PendingController(this._pending); + + final Completer _pending; + + SyncRunLogRepositoryState _current = const SyncRunLogRepositoryState( + pinned: [], + history: [], + nextBefore: null, + isLoading: true, + errorMessage: null, + ); + + @override + SyncRunLogRepositoryState get state => _current; + + @override + Future loadInitial({int limit = 50}) async { + _current = await _pending.future; + return _current; + } + + @override + Future refresh({int limit = 50}) => + loadInitial(limit: limit); + + @override + Future loadMore({int limit = 50}) async => + _current; + + @override + Future triggerResync() async => _current; + + @override + Future triggerCleanup({bool archive = false}) async => + _current; +} diff --git a/test/admin/services/admin_access_service_test.dart b/test/admin/services/admin_access_service_test.dart new file mode 100644 index 0000000..e0ec365 --- /dev/null +++ b/test/admin/services/admin_access_service_test.dart @@ -0,0 +1,33 @@ +import 'package:cyberhybridhub/admin/services/admin_access_service.dart'; +import 'package:cyberhybridhub/admin/services/market_history_admin_api.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:http/http.dart' as http; +import 'package:http/testing.dart'; + +void main() { + setUp(AdminAccessService.instance.reset); + + test('refresh marks authorized when probe succeeds', () async { + final MarketHistoryAdminApi api = MarketHistoryAdminApi( + tokenProvider: () async => 'token', + client: MockClient( + (http.Request request) async => http.Response('{"runs":[],"pinned":[]}', 200), + ), + ); + + await AdminAccessService.instance.refresh(api: api); + expect(AdminAccessService.instance.status.value, AdminAccessStatus.authorized); + }); + + test('refresh marks forbidden on 403', () async { + final MarketHistoryAdminApi api = MarketHistoryAdminApi( + tokenProvider: () async => 'token', + client: MockClient( + (http.Request request) async => http.Response('forbidden', 403), + ), + ); + + await AdminAccessService.instance.refresh(api: api); + expect(AdminAccessService.instance.status.value, AdminAccessStatus.forbidden); + }); +} diff --git a/test/admin/services/market_history_admin_api_test.dart b/test/admin/services/market_history_admin_api_test.dart new file mode 100644 index 0000000..1ad326b --- /dev/null +++ b/test/admin/services/market_history_admin_api_test.dart @@ -0,0 +1,256 @@ +import 'dart:convert'; + +import 'package:cyberhybridhub/admin/models/market_history_week_coverage.dart'; +import 'package:cyberhybridhub/admin/models/question_audit_asset.dart'; +import 'package:cyberhybridhub/admin/models/sync_run_event.dart'; +import 'package:cyberhybridhub/admin/services/market_history_admin_api.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:http/http.dart' as http; +import 'package:http/testing.dart'; + +void main() { + test('sends bearer token and query params, parses response', () async { + late Uri seenUri; + late String seenAuth; + final MarketHistoryAdminApi api = MarketHistoryAdminApi( + baseUrl: 'http://localhost:3000', + tokenProvider: () async => 'test-token', + client: MockClient((http.Request request) async { + seenUri = request.url; + seenAuth = request.headers['Authorization'] ?? ''; + return http.Response( + jsonEncode({ + 'runs': >[ + { + 'id': 1, + 'kind': 'cleanup', + 'startedAt': '2026-05-27T00:00:00Z', + 'finishedAt': '2026-05-27T00:00:05Z', + 'rowsWritten': 0, + 'rowsRemoved': 120, + 'error': null, + 'severity': 'ok', + 'status': 'success', + 'durationMs': 5000, + 'summary': 'Removed 120 rows' + } + ], + 'pinned': >[], + 'nextBefore': '2026-05-26T00:00:00Z', + }), + 200, + headers: {'content-type': 'application/json'}, + ); + }), + ); + + final result = await api.fetchSyncRuns( + limit: 10, + before: DateTime.utc(2026, 5, 27), + kind: 'cleanup', + ); + + expect(seenAuth, 'Bearer test-token'); + expect(seenUri.queryParameters['limit'], '10'); + expect(seenUri.queryParameters['kind'], 'cleanup'); + expect(result.runs, hasLength(1)); + expect(result.nextBefore, DateTime.utc(2026, 5, 26)); + }); + + test('throws on non-200 responses', () async { + final MarketHistoryAdminApi api = MarketHistoryAdminApi( + baseUrl: 'http://localhost:3000', + tokenProvider: () async => 'test-token', + client: MockClient( + (http.Request request) async => http.Response('forbidden', 403), + ), + ); + + expect( + () => api.fetchSyncRuns(), + throwsA(isA()), + ); + }); + + test('POST resync sends bearer token and parses runIds', () async { + late Uri seenUri; + late String seenAuth; + final MarketHistoryAdminApi api = MarketHistoryAdminApi( + baseUrl: 'http://localhost:3000', + tokenProvider: () async => 'test-token', + client: MockClient((http.Request request) async { + seenUri = request.url; + seenAuth = request.headers['Authorization'] ?? ''; + return http.Response( + jsonEncode({'runIds': [1, 2]}), + 202, + headers: {'content-type': 'application/json'}, + ); + }), + ); + + final AdminTriggerResponse result = await api.triggerResync(); + expect(seenAuth, 'Bearer test-token'); + expect(seenUri.path, '/v1/admin/market-data/resync'); + expect(result.runIds, [1, 2]); + }); + + test('POST cleanup sends archive query param', () async { + late Uri seenUri; + final MarketHistoryAdminApi api = MarketHistoryAdminApi( + baseUrl: 'http://localhost:3000', + tokenProvider: () async => 'test-token', + client: MockClient((http.Request request) async { + seenUri = request.url; + return http.Response( + jsonEncode({'runIds': [3]}), + 202, + ); + }), + ); + + await api.triggerCleanup(archive: true); + expect(seenUri.queryParameters['archive'], 'true'); + }); + + test('handles malformed sync-runs payload safely', () async { + final MarketHistoryAdminApi api = MarketHistoryAdminApi( + baseUrl: 'http://localhost:3000', + tokenProvider: () async => 'test-token', + client: MockClient( + (http.Request request) async => http.Response('{"runs":"bad"}', 200), + ), + ); + + expect(() => api.fetchSyncRuns(), throwsA(isA())); + }); + + test('parses config from sync-runs response', () async { + final MarketHistoryAdminApi api = MarketHistoryAdminApi( + baseUrl: 'http://localhost:3000', + tokenProvider: () async => 'test-token', + client: MockClient( + (http.Request request) async => http.Response( + jsonEncode({ + 'runs': [], + 'pinned': [], + 'config': { + 'archiveEnabled': true, + 'windowDays': 7, + 'retentionDays': 14, + 'syncEnabled': true, + }, + }), + 200, + ), + ), + ); + + final SyncRunLogPage page = await api.fetchSyncRuns(); + expect(page.config.archiveEnabled, isTrue); + expect(page.config.retentionDays, 14); + }); + + test('fetchWeekCoverage parses week report', () async { + final MarketHistoryAdminApi api = MarketHistoryAdminApi( + baseUrl: 'http://localhost:3000', + tokenProvider: () async => 'test-token', + client: MockClient((http.Request request) async { + expect(request.url.path, '/v1/admin/market-history/week-coverage'); + return http.Response( + jsonEncode({ + 'asOf': '2026-05-30T15:30:00Z', + 'windowDays': 7, + 'slotsPerDay': 6, + 'symbolCount': 2, + 'isConsistent': false, + 'days': >[ + { + 'date': '2026-05-30', + 'slotsPerDay': 6, + 'completedSlots': 4, + 'fullySyncedSlots': 3, + 'slots': >[ + { + 'slotStart': '2026-05-30T00:00:00Z', + 'completed': true, + 'fullySynced': true, + 'syncedSymbolCount': 2, + 'expectedSymbolCount': 2, + }, + ], + }, + ], + }), + 200, + headers: {'content-type': 'application/json'}, + ); + }), + ); + + final report = await api.fetchWeekCoverage(); + expect(report.symbolCount, 2); + expect(report.isConsistent, isFalse); + expect(report.days.single.fullySyncedSlots, 3); + }); + + test('fetchQuestionAudit parses asset deltas', () async { + final MarketHistoryAdminApi api = MarketHistoryAdminApi( + baseUrl: 'http://localhost:3000', + tokenProvider: () async => 'test-token', + client: MockClient((http.Request request) async { + expect(request.url.path, '/v1/admin/market-history/question-audit'); + return http.Response( + jsonEncode({ + 'compareUntil': '2026-05-30T16:00:00Z', + 'newerSlotStart': '2026-05-30T12:00:00Z', + 'olderSlotStart': '2026-05-30T08:00:00Z', + 'windowDays': 7, + 'canStepOlder': true, + 'canStepNewer': false, + 'stepOlderCompareUntil': '2026-05-30T12:00:00Z', + 'assets': >[ + { + 'symbol': 'AAA', + 'priceDelta': 2.5, + 'volumeDelta': 50, + 'olderSlot': { + 'asOf': '2026-05-30T08:00:00Z', + 'open': 10, + 'high': 12, + 'low': 8, + 'close': 10, + 'avgPrice': 10, + 'volume': 100, + }, + 'newerSlot': { + 'asOf': '2026-05-30T12:00:00Z', + 'open': 12, + 'high': 14, + 'low': 10, + 'close': 12, + 'avgPrice': 12.5, + 'volume': 150, + }, + }, + ], + }), + 200, + headers: {'content-type': 'application/json'}, + ); + }), + ); + + final QuestionAuditReport report = await api.fetchQuestionAudit(); + expect(report.windowDays, 7); + expect(report.canStepOlder, isTrue); + expect(report.canStepNewer, isFalse); + expect(report.stepOlderCompareUntil, DateTime.utc(2026, 5, 30, 12)); + expect(report.assets, hasLength(1)); + expect(report.assets.single.symbol, 'AAA'); + expect(report.assets.single.priceDelta, 2.5); + expect(report.assets.single.volumeDelta, 50); + expect(report.assets.single.olderSlot.avgPrice, 10); + expect(report.assets.single.newerSlot.close, 12); + }); +} diff --git a/test/admin/utils/sync_run_formatters_test.dart b/test/admin/utils/sync_run_formatters_test.dart new file mode 100644 index 0000000..f246259 --- /dev/null +++ b/test/admin/utils/sync_run_formatters_test.dart @@ -0,0 +1,36 @@ +import 'package:cyberhybridhub/admin/utils/sync_run_formatters.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + test('formatMarketHistorySlotWire matches server Alpaca start form', () { + expect( + formatMarketHistorySlotWire(DateTime.utc(2026, 5, 26, 8)), + '2026-05-26T08:00:00Z', + ); + expect( + formatMarketHistorySlotWire(DateTime.utc(2026, 5, 26, 10, 30)), + '2026-05-26T08:00:00Z', + ); + }); + + test('formatLocalTimestamp converts UTC to local display', () { + final DateTime utc = DateTime.utc(2026, 5, 27, 14, 30); + final String formatted = formatLocalTimestamp(utc); + expect(formatted, contains('2026-05-27')); + expect(formatted, contains(':')); + }); + + test('formatRelativeTime handles rate-limit style durations', () { + final DateTime started = DateTime.utc(2026, 5, 27, 10); + final DateTime now = DateTime.utc(2026, 5, 27, 12, 30); + expect(formatRelativeTime(started, now: now), '2h ago'); + }); + + test('parseHttpStatus finds status code in error text', () { + expect( + parseHttpStatus('getBarsRange rate limited: 429 Too Many Requests'), + '429', + ); + expect(parseHttpStatus('no status here'), isNull); + }); +} diff --git a/test/admin/widgets/admin_app_bar_action_test.dart b/test/admin/widgets/admin_app_bar_action_test.dart new file mode 100644 index 0000000..8be2d1f --- /dev/null +++ b/test/admin/widgets/admin_app_bar_action_test.dart @@ -0,0 +1,85 @@ +import 'package:cyberhybridhub/admin/services/admin_access_service.dart'; +import 'package:cyberhybridhub/admin/services/market_history_admin_api.dart'; +import 'package:cyberhybridhub/admin/widgets/admin_app_bar_action.dart'; +import 'package:cyberhybridhub/theme/app_theme.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:http/http.dart' as http; +import 'package:http/testing.dart'; + +MarketHistoryAdminApi _authorizedApi() { + return MarketHistoryAdminApi( + tokenProvider: () async => 'token', + client: MockClient( + (http.Request request) async => http.Response('{"runs":[],"pinned":[]}', 200), + ), + ); +} + +MarketHistoryAdminApi _forbiddenApi() { + return MarketHistoryAdminApi( + tokenProvider: () async => 'token', + client: MockClient( + (http.Request request) async => http.Response('forbidden', 403), + ), + ); +} + +void main() { + setUp(AdminAccessService.instance.reset); + + testWidgets('shows admin icon when allowlisted user probe succeeds', ( + WidgetTester tester, + ) async { + await tester.pumpWidget( + MaterialApp( + theme: buildAppTheme(), + home: Scaffold( + appBar: AppBar(actions: [AdminAppBarAction(api: _authorizedApi())]), + ), + ), + ); + await tester.pumpAndSettle(); + + expect(find.byKey(const Key('admin-app-bar-action')), findsOneWidget); + expect(find.byIcon(Icons.admin_panel_settings_outlined), findsOneWidget); + }); + + testWidgets('hides admin icon when user is not on allowlist', ( + WidgetTester tester, + ) async { + await tester.pumpWidget( + MaterialApp( + theme: buildAppTheme(), + home: Scaffold( + appBar: AppBar(actions: [AdminAppBarAction(api: _forbiddenApi())]), + ), + ), + ); + await tester.pumpAndSettle(); + + expect(find.byKey(const Key('admin-app-bar-action')), findsNothing); + }); + + testWidgets('navigates to admin portal when icon is tapped', ( + WidgetTester tester, + ) async { + await tester.pumpWidget( + MaterialApp( + theme: buildAppTheme(), + routes: { + '/admin/market-history': (_) => const Scaffold(body: Text('admin log')), + }, + home: Scaffold( + appBar: AppBar(actions: [AdminAppBarAction(api: _authorizedApi())]), + ), + ), + ); + await tester.pumpAndSettle(); + + await tester.tap(find.byKey(const Key('admin-app-bar-action'))); + await tester.pumpAndSettle(); + + expect(find.text('admin log'), findsOneWidget); + }); +} diff --git a/test/admin/widgets/admin_gate_test.dart b/test/admin/widgets/admin_gate_test.dart new file mode 100644 index 0000000..38ab8d5 --- /dev/null +++ b/test/admin/widgets/admin_gate_test.dart @@ -0,0 +1,61 @@ +import 'dart:convert'; + +import 'package:cyberhybridhub/admin/services/market_history_admin_api.dart'; +import 'package:cyberhybridhub/admin/widgets/admin_gate.dart'; +import 'package:cyberhybridhub/theme/app_theme.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:http/http.dart' as http; +import 'package:http/testing.dart'; + +MarketHistoryAdminApi gateApiWithStatus(int status) { + return MarketHistoryAdminApi( + tokenProvider: () async => 'token', + client: MockClient( + (http.Request request) async => http.Response( + status == 200 + ? jsonEncode({ + 'runs': [], + 'pinned': [], + 'nextBefore': null, + }) + : 'denied', + status, + ), + ), + ); +} + +void main() { + testWidgets('shows child when admin probe succeeds', (WidgetTester tester) async { + await tester.pumpWidget( + MaterialApp( + theme: buildAppTheme(), + home: AdminGate( + api: gateApiWithStatus(200), + child: const Scaffold(body: Text('admin content')), + ), + ), + ); + await tester.pumpAndSettle(); + expect(find.text('admin content'), findsOneWidget); + }); + + testWidgets('shows forbidden when admin probe returns 403', ( + WidgetTester tester, + ) async { + await tester.pumpWidget( + MaterialApp( + theme: buildAppTheme(), + home: AdminGate( + api: gateApiWithStatus(403), + child: const Scaffold(body: Text('admin content')), + ), + ), + ); + await tester.pumpAndSettle(); + expect(find.byKey(const Key('admin-gate-forbidden')), findsOneWidget); + expect(find.text('Not authorized'), findsOneWidget); + expect(find.text('admin content'), findsNothing); + }); +} diff --git a/test/admin/widgets/market_history_question_audit_sheet_test.dart b/test/admin/widgets/market_history_question_audit_sheet_test.dart new file mode 100644 index 0000000..f08c853 --- /dev/null +++ b/test/admin/widgets/market_history_question_audit_sheet_test.dart @@ -0,0 +1,151 @@ +import 'package:cyberhybridhub/admin/models/question_audit_asset.dart'; +import 'package:cyberhybridhub/admin/services/market_history_admin_api.dart'; +import 'package:cyberhybridhub/admin/widgets/market_history_question_audit_sheet.dart'; +import 'package:cyberhybridhub/theme/app_theme.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:http/http.dart' as http; +import 'package:http/testing.dart'; + +QuestionAuditAsset _sampleAsset() { + return QuestionAuditAsset( + symbol: 'AAA', + priceDelta: 2.5, + volumeDelta: -40, + olderSlot: QuestionAuditBarSlot( + asOf: DateTime.utc(2026, 5, 30, 8), + open: 10, + high: 12, + low: 8, + close: 10, + avgPrice: 10, + volume: 100, + ), + newerSlot: QuestionAuditBarSlot( + asOf: DateTime.utc(2026, 5, 30, 12), + open: 12, + high: 14, + low: 10, + close: 12.5, + avgPrice: 12.5, + volume: 60, + ), + ); +} + +QuestionAuditReport _sampleReport({ + required bool canStepOlder, + required bool canStepNewer, + DateTime? compareUntil, + DateTime? stepOlderCompareUntil, +}) { + final DateTime until = compareUntil ?? DateTime.utc(2026, 5, 30, 16); + final DateTime newer = until.subtract(const Duration(hours: 4)); + final DateTime older = newer.subtract(const Duration(hours: 4)); + return QuestionAuditReport( + compareUntil: until, + newerSlotStart: newer, + olderSlotStart: older, + windowDays: 7, + canStepOlder: canStepOlder, + canStepNewer: canStepNewer, + stepOlderCompareUntil: stepOlderCompareUntil, + assets: [_sampleAsset()], + ); +} + +class _FakeAuditApi extends MarketHistoryAdminApi { + _FakeAuditApi(this._pages) + : super( + baseUrl: 'http://localhost:3000', + tokenProvider: () async => 'token', + client: MockClient((http.Request request) async => http.Response('', 500)), + ); + + final List _pages; + int _index = 0; + final List requestedAsOf = []; + + @override + Future fetchQuestionAudit({DateTime? asOf}) async { + requestedAsOf.add(asOf); + if (asOf != null && _index < _pages.length - 1) { + _index++; + } + return _pages[_index]; + } +} + +Future _pumpSheet(WidgetTester tester, MarketHistoryAdminApi api) async { + await tester.pumpWidget( + MaterialApp( + theme: buildAppTheme(), + home: Scaffold( + body: SizedBox( + height: 500, + child: MarketHistoryQuestionAuditSheet(api: api), + ), + ), + ), + ); + await tester.pumpAndSettle(); +} + +void main() { + testWidgets('shows slot range between arrows and symbol count', ( + WidgetTester tester, + ) async { + await _pumpSheet( + tester, + _FakeAuditApi([ + _sampleReport(canStepOlder: true, canStepNewer: false), + ]), + ); + + expect(find.byKey(const Key('question-audit-slot-range')), findsOneWidget); + expect(find.text('05/30 08:00 – 05/30 12:00 UTC'), findsOneWidget); + expect(find.text('1 symbol'), findsOneWidget); + expect(find.byKey(const Key('question-audit-step-older')), findsOneWidget); + expect(find.byKey(const Key('question-audit-step-newer')), findsOneWidget); + }); + + testWidgets('older arrow loads stepped compareUntil', ( + WidgetTester tester, + ) async { + final DateTime olderBound = DateTime.utc(2026, 5, 30, 12); + final _FakeAuditApi api = _FakeAuditApi([ + _sampleReport( + canStepOlder: true, + canStepNewer: false, + stepOlderCompareUntil: olderBound, + ), + _sampleReport( + canStepOlder: false, + canStepNewer: true, + compareUntil: olderBound, + ), + ]); + await _pumpSheet(tester, api); + + await tester.tap(find.byKey(const Key('question-audit-step-older'))); + await tester.pumpAndSettle(); + + expect(api.requestedAsOf.last, olderBound); + expect(find.text('05/30 04:00 – 05/30 08:00 UTC'), findsOneWidget); + }); + + testWidgets('tap expands slot detail panels', (WidgetTester tester) async { + await _pumpSheet( + tester, + _FakeAuditApi([ + _sampleReport(canStepOlder: false, canStepNewer: false), + ]), + ); + + await tester.tap(find.byKey(const Key('question-audit-tile-AAA'))); + await tester.pumpAndSettle(); + + expect(find.byKey(const Key('question-audit-slot-older-AAA')), findsOneWidget); + expect(find.byKey(const Key('question-audit-slot-newer-AAA')), findsOneWidget); + }); +} diff --git a/test/admin/widgets/sync_run_expansion_tile_test.dart b/test/admin/widgets/sync_run_expansion_tile_test.dart new file mode 100644 index 0000000..f5d1105 --- /dev/null +++ b/test/admin/widgets/sync_run_expansion_tile_test.dart @@ -0,0 +1,148 @@ +import 'package:cyberhybridhub/admin/models/backfill_sync_item.dart'; +import 'package:cyberhybridhub/admin/models/sync_run_event.dart'; +import 'package:cyberhybridhub/admin/repositories/sync_run_log_repository.dart'; +import 'package:cyberhybridhub/admin/widgets/sync_run_expansion_tile.dart'; +import 'package:cyberhybridhub/theme/app_theme.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; + +SyncRunEvent _event({ + required int id, + required String kind, + required DateTime startedAt, + String? error, + SyncRunSeverity severity = SyncRunSeverity.ok, + SyncRunStatus status = SyncRunStatus.success, +}) { + return SyncRunEvent( + id: id, + kind: kind, + startedAt: startedAt, + finishedAt: startedAt.add(const Duration(minutes: 1)), + rowsWritten: kind == 'cleanup' ? 0 : 100, + rowsRemoved: kind == 'cleanup' ? 4200 : 0, + error: error, + severity: severity, + status: status, + durationMs: 60000, + summary: error ?? '100 bar rows written', + ); +} + +void main() { + testWidgets('shows severity chip and full error text when expanded', ( + WidgetTester tester, + ) async { + const String errorText = + 'AlpacaMarketDataException: getBarsRange rate limited: 429'; + final SyncRunEvent event = _event( + id: 7, + kind: 'backfill', + startedAt: DateTime.utc(2026, 5, 27, 10), + error: errorText, + severity: SyncRunSeverity.rateLimit, + status: SyncRunStatus.failed, + ); + + await tester.pumpWidget( + MaterialApp( + theme: buildAppTheme(), + home: Scaffold( + body: SyncRunExpansionTile( + event: event, + now: DateTime.utc(2026, 5, 27, 12), + ), + ), + ), + ); + + expect(find.text('Partial success'), findsNothing); + expect(find.textContaining('Failed'), findsWidgets); + expect(find.textContaining('Rate limited'), findsNothing); + + await tester.tap(find.byType(ExpansionTile)); + await tester.pumpAndSettle(); + + expect(find.byKey(const Key('sync-run-error-7')), findsOneWidget); + expect(find.text(errorText), findsWidgets); + expect(find.textContaining('raw'), findsNothing); + expect(find.text('HTTP status'), findsOneWidget); + expect(find.text('429'), findsOneWidget); + }); + + testWidgets('renders very long error text without raw field labels', ( + WidgetTester tester, + ) async { + tester.view.physicalSize = const Size(800, 1600); + tester.view.devicePixelRatio = 1.0; + addTearDown(tester.view.reset); + + final String longError = 'E' * 500; + final SyncRunEvent event = _event( + id: 8, + kind: 'backfill', + startedAt: DateTime.utc(2026, 5, 27, 10), + error: longError, + severity: SyncRunSeverity.error, + status: SyncRunStatus.failed, + ); + + await tester.pumpWidget( + MaterialApp( + theme: buildAppTheme(), + home: Scaffold( + body: SyncRunExpansionTile(event: event), + ), + ), + ); + await tester.tap(find.byType(ExpansionTile)); + await tester.pumpAndSettle(); + + expect(find.text(longError), findsWidgets); + expect(find.textContaining('"raw"'), findsNothing); + }); + + testWidgets('backfill items show Alpaca start wire string for DB spot checks', ( + WidgetTester tester, + ) async { + const String slotWire = '2026-05-26T08:00:00Z'; + final SyncRunEvent event = SyncRunEvent( + id: 9, + kind: 'backfill', + startedAt: DateTime.utc(2026, 5, 27, 10), + finishedAt: DateTime.utc(2026, 5, 27, 10, 1), + rowsWritten: 0, + rowsRemoved: 0, + severity: SyncRunSeverity.warning, + status: SyncRunStatus.partial, + durationMs: 60000, + summary: '0 bar rows written', + error: 'partial sync saved (0 rows)', + backfillItems: [ + BackfillSyncItem( + slotStart: DateTime.utc(2026, 5, 26, 8), + slotStartWire: slotWire, + symbols: ['A', 'AA'], + ), + ], + ); + + await tester.pumpWidget( + MaterialApp( + theme: buildAppTheme(), + home: Scaffold( + body: SyncRunExpansionTile(event: event), + ), + ), + ); + await tester.tap(find.byType(ExpansionTile)); + await tester.pumpAndSettle(); + + expect(find.text(slotWire), findsOneWidget); + expect(find.textContaining('2 assets: A, AA'), findsOneWidget); + expect( + find.textContaining('Backfill fetches (Alpaca start / raw.slot_start)'), + findsOneWidget, + ); + }); +}