cyberhybridhub/server/test/integration/market_data_retention_test.dart

@Tags(['integration', 'postgres'])
library;

import 'package:cyberhybridhub_server/trading/market_data_db.dart';
import 'package:cyberhybridhub_server/trading/market_data_retention.dart';
import 'package:postgres/postgres.dart';
import 'package:test/test.dart';

import '../helpers/test_db.dart';

void main() {
  TestDb? testDb;

  setUpAll(() async {
    testDb = await TestDb.open();
  });

  tearDown(() async {
    if (testDb != null) {
      await testDb!.connection.execute('TRUNCATE TABLE market_data_archive');
      await testDb!.truncateTradingTables();
    }
  });

  tearDownAll(() async {
    await testDb?.close();
  });

  MarketDataRetention retention({void Function(String sql)? onExecute}) {
    return MarketDataRetention(
      connection: testDb!.connection,
      onExecute: onExecute,
    );
  }

  group('runCleanup (hard delete)', () {
    test('deletes rows older than window and keeps rows within window',
        () async {
      if (testDb == null) {
        markTestSkipped(
          'Set DATABASE_URL or TEST_DATABASE_URL for integration tests',
        );
        return;
      }

      final MarketDataDb db = testDb!.marketDataDb;
      final DateTime now = DateTime.utc(2026, 5, 26, 12);
      final DateTime cutoff = now.subtract(const Duration(days: 7));

      int removedCount = 0;
      int keptCount = 0;

      for (int day = 0; day < 10; day++) {
        final DateTime asOf = now.subtract(Duration(days: 14 - day));
        await db.upsertSnapshot(
          symbol: 'SPY',
          metric: 'bar',
          timeframe: '1Day',
          asOf: asOf,
          price: 490 + day,
        );
        if (asOf.isBefore(cutoff)) {
          removedCount++;
        } else {
          keptCount++;
        }
      }

      final MarketDataRetentionResult result = await retention().runCleanup(
        now: now,
      );

      expect(result.error, isNull);
      expect(result.rowsRemoved, removedCount);
      expect(keptCount, greaterThan(0));

      final Result remaining = await testDb!.connection.execute(
        'SELECT COUNT(*)::int FROM market_data_snapshots',
      );
      expect((remaining.first[0]! as num).toInt(), keptCount);
    });

    test('empty table returns rowsRemoved 0 without throwing', () async {
      if (testDb == null) {
        markTestSkipped(
          'Set DATABASE_URL or TEST_DATABASE_URL for integration tests',
        );
        return;
      }

      final MarketDataRetentionResult result = await retention().runCleanup(
        now: DateTime.utc(2026, 5, 26, 12),
      );

      expect(result.rowsRemoved, 0);
      expect(result.error, isNull);
    });

    test('batchSize issues multiple DELETE statements for large backlog',
        () async {
      if (testDb == null) {
        markTestSkipped(
          'Set DATABASE_URL or TEST_DATABASE_URL for integration tests',
        );
        return;
      }

      const int rowCount = 5000;
      const int batchSize = 1000;
      final DateTime now = DateTime.utc(2026, 5, 26, 12);
      final DateTime oldAsOf = now.subtract(const Duration(days: 14));

      await testDb!.connection.execute(
        Sql.named(
          '''
          INSERT INTO market_data_snapshots (symbol, metric, timeframe, as_of, price)
          SELECT 'SYM' || g::text, 'bar', '1Day', @as_of, 100
          FROM generate_series(1, @count) AS g
          ''',
        ),
        parameters: <String, dynamic>{
          'as_of': oldAsOf,
          'count': rowCount,
        },
      );

      int deleteStatements = 0;
      final MarketDataRetentionResult result = await MarketDataRetention(
        connection: testDb!.connection,
        batchSize: batchSize,
        onExecute: (String sql) {
          if (sql.toUpperCase().contains('DELETE FROM MARKET_DATA_SNAPSHOTS')) {
            deleteStatements++;
          }
        },
      ).runCleanup(now: now);

      expect(result.rowsRemoved, rowCount);
      expect(deleteStatements, greaterThanOrEqualTo(5));
    });

    test('writes market_data_sync_runs row with kind cleanup and rows_removed',
        () async {
      if (testDb == null) {
        markTestSkipped(
          'Set DATABASE_URL or TEST_DATABASE_URL for integration tests',
        );
        return;
      }

      final DateTime now = DateTime.utc(2026, 5, 26, 12);
      await testDb!.marketDataDb.upsertSnapshot(
        symbol: 'SPY',
        metric: 'bar',
        timeframe: '1Day',
        asOf: now.subtract(const Duration(days: 10)),
        price: 480,
      );

      await retention().runCleanup(now: now);

      final Result runs = await testDb!.connection.execute(
        '''
        SELECT kind, rows_removed, finished_at, error
        FROM market_data_sync_runs
        ORDER BY id DESC
        LIMIT 1
        ''',
      );
      expect(runs, hasLength(1));
      expect(runs.first[0], 'cleanup');
      expect((runs.first[1]! as num).toInt(), 1);
      expect(runs.first[2], isNotNull);
      expect(runs.first[3], isNull);
    });

    test('rows within window are never deleted (by id)', () async {
      if (testDb == null) {
        markTestSkipped(
          'Set DATABASE_URL or TEST_DATABASE_URL for integration tests',
        );
        return;
      }

      final MarketDataDb db = testDb!.marketDataDb;
      final DateTime now = DateTime.utc(2026, 5, 26, 12);

      final MarketDataSnapshot kept = await db.upsertSnapshot(
        symbol: 'SPY',
        metric: 'bar',
        timeframe: '1Day',
        asOf: now.subtract(const Duration(days: 2)),
        price: 500,
      );
      await db.upsertSnapshot(
        symbol: 'SPY',
        metric: 'bar',
        timeframe: '1Day',
        asOf: now.subtract(const Duration(days: 10)),
        price: 480,
      );

      await retention().runCleanup(now: now);

      final Result row = await testDb!.connection.execute(
        Sql.named('SELECT id FROM market_data_snapshots WHERE id = @id'),
        parameters: <String, dynamic>{'id': kept.id},
      );
      expect(row, hasLength(1));
    });
  });

  group('runArchiveAndCleanup', () {
    test('archiveEnabled copies expired rows then deletes them', () async {
      if (testDb == null) {
        markTestSkipped(
          'Set DATABASE_URL or TEST_DATABASE_URL for integration tests',
        );
        return;
      }

      final DateTime now = DateTime.utc(2026, 5, 26, 12);
      await testDb!.marketDataDb.upsertSnapshot(
        symbol: 'SPY',
        metric: 'bar',
        timeframe: '1Day',
        asOf: now.subtract(const Duration(days: 10)),
        price: 480,
        volume: 1000,
        raw: <String, dynamic>{'c': 480},
      );
      await testDb!.marketDataDb.upsertSnapshot(
        symbol: 'SPY',
        metric: 'bar',
        timeframe: '1Day',
        asOf: now.subtract(const Duration(days: 2)),
        price: 500,
      );

      final MarketDataRetentionResult result =
          await retention().runArchiveAndCleanup(now: now);

      expect(result.error, isNull);
      expect(result.rowsRemoved, 1);

      final Result archiveCount = await testDb!.connection.execute(
        'SELECT COUNT(*)::int FROM market_data_archive',
      );
      expect((archiveCount.first[0]! as num).toInt(), 1);

      final Result liveCount = await testDb!.connection.execute(
        'SELECT COUNT(*)::int FROM market_data_snapshots',
      );
      expect((liveCount.first[0]! as num).toInt(), 1);
    });

    test('archive failure rolls back delete and records error', () async {
      if (testDb == null) {
        markTestSkipped(
          'Set DATABASE_URL or TEST_DATABASE_URL for integration tests',
        );
        return;
      }

      final DateTime now = DateTime.utc(2026, 5, 26, 12);
      await testDb!.marketDataDb.upsertSnapshot(
        symbol: 'SPY',
        metric: 'bar',
        timeframe: '1Day',
        asOf: now.subtract(const Duration(days: 10)),
        price: 480,
      );

      await testDb!.connection.execute(
        'ALTER TABLE market_data_archive RENAME TO market_data_archive_hidden',
      );

      final MarketDataRetentionResult result =
          await retention().runArchiveAndCleanup(now: now);

      expect(result.error, isNotNull);
      expect(result.rowsRemoved, 0);

      final Result liveCount = await testDb!.connection.execute(
        'SELECT COUNT(*)::int FROM market_data_snapshots',
      );
      expect((liveCount.first[0]! as num).toInt(), 1);

      await testDb!.connection.execute(
        'ALTER TABLE market_data_archive_hidden RENAME TO market_data_archive',
      );
    });

    test('runCleanup does not write to archive table', () async {
      if (testDb == null) {
        markTestSkipped(
          'Set DATABASE_URL or TEST_DATABASE_URL for integration tests',
        );
        return;
      }

      final DateTime now = DateTime.utc(2026, 5, 26, 12);
      await testDb!.marketDataDb.upsertSnapshot(
        symbol: 'SPY',
        metric: 'bar',
        timeframe: '1Day',
        asOf: now.subtract(const Duration(days: 10)),
        price: 480,
      );

      await retention().runCleanup(now: now);

      final Result archiveCount = await testDb!.connection.execute(
        'SELECT COUNT(*)::int FROM market_data_archive',
      );
      expect((archiveCount.first[0]! as num).toInt(), 0);
    });
  });
}