{ "driver": "scheduler", "faults": [ { "command": "taskset -c 0 stress_fixture --db /home/runner/work/monokulo/monokulo/target/coverage/stress/fault-rpc.db --driver scheduler --tenants 16 --orders 2 --large-window-orders 300 --seed 24601 --background-readers 2 --background-http-readers 2 --background-writers 1 --warmup 0 --measured 2 --drain 6 --write-lock-ms 25 --rpc-delay-ms 2 --rpc-fail-until-height 2 --rpc-fail-every 3", "detail": "6 injected RPC failures", "file": "fault-rpc", "fixture": { "background_admin_writers": 1, "background_http_readers": 2, "background_http_reads_completed": 32, "background_readers": 2, "background_reads_completed": 68, "background_writes_completed": 15, "checkpoint_busy": 0, "custody_delay_ms": 0, "custody_max_wait_us": 0, "custody_scans_completed": 0, "custody_slots": 0, "database_bytes": 471040, "db_jobs_completed": 175, "db_write_query_max_us": 33494, "db_write_queue_wait_max_us": 30182, "drain_ticks": 6, "driver": "scheduler", "http_max_latency_us": 4379, "large_window_orders": 300, "measured_duration_ms": 107, "measured_ticks": 2, "orders_per_tenant": 2, "points": [ { "duration_ms": 65, "error": null, "highwater": 1, "lagging_tenants": 0, "min_tenant_cursor": 1, "ok": true, "oldest_lag_blocks": 0, "phase": "measured", "process_memory": { "peak_resident_bytes": 15716352, "resident_bytes": 15716352 }, "progressed_tenants": 0, "tick": 0 }, { "duration_ms": 42, "error": null, "highwater": 1, "lagging_tenants": 0, "min_tenant_cursor": 1, "ok": true, "oldest_lag_blocks": 0, "phase": "measured", "process_memory": { "peak_resident_bytes": 15863808, "resident_bytes": 15863808 }, "progressed_tenants": 0, "tick": 1 }, { "duration_ms": 55, "error": null, "highwater": 3, "lagging_tenants": 0, "min_tenant_cursor": 3, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16039936, "resident_bytes": 16039936 }, "progressed_tenants": 16, "tick": 2 }, { "duration_ms": 49, "error": null, "highwater": 4, "lagging_tenants": 0, "min_tenant_cursor": 4, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16093184, "resident_bytes": 16093184 }, "progressed_tenants": 16, "tick": 3 }, { "duration_ms": 38, "error": null, "highwater": 5, "lagging_tenants": 0, "min_tenant_cursor": 5, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16109568, "resident_bytes": 16109568 }, "progressed_tenants": 16, "tick": 4 }, { "duration_ms": 49, "error": null, "highwater": 6, "lagging_tenants": 0, "min_tenant_cursor": 6, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16150528, "resident_bytes": 16150528 }, "progressed_tenants": 16, "tick": 5 }, { "duration_ms": 36, "error": null, "highwater": 7, "lagging_tenants": 0, "min_tenant_cursor": 7, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16162816, "resident_bytes": 16162816 }, "progressed_tenants": 16, "tick": 6 }, { "duration_ms": 38, "error": null, "highwater": 8, "lagging_tenants": 0, "min_tenant_cursor": 8, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16187392, "resident_bytes": 16187392 }, "progressed_tenants": 16, "tick": 7 } ], "read_max_latency_us": 2660, "rpc_calls": 64, "rpc_delay_ms": 2, "rpc_fail_every": 3, "rpc_fail_until_height": 2, "rpc_failures": 6, "schema_version": 3, "seed": 24601, "sqlite_version": "3.53.2", "tenants": 16, "timer_max_delay_us": 4990, "wal_bytes": 4185952, "wal_checkpointed_pages": 299, "wal_log_pages": 299, "warmup_ticks": 0, "write_lock_hold_ms_per_tick": 25, "write_lock_until_tick": 18446744073709551615, "write_max_latency_us": 33944 }, "min_tenant_cursor": 8, "name": "transient daemon RPC failures under SQLite contention", "scenario_checksum": "d61357425730fcd865c51b5b530f3211f9aae8002453aaafb8f1f4a15bad3f27", "scenario_version": 3, "status": "recovered", "tenants": 16 }, { "command": "taskset -c 0 stress_fixture --db /home/runner/work/monokulo/monokulo/target/coverage/stress/fault-sqlite-lock.db --driver scheduler --tenants 16 --orders 2 --large-window-orders 300 --seed 24601 --background-readers 2 --background-http-readers 2 --background-writers 1 --warmup 0 --measured 2 --drain 6 --write-lock-ms 1500 --write-lock-until-tick 2", "detail": "1500 ms per tick for two ticks", "file": "fault-sqlite-lock", "fixture": { "background_admin_writers": 1, "background_http_readers": 2, "background_http_reads_completed": 294, "background_readers": 2, "background_reads_completed": 566, "background_writes_completed": 9, "checkpoint_busy": 0, "custody_delay_ms": 0, "custody_max_wait_us": 0, "custody_scans_completed": 0, "custody_slots": 0, "database_bytes": 471040, "db_jobs_completed": 172, "db_write_query_max_us": 1530408, "db_write_queue_wait_max_us": 1529536, "drain_ticks": 6, "driver": "scheduler", "http_max_latency_us": 1659, "large_window_orders": 300, "measured_duration_ms": 3095, "measured_ticks": 2, "orders_per_tenant": 2, "points": [ { "duration_ms": 1556, "error": null, "highwater": 1, "lagging_tenants": 0, "min_tenant_cursor": 1, "ok": true, "oldest_lag_blocks": 0, "phase": "measured", "process_memory": { "peak_resident_bytes": 15859712, "resident_bytes": 15859712 }, "progressed_tenants": 0, "tick": 0 }, { "duration_ms": 1539, "error": null, "highwater": 2, "lagging_tenants": 0, "min_tenant_cursor": 2, "ok": true, "oldest_lag_blocks": 0, "phase": "measured", "process_memory": { "peak_resident_bytes": 15994880, "resident_bytes": 15994880 }, "progressed_tenants": 16, "tick": 1 }, { "duration_ms": 8, "error": null, "highwater": 3, "lagging_tenants": 0, "min_tenant_cursor": 3, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16019456, "resident_bytes": 16019456 }, "progressed_tenants": 16, "tick": 2 }, { "duration_ms": 8, "error": null, "highwater": 4, "lagging_tenants": 0, "min_tenant_cursor": 4, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16039936, "resident_bytes": 16039936 }, "progressed_tenants": 16, "tick": 3 }, { "duration_ms": 8, "error": null, "highwater": 5, "lagging_tenants": 0, "min_tenant_cursor": 5, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16072704, "resident_bytes": 16072704 }, "progressed_tenants": 16, "tick": 4 }, { "duration_ms": 8, "error": null, "highwater": 6, "lagging_tenants": 0, "min_tenant_cursor": 6, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16080896, "resident_bytes": 16080896 }, "progressed_tenants": 16, "tick": 5 }, { "duration_ms": 8, "error": null, "highwater": 7, "lagging_tenants": 0, "min_tenant_cursor": 7, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16105472, "resident_bytes": 16105472 }, "progressed_tenants": 16, "tick": 6 }, { "duration_ms": 8, "error": null, "highwater": 8, "lagging_tenants": 0, "min_tenant_cursor": 8, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16117760, "resident_bytes": 16117760 }, "progressed_tenants": 16, "tick": 7 } ], "read_max_latency_us": 1410, "rpc_calls": 326, "rpc_delay_ms": 0, "rpc_fail_every": 0, "rpc_fail_until_height": 0, "rpc_failures": 0, "schema_version": 3, "seed": 24601, "sqlite_version": "3.53.2", "tenants": 16, "timer_max_delay_us": 1966, "wal_bytes": 4185952, "wal_checkpointed_pages": 318, "wal_log_pages": 318, "warmup_ticks": 0, "write_lock_hold_ms_per_tick": 1500, "write_lock_until_tick": 2, "write_max_latency_us": 1530658 }, "min_tenant_cursor": 8, "name": "blocked SQLite writer, then recovery", "scenario_checksum": "d61357425730fcd865c51b5b530f3211f9aae8002453aaafb8f1f4a15bad3f27", "scenario_version": 3, "status": "recovered", "tenants": 16 }, { "command": "taskset -c 0 stress_fixture --db /home/runner/work/monokulo/monokulo/target/coverage/stress/fault-custody.db --driver scheduler --tenants 16 --orders 2 --large-window-orders 300 --seed 24601 --background-readers 2 --background-http-readers 2 --background-writers 1 --warmup 0 --measured 2 --drain 6 --write-lock-ms 25 --custody-slots 1 --custody-delay-ms 5", "detail": "128 scans; max slot wait 107843 µs", "file": "fault-custody", "fixture": { "background_admin_writers": 1, "background_http_readers": 2, "background_http_reads_completed": 110, "background_readers": 2, "background_reads_completed": 208, "background_writes_completed": 82, "checkpoint_busy": 0, "custody_delay_ms": 5, "custody_max_wait_us": 107843, "custody_scans_completed": 128, "custody_slots": 1, "database_bytes": 471040, "db_jobs_completed": 245, "db_write_query_max_us": 33533, "db_write_queue_wait_max_us": 33558, "drain_ticks": 6, "driver": "scheduler", "http_max_latency_us": 1654, "large_window_orders": 300, "measured_duration_ms": 300, "measured_ticks": 2, "orders_per_tenant": 2, "points": [ { "duration_ms": 157, "error": null, "highwater": 1, "lagging_tenants": 0, "min_tenant_cursor": 1, "ok": true, "oldest_lag_blocks": 0, "phase": "measured", "process_memory": { "peak_resident_bytes": 15613952, "resident_bytes": 15613952 }, "progressed_tenants": 0, "tick": 0 }, { "duration_ms": 143, "error": null, "highwater": 2, "lagging_tenants": 0, "min_tenant_cursor": 2, "ok": true, "oldest_lag_blocks": 0, "phase": "measured", "process_memory": { "peak_resident_bytes": 15761408, "resident_bytes": 15761408 }, "progressed_tenants": 16, "tick": 1 }, { "duration_ms": 140, "error": null, "highwater": 3, "lagging_tenants": 0, "min_tenant_cursor": 3, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 15798272, "resident_bytes": 15798272 }, "progressed_tenants": 16, "tick": 2 }, { "duration_ms": 141, "error": null, "highwater": 4, "lagging_tenants": 0, "min_tenant_cursor": 4, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 15822848, "resident_bytes": 15822848 }, "progressed_tenants": 16, "tick": 3 }, { "duration_ms": 141, "error": null, "highwater": 5, "lagging_tenants": 0, "min_tenant_cursor": 5, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 15867904, "resident_bytes": 15867904 }, "progressed_tenants": 16, "tick": 4 }, { "duration_ms": 143, "error": null, "highwater": 6, "lagging_tenants": 0, "min_tenant_cursor": 6, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 15872000, "resident_bytes": 15872000 }, "progressed_tenants": 16, "tick": 5 }, { "duration_ms": 141, "error": null, "highwater": 7, "lagging_tenants": 0, "min_tenant_cursor": 7, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 15896576, "resident_bytes": 15896576 }, "progressed_tenants": 16, "tick": 6 }, { "duration_ms": 142, "error": null, "highwater": 8, "lagging_tenants": 0, "min_tenant_cursor": 8, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 15900672, "resident_bytes": 15900672 }, "progressed_tenants": 16, "tick": 7 } ], "read_max_latency_us": 1150, "rpc_calls": 142, "rpc_delay_ms": 0, "rpc_fail_every": 0, "rpc_fail_until_height": 0, "rpc_failures": 0, "schema_version": 3, "seed": 24601, "sqlite_version": "3.53.2", "tenants": 16, "timer_max_delay_us": 1917, "wal_bytes": 4185952, "wal_checkpointed_pages": 383, "wal_log_pages": 383, "warmup_ticks": 0, "write_lock_hold_ms_per_tick": 25, "write_lock_until_tick": 18446744073709551615, "write_max_latency_us": 33771 }, "min_tenant_cursor": 8, "name": "single custody scan slot under concurrent tenant scans", "scenario_checksum": "d61357425730fcd865c51b5b530f3211f9aae8002453aaafb8f1f4a15bad3f27", "scenario_version": 3, "status": "recovered", "tenants": 16 } ], "hardware": "hardware.json", "profile": "ci", "results": [ { "command": "taskset -c 0 stress_fixture --db /home/runner/work/monokulo/monokulo/target/coverage/stress/point-32.db --driver scheduler --tenants 32 --orders 2 --large-window-orders 300 --seed 24601 --background-readers 2 --background-http-readers 2 --background-writers 1 --warmup 2 --measured 4 --drain 3 --write-lock-ms 25", "exit_code": 0, "final_lagging_tenants": 0, "final_oldest_lag_blocks": 0, "fixture": { "background_admin_writers": 1, "background_http_readers": 2, "background_http_reads_completed": 42, "background_readers": 2, "background_reads_completed": 80, "background_writes_completed": 19, "checkpoint_busy": 0, "custody_delay_ms": 0, "custody_max_wait_us": 0, "custody_scans_completed": 0, "custody_slots": 0, "database_bytes": 499712, "db_jobs_completed": 211, "db_write_query_max_us": 33572, "db_write_queue_wait_max_us": 33453, "drain_ticks": 3, "driver": "scheduler", "http_max_latency_us": 1193, "large_window_orders": 300, "measured_duration_ms": 184, "measured_ticks": 4, "orders_per_tenant": 2, "points": [ { "duration_ms": 65, "error": null, "highwater": 1, "lagging_tenants": 0, "min_tenant_cursor": 1, "ok": true, "oldest_lag_blocks": 0, "phase": "warmup", "process_memory": { "peak_resident_bytes": 15847424, "resident_bytes": 15847424 }, "progressed_tenants": 0, "tick": 0 }, { "duration_ms": 46, "error": null, "highwater": 2, "lagging_tenants": 0, "min_tenant_cursor": 2, "ok": true, "oldest_lag_blocks": 0, "phase": "warmup", "process_memory": { "peak_resident_bytes": 16007168, "resident_bytes": 16007168 }, "progressed_tenants": 32, "tick": 1 }, { "duration_ms": 46, "error": null, "highwater": 3, "lagging_tenants": 0, "min_tenant_cursor": 3, "ok": true, "oldest_lag_blocks": 0, "phase": "measured", "process_memory": { "peak_resident_bytes": 16084992, "resident_bytes": 16084992 }, "progressed_tenants": 32, "tick": 2 }, { "duration_ms": 46, "error": null, "highwater": 4, "lagging_tenants": 0, "min_tenant_cursor": 4, "ok": true, "oldest_lag_blocks": 0, "phase": "measured", "process_memory": { "peak_resident_bytes": 16097280, "resident_bytes": 16097280 }, "progressed_tenants": 32, "tick": 3 }, { "duration_ms": 46, "error": null, "highwater": 5, "lagging_tenants": 0, "min_tenant_cursor": 5, "ok": true, "oldest_lag_blocks": 0, "phase": "measured", "process_memory": { "peak_resident_bytes": 16220160, "resident_bytes": 16220160 }, "progressed_tenants": 32, "tick": 4 }, { "duration_ms": 46, "error": null, "highwater": 6, "lagging_tenants": 0, "min_tenant_cursor": 6, "ok": true, "oldest_lag_blocks": 0, "phase": "measured", "process_memory": { "peak_resident_bytes": 16248832, "resident_bytes": 16248832 }, "progressed_tenants": 32, "tick": 5 }, { "duration_ms": 46, "error": null, "highwater": 7, "lagging_tenants": 0, "min_tenant_cursor": 7, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16273408, "resident_bytes": 16273408 }, "progressed_tenants": 32, "tick": 6 }, { "duration_ms": 46, "error": null, "highwater": 8, "lagging_tenants": 0, "min_tenant_cursor": 8, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16306176, "resident_bytes": 16306176 }, "progressed_tenants": 32, "tick": 7 }, { "duration_ms": 46, "error": null, "highwater": 9, "lagging_tenants": 0, "min_tenant_cursor": 9, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16318464, "resident_bytes": 16318464 }, "progressed_tenants": 32, "tick": 8 } ], "read_max_latency_us": 1219, "rpc_calls": 78, "rpc_delay_ms": 0, "rpc_fail_every": 0, "rpc_fail_until_height": 0, "rpc_failures": 0, "schema_version": 3, "seed": 24601, "sqlite_version": "3.53.2", "tenants": 32, "timer_max_delay_us": 1845, "wal_bytes": 4185952, "wal_checkpointed_pages": 420, "wal_log_pages": 420, "warmup_ticks": 2, "write_lock_hold_ms_per_tick": 25, "write_lock_until_tick": 18446744073709551615, "write_max_latency_us": 33531 }, "hardware": "hardware.json", "measured_duration_ms": 184, "min_tenant_cursor": 9, "peak_resident_bytes": 16318464, "progressed_tenants": 32, "responsive": true, "scenario_checksum": "d61357425730fcd865c51b5b530f3211f9aae8002453aaafb8f1f4a15bad3f27", "scenario_version": 3, "seed": 24601, "status": "sustainable", "tenants": 32, "timer_max_delay_us": 1845 }, { "command": "taskset -c 0 stress_fixture --db /home/runner/work/monokulo/monokulo/target/coverage/stress/point-64.db --driver scheduler --tenants 64 --orders 2 --large-window-orders 300 --seed 24601 --background-readers 2 --background-http-readers 2 --background-writers 1 --warmup 2 --measured 4 --drain 3 --write-lock-ms 25", "exit_code": 0, "final_lagging_tenants": 0, "final_oldest_lag_blocks": 0, "fixture": { "background_admin_writers": 1, "background_http_readers": 2, "background_http_reads_completed": 50, "background_readers": 2, "background_reads_completed": 96, "background_writes_completed": 28, "checkpoint_busy": 0, "custody_delay_ms": 0, "custody_max_wait_us": 0, "custody_scans_completed": 0, "custody_slots": 0, "database_bytes": 569344, "db_jobs_completed": 229, "db_write_query_max_us": 33591, "db_write_queue_wait_max_us": 33460, "drain_ticks": 3, "driver": "scheduler", "http_max_latency_us": 2620, "large_window_orders": 300, "measured_duration_ms": 221, "measured_ticks": 4, "orders_per_tenant": 2, "points": [ { "duration_ms": 80, "error": null, "highwater": 1, "lagging_tenants": 0, "min_tenant_cursor": 1, "ok": true, "oldest_lag_blocks": 0, "phase": "warmup", "process_memory": { "peak_resident_bytes": 15904768, "resident_bytes": 15904768 }, "progressed_tenants": 0, "tick": 0 }, { "duration_ms": 56, "error": null, "highwater": 2, "lagging_tenants": 0, "min_tenant_cursor": 2, "ok": true, "oldest_lag_blocks": 0, "phase": "warmup", "process_memory": { "peak_resident_bytes": 16150528, "resident_bytes": 16150528 }, "progressed_tenants": 64, "tick": 1 }, { "duration_ms": 56, "error": null, "highwater": 3, "lagging_tenants": 0, "min_tenant_cursor": 3, "ok": true, "oldest_lag_blocks": 0, "phase": "measured", "process_memory": { "peak_resident_bytes": 16199680, "resident_bytes": 16199680 }, "progressed_tenants": 64, "tick": 2 }, { "duration_ms": 55, "error": null, "highwater": 4, "lagging_tenants": 0, "min_tenant_cursor": 4, "ok": true, "oldest_lag_blocks": 0, "phase": "measured", "process_memory": { "peak_resident_bytes": 16293888, "resident_bytes": 16293888 }, "progressed_tenants": 64, "tick": 3 }, { "duration_ms": 55, "error": null, "highwater": 5, "lagging_tenants": 0, "min_tenant_cursor": 5, "ok": true, "oldest_lag_blocks": 0, "phase": "measured", "process_memory": { "peak_resident_bytes": 16326656, "resident_bytes": 16326656 }, "progressed_tenants": 64, "tick": 4 }, { "duration_ms": 55, "error": null, "highwater": 6, "lagging_tenants": 0, "min_tenant_cursor": 6, "ok": true, "oldest_lag_blocks": 0, "phase": "measured", "process_memory": { "peak_resident_bytes": 16363520, "resident_bytes": 16363520 }, "progressed_tenants": 64, "tick": 5 }, { "duration_ms": 55, "error": null, "highwater": 7, "lagging_tenants": 0, "min_tenant_cursor": 7, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16388096, "resident_bytes": 16388096 }, "progressed_tenants": 64, "tick": 6 }, { "duration_ms": 55, "error": null, "highwater": 8, "lagging_tenants": 0, "min_tenant_cursor": 8, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16408576, "resident_bytes": 16408576 }, "progressed_tenants": 64, "tick": 7 }, { "duration_ms": 55, "error": null, "highwater": 9, "lagging_tenants": 0, "min_tenant_cursor": 9, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16416768, "resident_bytes": 16416768 }, "progressed_tenants": 64, "tick": 8 } ], "read_max_latency_us": 1301, "rpc_calls": 86, "rpc_delay_ms": 0, "rpc_fail_every": 0, "rpc_fail_until_height": 0, "rpc_failures": 0, "schema_version": 3, "seed": 24601, "sqlite_version": "3.53.2", "tenants": 64, "timer_max_delay_us": 2079, "wal_bytes": 4185952, "wal_checkpointed_pages": 506, "wal_log_pages": 506, "warmup_ticks": 2, "write_lock_hold_ms_per_tick": 25, "write_lock_until_tick": 18446744073709551615, "write_max_latency_us": 33915 }, "hardware": "hardware.json", "measured_duration_ms": 221, "min_tenant_cursor": 9, "peak_resident_bytes": 16416768, "progressed_tenants": 64, "responsive": true, "scenario_checksum": "d61357425730fcd865c51b5b530f3211f9aae8002453aaafb8f1f4a15bad3f27", "scenario_version": 3, "seed": 24601, "status": "sustainable", "tenants": 64, "timer_max_delay_us": 2079 }, { "command": "taskset -c 0 stress_fixture --db /home/runner/work/monokulo/monokulo/target/coverage/stress/point-128.db --driver scheduler --tenants 128 --orders 2 --large-window-orders 300 --seed 24601 --background-readers 2 --background-http-readers 2 --background-writers 1 --warmup 2 --measured 4 --drain 3 --write-lock-ms 25", "exit_code": 0, "final_lagging_tenants": 0, "final_oldest_lag_blocks": 0, "fixture": { "background_admin_writers": 1, "background_http_readers": 2, "background_http_reads_completed": 68, "background_readers": 2, "background_reads_completed": 130, "background_writes_completed": 40, "checkpoint_busy": 0, "custody_delay_ms": 0, "custody_max_wait_us": 0, "custody_scans_completed": 0, "custody_slots": 0, "database_bytes": 704512, "db_jobs_completed": 259, "db_write_query_max_us": 33940, "db_write_queue_wait_max_us": 32943, "drain_ticks": 3, "driver": "scheduler", "http_max_latency_us": 3232, "large_window_orders": 300, "measured_duration_ms": 298, "measured_ticks": 4, "orders_per_tenant": 2, "points": [ { "duration_ms": 107, "error": null, "highwater": 1, "lagging_tenants": 0, "min_tenant_cursor": 1, "ok": true, "oldest_lag_blocks": 0, "phase": "warmup", "process_memory": { "peak_resident_bytes": 16191488, "resident_bytes": 16191488 }, "progressed_tenants": 0, "tick": 0 }, { "duration_ms": 76, "error": null, "highwater": 2, "lagging_tenants": 0, "min_tenant_cursor": 2, "ok": true, "oldest_lag_blocks": 0, "phase": "warmup", "process_memory": { "peak_resident_bytes": 16478208, "resident_bytes": 16478208 }, "progressed_tenants": 128, "tick": 1 }, { "duration_ms": 75, "error": null, "highwater": 3, "lagging_tenants": 0, "min_tenant_cursor": 3, "ok": true, "oldest_lag_blocks": 0, "phase": "measured", "process_memory": { "peak_resident_bytes": 16613376, "resident_bytes": 16613376 }, "progressed_tenants": 128, "tick": 2 }, { "duration_ms": 74, "error": null, "highwater": 4, "lagging_tenants": 0, "min_tenant_cursor": 4, "ok": true, "oldest_lag_blocks": 0, "phase": "measured", "process_memory": { "peak_resident_bytes": 16670720, "resident_bytes": 16670720 }, "progressed_tenants": 128, "tick": 3 }, { "duration_ms": 74, "error": null, "highwater": 5, "lagging_tenants": 0, "min_tenant_cursor": 5, "ok": true, "oldest_lag_blocks": 0, "phase": "measured", "process_memory": { "peak_resident_bytes": 16719872, "resident_bytes": 16719872 }, "progressed_tenants": 128, "tick": 4 }, { "duration_ms": 75, "error": null, "highwater": 6, "lagging_tenants": 0, "min_tenant_cursor": 6, "ok": true, "oldest_lag_blocks": 0, "phase": "measured", "process_memory": { "peak_resident_bytes": 16748544, "resident_bytes": 16748544 }, "progressed_tenants": 128, "tick": 5 }, { "duration_ms": 74, "error": null, "highwater": 7, "lagging_tenants": 0, "min_tenant_cursor": 7, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16801792, "resident_bytes": 16801792 }, "progressed_tenants": 128, "tick": 6 }, { "duration_ms": 76, "error": null, "highwater": 8, "lagging_tenants": 0, "min_tenant_cursor": 8, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16850944, "resident_bytes": 16850944 }, "progressed_tenants": 128, "tick": 7 }, { "duration_ms": 75, "error": null, "highwater": 9, "lagging_tenants": 0, "min_tenant_cursor": 9, "ok": true, "oldest_lag_blocks": 0, "phase": "drain", "process_memory": { "peak_resident_bytes": 16859136, "resident_bytes": 16859136 }, "progressed_tenants": 128, "tick": 8 } ], "read_max_latency_us": 1662, "rpc_calls": 104, "rpc_delay_ms": 0, "rpc_fail_every": 0, "rpc_fail_until_height": 0, "rpc_failures": 0, "schema_version": 3, "seed": 24601, "sqlite_version": "3.53.2", "tenants": 128, "timer_max_delay_us": 2055, "wal_bytes": 4185952, "wal_checkpointed_pages": 672, "wal_log_pages": 672, "warmup_ticks": 2, "write_lock_hold_ms_per_tick": 25, "write_lock_until_tick": 18446744073709551615, "write_max_latency_us": 34488 }, "hardware": "hardware.json", "measured_duration_ms": 298, "min_tenant_cursor": 9, "peak_resident_bytes": 16859136, "progressed_tenants": 128, "responsive": true, "scenario_checksum": "d61357425730fcd865c51b5b530f3211f9aae8002453aaafb8f1f4a15bad3f27", "scenario_version": 3, "seed": 24601, "status": "sustainable", "tenants": 128, "timer_max_delay_us": 2055 } ], "scenario": { "background_admin_writers": 1, "background_http_readers": 2, "background_readers": 2, "ci_tenant_points": [ 32, 64, 128 ], "drain_ticks": 3, "fault_custody_delay_ms": 5, "fault_custody_slots": 1, "fault_drain_ticks": 6, "fault_rpc_delay_ms": 2, "fault_rpc_fail_every": 3, "fault_rpc_fail_until_height": 2, "fault_sqlite_lock_ms": 1500, "fault_tenants": 16, "full_tenant_points": [ 32, 64, 128, 256, 512 ], "large_window_orders": 300, "max_http_latency_ms": 250, "max_oldest_lag_blocks": 3, "max_timer_delay_ms": 100, "measured_ticks": 4, "orders_per_tenant": 2, "poll_interval_ms": 5000, "schema_version": 3, "seed": 24601, "warmup_ticks": 2, "write_lock_hold_ms_per_tick": 25 }, "scenario_checksum": "d61357425730fcd865c51b5b530f3211f9aae8002453aaafb8f1f4a15bad3f27", "started_at_utc": 1792147501 }