Failed upload breaks compaction

0.5.6/0.5.8

time=2026-02-20T03:03:40.246+01:00 level=INFO msg="compaction complete" level=1 txid.min=000000000004b197 txid.max=000000000004b19a size=457940
time=2026-02-20T03:03:51.940+01:00 level=INFO msg="l0 retention enforced" db=state.db deleted_count=1 max_l1_txid=000000000004b19a
time=2026-02-20T03:04:16.195+01:00 level=ERROR msg="compaction failed" level=3 error="open ltx file: s3: get object state/0002/0000000000034936-000000000003498a.ltx: operation error S3: GetObject, https response error StatusCode: 504, RequestID: N/A, HostID: N/A, api error GatewayTimeout: The server did not respond in time.\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
time=2026-02-20T03:05:08.007+01:00 level=ERROR msg="enforce L0 time retention" db=state.db error="remove expired l0 files: s3: delete batch of 1 objects: operation error S3: DeleteObjects, https response error StatusCode: 200, RequestID: tx00000801ec216c637ecf6-006997c117-1a354df3-hel1-prod1-ceph3, HostID: , deserialization failed, failed to decode response body, stream error: stream ID 269099; CANCEL; received from peer"
time=2026-02-20T03:05:08.007+01:00 level=INFO msg="compaction complete" level=1 txid.min=000000000004b19b txid.max=000000000004b19d size=961373
time=2026-02-20T03:05:08.787+01:00 level=ERROR msg="l0 retention enforcement failed" path=/var/lib/rancher/k3s/server/db/state.db error="remove expired l0 files: s3: delete batch of 1 objects: operation error S3: DeleteObjects, https response error StatusCode: 200, RequestID: tx000005ff60d6a25fec950-006997c118-1a3f4e79-hel1-prod1-ceph3, HostID: , deserialization failed, failed to decode response body, stream error: stream ID 269117; CANCEL; received from peer"
time=2026-02-20T03:05:20.142+01:00 level=INFO msg="compaction complete" level=2 txid.min=000000000004b17a txid.max=000000000004b19d size=7154591
time=2026-02-20T03:06:46.721+01:00 level=INFO msg="l0 retention enforced" db=state.db deleted_count=9 max_l1_txid=000000000004b19d
time=2026-02-20T03:06:48.130+01:00 level=INFO msg="l0 retention enforced" db=state.db deleted_count=9 max_l1_txid=000000000004b1ab
time=2026-02-20T03:06:48.130+01:00 level=INFO msg="compaction complete" level=1 txid.min=000000000004b19e txid.max=000000000004b1ab size=1821755
time=2026-02-20T03:06:52.372+01:00 level=INFO msg="l0 retention enforced" db=state.db deleted_count=5 max_l1_txid=000000000004b1ab
time=2026-02-20T03:08:15.130+01:00 level=INFO msg="l0 retention enforced" db=state.db deleted_count=15 max_l1_txid=000000000004b1ab
time=2026-02-20T03:09:19.671+01:00 level=ERROR msg="monitor error" db=state.db replica=s3 error="write ltx file: s3: upload to state/0000/000000000004b1c7-000000000004b1c7.ltx: operation error S3: PutObject, https response error StatusCode: 504, RequestID: N/A, HostID: N/A, api error GatewayTimeout: The server did not respond in time.\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" consecutive_errors=1 backoff=1s
time=2026-02-20T03:10:09.401+01:00 level=INFO msg="compaction complete" level=2 txid.min=000000000004b19e txid.max=000000000004b1ab size=1821755
time=2026-02-20T03:10:20.673+01:00 level=ERROR msg="monitor error" db=state.db replica=s3 error="calc pos: max ltx file: operation error S3: ListObjectsV2, https response error StatusCode: 504, RequestID: N/A, HostID: N/A, api error GatewayTimeout: The server did not respond in time.\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested" consecutive_errors=2 backoff=2s
time=2026-02-20T03:11:30.851+01:00 level=ERROR msg="compaction failed" level=1 error="write ltx file: extract timestamp from LTX header: non-contiguous transaction ids in input files: (000000000004b1c6,000000000004b1c6) -> (000000000004b1c8,000000000004b1c8)"
time=2026-02-20T03:11:37.160+01:00 level=INFO msg="l0 retention enforced" db=state.db deleted_count=19 max_l1_txid=000000000004b1ab
time=2026-02-20T03:11:53.183+01:00 level=INFO msg="replica sync recovered" db=state.db replica=s3 previous_errors=2
time=2026-02-20T03:12:00.987+01:00 level=ERROR msg="compaction failed" level=1 error="write ltx file: extract timestamp from LTX header: non-contiguous transaction ids in input files: (000000000004b1c6,000000000004b1c6) -> (000000000004b1c8,000000000004b1c8)"
time=2026-02-20T03:12:30.318+01:00 level=ERROR msg="compaction failed" level=1 error="write ltx file: extract timestamp from LTX header: non-contiguous transaction ids in input files: (000000000004b1c6,000000000004b1c6) -> (000000000004b1c8,000000000004b1c8)"
time=2026-02-20T03:13:00.597+01:00 level=ERROR msg="compaction failed" level=1 error="write ltx file: extract timestamp from LTX header: non-contiguous transaction ids in input files: (000000000004b1c6,000000000004b1c6) -> (000000000004b1c8,000000000004b1c8)"
time=2026-02-20T03:13:30.551+01:00 level=ERROR msg="compaction failed" level=1 error="write ltx file: extract timestamp from LTX header: non-contiguous transaction ids in input files: (000000000004b1c6,000000000004b1c6) -> (000000000004b1c8,000000000004b1c8)"
time=2026-02-20T03:14:01.312+01:00 level=ERROR msg="compaction failed" level=1 error="write ltx file: extract timestamp from LTX header: non-contiguous transaction ids in input files: (000000000004b1c6,000000000004b1c6) -> (000000000004b1c8,000000000004b1c8)"
time=2026-02-20T03:14:31.243+01:00 level=ERROR msg="compaction failed" level=1 error="write ltx file: extract timestamp from LTX header: non-contiguous transaction ids in input files: (000000000004b1c6,000000000004b1c6) -> (000000000004b1c8,000000000004b1c8)"