fix: stream scan writes instead of accumulating in memory

This commit is contained in:
2026-04-21 15:53:50 +08:00
parent 8070f2f99b
commit b135a49de5

View File

@@ -1,10 +1,11 @@
import csv
import time import time
from datetime import datetime, timezone from datetime import datetime, timezone
from pathlib import Path from pathlib import Path
from azure.data.tables import TableClient from azure.data.tables import TableClient
from order_bak.csv_utils import CSV_COLUMNS, entity_to_row, write_csv from order_bak.csv_utils import CSV_COLUMNS
def scan_orders( def scan_orders(
@@ -22,16 +23,26 @@ def scan_orders(
paged = table.list_entities(results_per_page=page_size) paged = table.list_entities(results_per_page=page_size)
total_scanned = 0 total_scanned = 0
entities_batch: list[dict] = []
with open(csv_path, "w", newline="") as f:
writer = csv.DictWriter(f, fieldnames=CSV_COLUMNS)
writer.writeheader()
for page in paged.by_page(): for page in paged.by_page():
for entity in page: for entity in page:
entities_batch.append(entity) writer.writerow({col: _fmt(entity.get(col)) for col in CSV_COLUMNS})
total_scanned += 1 total_scanned += 1
if delay_ms > 0: if delay_ms > 0:
time.sleep(delay_ms / 1000.0) time.sleep(delay_ms / 1000.0)
write_csv(csv_path, entities_batch)
print(f"Scan complete: {total_scanned} orders saved to {csv_path}") print(f"Scan complete: {total_scanned} orders saved to {csv_path}")
return csv_path return csv_path
def _fmt(val):
if val is None:
return ""
if isinstance(val, datetime):
return val.isoformat()
return str(val)