fix: stream scan writes instead of accumulating in memory

This commit is contained in:
2026-04-21 15:53:50 +08:00
parent 8070f2f99b
commit b135a49de5

View File

@@ -1,10 +1,11 @@
import csv
import time
from datetime import datetime, timezone
from pathlib import Path
from azure.data.tables import TableClient
from order_bak.csv_utils import CSV_COLUMNS, entity_to_row, write_csv
from order_bak.csv_utils import CSV_COLUMNS
def scan_orders(
@@ -22,16 +23,26 @@ def scan_orders(
paged = table.list_entities(results_per_page=page_size)
total_scanned = 0
entities_batch: list[dict] = []
with open(csv_path, "w", newline="") as f:
writer = csv.DictWriter(f, fieldnames=CSV_COLUMNS)
writer.writeheader()
for page in paged.by_page():
for entity in page:
entities_batch.append(entity)
writer.writerow({col: _fmt(entity.get(col)) for col in CSV_COLUMNS})
total_scanned += 1
if delay_ms > 0:
time.sleep(delay_ms / 1000.0)
write_csv(csv_path, entities_batch)
print(f"Scan complete: {total_scanned} orders saved to {csv_path}")
return csv_path
def _fmt(val):
if val is None:
return ""
if isinstance(val, datetime):
return val.isoformat()
return str(val)