fix: stream scan writes instead of accumulating in memory
This commit is contained in:
@@ -1,10 +1,11 @@
|
||||
import csv
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from azure.data.tables import TableClient
|
||||
|
||||
from order_bak.csv_utils import CSV_COLUMNS, entity_to_row, write_csv
|
||||
from order_bak.csv_utils import CSV_COLUMNS
|
||||
|
||||
|
||||
def scan_orders(
|
||||
@@ -22,16 +23,26 @@ def scan_orders(
|
||||
paged = table.list_entities(results_per_page=page_size)
|
||||
|
||||
total_scanned = 0
|
||||
entities_batch: list[dict] = []
|
||||
|
||||
with open(csv_path, "w", newline="") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=CSV_COLUMNS)
|
||||
writer.writeheader()
|
||||
|
||||
for page in paged.by_page():
|
||||
for entity in page:
|
||||
entities_batch.append(entity)
|
||||
writer.writerow({col: _fmt(entity.get(col)) for col in CSV_COLUMNS})
|
||||
total_scanned += 1
|
||||
|
||||
if delay_ms > 0:
|
||||
time.sleep(delay_ms / 1000.0)
|
||||
|
||||
write_csv(csv_path, entities_batch)
|
||||
print(f"Scan complete: {total_scanned} orders saved to {csv_path}")
|
||||
return csv_path
|
||||
|
||||
|
||||
def _fmt(val):
|
||||
if val is None:
|
||||
return ""
|
||||
if isinstance(val, datetime):
|
||||
return val.isoformat()
|
||||
return str(val)
|
||||
|
||||
Reference in New Issue
Block a user