feat: scan command — full table scan to CSV with throttling
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -1,9 +1,30 @@
|
|||||||
|
from pathlib import Path
|
||||||
|
|
||||||
import click
|
import click
|
||||||
|
|
||||||
|
from order_bak.config import load_config
|
||||||
|
|
||||||
|
|
||||||
@click.group()
|
@click.group()
|
||||||
def main():
|
@click.option("--config", "-c", default="~/.config/order-bak/config.toml", type=click.Path())
|
||||||
pass
|
@click.pass_context
|
||||||
|
def main(ctx, config):
|
||||||
|
ctx.ensure_object(dict)
|
||||||
|
ctx.obj["config_path"] = Path(config).expanduser()
|
||||||
|
|
||||||
|
|
||||||
|
@main.command()
|
||||||
|
@click.pass_context
|
||||||
|
def scan(ctx):
|
||||||
|
cfg = load_config(ctx.obj["config_path"])
|
||||||
|
from order_bak.scan import scan_orders
|
||||||
|
|
||||||
|
scan_orders(
|
||||||
|
connection_string=cfg.azure_connection_string,
|
||||||
|
page_size=cfg.scan_page_size,
|
||||||
|
delay_ms=cfg.scan_delay_ms,
|
||||||
|
scan_dir=cfg.scan_dir,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
37
src/order_bak/scan.py
Normal file
37
src/order_bak/scan.py
Normal file
@@ -0,0 +1,37 @@
|
|||||||
|
import time
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from azure.data.tables import TableClient
|
||||||
|
|
||||||
|
from order_bak.csv_utils import CSV_COLUMNS, entity_to_row, write_csv
|
||||||
|
|
||||||
|
|
||||||
|
def scan_orders(
|
||||||
|
connection_string: str,
|
||||||
|
page_size: int,
|
||||||
|
delay_ms: int,
|
||||||
|
scan_dir: Path,
|
||||||
|
) -> Path:
|
||||||
|
table = TableClient.from_connection_string(connection_string, table_name="order")
|
||||||
|
scan_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
csv_path = scan_dir / f"scan-{now.strftime('%Y-%m-%d')}.csv"
|
||||||
|
|
||||||
|
paged = table.list_entities(results_per_page=page_size)
|
||||||
|
|
||||||
|
total_scanned = 0
|
||||||
|
entities_batch: list[dict] = []
|
||||||
|
|
||||||
|
for page in paged.by_page():
|
||||||
|
for entity in page:
|
||||||
|
entities_batch.append(entity)
|
||||||
|
total_scanned += 1
|
||||||
|
|
||||||
|
if delay_ms > 0:
|
||||||
|
time.sleep(delay_ms / 1000.0)
|
||||||
|
|
||||||
|
write_csv(csv_path, entities_batch)
|
||||||
|
print(f"Scan complete: {total_scanned} orders saved to {csv_path}")
|
||||||
|
return csv_path
|
||||||
Reference in New Issue
Block a user