feat: scan command — full table scan to CSV with throttling
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -1,9 +1,30 @@
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
|
||||
from order_bak.config import load_config
|
||||
|
||||
|
||||
@click.group()
|
||||
def main():
|
||||
pass
|
||||
@click.option("--config", "-c", default="~/.config/order-bak/config.toml", type=click.Path())
|
||||
@click.pass_context
|
||||
def main(ctx, config):
|
||||
ctx.ensure_object(dict)
|
||||
ctx.obj["config_path"] = Path(config).expanduser()
|
||||
|
||||
|
||||
@main.command()
|
||||
@click.pass_context
|
||||
def scan(ctx):
|
||||
cfg = load_config(ctx.obj["config_path"])
|
||||
from order_bak.scan import scan_orders
|
||||
|
||||
scan_orders(
|
||||
connection_string=cfg.azure_connection_string,
|
||||
page_size=cfg.scan_page_size,
|
||||
delay_ms=cfg.scan_delay_ms,
|
||||
scan_dir=cfg.scan_dir,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
37
src/order_bak/scan.py
Normal file
37
src/order_bak/scan.py
Normal file
@@ -0,0 +1,37 @@
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from azure.data.tables import TableClient
|
||||
|
||||
from order_bak.csv_utils import CSV_COLUMNS, entity_to_row, write_csv
|
||||
|
||||
|
||||
def scan_orders(
|
||||
connection_string: str,
|
||||
page_size: int,
|
||||
delay_ms: int,
|
||||
scan_dir: Path,
|
||||
) -> Path:
|
||||
table = TableClient.from_connection_string(connection_string, table_name="order")
|
||||
scan_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
csv_path = scan_dir / f"scan-{now.strftime('%Y-%m-%d')}.csv"
|
||||
|
||||
paged = table.list_entities(results_per_page=page_size)
|
||||
|
||||
total_scanned = 0
|
||||
entities_batch: list[dict] = []
|
||||
|
||||
for page in paged.by_page():
|
||||
for entity in page:
|
||||
entities_batch.append(entity)
|
||||
total_scanned += 1
|
||||
|
||||
if delay_ms > 0:
|
||||
time.sleep(delay_ms / 1000.0)
|
||||
|
||||
write_csv(csv_path, entities_batch)
|
||||
print(f"Scan complete: {total_scanned} orders saved to {csv_path}")
|
||||
return csv_path
|
||||
Reference in New Issue
Block a user