Compare commits
8 Commits
c5897e5d0d
...
c4290b830e
| Author | SHA1 | Date | |
|---|---|---|---|
| c4290b830e | |||
| dba7f7bdc9 | |||
| 9997ee6f32 | |||
| d3ff2dae98 | |||
| e0eff89e8c | |||
| fac909ca8b | |||
| 34db3dc439 | |||
| ef449729d7 |
12
assets/sandbox.toml
Normal file
12
assets/sandbox.toml
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
|
||||||
|
[azure]
|
||||||
|
connection_string = "DefaultEndpointsProtocol=https;AccountName=n3ordersmokestorage;AccountKey=aETnv18XBLUDVg7W1fu9ilnKchdLRy2sI2pJJyzqG6R/fqwcnIoSm7eM8ObyXFQAsvZrddZZTBN0+AStnOK/wg==;EndpointSuffix=core.windows.net;"
|
||||||
|
|
||||||
|
[scan]
|
||||||
|
page_size = 500
|
||||||
|
delay_ms = 200
|
||||||
|
scan_dir = "./scan"
|
||||||
|
|
||||||
|
[delete]
|
||||||
|
batch_size = 100
|
||||||
|
delay_ms = 300
|
||||||
282
docs/superpowers/plans/2026-04-21-backup-days-param.md
Normal file
282
docs/superpowers/plans/2026-04-21-backup-days-param.md
Normal file
@@ -0,0 +1,282 @@
|
|||||||
|
# Backup --days Parameter Implementation Plan
|
||||||
|
|
||||||
|
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||||
|
|
||||||
|
**Goal:** Replace `--start`/`--end` date range params on the `backup` command with a single `--days N` param that backs up all orders older than N days.
|
||||||
|
|
||||||
|
**Architecture:** The `backup_orders` function currently takes `start`/`end` datetime pair and filters `[start, end)`. We change it to accept a single `cutoff` datetime and filter `OrderTime < cutoff`. The CLI computes `cutoff = today_00_utc - timedelta(days=N)`.
|
||||||
|
|
||||||
|
**Tech Stack:** Python, Click, pytest
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 1: Update `backup.py` — change function signature and filter logic
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `src/order_bak/backup.py`
|
||||||
|
|
||||||
|
- [ ] **Step 1: Write the failing test**
|
||||||
|
|
||||||
|
In `tests/test_backup.py`, add a new test that uses the `cutoff` signature:
|
||||||
|
|
||||||
|
```python
|
||||||
|
def test_backup_filters_by_cutoff(tmp_path):
|
||||||
|
scan_dir = tmp_path / "scan"
|
||||||
|
backup_dir = tmp_path / "backups"
|
||||||
|
scan_dir.mkdir()
|
||||||
|
backup_dir.mkdir()
|
||||||
|
|
||||||
|
entities = [
|
||||||
|
_make_entity("u1", "o1", datetime(2025, 1, 1, 10, tzinfo=timezone.utc), 1),
|
||||||
|
_make_entity("u2", "o2", datetime(2025, 1, 2, 8, tzinfo=timezone.utc), 1),
|
||||||
|
_make_entity("u3", "o3", datetime(2025, 1, 3, 8, tzinfo=timezone.utc), 1), # at/after cutoff
|
||||||
|
]
|
||||||
|
scan_csv = scan_dir / "scan-2025-01-03.csv"
|
||||||
|
write_csv(scan_csv, entities)
|
||||||
|
|
||||||
|
cutoff = datetime(2025, 1, 3, tzinfo=timezone.utc)
|
||||||
|
result = backup_orders(
|
||||||
|
scan_path=scan_csv,
|
||||||
|
backup_dir=backup_dir,
|
||||||
|
cutoff=cutoff,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result == 2
|
||||||
|
assert (backup_dir / "2025-01-01.csv").exists()
|
||||||
|
assert (backup_dir / "2025-01-02.csv").exists()
|
||||||
|
assert not (backup_dir / "2025-01-03.csv").exists()
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: Run test to verify it fails**
|
||||||
|
|
||||||
|
Run: `uv run pytest tests/test_backup.py::test_backup_filters_by_cutoff -v`
|
||||||
|
Expected: FAIL — `backup_orders()` got unexpected keyword `cutoff`
|
||||||
|
|
||||||
|
- [ ] **Step 3: Update `backup_orders` signature and filter**
|
||||||
|
|
||||||
|
In `src/order_bak/backup.py`, replace the function:
|
||||||
|
|
||||||
|
```python
|
||||||
|
def backup_orders(
|
||||||
|
scan_path: Path,
|
||||||
|
backup_dir: Path,
|
||||||
|
cutoff: datetime,
|
||||||
|
) -> int:
|
||||||
|
backup_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
by_date: dict[str, list[dict]] = defaultdict(list)
|
||||||
|
|
||||||
|
with open(scan_path, newline="") as f:
|
||||||
|
reader = csv.DictReader(f)
|
||||||
|
for raw_row in reader:
|
||||||
|
entity = parse_row(raw_row)
|
||||||
|
if entity["state"] != 1:
|
||||||
|
continue
|
||||||
|
order_time = entity["OrderTime"]
|
||||||
|
if order_time is None:
|
||||||
|
continue
|
||||||
|
if order_time >= cutoff:
|
||||||
|
continue
|
||||||
|
date_key = order_time.strftime("%Y-%m-%d")
|
||||||
|
by_date[date_key].append(entity)
|
||||||
|
|
||||||
|
total = 0
|
||||||
|
for date_key, rows in sorted(by_date.items()):
|
||||||
|
csv_path = backup_dir / f"{date_key}.csv"
|
||||||
|
with open(csv_path, "w", newline="") as f:
|
||||||
|
writer = csv.DictWriter(f, fieldnames=CSV_COLUMNS)
|
||||||
|
writer.writeheader()
|
||||||
|
writer.writerows(entity_to_row(r) for r in rows)
|
||||||
|
total += len(rows)
|
||||||
|
print(f" {date_key}: {len(rows)} orders")
|
||||||
|
|
||||||
|
print(f"Backup complete: {total} orders across {len(by_date)} days")
|
||||||
|
return total
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 4: Run test to verify it passes**
|
||||||
|
|
||||||
|
Run: `uv run pytest tests/test_backup.py::test_backup_filters_by_cutoff -v`
|
||||||
|
Expected: PASS
|
||||||
|
|
||||||
|
- [ ] **Step 5: Update the old test to use `cutoff`**
|
||||||
|
|
||||||
|
In `tests/test_backup.py`, replace `test_backup_filters_by_state_and_date`:
|
||||||
|
|
||||||
|
```python
|
||||||
|
def test_backup_filters_by_state_and_date(tmp_path):
|
||||||
|
scan_dir = tmp_path / "scan"
|
||||||
|
backup_dir = tmp_path / "backups"
|
||||||
|
scan_dir.mkdir()
|
||||||
|
backup_dir.mkdir()
|
||||||
|
|
||||||
|
entities = [
|
||||||
|
_make_entity("u1", "o1", datetime(2025, 1, 1, 10, tzinfo=timezone.utc), 1),
|
||||||
|
_make_entity("u2", "o2", datetime(2025, 1, 1, 12, tzinfo=timezone.utc), 0), # Payed, skip
|
||||||
|
_make_entity("u3", "o3", datetime(2025, 1, 2, 8, tzinfo=timezone.utc), 1),
|
||||||
|
_make_entity("u4", "o4", datetime(2025, 1, 3, 8, tzinfo=timezone.utc), 1), # at cutoff, skip
|
||||||
|
]
|
||||||
|
scan_csv = scan_dir / "scan-2025-01-03.csv"
|
||||||
|
write_csv(scan_csv, entities)
|
||||||
|
|
||||||
|
result = backup_orders(
|
||||||
|
scan_path=scan_csv,
|
||||||
|
backup_dir=backup_dir,
|
||||||
|
cutoff=datetime(2025, 1, 3, tzinfo=timezone.utc),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result == 2
|
||||||
|
jan1 = backup_dir / "2025-01-01.csv"
|
||||||
|
jan2 = backup_dir / "2025-01-02.csv"
|
||||||
|
assert jan1.exists()
|
||||||
|
assert jan2.exists()
|
||||||
|
assert not (backup_dir / "2025-01-03.csv").exists()
|
||||||
|
|
||||||
|
from order_bak.csv_utils import read_csv
|
||||||
|
jan1_rows = list(read_csv(jan1))
|
||||||
|
assert len(jan1_rows) == 1
|
||||||
|
assert jan1_rows[0]["RowKey"] == "o1"
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 6: Run all backup tests**
|
||||||
|
|
||||||
|
Run: `uv run pytest tests/test_backup.py -v`
|
||||||
|
Expected: All PASS
|
||||||
|
|
||||||
|
- [ ] **Step 7: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add src/order_bak/backup.py tests/test_backup.py
|
||||||
|
git commit -m "refactor: change backup_orders to use cutoff param instead of start/end"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 2: Update `cli.py` — replace `--start`/`--end` with `--days`
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `src/order_bak/cli.py`
|
||||||
|
|
||||||
|
- [ ] **Step 1: Update the backup command in `cli.py`**
|
||||||
|
|
||||||
|
Replace the `backup` command function in `src/order_bak/cli.py`:
|
||||||
|
|
||||||
|
```python
|
||||||
|
@main.command()
|
||||||
|
@click.option("--days", required=True, type=int, help="Backup orders older than N days (min 1)")
|
||||||
|
@click.option("--scan", "scan_path", default=None, type=click.Path(), help="Path to scan CSV (default: latest in scan_dir)")
|
||||||
|
@click.pass_context
|
||||||
|
def backup(ctx, days, scan_path):
|
||||||
|
if days < 1:
|
||||||
|
click.echo("Error: --days must be at least 1")
|
||||||
|
raise SystemExit(1)
|
||||||
|
|
||||||
|
cfg = load_config(ctx.obj["config_path"])
|
||||||
|
from order_bak.backup import backup_orders
|
||||||
|
|
||||||
|
if scan_path:
|
||||||
|
scan_file = Path(scan_path)
|
||||||
|
else:
|
||||||
|
scan_files = sorted(cfg.scan_dir.glob("scan-*.csv"))
|
||||||
|
if not scan_files:
|
||||||
|
click.echo("No scan files found in scan_dir")
|
||||||
|
raise SystemExit(1)
|
||||||
|
scan_file = scan_files[-1]
|
||||||
|
|
||||||
|
from datetime import timedelta
|
||||||
|
today_utc = datetime.now(timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0)
|
||||||
|
cutoff = today_utc - timedelta(days=days)
|
||||||
|
|
||||||
|
click.echo(f"Backing up orders before {cutoff.strftime('%Y-%m-%d')} UTC")
|
||||||
|
backup_orders(
|
||||||
|
scan_path=scan_file,
|
||||||
|
backup_dir=Path("./backups"),
|
||||||
|
cutoff=cutoff,
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
Add `from datetime import datetime, timedelta, timezone` to the top-level imports (replace the existing `from datetime import timezone`).
|
||||||
|
|
||||||
|
- [ ] **Step 2: Run all tests**
|
||||||
|
|
||||||
|
Run: `uv run pytest tests/ -v`
|
||||||
|
Expected: All PASS (backup tests already updated in Task 1)
|
||||||
|
|
||||||
|
- [ ] **Step 3: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add src/order_bak/cli.py
|
||||||
|
git commit -m "feat: replace --start/--end with --days param on backup command"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 3: Add edge case tests
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `tests/test_backup.py`
|
||||||
|
|
||||||
|
- [ ] **Step 1: Add edge case tests**
|
||||||
|
|
||||||
|
Append to `tests/test_backup.py`:
|
||||||
|
|
||||||
|
```python
|
||||||
|
def test_backup_cutoff_at_midnight(tmp_path):
|
||||||
|
"""Order exactly at cutoff midnight should be excluded."""
|
||||||
|
scan_dir = tmp_path / "scan"
|
||||||
|
backup_dir = tmp_path / "backups"
|
||||||
|
scan_dir.mkdir()
|
||||||
|
backup_dir.mkdir()
|
||||||
|
|
||||||
|
entities = [
|
||||||
|
_make_entity("u1", "o1", datetime(2025, 1, 3, 0, 0, 0, tzinfo=timezone.utc), 1), # exactly at cutoff
|
||||||
|
_make_entity("u2", "o2", datetime(2025, 1, 2, 23, 59, 59, tzinfo=timezone.utc), 1), # just before
|
||||||
|
]
|
||||||
|
scan_csv = scan_dir / "scan.csv"
|
||||||
|
write_csv(scan_csv, entities)
|
||||||
|
|
||||||
|
result = backup_orders(
|
||||||
|
scan_path=scan_csv,
|
||||||
|
backup_dir=backup_dir,
|
||||||
|
cutoff=datetime(2025, 1, 3, tzinfo=timezone.utc),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result == 1
|
||||||
|
assert (backup_dir / "2025-01-02.csv").exists()
|
||||||
|
assert not (backup_dir / "2025-01-03.csv").exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_backup_empty_result(tmp_path):
|
||||||
|
"""No orders before cutoff returns 0 and no files."""
|
||||||
|
scan_dir = tmp_path / "scan"
|
||||||
|
backup_dir = tmp_path / "backups"
|
||||||
|
scan_dir.mkdir()
|
||||||
|
backup_dir.mkdir()
|
||||||
|
|
||||||
|
entities = [
|
||||||
|
_make_entity("u1", "o1", datetime(2025, 1, 5, 10, tzinfo=timezone.utc), 1),
|
||||||
|
]
|
||||||
|
scan_csv = scan_dir / "scan.csv"
|
||||||
|
write_csv(scan_csv, entities)
|
||||||
|
|
||||||
|
result = backup_orders(
|
||||||
|
scan_path=scan_csv,
|
||||||
|
backup_dir=backup_dir,
|
||||||
|
cutoff=datetime(2025, 1, 1, tzinfo=timezone.utc),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result == 0
|
||||||
|
assert list(backup_dir.glob("*.csv")) == []
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: Run all tests**
|
||||||
|
|
||||||
|
Run: `uv run pytest tests/test_backup.py -v`
|
||||||
|
Expected: All 4 tests PASS
|
||||||
|
|
||||||
|
- [ ] **Step 3: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add tests/test_backup.py
|
||||||
|
git commit -m "test: add edge cases for backup cutoff filtering"
|
||||||
|
```
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
# Backup Days Parameter Design
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
Replace the `--start` / `--end` date range parameters on the `backup` command with a single `--days N` parameter that backs up all orders older than N days. The minimum value is 1 (today's data is never backed up).
|
||||||
|
|
||||||
|
## Current Behavior
|
||||||
|
|
||||||
|
```
|
||||||
|
order-bak backup --start 2025-01-01 --end 2025-02-01
|
||||||
|
```
|
||||||
|
|
||||||
|
Backs up orders where `start <= OrderTime < end` and `state == 1`.
|
||||||
|
|
||||||
|
## New Behavior
|
||||||
|
|
||||||
|
```
|
||||||
|
order-bak backup --days 7
|
||||||
|
```
|
||||||
|
|
||||||
|
Backs up all orders where `OrderTime < cutoff` and `state == 1`, where `cutoff = today_utc - timedelta(days=N)`.
|
||||||
|
|
||||||
|
Example: `--days 7` on April 21 → cutoff is April 14 00:00 UTC → backs up all orders with OrderTime before April 14.
|
||||||
|
|
||||||
|
## Changes
|
||||||
|
|
||||||
|
### cli.py
|
||||||
|
|
||||||
|
- Remove `--start` and `--end` options
|
||||||
|
- Add `--days` option: required, int, must be >= 1
|
||||||
|
- Compute `cutoff = datetime.now(timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0) - timedelta(days=days)`
|
||||||
|
- Pass `cutoff` to `backup_orders()`
|
||||||
|
|
||||||
|
### backup.py
|
||||||
|
|
||||||
|
- Change signature from `backup_orders(scan_path, backup_dir, start, end)` to `backup_orders(scan_path, backup_dir, cutoff)`
|
||||||
|
- Replace filter `if order_time < start or order_time >= end` with `if order_time >= cutoff`
|
||||||
|
|
||||||
|
### test_backup.py
|
||||||
|
|
||||||
|
- Update all existing tests to pass `cutoff` instead of `start`/`end`
|
||||||
|
- Add edge cases: cutoff exactly at midnight, N=1 (only today excluded)
|
||||||
|
|
||||||
|
## Unchanged
|
||||||
|
|
||||||
|
- `scan` command, `delete` command, config, csv_utils
|
||||||
|
- State filtering (state == 1), OrderTime null check
|
||||||
|
- Output format (per-date CSV files in backups/)
|
||||||
@@ -1,35 +1,26 @@
|
|||||||
"""Seed test data into Azure Table emulator for integration testing."""
|
"""Seed test data into Azure Table emulator for integration testing."""
|
||||||
|
|
||||||
|
import random
|
||||||
import sys
|
import sys
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
from azure.data.tables import TableServiceClient
|
from azure.data.tables import TableServiceClient
|
||||||
|
|
||||||
DEFAULT_CONNECTION_STRING = (
|
from order_bak.config import load_config
|
||||||
"AccountName=devstoreaccount1;"
|
|
||||||
"AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/"
|
|
||||||
"K1SZFPTOtr/KBHBeksoGMGw==;"
|
|
||||||
"DefaultEndpointsProtocol=http;"
|
|
||||||
"BlobEndpoint=http://192.168.9.101:10000/devstoreaccount1;"
|
|
||||||
"QueueEndpoint=http://192.168.9.101:10001/devstoreaccount1;"
|
|
||||||
"TableEndpoint=http://192.168.9.101:10002/devstoreaccount1;"
|
|
||||||
)
|
|
||||||
|
|
||||||
TABLE_NAME = "order"
|
TABLE_NAME = "order"
|
||||||
|
|
||||||
# Test users
|
USER_COUNT = 1000
|
||||||
USERS = ["user001", "user002", "user003", "user004", "user005"]
|
ENTRIES_PER_USER_PER_DAY = 2
|
||||||
|
PAYED_PER_USER_PER_DAY = 1
|
||||||
# 3 days of data: Jan 1-3, 2025
|
DATE_START = datetime(2026, 1, 1, tzinfo=timezone.utc)
|
||||||
DATES = [
|
DATE_END = datetime.now(timezone.utc)
|
||||||
datetime(2025, 1, 1, tzinfo=timezone.utc),
|
|
||||||
datetime(2025, 1, 2, tzinfo=timezone.utc),
|
|
||||||
datetime(2025, 1, 3, tzinfo=timezone.utc),
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def seed(connection_string: str) -> None:
|
def seed(config_path: str) -> None:
|
||||||
service = TableServiceClient.from_connection_string(connection_string)
|
cfg = load_config(Path(config_path))
|
||||||
|
service = TableServiceClient.from_connection_string(cfg.azure_connection_string)
|
||||||
service.create_table_if_not_exists(table_name=TABLE_NAME)
|
service.create_table_if_not_exists(table_name=TABLE_NAME)
|
||||||
client = service.get_table_client(table_name=TABLE_NAME)
|
client = service.get_table_client(table_name=TABLE_NAME)
|
||||||
|
|
||||||
@@ -49,36 +40,43 @@ def seed(connection_string: str) -> None:
|
|||||||
if existing:
|
if existing:
|
||||||
print(f"Cleared {len(existing)} existing records")
|
print(f"Cleared {len(existing)} existing records")
|
||||||
|
|
||||||
|
# Generate random users and dates
|
||||||
|
users = [f"user{i:03d}" for i in random.sample(range(1, USER_COUNT + 1), k=min(USER_COUNT, USER_COUNT))]
|
||||||
|
date_range_seconds = int((DATE_END - DATE_START).total_seconds())
|
||||||
|
|
||||||
# Seed new data
|
# Seed new data
|
||||||
entities = []
|
entities = []
|
||||||
order_idx = 0
|
order_idx = 0
|
||||||
for date in DATES:
|
for _ in range(len(users)):
|
||||||
for user in USERS:
|
user = random.choice(users)
|
||||||
# 2 Recived orders per user per day
|
# Random timestamp in [DATE_START, DATE_END)
|
||||||
for _ in range(2):
|
offset = random.randint(0, max(date_range_seconds - 1, 0))
|
||||||
hour = 8 + (order_idx % 12)
|
dt = DATE_START + timedelta(seconds=offset)
|
||||||
|
|
||||||
|
# Received orders (state=1)
|
||||||
|
for _ in range(ENTRIES_PER_USER_PER_DAY):
|
||||||
entities.append({
|
entities.append({
|
||||||
"PartitionKey": user,
|
"PartitionKey": user,
|
||||||
"RowKey": f"test-order-{order_idx:04d}",
|
"RowKey": f"test-order-{order_idx:04d}",
|
||||||
"OrderTime": date.replace(hour=hour, minute=order_idx % 60),
|
"OrderTime": dt.replace(hour=8 + (order_idx % 12), minute=order_idx % 60),
|
||||||
"ReceiveTime": date.replace(hour=hour + 1, minute=order_idx % 60),
|
"ReceiveTime": dt.replace(hour=9 + (order_idx % 12), minute=order_idx % 60),
|
||||||
"revenue": 10.0 + order_idx,
|
"revenue": 10.0 + order_idx,
|
||||||
"platformOrder": f"plat-{order_idx:04d}",
|
"platformOrder": f"plat-{order_idx:04d}",
|
||||||
"prodCount": 1 + (order_idx % 5),
|
"prodCount": 1 + (order_idx % 5),
|
||||||
"pfId": user,
|
"pfId": user,
|
||||||
"prodId": f"prod-{order_idx % 3}",
|
"prodId": f"prod-{order_idx % 3}",
|
||||||
"prodPrice": 10.0 + (order_idx % 10),
|
"prodPrice": 10.0 + (order_idx % 10),
|
||||||
"state": 1, # Recived
|
"state": 1, # Received
|
||||||
"ver": 3,
|
"ver": 3,
|
||||||
"customData": None,
|
"customData": None,
|
||||||
})
|
})
|
||||||
order_idx += 1
|
order_idx += 1
|
||||||
|
|
||||||
# 1 Payed order per user per day (should be skipped by backup)
|
# Payed order (state=0, should be skipped by backup)
|
||||||
entities.append({
|
entities.append({
|
||||||
"PartitionKey": user,
|
"PartitionKey": user,
|
||||||
"RowKey": f"test-order-{order_idx:04d}",
|
"RowKey": f"test-order-{order_idx:04d}",
|
||||||
"OrderTime": date.replace(hour=14, minute=30),
|
"OrderTime": dt.replace(hour=14, minute=30),
|
||||||
"ReceiveTime": None,
|
"ReceiveTime": None,
|
||||||
"revenue": 5.0,
|
"revenue": 5.0,
|
||||||
"platformOrder": f"plat-payed-{order_idx:04d}",
|
"platformOrder": f"plat-payed-{order_idx:04d}",
|
||||||
@@ -105,11 +103,13 @@ def seed(connection_string: str) -> None:
|
|||||||
client.submit_transaction(ops)
|
client.submit_transaction(ops)
|
||||||
total += len(batch)
|
total += len(batch)
|
||||||
|
|
||||||
print(f"Seeded {total} orders across {len(by_pk)} partitions, {len(DATES)} days")
|
print(f"Seeded {total} orders across {len(by_pk)} partitions")
|
||||||
print(f" Recived (state=1): {len([e for e in entities if e['state'] == 1])}")
|
print(f" Received (state=1): {len([e for e in entities if e['state'] == 1])}")
|
||||||
print(f" Payed (state=0): {len([e for e in entities if e['state'] == 0])}")
|
print(f" Payed (state=0): {len([e for e in entities if e['state'] == 0])}")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
conn_str = sys.argv[1] if len(sys.argv) > 1 else DEFAULT_CONNECTION_STRING
|
config_path = sys.argv[1] if len(sys.argv) > 1 else str(
|
||||||
seed(conn_str)
|
Path.home() / ".config" / "order-bak" / "config.toml"
|
||||||
|
)
|
||||||
|
seed(config_path)
|
||||||
|
|||||||
@@ -3,14 +3,13 @@ from collections import defaultdict
|
|||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from order_bak.csv_utils import CSV_COLUMNS, parse_row
|
from order_bak.csv_utils import CSV_COLUMNS, entity_to_row, parse_row
|
||||||
|
|
||||||
|
|
||||||
def backup_orders(
|
def backup_orders(
|
||||||
scan_path: Path,
|
scan_path: Path,
|
||||||
backup_dir: Path,
|
backup_dir: Path,
|
||||||
start: datetime,
|
cutoff: datetime,
|
||||||
end: datetime,
|
|
||||||
) -> int:
|
) -> int:
|
||||||
backup_dir.mkdir(parents=True, exist_ok=True)
|
backup_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
@@ -25,10 +24,10 @@ def backup_orders(
|
|||||||
order_time = entity["OrderTime"]
|
order_time = entity["OrderTime"]
|
||||||
if order_time is None:
|
if order_time is None:
|
||||||
continue
|
continue
|
||||||
if order_time < start or order_time >= end:
|
if order_time >= cutoff:
|
||||||
continue
|
continue
|
||||||
date_key = order_time.strftime("%Y-%m-%d")
|
date_key = order_time.strftime("%Y-%m-%d")
|
||||||
by_date[date_key].append(raw_row)
|
by_date[date_key].append(entity)
|
||||||
|
|
||||||
total = 0
|
total = 0
|
||||||
for date_key, rows in sorted(by_date.items()):
|
for date_key, rows in sorted(by_date.items()):
|
||||||
@@ -36,7 +35,7 @@ def backup_orders(
|
|||||||
with open(csv_path, "w", newline="") as f:
|
with open(csv_path, "w", newline="") as f:
|
||||||
writer = csv.DictWriter(f, fieldnames=CSV_COLUMNS)
|
writer = csv.DictWriter(f, fieldnames=CSV_COLUMNS)
|
||||||
writer.writeheader()
|
writer.writeheader()
|
||||||
writer.writerows(rows)
|
writer.writerows(entity_to_row(r) for r in rows)
|
||||||
total += len(rows)
|
total += len(rows)
|
||||||
print(f" {date_key}: {len(rows)} orders")
|
print(f" {date_key}: {len(rows)} orders")
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
from datetime import timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import click
|
import click
|
||||||
@@ -31,11 +31,14 @@ def scan(ctx, count):
|
|||||||
|
|
||||||
|
|
||||||
@main.command()
|
@main.command()
|
||||||
@click.option("--start", required=True, type=click.DateTime(formats=["%Y-%m-%d"]), help="Start date (UTC, inclusive)")
|
@click.option("--days", required=True, type=int, help="Backup orders older than N days (min 1)")
|
||||||
@click.option("--end", required=True, type=click.DateTime(formats=["%Y-%m-%d"]), help="End date (UTC, exclusive)")
|
|
||||||
@click.option("--scan", "scan_path", default=None, type=click.Path(), help="Path to scan CSV (default: latest in scan_dir)")
|
@click.option("--scan", "scan_path", default=None, type=click.Path(), help="Path to scan CSV (default: latest in scan_dir)")
|
||||||
@click.pass_context
|
@click.pass_context
|
||||||
def backup(ctx, start, end, scan_path):
|
def backup(ctx, days, scan_path):
|
||||||
|
if days < 1:
|
||||||
|
click.echo("Error: --days must be at least 1")
|
||||||
|
raise SystemExit(1)
|
||||||
|
|
||||||
cfg = load_config(ctx.obj["config_path"])
|
cfg = load_config(ctx.obj["config_path"])
|
||||||
from order_bak.backup import backup_orders
|
from order_bak.backup import backup_orders
|
||||||
|
|
||||||
@@ -48,18 +51,14 @@ def backup(ctx, start, end, scan_path):
|
|||||||
raise SystemExit(1)
|
raise SystemExit(1)
|
||||||
scan_file = scan_files[-1]
|
scan_file = scan_files[-1]
|
||||||
|
|
||||||
start_utc = start.replace(tzinfo=timezone.utc)
|
today_utc = datetime.now(timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0)
|
||||||
end_utc = end.replace(tzinfo=timezone.utc)
|
cutoff = today_utc - timedelta(days=days)
|
||||||
|
|
||||||
if start_utc >= end_utc:
|
|
||||||
click.echo("Error: --start must be before --end")
|
|
||||||
raise SystemExit(1)
|
|
||||||
|
|
||||||
|
click.echo(f"Backing up orders before {cutoff.strftime('%Y-%m-%d')} UTC")
|
||||||
backup_orders(
|
backup_orders(
|
||||||
scan_path=scan_file,
|
scan_path=scan_file,
|
||||||
backup_dir=Path("./backups"),
|
backup_dir=Path("./backups"),
|
||||||
start=start_utc,
|
cutoff=cutoff,
|
||||||
end=end_utc,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -43,8 +43,14 @@ def parse_row(raw: dict[str, str]) -> dict[str, Any]:
|
|||||||
result[col] = datetime.fromisoformat(val)
|
result[col] = datetime.fromisoformat(val)
|
||||||
elif col in ("revenue", "prodPrice"):
|
elif col in ("revenue", "prodPrice"):
|
||||||
result[col] = float(val)
|
result[col] = float(val)
|
||||||
elif col in ("prodCount", "state", "ver"):
|
elif col in ("prodCount", "ver"):
|
||||||
result[col] = int(val)
|
result[col] = int(val)
|
||||||
|
elif col == "state":
|
||||||
|
try:
|
||||||
|
result[col] = int(val)
|
||||||
|
except ValueError:
|
||||||
|
_STATE_MAP = {"Payed": 0, "Recived": 1}
|
||||||
|
result[col] = _STATE_MAP.get(val, val)
|
||||||
else:
|
else:
|
||||||
result[col] = val
|
result[col] = val
|
||||||
return result
|
return result
|
||||||
|
|||||||
@@ -1,6 +1,4 @@
|
|||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from order_bak.csv_utils import write_csv
|
from order_bak.csv_utils import write_csv
|
||||||
from order_bak.backup import backup_orders
|
from order_bak.backup import backup_orders
|
||||||
|
|
||||||
@@ -33,7 +31,7 @@ def test_backup_filters_by_state_and_date(tmp_path):
|
|||||||
_make_entity("u1", "o1", datetime(2025, 1, 1, 10, tzinfo=timezone.utc), 1),
|
_make_entity("u1", "o1", datetime(2025, 1, 1, 10, tzinfo=timezone.utc), 1),
|
||||||
_make_entity("u2", "o2", datetime(2025, 1, 1, 12, tzinfo=timezone.utc), 0), # Payed, skip
|
_make_entity("u2", "o2", datetime(2025, 1, 1, 12, tzinfo=timezone.utc), 0), # Payed, skip
|
||||||
_make_entity("u3", "o3", datetime(2025, 1, 2, 8, tzinfo=timezone.utc), 1),
|
_make_entity("u3", "o3", datetime(2025, 1, 2, 8, tzinfo=timezone.utc), 1),
|
||||||
_make_entity("u4", "o4", datetime(2025, 1, 3, 8, tzinfo=timezone.utc), 1), # outside range
|
_make_entity("u4", "o4", datetime(2025, 1, 3, 8, tzinfo=timezone.utc), 1), # at cutoff, skip
|
||||||
]
|
]
|
||||||
scan_csv = scan_dir / "scan-2025-01-03.csv"
|
scan_csv = scan_dir / "scan-2025-01-03.csv"
|
||||||
write_csv(scan_csv, entities)
|
write_csv(scan_csv, entities)
|
||||||
@@ -41,8 +39,7 @@ def test_backup_filters_by_state_and_date(tmp_path):
|
|||||||
result = backup_orders(
|
result = backup_orders(
|
||||||
scan_path=scan_csv,
|
scan_path=scan_csv,
|
||||||
backup_dir=backup_dir,
|
backup_dir=backup_dir,
|
||||||
start=datetime(2025, 1, 1, tzinfo=timezone.utc),
|
cutoff=datetime(2025, 1, 3, tzinfo=timezone.utc),
|
||||||
end=datetime(2025, 1, 3, tzinfo=timezone.utc),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
assert result == 2
|
assert result == 2
|
||||||
@@ -56,3 +53,78 @@ def test_backup_filters_by_state_and_date(tmp_path):
|
|||||||
jan1_rows = list(read_csv(jan1))
|
jan1_rows = list(read_csv(jan1))
|
||||||
assert len(jan1_rows) == 1
|
assert len(jan1_rows) == 1
|
||||||
assert jan1_rows[0]["RowKey"] == "o1"
|
assert jan1_rows[0]["RowKey"] == "o1"
|
||||||
|
|
||||||
|
|
||||||
|
def test_backup_filters_by_cutoff(tmp_path):
|
||||||
|
scan_dir = tmp_path / "scan"
|
||||||
|
backup_dir = tmp_path / "backups"
|
||||||
|
scan_dir.mkdir()
|
||||||
|
backup_dir.mkdir()
|
||||||
|
|
||||||
|
entities = [
|
||||||
|
_make_entity("u1", "o1", datetime(2025, 1, 1, 10, tzinfo=timezone.utc), 1),
|
||||||
|
_make_entity("u2", "o2", datetime(2025, 1, 2, 8, tzinfo=timezone.utc), 1),
|
||||||
|
_make_entity("u3", "o3", datetime(2025, 1, 3, 8, tzinfo=timezone.utc), 1), # at/after cutoff
|
||||||
|
]
|
||||||
|
scan_csv = scan_dir / "scan-2025-01-03.csv"
|
||||||
|
write_csv(scan_csv, entities)
|
||||||
|
|
||||||
|
cutoff = datetime(2025, 1, 3, tzinfo=timezone.utc)
|
||||||
|
result = backup_orders(
|
||||||
|
scan_path=scan_csv,
|
||||||
|
backup_dir=backup_dir,
|
||||||
|
cutoff=cutoff,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result == 2
|
||||||
|
assert (backup_dir / "2025-01-01.csv").exists()
|
||||||
|
assert (backup_dir / "2025-01-02.csv").exists()
|
||||||
|
assert not (backup_dir / "2025-01-03.csv").exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_backup_cutoff_at_midnight(tmp_path):
|
||||||
|
"""Order exactly at cutoff midnight should be excluded."""
|
||||||
|
scan_dir = tmp_path / "scan"
|
||||||
|
backup_dir = tmp_path / "backups"
|
||||||
|
scan_dir.mkdir()
|
||||||
|
backup_dir.mkdir()
|
||||||
|
|
||||||
|
entities = [
|
||||||
|
_make_entity("u1", "o1", datetime(2025, 1, 3, 0, 0, 0, tzinfo=timezone.utc), 1), # exactly at cutoff
|
||||||
|
_make_entity("u2", "o2", datetime(2025, 1, 2, 23, 59, 59, tzinfo=timezone.utc), 1), # just before
|
||||||
|
]
|
||||||
|
scan_csv = scan_dir / "scan.csv"
|
||||||
|
write_csv(scan_csv, entities)
|
||||||
|
|
||||||
|
result = backup_orders(
|
||||||
|
scan_path=scan_csv,
|
||||||
|
backup_dir=backup_dir,
|
||||||
|
cutoff=datetime(2025, 1, 3, tzinfo=timezone.utc),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result == 1
|
||||||
|
assert (backup_dir / "2025-01-02.csv").exists()
|
||||||
|
assert not (backup_dir / "2025-01-03.csv").exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_backup_empty_result(tmp_path):
|
||||||
|
"""No orders before cutoff returns 0 and no files."""
|
||||||
|
scan_dir = tmp_path / "scan"
|
||||||
|
backup_dir = tmp_path / "backups"
|
||||||
|
scan_dir.mkdir()
|
||||||
|
backup_dir.mkdir()
|
||||||
|
|
||||||
|
entities = [
|
||||||
|
_make_entity("u1", "o1", datetime(2025, 1, 5, 10, tzinfo=timezone.utc), 1),
|
||||||
|
]
|
||||||
|
scan_csv = scan_dir / "scan.csv"
|
||||||
|
write_csv(scan_csv, entities)
|
||||||
|
|
||||||
|
result = backup_orders(
|
||||||
|
scan_path=scan_csv,
|
||||||
|
backup_dir=backup_dir,
|
||||||
|
cutoff=datetime(2025, 1, 1, tzinfo=timezone.utc),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result == 0
|
||||||
|
assert list(backup_dir.glob("*.csv")) == []
|
||||||
|
|||||||
@@ -123,13 +123,12 @@ def test_full_pipeline(table_client, tmp_path):
|
|||||||
scan_rows = list(read_csv(scan_path))
|
scan_rows = list(read_csv(scan_path))
|
||||||
assert len(scan_rows) == len(orders) # all orders, no filtering
|
assert len(scan_rows) == len(orders) # all orders, no filtering
|
||||||
|
|
||||||
# --- Backup (Jan 1-2 only) ---
|
# --- Backup (before Jan 3) ---
|
||||||
backup_dir = tmp_path / "backups"
|
backup_dir = tmp_path / "backups"
|
||||||
count = backup_orders(
|
count = backup_orders(
|
||||||
scan_path=scan_path,
|
scan_path=scan_path,
|
||||||
backup_dir=backup_dir,
|
backup_dir=backup_dir,
|
||||||
start=datetime(2025, 1, 1, tzinfo=timezone.utc),
|
cutoff=datetime(2025, 1, 3, tzinfo=timezone.utc),
|
||||||
end=datetime(2025, 1, 3, tzinfo=timezone.utc),
|
|
||||||
)
|
)
|
||||||
# 2 days × 2 users × 2 Recived = 8 orders
|
# 2 days × 2 users × 2 Recived = 8 orders
|
||||||
assert count == 8
|
assert count == 8
|
||||||
|
|||||||
Reference in New Issue
Block a user