Developer/order-bak
11
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: read config from file, randomize users and dates in seed script

Browse Source 
Replace hardcoded connection string with config file loading via
load_config(). Users now randomly chosen from user001-user1000, dates
randomly sampled between 2026-01-01 and UTC now.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
tech
2026-04-22 00:04:32 +08:00
parent dba7f7bdc9
commit c4290b830e
1 changed files with 56 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
scripts/seed_test_data.py
View File

@@ -1,35 +1,26 @@
"""Seed test data into Azure Table emulator for integration testing.""" """Seed test data into Azure Table emulator for integration testing."""
import random
import sys import sys
from datetime import datetime, timezone from datetime import datetime, timedelta, timezone
from pathlib import Path
from azure.data.tables import TableServiceClient from azure.data.tables import TableServiceClient
DEFAULT_CONNECTION_STRING = ( from order_bak.config import load_config
"AccountName=devstoreaccount1;"
"AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/"
"K1SZFPTOtr/KBHBeksoGMGw==;"
"DefaultEndpointsProtocol=http;"
"BlobEndpoint=http://192.168.9.101:10000/devstoreaccount1;"
"QueueEndpoint=http://192.168.9.101:10001/devstoreaccount1;"
"TableEndpoint=http://192.168.9.101:10002/devstoreaccount1;"
)
TABLE_NAME = "order" TABLE_NAME = "order"
# Test users USER_COUNT = 1000
USERS = ["user001", "user002", "user003", "user004", "user005"] ENTRIES_PER_USER_PER_DAY = 2
PAYED_PER_USER_PER_DAY = 1
# 3 days of data: Jan 1-3, 2025 DATE_START = datetime(2026, 1, 1, tzinfo=timezone.utc)
DATES = [ DATE_END = datetime.now(timezone.utc)
datetime(2025, 1, 1, tzinfo=timezone.utc),
datetime(2025, 1, 2, tzinfo=timezone.utc),
datetime(2025, 1, 3, tzinfo=timezone.utc),
]
def seed(connection_string: str) -> None: def seed(config_path: str) -> None:
service = TableServiceClient.from_connection_string(connection_string) cfg = load_config(Path(config_path))
service = TableServiceClient.from_connection_string(cfg.azure_connection_string)
service.create_table_if_not_exists(table_name=TABLE_NAME) service.create_table_if_not_exists(table_name=TABLE_NAME)
client = service.get_table_client(table_name=TABLE_NAME) client = service.get_table_client(table_name=TABLE_NAME)
@@ -49,49 +40,56 @@ def seed(connection_string: str) -> None:
if existing: if existing:
print(f"Cleared {len(existing)} existing records") print(f"Cleared {len(existing)} existing records")
# Generate random users and dates
users = [f"user{i:03d}" for i in random.sample(range(1, USER_COUNT + 1), k=min(USER_COUNT, USER_COUNT))]
date_range_seconds = int((DATE_END - DATE_START).total_seconds())
# Seed new data # Seed new data
entities = [] entities = []
order_idx = 0 order_idx = 0
for date in DATES: for _ in range(len(users)):
for user in USERS: user = random.choice(users)
# 2 Recived orders per user per day # Random timestamp in [DATE_START, DATE_END)
for _ in range(2): offset = random.randint(0, max(date_range_seconds - 1, 0))
hour = 8 + (order_idx % 12) dt = DATE_START + timedelta(seconds=offset)
entities.append({
"PartitionKey": user,
"RowKey": f"test-order-{order_idx:04d}",
"OrderTime": date.replace(hour=hour, minute=order_idx % 60),
"ReceiveTime": date.replace(hour=hour + 1, minute=order_idx % 60),
"revenue": 10.0 + order_idx,
"platformOrder": f"plat-{order_idx:04d}",
"prodCount": 1 + (order_idx % 5),
"pfId": user,
"prodId": f"prod-{order_idx % 3}",
"prodPrice": 10.0 + (order_idx % 10),
"state": 1, # Recived
"ver": 3,
"customData": None,
})
order_idx += 1
# 1 Payed order per user per day (should be skipped by backup) # Received orders (state=1)
for _ in range(ENTRIES_PER_USER_PER_DAY):
entities.append({ entities.append({
"PartitionKey": user, "PartitionKey": user,
"RowKey": f"test-order-{order_idx:04d}", "RowKey": f"test-order-{order_idx:04d}",
"OrderTime": date.replace(hour=14, minute=30), "OrderTime": dt.replace(hour=8 + (order_idx % 12), minute=order_idx % 60),
"ReceiveTime": None, "ReceiveTime": dt.replace(hour=9 + (order_idx % 12), minute=order_idx % 60),
"revenue": 5.0, "revenue": 10.0 + order_idx,
"platformOrder": f"plat-payed-{order_idx:04d}", "platformOrder": f"plat-{order_idx:04d}",
"prodCount": 1, "prodCount": 1 + (order_idx % 5),
"pfId": user, "pfId": user,
"prodId": "prod-pay", "prodId": f"prod-{order_idx % 3}",
"prodPrice": 5.0, "prodPrice": 10.0 + (order_idx % 10),
"state": 0, # Payed "state": 1, # Received
"ver": 3, "ver": 3,
"customData": None, "customData": None,
}) })
order_idx += 1 order_idx += 1
# Payed order (state=0, should be skipped by backup)
entities.append({
"PartitionKey": user,
"RowKey": f"test-order-{order_idx:04d}",
"OrderTime": dt.replace(hour=14, minute=30),
"ReceiveTime": None,
"revenue": 5.0,
"platformOrder": f"plat-payed-{order_idx:04d}",
"prodCount": 1,
"pfId": user,
"prodId": "prod-pay",
"prodPrice": 5.0,
"state": 0, # Payed
"ver": 3,
"customData": None,
})
order_idx += 1
# Insert in batches grouped by PartitionKey # Insert in batches grouped by PartitionKey
total = 0 total = 0
by_pk: dict[str, list[dict]] = {} by_pk: dict[str, list[dict]] = {}
@@ -105,11 +103,13 @@ def seed(connection_string: str) -> None:
client.submit_transaction(ops) client.submit_transaction(ops)
total += len(batch) total += len(batch)
print(f"Seeded {total} orders across {len(by_pk)} partitions, {len(DATES)} days") print(f"Seeded {total} orders across {len(by_pk)} partitions")
print(f" Recived (state=1): {len([e for e in entities if e['state'] == 1])}") print(f" Received (state=1): {len([e for e in entities if e['state'] == 1])}")
print(f" Payed (state=0): {len([e for e in entities if e['state'] == 0])}") print(f" Payed (state=0): {len([e for e in entities if e['state'] == 0])}")
if __name__ == "__main__": if __name__ == "__main__":
conn_str = sys.argv[1] if len(sys.argv) > 1 else DEFAULT_CONNECTION_STRING config_path = sys.argv[1] if len(sys.argv) > 1 else str(
seed(conn_str) Path.home() / ".config" / "order-bak" / "config.toml"
)
seed(config_path)