This commit is contained in:
Shroominic
2025-10-18 13:29:49 +08:00
parent 229702a983
commit 24aceb2b07
7 changed files with 81 additions and 91 deletions
@@ -549,9 +549,9 @@ class TestPerformance:
max_time = max(times)
# Average should be well under 100ms
assert avg_time < 100, (
f"{op_type} average time {avg_time}ms exceeds 100ms"
)
assert (
avg_time < 100
), f"{op_type} average time {avg_time}ms exceeds 100ms"
# No single operation should exceed 200ms
assert max_time < 200, f"{op_type} max time {max_time}ms exceeds 200ms"
@@ -359,15 +359,15 @@ async def test_all_info_endpoints_no_database_changes(
# Check no database changes after each request
diff = await db_snapshot.diff()
assert len(diff["api_keys"]["added"]) == 0, (
f"Endpoint {endpoint} added API keys"
)
assert len(diff["api_keys"]["removed"]) == 0, (
f"Endpoint {endpoint} removed API keys"
)
assert len(diff["api_keys"]["modified"]) == 0, (
f"Endpoint {endpoint} modified API keys"
)
assert (
len(diff["api_keys"]["added"]) == 0
), f"Endpoint {endpoint} added API keys"
assert (
len(diff["api_keys"]["removed"]) == 0
), f"Endpoint {endpoint} removed API keys"
assert (
len(diff["api_keys"]["modified"]) == 0
), f"Endpoint {endpoint} modified API keys"
# Final verification - database state should be identical
final_state = await db_snapshot.capture()
+27 -27
View File
@@ -131,9 +131,9 @@ class TestPerformanceBaseline:
# Verify 95th percentile < 500ms
p95 = sorted(response_times)[int(len(response_times) * 0.95)]
assert p95 < 500, (
f"{method} {path} p95 response time {p95}ms exceeds 500ms limit"
)
assert (
p95 < 500
), f"{method} {path} p95 response time {p95}ms exceeds 500ms limit"
print(f"\n{method} {path}:")
print(f" Mean: {statistics.mean(response_times):.2f}ms")
@@ -175,9 +175,9 @@ class TestPerformanceBaseline:
query_times.append(duration)
# All queries should complete < 100ms
assert max(query_times) < 100, (
f"Max query time {max(query_times)}ms exceeds 100ms limit"
)
assert (
max(query_times) < 100
), f"Max query time {max(query_times)}ms exceeds 100ms limit"
print("\nDatabase query performance:")
print(f" Mean: {statistics.mean(query_times):.2f}ms")
print(f" Max: {max(query_times):.2f}ms")
@@ -272,9 +272,9 @@ class TestLoadScenarios:
# Performance requirements
assert summary["error_rate"] < 0.05, "Error rate exceeds 5%"
assert summary["response_times"]["p95"] < 2.0, (
"P95 response time exceeds 2 seconds"
)
assert (
summary["response_times"]["p95"] < 2.0
), "P95 response time exceeds 2 seconds"
@pytest.mark.asyncio
async def test_sustained_load_1000_rpm(
@@ -354,13 +354,13 @@ class TestLoadScenarios:
)
# Verify performance
assert actual_rps >= target_rps * 0.95, (
f"Could not sustain target rate (achieved {actual_rps:.2f} req/s)"
)
assert (
actual_rps >= target_rps * 0.95
), f"Could not sustain target rate (achieved {actual_rps:.2f} req/s)"
assert summary["error_rate"] < 0.01, "Error rate exceeds 1%"
assert summary["response_times"]["p95"] < 1.0, (
"P95 response time exceeds 1 second"
)
assert (
summary["response_times"]["p95"] < 1.0
), "P95 response time exceeds 1 second"
@pytest.mark.integration
@@ -418,12 +418,12 @@ class TestMemoryLeaks:
# Check for significant memory leaks
# Allow some growth but not more than 20% or 50MB total
assert memory_growth < 50, (
f"Memory grew by {memory_growth} MB, indicating a potential leak"
)
assert memory_samples[-1] < memory_samples[0] * 1.2, (
"Memory grew by more than 20%"
)
assert (
memory_growth < 50
), f"Memory grew by {memory_growth} MB, indicating a potential leak"
assert (
memory_samples[-1] < memory_samples[0] * 1.2
), "Memory grew by more than 20%"
@pytest.mark.integration
@@ -471,9 +471,9 @@ class TestPerformanceRegression:
print(f" Current: {mean_time * 1000:.1f}ms")
print(f" Difference: {((mean_time / baseline - 1) * 100):.1f}%")
assert mean_time <= max_allowed, (
f"{endpoint} performance degraded by more than 20% (baseline: {baseline}s, current: {mean_time}s)"
)
assert (
mean_time <= max_allowed
), f"{endpoint} performance degraded by more than 20% (baseline: {baseline}s, current: {mean_time}s)"
# Get overall validation results
results = {}
@@ -482,9 +482,9 @@ class TestPerformanceRegression:
endpoint, max_duration=baselines[endpoint] * 1.2, percentile=0.95
)
results[endpoint] = result
assert result["valid"], (
f"Performance validation failed for {endpoint}: {result}"
)
assert result[
"valid"
], f"Performance validation failed for {endpoint}: {result}"
# Performance test utilities
@@ -42,12 +42,12 @@ async def test_reserved_balance_never_negative(integration_client: AsyncClient)
async with create_session() as session:
key = await session.get(ApiKey, "test_reserved_balance_key")
assert key is not None
assert key.reserved_balance >= 0, (
f"Reserved balance went negative: {key.reserved_balance}"
)
assert key.balance == 1000, (
"Balance should remain unchanged after failed request"
)
assert (
key.reserved_balance >= 0
), f"Reserved balance went negative: {key.reserved_balance}"
assert (
key.balance == 1000
), "Balance should remain unchanged after failed request"
# Test 2: Simulate concurrent failed requests
# This tests the race condition protection
@@ -71,9 +71,9 @@ async def test_reserved_balance_never_negative(integration_client: AsyncClient)
async with create_session() as session:
key = await session.get(ApiKey, "test_reserved_balance_key")
assert key is not None
assert key.reserved_balance >= 0, (
f"Reserved balance went negative after concurrent requests: {key.reserved_balance}"
)
assert (
key.reserved_balance >= 0
), f"Reserved balance went negative after concurrent requests: {key.reserved_balance}"
print(f"Final state - Balance: {key.balance}, Reserved: {key.reserved_balance}")
@@ -113,20 +113,20 @@ async def test_reserved_balance_with_successful_requests(
async with create_session() as session:
key = await session.get(ApiKey, unique_key)
assert key is not None
assert key.reserved_balance >= 0, (
f"Reserved balance went negative: {key.reserved_balance}"
)
assert (
key.reserved_balance >= 0
), f"Reserved balance went negative: {key.reserved_balance}"
# Check if the request was processed (might fail due to model pricing in test env)
# The important part is that reserved_balance doesn't go negative
if key.total_spent > 0:
assert key.balance < 100000, (
"Balance should decrease after successful request"
)
assert (
key.balance < 100000
), "Balance should decrease after successful request"
else:
# Request failed, but reserved balance should still be non-negative
assert key.balance == 100000, (
"Balance should remain unchanged if request failed"
)
assert (
key.balance == 100000
), "Balance should remain unchanged if request failed"
print(
f"After successful request - Balance: {key.balance}, Reserved: {key.reserved_balance}, Spent: {key.total_spent}"
)
@@ -157,9 +157,9 @@ async def test_insufficient_reserved_balance_for_revert(
await integration_session.refresh(test_key)
# Current implementation allows negative reserved balance
assert test_key.reserved_balance == -100, (
f"Expected reserved_balance to be -100, got: {test_key.reserved_balance}"
)
assert test_key.total_requests == -1, (
f"Expected total_requests to be -1, got: {test_key.total_requests}"
)
assert (
test_key.reserved_balance == -100
), f"Expected reserved_balance to be -100, got: {test_key.reserved_balance}"
assert (
test_key.total_requests == -1
), f"Expected total_requests to be -1, got: {test_key.total_requests}"
@@ -94,9 +94,9 @@ async def test_api_key_generation_invalid_token(
response = await integration_client.get("/v1/wallet/info")
# Should fail with 401
assert response.status_code == 401, (
f"Token {invalid_token[:20]}... should be invalid"
)
assert (
response.status_code == 401
), f"Token {invalid_token[:20]}... should be invalid"
# Validate error response
validator = ResponseValidator()
@@ -198,9 +198,9 @@ async def test_authorization_header_validation(
# Make request to protected endpoint
response = await integration_client.get("/v1/wallet/")
assert response.status_code == expected_status, (
f"{description}: Expected {expected_status}, got {response.status_code}"
)
assert (
response.status_code == expected_status
), f"{description}: Expected {expected_status}, got {response.status_code}"
if expected_status == 401:
assert "detail" in response.json()
+3 -3
View File
@@ -133,9 +133,9 @@ async def test_topup_with_invalid_token(
)
# Should fail with 400
assert response.status_code == 400, (
f"Token {invalid_token[:20]}... should be invalid"
)
assert (
response.status_code == 400
), f"Token {invalid_token[:20]}... should be invalid"
# Validate error response
validator = ResponseValidator()
+9 -19
View File
@@ -96,7 +96,9 @@ def test_exchange_fee_applied_to_usd_pricing(standard_model: Model) -> None:
# Verify all pricing fields have exchange fee applied
assert pricing["prompt"] == pytest.approx(0.001 * exchange_fee, rel=1e-9)
assert pricing["completion"] == pytest.approx(0.002 * exchange_fee, rel=1e-9)
assert pricing["completion"] == pytest.approx(
0.002 * exchange_fee, rel=1e-9
)
assert pricing["request"] == pytest.approx(0.01 * exchange_fee, rel=1e-9)
assert pricing["image"] == pytest.approx(0.05 * exchange_fee, rel=1e-9)
assert pricing["web_search"] == pytest.approx(0.03 * exchange_fee, rel=1e-9)
@@ -227,17 +229,11 @@ def test_default_fee_values_from_settings(standard_model: Model) -> None:
assert pricing["request"] > standard_model.pricing.request
# Verify exact values with default fees
assert pricing["prompt"] == pytest.approx(
0.001 * expected_multiplier, rel=1e-9
)
assert pricing["completion"] == pytest.approx(
0.002 * expected_multiplier, rel=1e-9
)
assert pricing["prompt"] == pytest.approx(0.001 * expected_multiplier, rel=1e-9)
assert pricing["completion"] == pytest.approx(0.002 * expected_multiplier, rel=1e-9)
assert pricing["request"] == pytest.approx(0.01 * expected_multiplier, rel=1e-9)
assert pricing["image"] == pytest.approx(0.05 * expected_multiplier, rel=1e-9)
assert pricing["web_search"] == pytest.approx(
0.03 * expected_multiplier, rel=1e-9
)
assert pricing["web_search"] == pytest.approx(0.03 * expected_multiplier, rel=1e-9)
assert pricing["internal_reasoning"] == pytest.approx(
0.015 * expected_multiplier, rel=1e-9
)
@@ -727,20 +723,14 @@ def test_fee_consistency_across_all_fields(standard_model: Model) -> None:
# All multipliers should be identical and equal to expected multiplier
assert prompt_multiplier == pytest.approx(expected_multiplier, rel=1e-9)
assert completion_multiplier == pytest.approx(
expected_multiplier, rel=1e-9
)
assert completion_multiplier == pytest.approx(expected_multiplier, rel=1e-9)
assert request_multiplier == pytest.approx(expected_multiplier, rel=1e-9)
assert image_multiplier == pytest.approx(expected_multiplier, rel=1e-9)
assert web_search_multiplier == pytest.approx(
expected_multiplier, rel=1e-9
)
assert web_search_multiplier == pytest.approx(expected_multiplier, rel=1e-9)
assert internal_reasoning_multiplier == pytest.approx(
expected_multiplier, rel=1e-9
)
assert max_prompt_multiplier == pytest.approx(
expected_multiplier, rel=1e-9
)
assert max_prompt_multiplier == pytest.approx(expected_multiplier, rel=1e-9)
assert max_completion_multiplier == pytest.approx(
expected_multiplier, rel=1e-9
)