feat(agent): add result-verifier for blind visual comparison
Root cause: test-runner was giving overly optimistic results due to: 1. Context bias - knew the implementation, tended to defend it 2. No actual visual comparison - just wrote 'ACCEPTABLE' without looking 3. No structural validation - accepted 35x scale differences as 'acceptable' Solution: - New result-verifier agent that performs blind visual comparison - Strict pass/fail criteria for structural validation - Updated test-runner to use result-verifier for each figure - Clear guidelines: structural mismatches = FAIL, not ACCEPTABLE Test result: verifier correctly identified Fig3 as FAIL with 7 specific issues: - Wrong X-axis variable (channels vs power) - Wrong Y-axis scale (5x difference) - Wrong curve count (5 vs 4) - etc.
This commit is contained in:
@@ -0,0 +1,44 @@
|
||||
"""
|
||||
tests/test_allocator.py
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import numpy as np
|
||||
from src.models.environment import EnvironmentConfig
|
||||
from src.models.semantic_model import SemanticSurrogate
|
||||
from src.models.allocator import ResourceAllocator
|
||||
|
||||
|
||||
def test_allocator_initialization():
|
||||
env_config = EnvironmentConfig(num_users=3, num_channels=4, bandwidth=1e6)
|
||||
surrogate = SemanticSurrogate(L=12)
|
||||
allocator = ResourceAllocator(surrogate, env_config, K_max=8)
|
||||
|
||||
assert allocator.K_max == 8
|
||||
|
||||
|
||||
def test_optimize_semantic_aware():
|
||||
env_config = EnvironmentConfig(num_users=2, num_channels=3, bandwidth=1e6)
|
||||
surrogate = SemanticSurrogate(L=12)
|
||||
allocator = ResourceAllocator(surrogate, env_config, K_max=4)
|
||||
|
||||
# Mock SNR values
|
||||
snr_linear = np.array([[10.0, 100.0, 1000.0], [50.0, 20.0, 500.0]])
|
||||
|
||||
optimal_k, assignment, total_sse = allocator.optimize_semantic_aware(snr_linear)
|
||||
|
||||
assert optimal_k.shape == (2,)
|
||||
assert assignment.shape == (2, 3)
|
||||
assert np.all(assignment.sum(axis=1) == 1) # Each user gets 1 channel
|
||||
assert np.all(assignment.sum(axis=0) <= 1) # Each channel used by max 1 user
|
||||
|
||||
|
||||
def test_evaluate_fixed_k():
|
||||
env_config = EnvironmentConfig(num_users=2, num_channels=2, bandwidth=1e6)
|
||||
surrogate = SemanticSurrogate(L=12)
|
||||
allocator = ResourceAllocator(surrogate, env_config, K_max=4)
|
||||
|
||||
snr_linear = np.array([[10.0, 100.0], [50.0, 20.0]])
|
||||
|
||||
total_sse = allocator.evaluate_fixed_k(snr_linear, k_n=4)
|
||||
assert total_sse > 0
|
||||
@@ -0,0 +1,26 @@
|
||||
"""
|
||||
tests/test_baselines.py
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import numpy as np
|
||||
from src.models.baselines import BaselineModels
|
||||
|
||||
|
||||
def test_baselines_initialization():
|
||||
models = BaselineModels(mu=19.0, L=12)
|
||||
assert models.mu == 19.0
|
||||
|
||||
|
||||
def test_calculate_baseline_sse():
|
||||
models = BaselineModels(mu=19.0, L=12)
|
||||
|
||||
snr_linear = np.array([10.0, 100.0, 1000.0])
|
||||
|
||||
sse_ideal = models.calculate_baseline_sse(snr_linear, "ideal")
|
||||
sse_5g = models.calculate_baseline_sse(snr_linear, "5G")
|
||||
sse_4g = models.calculate_baseline_sse(snr_linear, "4G")
|
||||
|
||||
assert len(sse_ideal) == 3
|
||||
assert np.all(sse_ideal > sse_5g)
|
||||
assert np.all(sse_5g > sse_4g)
|
||||
@@ -0,0 +1,41 @@
|
||||
"""
|
||||
tests/test_environment.py
|
||||
|
||||
Tests for Module 1: Environment & Channel Simulator
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import numpy as np
|
||||
from src.models.environment import EnvironmentConfig, ChannelSimulator
|
||||
|
||||
|
||||
def test_environment_initialization():
|
||||
config = EnvironmentConfig(num_users=10, num_channels=10, bandwidth=1e6)
|
||||
env = ChannelSimulator(config)
|
||||
assert env.config.num_users == 10
|
||||
assert env.config.num_channels == 10
|
||||
assert env.config.bandwidth == 1e6
|
||||
|
||||
|
||||
def test_path_loss_calculation():
|
||||
config = EnvironmentConfig(num_users=1, num_channels=1)
|
||||
env = ChannelSimulator(config)
|
||||
|
||||
# 1 km distance
|
||||
distances = np.array([1.0])
|
||||
path_loss = env._calculate_pathloss(distances)
|
||||
# PL = 128.1 + 37.6 * log10(d)
|
||||
expected_pl = 128.1
|
||||
np.testing.assert_allclose(path_loss, expected_pl, atol=1e-5)
|
||||
|
||||
|
||||
def test_snr_generation():
|
||||
config = EnvironmentConfig(num_users=5, num_channels=3)
|
||||
env = ChannelSimulator(config)
|
||||
|
||||
transmit_power_dbm = 10.0
|
||||
snr_db, snr_linear = env.generate_channels(transmit_power_dbm)
|
||||
|
||||
assert snr_db.shape == (5, 3)
|
||||
assert snr_linear.shape == (5, 3)
|
||||
assert np.all(snr_linear > 0)
|
||||
@@ -0,0 +1,27 @@
|
||||
"""
|
||||
tests/test_semantic_model.py
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import numpy as np
|
||||
from src.models.semantic_model import SemanticSurrogate
|
||||
|
||||
|
||||
def test_semantic_surrogate():
|
||||
surrogate = SemanticSurrogate()
|
||||
|
||||
# Test bounds
|
||||
snr_linear = np.array([10.0, 100.0, 1000.0])
|
||||
k_n = 4
|
||||
sim = surrogate.get_similarity(snr_linear, k_n)
|
||||
|
||||
assert np.all(sim >= 0) and np.all(sim <= 1)
|
||||
|
||||
# Test monotonicity with SNR
|
||||
assert sim[0] < sim[1] < sim[2]
|
||||
|
||||
# Test monotonicity with k_n
|
||||
sim_k4 = surrogate.get_similarity(snr_linear, 4)
|
||||
sim_k6 = surrogate.get_similarity(snr_linear, 6)
|
||||
|
||||
assert np.all(sim_k4 < sim_k6)
|
||||
Reference in New Issue
Block a user