feat(agent): add result-verifier for blind visual comparison

Root cause: test-runner was giving overly optimistic results due to:
1. Context bias - knew the implementation, tended to defend it
2. No actual visual comparison - just wrote 'ACCEPTABLE' without looking
3. No structural validation - accepted 35x scale differences as 'acceptable'

Solution:
- New result-verifier agent that performs blind visual comparison
- Strict pass/fail criteria for structural validation
- Updated test-runner to use result-verifier for each figure
- Clear guidelines: structural mismatches = FAIL, not ACCEPTABLE

Test result: verifier correctly identified Fig3 as FAIL with 7 specific issues:
- Wrong X-axis variable (channels vs power)
- Wrong Y-axis scale (5x difference)
- Wrong curve count (5 vs 4)
- etc.
This commit is contained in:
hc
2026-03-31 23:56:36 +08:00
parent 3533e15995
commit ced50ea2b0
41 changed files with 2023 additions and 4 deletions
@@ -0,0 +1,44 @@
"""
tests/test_allocator.py
"""
import pytest
import numpy as np
from src.models.environment import EnvironmentConfig
from src.models.semantic_model import SemanticSurrogate
from src.models.allocator import ResourceAllocator
def test_allocator_initialization():
env_config = EnvironmentConfig(num_users=3, num_channels=4, bandwidth=1e6)
surrogate = SemanticSurrogate(L=12)
allocator = ResourceAllocator(surrogate, env_config, K_max=8)
assert allocator.K_max == 8
def test_optimize_semantic_aware():
env_config = EnvironmentConfig(num_users=2, num_channels=3, bandwidth=1e6)
surrogate = SemanticSurrogate(L=12)
allocator = ResourceAllocator(surrogate, env_config, K_max=4)
# Mock SNR values
snr_linear = np.array([[10.0, 100.0, 1000.0], [50.0, 20.0, 500.0]])
optimal_k, assignment, total_sse = allocator.optimize_semantic_aware(snr_linear)
assert optimal_k.shape == (2,)
assert assignment.shape == (2, 3)
assert np.all(assignment.sum(axis=1) == 1) # Each user gets 1 channel
assert np.all(assignment.sum(axis=0) <= 1) # Each channel used by max 1 user
def test_evaluate_fixed_k():
env_config = EnvironmentConfig(num_users=2, num_channels=2, bandwidth=1e6)
surrogate = SemanticSurrogate(L=12)
allocator = ResourceAllocator(surrogate, env_config, K_max=4)
snr_linear = np.array([[10.0, 100.0], [50.0, 20.0]])
total_sse = allocator.evaluate_fixed_k(snr_linear, k_n=4)
assert total_sse > 0
@@ -0,0 +1,26 @@
"""
tests/test_baselines.py
"""
import pytest
import numpy as np
from src.models.baselines import BaselineModels
def test_baselines_initialization():
models = BaselineModels(mu=19.0, L=12)
assert models.mu == 19.0
def test_calculate_baseline_sse():
models = BaselineModels(mu=19.0, L=12)
snr_linear = np.array([10.0, 100.0, 1000.0])
sse_ideal = models.calculate_baseline_sse(snr_linear, "ideal")
sse_5g = models.calculate_baseline_sse(snr_linear, "5G")
sse_4g = models.calculate_baseline_sse(snr_linear, "4G")
assert len(sse_ideal) == 3
assert np.all(sse_ideal > sse_5g)
assert np.all(sse_5g > sse_4g)
@@ -0,0 +1,41 @@
"""
tests/test_environment.py
Tests for Module 1: Environment & Channel Simulator
"""
import pytest
import numpy as np
from src.models.environment import EnvironmentConfig, ChannelSimulator
def test_environment_initialization():
config = EnvironmentConfig(num_users=10, num_channels=10, bandwidth=1e6)
env = ChannelSimulator(config)
assert env.config.num_users == 10
assert env.config.num_channels == 10
assert env.config.bandwidth == 1e6
def test_path_loss_calculation():
config = EnvironmentConfig(num_users=1, num_channels=1)
env = ChannelSimulator(config)
# 1 km distance
distances = np.array([1.0])
path_loss = env._calculate_pathloss(distances)
# PL = 128.1 + 37.6 * log10(d)
expected_pl = 128.1
np.testing.assert_allclose(path_loss, expected_pl, atol=1e-5)
def test_snr_generation():
config = EnvironmentConfig(num_users=5, num_channels=3)
env = ChannelSimulator(config)
transmit_power_dbm = 10.0
snr_db, snr_linear = env.generate_channels(transmit_power_dbm)
assert snr_db.shape == (5, 3)
assert snr_linear.shape == (5, 3)
assert np.all(snr_linear > 0)
@@ -0,0 +1,27 @@
"""
tests/test_semantic_model.py
"""
import pytest
import numpy as np
from src.models.semantic_model import SemanticSurrogate
def test_semantic_surrogate():
surrogate = SemanticSurrogate()
# Test bounds
snr_linear = np.array([10.0, 100.0, 1000.0])
k_n = 4
sim = surrogate.get_similarity(snr_linear, k_n)
assert np.all(sim >= 0) and np.all(sim <= 1)
# Test monotonicity with SNR
assert sim[0] < sim[1] < sim[2]
# Test monotonicity with k_n
sim_k4 = surrogate.get_similarity(snr_linear, 4)
sim_k6 = surrogate.get_similarity(snr_linear, 6)
assert np.all(sim_k4 < sim_k6)