"""Offline comparison of TRACK recovery with and without PTZ17 telemetry.""" from contextlib import ExitStack import importlib.util import json import math import os from pathlib import Path import sys from types import SimpleNamespace from unittest.mock import Mock, patch import numpy as np import pytest import torch HERE = Path(__file__).resolve().parent TRUSTED = (620, 350, 640, 364) REJECTED = (570, 325, 680, 382) GOOD = [622, 351, 643, 365] WRONG = [590, 335, 670, 372] MOVED = [760, 350, 780, 364] @pytest.fixture(scope="module") def versions(): runtime = Path(os.environ["PTZ17_TEST_RUNTIME"]) package = "_ptz17_recovery_runtime" paths = [runtime / "__init__.py", Path(os.environ["PTZ17_TEST_BASELINE"]), Path(os.environ["PTZ17_TEST_CANDIDATE"])] modules = [] for name, path in zip((package, package + ".baseline", package + ".candidate"), paths, strict=True): spec = importlib.util.spec_from_file_location( name, path, submodule_search_locations=[str(runtime)] if name == package else None, ) assert spec and spec.loader module = importlib.util.module_from_spec(spec) sys.modules[name] = module spec.loader.exec_module(module) modules.append(module) return modules[1:] class Boxes: def __init__(self, observations): self.xyxy = torch.tensor([box for box, _, _ in observations], dtype=torch.float32) self.conf = torch.tensor([confidence for _, confidence, _ in observations]) self.cls = torch.tensor([class_id for _, _, class_id in observations]) def __len__(self): return len(self.conf) def exercise(module, scenario, *, cam=6, enabled=True): st = module.state._default_ptz_state(cam) st.update(tracking_active=True, mode="TRACK", ptz_generation=31, track_started_at=90.0, last_seen=99.8, last_zoom_seen=99.8, last_bbox=TRUSTED, track_source="visual", hfov_deg=59.0, zoom_status_at=99.95, abs_zoom=1.0, identity_class=0, visual_tracker_last_detector_at=99.9, visual_tracker_last_zoom_detector_at=99.9, visual_tracker_last_detector_conf=0.80) result_type = module.visual_tracker_manager.update.__globals__["VisualTrackResult"] frame = np.zeros((720, 1280, 3), dtype=np.uint8) operations, telemetry = [], [] def emit(_config, _cam, owner, name, at, **fields): json.dumps(fields, allow_nan=False) if name == "recovery_frame": telemetry.append((owner, at, fields)) with ExitStack() as stack: stack.enter_context(patch.dict(module.state.ptz_states, {cam: st})) for key, value in { "PTZ_VISUAL_CONTROL_ENABLE": True, "PREDICTIVE_ZOOM_TELEMETRY_DIR": "offline" if enabled else "", "DETECT_CONF_BY_CLASS": {0: .65, 1: .65}, "DETECT_CONF": .65, "DRONE_ONLY": False, "MOTION_DETECT_ENABLE": False, "BT_ENABLE": True, "VISUAL_TRACKER_MATCH_CENTER_NORM": .10, "VISUAL_TRACKER_LOSS_REACQUIRE_CENTER_BASE": .15, "VISUAL_TRACKER_LOSS_REACQUIRE_CONFIRM_MAX_GAP_SEC": .65, }.items(): stack.enter_context(patch.object(module.config, key, value, create=True)) clock = stack.enter_context(patch.object(module.time, "time")) update = stack.enter_context(patch.object(module.visual_tracker_manager, "update")) for name in ("_publish_with_boxes", "notify_detected", "call_autofocus", "return_to_preset"): stack.enter_context(patch.object(module, name)) for name in ("stop_ptz", "move_ptz_bounded"): stack.enter_context(patch.object(module, name, side_effect=lambda *a, key=name, **kw: operations.append((key, a, kw)))) stack.enter_context(patch.object(module.alarms, "is_armed", return_value=False)) stack.enter_context(patch.object(module.alarms, "ingest_detection", side_effect=lambda *a: operations.append(("alarm", a, {})))) stack.enter_context(patch.object(module.visual_tracker_manager, "record_detector_validation")) stack.enter_context(patch.object(module.visual_tracker_manager, "reset")) stack.enter_context(patch.object(module.visual_tracker_manager, "should_refresh", return_value=False)) stack.enter_context(patch.object(module, "_episode_telemetry", side_effect=emit)) zoom_module = sys.modules[module.record_zoom_blocks.__module__] stack.enter_context(patch.object(zoom_module, "_episode_telemetry", side_effect=emit)) def start(cam_idx, _image, box, *, target_class, detector_conf, now, source_seq): operations.append(("seed", (cam_idx, tuple(box), target_class, detector_conf, now, source_seq), {})) ok = scenario != "seed_failed" return result_type(ok, tuple(box) if ok else None, .8, target_class, detector_conf, 0, "initialized" if ok else "no template", source_seq) stack.enter_context(patch.object(module.visual_tracker_manager, "start", side_effect=start)) def frame_step(seq, at, tracker_box, reason, observations=(), *, paired=False, visual_only=False, tracker_seq=None): ok = reason in ("tracked", "recovered") result = result_type(ok, tracker_box, .80 if ok else .29, 0, .80, 40, reason, seq if tracker_seq is None else tracker_seq) update.return_value = result clock.return_value = at sample = module.visual_control.VisualSample(module.visual_control.owner(st), seq, at - .010, result) module.postprocess_control(cam, frame, SimpleNamespace(boxes=Boxes(observations)), True, seq, visual_sample=sample if paired or visual_only else None, visual_only=visual_only) if scenario in {"trusted_pair", "duplicate", "gap", "interrupted", "mismatched_tracker", "weak"}: good = (GOOD, .51 if scenario != "weak" else .15, 0) rejected_reason = "validated area ratio 11.34" frame_step(41, 100.0, REJECTED, rejected_reason, [good, (WRONG, .95, 0)]) if scenario == "interrupted": frame_step(42, 100.1, REJECTED, rejected_reason, [(WRONG, .95, 0)]) if scenario == "mismatched_tracker": frame_step(42, 100.1, REJECTED, rejected_reason, [good], tracker_seq=43) frame_step(41 if scenario == "duplicate" else 43 if scenario == "interrupted" else 42, 100.7 if scenario == "gap" else 100.2, REJECTED, rejected_reason, [good], tracker_seq=43 if scenario == "mismatched_tracker" else None) elif scenario == "hold_then_strict": frame_step(41, 100.0, REJECTED, "validated area ratio 11.34", [(GOOD, .50, 0), (WRONG, .95, 0)]) frame_step(42, 100.2, REJECTED, "low score", [(GOOD, .51, 0)]) elif scenario == "wide_two_frames": frame_step(41, 100.0, None, "low score", [(MOVED, .70, 0)]) frame_step(42, 100.3, None, "low score", [(MOVED, .71, 0)]) elif scenario == "seed_failed": frame_step(41, 100.0, None, "low score", [(GOOD, .81, 0)]) elif scenario == "visual_recovers": frame_step(41, 100.0, REJECTED, "validated area ratio 11.34", visual_only=True) frame_step(42, 100.1, TRUSTED, "recovered", visual_only=True) elif scenario == "paired_hold": frame_step(41, 100.0, REJECTED, "validated area ratio 11.34", visual_only=True) frame_step(42, 100.1, TRUSTED, "tracked", [(GOOD, .81, 0)], paired=True) elif scenario == "geometry_two_frames": frame_step(41, 100.0, TRUSTED, "center jump", [(GOOD, .81, 0)]) frame_step(42, 100.2, TRUSTED, "center jump", [(GOOD, .82, 0)]) elif scenario == "ordinary_track": frame_step(41, 100.0, TRUSTED, "tracked", [(GOOD, .81, 0)]) else: raise ValueError(scenario) state_evidence = {key: st.get(key) for key in ( "track_source", "zoom_state", "zoom_rearm_epoch", "zoom_rearm_votes", "zoom_prev_cmd", "visual_reacquire_votes", "identity_state", "alarm_gate_eligible", "last_bbox", )} return operations, state_evidence, [fields for _, _, fields in telemetry] def test_trusted_geometry_recovers_on_second_detector_frame(versions): baseline, candidate = versions old_operations, _, old_rows = exercise(baseline, "trusted_pair") new_operations, _, new_rows = exercise(candidate, "trusted_pair") assert not any(action == "seed" for action, _, _ in old_operations) assert [row["outcome"] for row in old_rows] == ["geometry_trusted_area_rejected", "geometry_no_match"] assert [row["outcome"] for row in new_rows] == ["geometry_confirmation_pending", "geometry_reinitialized"] assert [row["reacquire_votes"] for row in new_rows] == [1, 2] assert new_rows[0]["association_reference_bbox_px"] == list(TRUSTED) assert new_rows[1]["yolo_selected"]["bbox_px"] == GOOD assert new_rows[1]["zoom_rearm_required"] assert [action for action, _, _ in new_operations].count("seed") == 1 assert next(action for action, _, _ in new_operations if action != "alarm") == "stop_ptz" assert not any(action == "move_ptz_bounded" and args[3] > 0 for action, args, _ in new_operations) assert not any(action == "alarm" and args[-1] for action, args, _ in new_operations) @pytest.mark.parametrize("scenario", ["duplicate", "gap", "interrupted", "mismatched_tracker", "weak"]) def test_rejected_evidence_cannot_recover(scenario, versions): operations, _, rows = exercise(versions[1], scenario) assert not any(action in {"seed", "move_ptz_bounded"} for action, _, _ in operations) assert all(row["reacquire_votes"] <= 1 for row in rows) assert any(action == "stop_ptz" for action, _, _ in operations) def test_trusted_geometry_change_is_limited_to_ptz17(versions): old, new = (exercise(module, "trusted_pair", cam=8) for module in versions) assert old == new @pytest.mark.parametrize("scenario,outcomes", [ ("hold_then_strict", ["geometry_confirmation_pending", "detector_reacquire"]), ("wide_two_frames", ["loss_confirmation_pending", "detector_reacquire"]), ("seed_failed", ["detector_fallback"]), ("visual_recovers", ["geometry_no_current_detector_frame", "visual_recovered"]), ("paired_hold", ["geometry_no_current_detector_frame", "validation_only"]), ("geometry_two_frames", ["geometry_confirmation_pending", "geometry_reinitialized"]), ("ordinary_track", []), ]) def test_command_and_alarm_parity(versions, scenario, outcomes): base, candidate = versions previous, previous_state, _ = exercise(base, scenario) actual, actual_state, rows = exercise(candidate, scenario) assert (actual, actual_state) == (previous, previous_state) assert [row["outcome"] for row in rows] == outcomes if scenario == "hold_then_strict": reject, recover = rows assert reject["trusted_bbox_px"] == list(TRUSTED) assert reject["association_reference_bbox_px"] == list(TRUSTED) assert reject["ostrack"]["bbox_px"] == list(REJECTED) assert reject["yolo_match"]["bbox_px"] == GOOD and reject["yolo_selected"] is None assert {tuple(box["bbox_px"]) for box in reject["yolo_reacquire"]} == {tuple(WRONG), tuple(GOOD)} assert recover["yolo_selected"]["bbox_px"] == GOOD assert recover["reinitialized_ostrack"]["bbox_px"] == GOOD if scenario == "wide_two_frames": assert [row["reacquire_votes"] for row in rows] == [1, 2] assert rows[0]["yolo_selected"] is None assert rows[1]["yolo_selected"]["bbox_px"] == MOVED if scenario == "paired_hold": paired = rows[-1] assert paired["frame_seq"] == paired["ostrack"]["frame_seq"] == 42 assert paired["observation_utc"] == pytest.approx(100.09) assert paired["input_kind"] == "paired_detector" assert paired["yolo_match"]["bbox_px"] == GOOD if scenario == "geometry_two_frames": assert [row["reacquire_votes"] for row in rows] == [1, 2] assert rows[-1]["ostrack"]["ok"] is False assert rows[-1]["reinitialized_ostrack"]["ok"] is True assert rows[-1]["reinitialized_ostrack"]["bbox_px"] == GOOD @pytest.mark.parametrize("cam,enabled", [(8, True), (6, False)]) def test_scope_and_disable(versions, cam, enabled): assert exercise(versions[1], "hold_then_strict", cam=cam, enabled=enabled)[2] == [] def test_bounds_and_finite_json(versions): module = versions[1] valid = module.visual_tracker_manager.update.__globals__["VisualTrackResult"] result = valid(False, (620, 350, 640, 364), math.nan, 0, .8, 40, "x" * 900, 17) candidates = [([k, 4, k + 20, 17], math.inf if k == 0 else .8, 0) for k in range(33)] selected = candidates[-1] state = dict(mode="TRACK", tracking_active=True, track_started_at=90.0, ptz_generation=7) sink = Mock() with patch.object(module, "_episode_telemetry", sink): module._record_ptz17_recovery( 6, state, 100.0, 17, 1280, 720, result, input_kind="detector_control", outcome="geometry_hold", trusted_bbox=TRUSTED, reference_bbox=result.bbox, validation_targets=candidates, reacquire_targets=candidates, raw_box_count=33, selected=selected, ) assert sink.call_args.args[1:4] == (6, (7, 90.0), "recovery_frame") row = sink.call_args.kwargs assert len(row["yolo_validation"]) == len(row["yolo_reacquire"]) == 12 assert row["yolo_validation_count"] == row["yolo_reacquire_count"] == 33 assert row["yolo_selected"]["bbox_px"] == selected[0] assert row["ostrack"]["score"] is None and row["yolo_validation"][0]["confidence"] is None assert len(row["ostrack"]["reason"]) == 160 json.dumps(row, allow_nan=False) def test_full_queue_drops_telemetry_without_waiting(versions, tmp_path): module = versions[1] log_module = sys.modules[module.record_zoom_blocks.__module__] with patch.object(log_module, "Thread"): writer = log_module.EpisodeTelemetryLogger(tmp_path, capacity=1) result_type = module.visual_tracker_manager.update.__globals__["VisualTrackResult"] result = result_type(False, REJECTED, .29, 0, .8, 40, "area ratio 11.34", 17) state = dict(mode="TRACK", tracking_active=True, track_started_at=90.0, ptz_generation=7) with patch.object(module.config, "PREDICTIVE_ZOOM_TELEMETRY_DIR", str(tmp_path), create=True): with patch.dict(log_module._telemetry_loggers, {str(tmp_path): writer}): for _ in range(2): module._record_ptz17_recovery( 6, state, 100.0, 17, 1280, 720, result, input_kind="visual_control", outcome="geometry_no_match", trusted_bbox=TRUSTED, reference_bbox=REJECTED, validation_targets=[], reacquire_targets=[], raw_box_count=0, ) assert writer.queue.qsize() == 1 assert writer.dropped == 1