You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
Chameleon/integration/ptz17_trusted_geometry_reco.../test_recovery.py

275 lines
15 KiB
Python

"""Offline comparison of TRACK recovery with and without PTZ17 telemetry."""
from contextlib import ExitStack
import importlib.util
import json
import math
import os
from pathlib import Path
import sys
from types import SimpleNamespace
from unittest.mock import Mock, patch
import numpy as np
import pytest
import torch
HERE = Path(__file__).resolve().parent
TRUSTED = (620, 350, 640, 364)
REJECTED = (570, 325, 680, 382)
GOOD = [622, 351, 643, 365]
WRONG = [590, 335, 670, 372]
MOVED = [760, 350, 780, 364]
@pytest.fixture(scope="module")
def versions():
runtime = Path(os.environ["PTZ17_TEST_RUNTIME"])
package = "_ptz17_recovery_runtime"
paths = [runtime / "__init__.py", Path(os.environ["PTZ17_TEST_BASELINE"]),
Path(os.environ["PTZ17_TEST_CANDIDATE"])]
modules = []
for name, path in zip((package, package + ".baseline", package + ".candidate"), paths, strict=True):
spec = importlib.util.spec_from_file_location(
name, path, submodule_search_locations=[str(runtime)] if name == package else None,
)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
sys.modules[name] = module
spec.loader.exec_module(module)
modules.append(module)
return modules[1:]
class Boxes:
def __init__(self, observations):
self.xyxy = torch.tensor([box for box, _, _ in observations], dtype=torch.float32)
self.conf = torch.tensor([confidence for _, confidence, _ in observations])
self.cls = torch.tensor([class_id for _, _, class_id in observations])
def __len__(self):
return len(self.conf)
def exercise(module, scenario, *, cam=6, enabled=True):
st = module.state._default_ptz_state(cam)
st.update(tracking_active=True, mode="TRACK", ptz_generation=31, track_started_at=90.0,
last_seen=99.8, last_zoom_seen=99.8, last_bbox=TRUSTED, track_source="visual",
hfov_deg=59.0, zoom_status_at=99.95, abs_zoom=1.0, identity_class=0,
visual_tracker_last_detector_at=99.9, visual_tracker_last_zoom_detector_at=99.9,
visual_tracker_last_detector_conf=0.80)
result_type = module.visual_tracker_manager.update.__globals__["VisualTrackResult"]
frame = np.zeros((720, 1280, 3), dtype=np.uint8)
operations, telemetry = [], []
def emit(_config, _cam, owner, name, at, **fields):
json.dumps(fields, allow_nan=False)
if name == "recovery_frame":
telemetry.append((owner, at, fields))
with ExitStack() as stack:
stack.enter_context(patch.dict(module.state.ptz_states, {cam: st}))
for key, value in {
"PTZ_VISUAL_CONTROL_ENABLE": True, "PREDICTIVE_ZOOM_TELEMETRY_DIR": "offline" if enabled else "",
"DETECT_CONF_BY_CLASS": {0: .65, 1: .65}, "DETECT_CONF": .65,
"DRONE_ONLY": False, "MOTION_DETECT_ENABLE": False, "BT_ENABLE": True,
"VISUAL_TRACKER_MATCH_CENTER_NORM": .10, "VISUAL_TRACKER_LOSS_REACQUIRE_CENTER_BASE": .15,
"VISUAL_TRACKER_LOSS_REACQUIRE_CONFIRM_MAX_GAP_SEC": .65,
}.items():
stack.enter_context(patch.object(module.config, key, value, create=True))
clock = stack.enter_context(patch.object(module.time, "time"))
update = stack.enter_context(patch.object(module.visual_tracker_manager, "update"))
for name in ("_publish_with_boxes", "notify_detected", "call_autofocus", "return_to_preset"):
stack.enter_context(patch.object(module, name))
for name in ("stop_ptz", "move_ptz_bounded"):
stack.enter_context(patch.object(module, name, side_effect=lambda *a, key=name, **kw:
operations.append((key, a, kw))))
stack.enter_context(patch.object(module.alarms, "is_armed", return_value=False))
stack.enter_context(patch.object(module.alarms, "ingest_detection", side_effect=lambda *a:
operations.append(("alarm", a, {}))))
stack.enter_context(patch.object(module.visual_tracker_manager, "record_detector_validation"))
stack.enter_context(patch.object(module.visual_tracker_manager, "reset"))
stack.enter_context(patch.object(module.visual_tracker_manager, "should_refresh", return_value=False))
stack.enter_context(patch.object(module, "_episode_telemetry", side_effect=emit))
zoom_module = sys.modules[module.record_zoom_blocks.__module__]
stack.enter_context(patch.object(zoom_module, "_episode_telemetry", side_effect=emit))
def start(cam_idx, _image, box, *, target_class, detector_conf, now, source_seq):
operations.append(("seed", (cam_idx, tuple(box), target_class, detector_conf, now, source_seq), {}))
ok = scenario != "seed_failed"
return result_type(ok, tuple(box) if ok else None, .8, target_class, detector_conf,
0, "initialized" if ok else "no template", source_seq)
stack.enter_context(patch.object(module.visual_tracker_manager, "start", side_effect=start))
def frame_step(seq, at, tracker_box, reason, observations=(), *, paired=False, visual_only=False,
tracker_seq=None):
ok = reason in ("tracked", "recovered")
result = result_type(ok, tracker_box, .80 if ok else .29, 0, .80, 40, reason,
seq if tracker_seq is None else tracker_seq)
update.return_value = result
clock.return_value = at
sample = module.visual_control.VisualSample(module.visual_control.owner(st), seq, at - .010, result)
module.postprocess_control(cam, frame, SimpleNamespace(boxes=Boxes(observations)), True, seq,
visual_sample=sample if paired or visual_only else None,
visual_only=visual_only)
if scenario in {"trusted_pair", "duplicate", "gap", "interrupted", "mismatched_tracker", "weak"}:
good = (GOOD, .51 if scenario != "weak" else .15, 0)
rejected_reason = "validated area ratio 11.34"
frame_step(41, 100.0, REJECTED, rejected_reason, [good, (WRONG, .95, 0)])
if scenario == "interrupted":
frame_step(42, 100.1, REJECTED, rejected_reason, [(WRONG, .95, 0)])
if scenario == "mismatched_tracker":
frame_step(42, 100.1, REJECTED, rejected_reason, [good], tracker_seq=43)
frame_step(41 if scenario == "duplicate" else 43 if scenario == "interrupted" else 42,
100.7 if scenario == "gap" else 100.2, REJECTED, rejected_reason, [good],
tracker_seq=43 if scenario == "mismatched_tracker" else None)
elif scenario == "hold_then_strict":
frame_step(41, 100.0, REJECTED, "validated area ratio 11.34",
[(GOOD, .50, 0), (WRONG, .95, 0)])
frame_step(42, 100.2, REJECTED, "low score", [(GOOD, .51, 0)])
elif scenario == "wide_two_frames":
frame_step(41, 100.0, None, "low score", [(MOVED, .70, 0)])
frame_step(42, 100.3, None, "low score", [(MOVED, .71, 0)])
elif scenario == "seed_failed":
frame_step(41, 100.0, None, "low score", [(GOOD, .81, 0)])
elif scenario == "visual_recovers":
frame_step(41, 100.0, REJECTED, "validated area ratio 11.34", visual_only=True)
frame_step(42, 100.1, TRUSTED, "recovered", visual_only=True)
elif scenario == "paired_hold":
frame_step(41, 100.0, REJECTED, "validated area ratio 11.34", visual_only=True)
frame_step(42, 100.1, TRUSTED, "tracked", [(GOOD, .81, 0)], paired=True)
elif scenario == "geometry_two_frames":
frame_step(41, 100.0, TRUSTED, "center jump", [(GOOD, .81, 0)])
frame_step(42, 100.2, TRUSTED, "center jump", [(GOOD, .82, 0)])
elif scenario == "ordinary_track":
frame_step(41, 100.0, TRUSTED, "tracked", [(GOOD, .81, 0)])
else:
raise ValueError(scenario)
state_evidence = {key: st.get(key) for key in (
"track_source", "zoom_state", "zoom_rearm_epoch", "zoom_rearm_votes", "zoom_prev_cmd",
"visual_reacquire_votes", "identity_state", "alarm_gate_eligible", "last_bbox",
)}
return operations, state_evidence, [fields for _, _, fields in telemetry]
def test_trusted_geometry_recovers_on_second_detector_frame(versions):
baseline, candidate = versions
old_operations, _, old_rows = exercise(baseline, "trusted_pair")
new_operations, _, new_rows = exercise(candidate, "trusted_pair")
assert not any(action == "seed" for action, _, _ in old_operations)
assert [row["outcome"] for row in old_rows] == ["geometry_trusted_area_rejected", "geometry_no_match"]
assert [row["outcome"] for row in new_rows] == ["geometry_confirmation_pending", "geometry_reinitialized"]
assert [row["reacquire_votes"] for row in new_rows] == [1, 2]
assert new_rows[0]["association_reference_bbox_px"] == list(TRUSTED)
assert new_rows[1]["yolo_selected"]["bbox_px"] == GOOD
assert new_rows[1]["zoom_rearm_required"]
assert [action for action, _, _ in new_operations].count("seed") == 1
assert next(action for action, _, _ in new_operations if action != "alarm") == "stop_ptz"
assert not any(action == "move_ptz_bounded" and args[3] > 0 for action, args, _ in new_operations)
assert not any(action == "alarm" and args[-1] for action, args, _ in new_operations)
@pytest.mark.parametrize("scenario", ["duplicate", "gap", "interrupted", "mismatched_tracker", "weak"])
def test_rejected_evidence_cannot_recover(scenario, versions):
operations, _, rows = exercise(versions[1], scenario)
assert not any(action in {"seed", "move_ptz_bounded"} for action, _, _ in operations)
assert all(row["reacquire_votes"] <= 1 for row in rows)
assert any(action == "stop_ptz" for action, _, _ in operations)
def test_trusted_geometry_change_is_limited_to_ptz17(versions):
old, new = (exercise(module, "trusted_pair", cam=8) for module in versions)
assert old == new
@pytest.mark.parametrize("scenario,outcomes", [
("hold_then_strict", ["geometry_confirmation_pending", "detector_reacquire"]),
("wide_two_frames", ["loss_confirmation_pending", "detector_reacquire"]),
("seed_failed", ["detector_fallback"]),
("visual_recovers", ["geometry_no_current_detector_frame", "visual_recovered"]),
("paired_hold", ["geometry_no_current_detector_frame", "validation_only"]),
("geometry_two_frames", ["geometry_confirmation_pending", "geometry_reinitialized"]),
("ordinary_track", []),
])
def test_command_and_alarm_parity(versions, scenario, outcomes):
base, candidate = versions
previous, previous_state, _ = exercise(base, scenario)
actual, actual_state, rows = exercise(candidate, scenario)
assert (actual, actual_state) == (previous, previous_state)
assert [row["outcome"] for row in rows] == outcomes
if scenario == "hold_then_strict":
reject, recover = rows
assert reject["trusted_bbox_px"] == list(TRUSTED)
assert reject["association_reference_bbox_px"] == list(TRUSTED)
assert reject["ostrack"]["bbox_px"] == list(REJECTED)
assert reject["yolo_match"]["bbox_px"] == GOOD and reject["yolo_selected"] is None
assert {tuple(box["bbox_px"]) for box in reject["yolo_reacquire"]} == {tuple(WRONG), tuple(GOOD)}
assert recover["yolo_selected"]["bbox_px"] == GOOD
assert recover["reinitialized_ostrack"]["bbox_px"] == GOOD
if scenario == "wide_two_frames":
assert [row["reacquire_votes"] for row in rows] == [1, 2]
assert rows[0]["yolo_selected"] is None
assert rows[1]["yolo_selected"]["bbox_px"] == MOVED
if scenario == "paired_hold":
paired = rows[-1]
assert paired["frame_seq"] == paired["ostrack"]["frame_seq"] == 42
assert paired["observation_utc"] == pytest.approx(100.09)
assert paired["input_kind"] == "paired_detector"
assert paired["yolo_match"]["bbox_px"] == GOOD
if scenario == "geometry_two_frames":
assert [row["reacquire_votes"] for row in rows] == [1, 2]
assert rows[-1]["ostrack"]["ok"] is False
assert rows[-1]["reinitialized_ostrack"]["ok"] is True
assert rows[-1]["reinitialized_ostrack"]["bbox_px"] == GOOD
@pytest.mark.parametrize("cam,enabled", [(8, True), (6, False)])
def test_scope_and_disable(versions, cam, enabled):
assert exercise(versions[1], "hold_then_strict", cam=cam, enabled=enabled)[2] == []
def test_bounds_and_finite_json(versions):
module = versions[1]
valid = module.visual_tracker_manager.update.__globals__["VisualTrackResult"]
result = valid(False, (620, 350, 640, 364), math.nan, 0, .8, 40, "x" * 900, 17)
candidates = [([k, 4, k + 20, 17], math.inf if k == 0 else .8, 0) for k in range(33)]
selected = candidates[-1]
state = dict(mode="TRACK", tracking_active=True, track_started_at=90.0, ptz_generation=7)
sink = Mock()
with patch.object(module, "_episode_telemetry", sink):
module._record_ptz17_recovery(
6, state, 100.0, 17, 1280, 720, result, input_kind="detector_control",
outcome="geometry_hold", trusted_bbox=TRUSTED, reference_bbox=result.bbox,
validation_targets=candidates, reacquire_targets=candidates, raw_box_count=33, selected=selected,
)
assert sink.call_args.args[1:4] == (6, (7, 90.0), "recovery_frame")
row = sink.call_args.kwargs
assert len(row["yolo_validation"]) == len(row["yolo_reacquire"]) == 12
assert row["yolo_validation_count"] == row["yolo_reacquire_count"] == 33
assert row["yolo_selected"]["bbox_px"] == selected[0]
assert row["ostrack"]["score"] is None and row["yolo_validation"][0]["confidence"] is None
assert len(row["ostrack"]["reason"]) == 160
json.dumps(row, allow_nan=False)
def test_full_queue_drops_telemetry_without_waiting(versions, tmp_path):
module = versions[1]
log_module = sys.modules[module.record_zoom_blocks.__module__]
with patch.object(log_module, "Thread"):
writer = log_module.EpisodeTelemetryLogger(tmp_path, capacity=1)
result_type = module.visual_tracker_manager.update.__globals__["VisualTrackResult"]
result = result_type(False, REJECTED, .29, 0, .8, 40, "area ratio 11.34", 17)
state = dict(mode="TRACK", tracking_active=True, track_started_at=90.0, ptz_generation=7)
with patch.object(module.config, "PREDICTIVE_ZOOM_TELEMETRY_DIR", str(tmp_path), create=True):
with patch.dict(log_module._telemetry_loggers, {str(tmp_path): writer}):
for _ in range(2):
module._record_ptz17_recovery(
6, state, 100.0, 17, 1280, 720, result, input_kind="visual_control",
outcome="geometry_no_match", trusted_bbox=TRUSTED, reference_bbox=REJECTED,
validation_targets=[], reacquire_targets=[], raw_box_count=0,
)
assert writer.queue.qsize() == 1
assert writer.dropped == 1