You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
Chameleon/integration/ptz17_focus_zoom/test_recovery_telemetry.py

232 lines
12 KiB
Python

"""Offline comparison of TRACK recovery with and without PTZ17 telemetry."""
from contextlib import ExitStack
import importlib.util
import json
import math
import os
from pathlib import Path
import sys
from types import SimpleNamespace
from unittest.mock import Mock, patch
import numpy as np
import pytest
import torch
HERE = Path(__file__).resolve().parent
TRUSTED = (620, 350, 640, 364)
REJECTED = (570, 325, 680, 382)
GOOD = [622, 351, 643, 365]
WRONG = [590, 335, 670, 372]
MOVED = [760, 350, 780, 364]
@pytest.fixture(scope="module")
def versions():
runtime = Path(os.environ["PTZ17_TEST_RUNTIME"])
package = "_ptz17_recovery_runtime"
paths = [runtime / "__init__.py", Path(os.environ["PTZ17_TEST_BASELINE"]),
Path(os.environ["PTZ17_TEST_CANDIDATE"])]
modules = []
for name, path in zip((package, package + ".baseline", package + ".candidate"), paths, strict=True):
spec = importlib.util.spec_from_file_location(
name, path, submodule_search_locations=[str(runtime)] if name == package else None,
)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
sys.modules[name] = module
spec.loader.exec_module(module)
modules.append(module)
return modules[1:]
class Boxes:
def __init__(self, observations):
self.xyxy = torch.tensor([box for box, _, _ in observations], dtype=torch.float32)
self.conf = torch.tensor([confidence for _, confidence, _ in observations])
self.cls = torch.tensor([class_id for _, _, class_id in observations])
def __len__(self):
return len(self.conf)
def exercise(module, scenario, *, cam=6, enabled=True):
st = module.state._default_ptz_state(cam)
st.update(tracking_active=True, mode="TRACK", ptz_generation=31, track_started_at=90.0,
last_seen=99.8, last_zoom_seen=99.8, last_bbox=TRUSTED, track_source="visual",
hfov_deg=59.0, zoom_status_at=99.95, abs_zoom=1.0, identity_class=0,
visual_tracker_last_detector_at=99.9, visual_tracker_last_zoom_detector_at=99.9,
visual_tracker_last_detector_conf=0.80)
result_type = module.visual_tracker_manager.update.__globals__["VisualTrackResult"]
frame = np.zeros((720, 1280, 3), dtype=np.uint8)
operations, telemetry = [], []
def emit(_config, _cam, owner, name, at, **fields):
json.dumps(fields, allow_nan=False)
if name == "recovery_frame":
telemetry.append((owner, at, fields))
with ExitStack() as stack:
stack.enter_context(patch.dict(module.state.ptz_states, {cam: st}))
for key, value in {
"PTZ_VISUAL_CONTROL_ENABLE": True, "PREDICTIVE_ZOOM_TELEMETRY_DIR": "offline" if enabled else "",
"DETECT_CONF_BY_CLASS": {0: .65, 1: .65}, "DETECT_CONF": .65,
"DRONE_ONLY": False, "MOTION_DETECT_ENABLE": False, "BT_ENABLE": True,
"VISUAL_TRACKER_MATCH_CENTER_NORM": .10, "VISUAL_TRACKER_LOSS_REACQUIRE_CENTER_BASE": .15,
"VISUAL_TRACKER_LOSS_REACQUIRE_CONFIRM_MAX_GAP_SEC": .65,
}.items():
stack.enter_context(patch.object(module.config, key, value, create=True))
clock = stack.enter_context(patch.object(module.time, "time"))
update = stack.enter_context(patch.object(module.visual_tracker_manager, "update"))
for name in ("_publish_with_boxes", "notify_detected", "call_autofocus", "return_to_preset"):
stack.enter_context(patch.object(module, name))
for name in ("stop_ptz", "move_ptz_bounded"):
stack.enter_context(patch.object(module, name, side_effect=lambda *a, key=name, **kw:
operations.append((key, a, kw))))
stack.enter_context(patch.object(module.alarms, "is_armed", return_value=False))
stack.enter_context(patch.object(module.alarms, "ingest_detection", side_effect=lambda *a:
operations.append(("alarm", a, {}))))
stack.enter_context(patch.object(module.visual_tracker_manager, "record_detector_validation"))
stack.enter_context(patch.object(module.visual_tracker_manager, "reset"))
stack.enter_context(patch.object(module.visual_tracker_manager, "should_refresh", return_value=False))
stack.enter_context(patch.object(module, "_episode_telemetry", side_effect=emit))
zoom_module = sys.modules[module.record_zoom_blocks.__module__]
stack.enter_context(patch.object(zoom_module, "_episode_telemetry", side_effect=emit))
def start(cam_idx, _image, box, *, target_class, detector_conf, now, source_seq):
operations.append(("seed", (cam_idx, tuple(box), target_class, detector_conf, now, source_seq), {}))
ok = scenario != "seed_failed"
return result_type(ok, tuple(box) if ok else None, .8, target_class, detector_conf,
0, "initialized" if ok else "no template", source_seq)
stack.enter_context(patch.object(module.visual_tracker_manager, "start", side_effect=start))
def frame_step(seq, at, tracker_box, reason, observations=(), *, paired=False, visual_only=False):
ok = reason in ("tracked", "recovered")
result = result_type(ok, tracker_box, .80 if ok else .29, 0, .80, 40, reason, seq)
update.return_value = result
clock.return_value = at
sample = module.visual_control.VisualSample(module.visual_control.owner(st), seq, at - .010, result)
module.postprocess_control(cam, frame, SimpleNamespace(boxes=Boxes(observations)), True, seq,
visual_sample=sample if paired or visual_only else None,
visual_only=visual_only)
if scenario == "hold_then_strict":
frame_step(41, 100.0, REJECTED, "validated area ratio 11.34",
[(GOOD, .50, 0), (WRONG, .95, 0)])
frame_step(42, 100.2, REJECTED, "low score", [(GOOD, .51, 0)])
elif scenario == "wide_two_frames":
frame_step(41, 100.0, None, "low score", [(MOVED, .70, 0)])
frame_step(42, 100.3, None, "low score", [(MOVED, .71, 0)])
elif scenario == "seed_failed":
frame_step(41, 100.0, None, "low score", [(GOOD, .81, 0)])
elif scenario == "visual_recovers":
frame_step(41, 100.0, REJECTED, "validated area ratio 11.34", visual_only=True)
frame_step(42, 100.1, TRUSTED, "recovered", visual_only=True)
elif scenario == "paired_hold":
frame_step(41, 100.0, REJECTED, "validated area ratio 11.34", visual_only=True)
frame_step(42, 100.1, TRUSTED, "tracked", [(GOOD, .81, 0)], paired=True)
elif scenario == "geometry_two_frames":
frame_step(41, 100.0, TRUSTED, "center jump", [(GOOD, .81, 0)])
frame_step(42, 100.2, TRUSTED, "center jump", [(GOOD, .82, 0)])
elif scenario == "ordinary_track":
frame_step(41, 100.0, TRUSTED, "tracked", [(GOOD, .81, 0)])
else:
raise ValueError(scenario)
state_evidence = {key: st.get(key) for key in (
"track_source", "zoom_state", "zoom_rearm_epoch", "zoom_rearm_votes", "zoom_prev_cmd",
"visual_reacquire_votes", "identity_state", "alarm_gate_eligible", "last_bbox",
)}
return operations, state_evidence, [fields for _, _, fields in telemetry]
@pytest.mark.parametrize("scenario,outcomes", [
("hold_then_strict", ["geometry_trusted_area_rejected", "detector_reacquire"]),
("wide_two_frames", ["loss_confirmation_pending", "detector_reacquire"]),
("seed_failed", ["detector_fallback"]),
("visual_recovers", ["geometry_no_match", "visual_recovered"]),
("paired_hold", ["geometry_no_match", "validation_only"]),
("geometry_two_frames", ["geometry_confirmation_pending", "geometry_reinitialized"]),
("ordinary_track", []),
])
def test_command_and_alarm_parity(versions, scenario, outcomes):
base, candidate = versions
previous, previous_state, _ = exercise(base, scenario)
actual, actual_state, rows = exercise(candidate, scenario)
assert (actual, actual_state) == (previous, previous_state)
assert [row["outcome"] for row in rows] == outcomes
if scenario == "hold_then_strict":
reject, recover = rows
assert reject["trusted_bbox_px"] == list(TRUSTED)
assert reject["association_reference_bbox_px"] == list(REJECTED)
assert reject["ostrack"]["bbox_px"] == list(REJECTED)
assert reject["yolo_match"]["bbox_px"] == WRONG and reject["yolo_selected"] is None
assert {tuple(box["bbox_px"]) for box in reject["yolo_reacquire"]} == {tuple(WRONG), tuple(GOOD)}
assert recover["yolo_selected"]["bbox_px"] == GOOD
assert recover["reinitialized_ostrack"]["bbox_px"] == GOOD
if scenario == "wide_two_frames":
assert [row["reacquire_votes"] for row in rows] == [1, 2]
assert rows[0]["yolo_selected"] is None
assert rows[1]["yolo_selected"]["bbox_px"] == MOVED
if scenario == "paired_hold":
paired = rows[-1]
assert paired["frame_seq"] == paired["ostrack"]["frame_seq"] == 42
assert paired["observation_utc"] == pytest.approx(100.09)
assert paired["input_kind"] == "paired_detector"
assert paired["yolo_match"]["bbox_px"] == GOOD
if scenario == "geometry_two_frames":
assert [row["reacquire_votes"] for row in rows] == [1, 2]
assert rows[-1]["ostrack"]["ok"] is False
assert rows[-1]["reinitialized_ostrack"]["ok"] is True
assert rows[-1]["reinitialized_ostrack"]["bbox_px"] == GOOD
@pytest.mark.parametrize("cam,enabled", [(8, True), (6, False)])
def test_scope_and_disable(versions, cam, enabled):
assert exercise(versions[1], "hold_then_strict", cam=cam, enabled=enabled)[2] == []
def test_bounds_and_finite_json(versions):
module = versions[1]
valid = module.visual_tracker_manager.update.__globals__["VisualTrackResult"]
result = valid(False, (620, 350, 640, 364), math.nan, 0, .8, 40, "x" * 900, 17)
candidates = [([k, 4, k + 20, 17], math.inf if k == 0 else .8, 0) for k in range(33)]
selected = candidates[-1]
state = dict(mode="TRACK", tracking_active=True, track_started_at=90.0, ptz_generation=7)
sink = Mock()
with patch.object(module, "_episode_telemetry", sink):
module._record_ptz17_recovery(
6, state, 100.0, 17, 1280, 720, result, input_kind="detector_control",
outcome="geometry_hold", trusted_bbox=TRUSTED, reference_bbox=result.bbox,
validation_targets=candidates, reacquire_targets=candidates, raw_box_count=33, selected=selected,
)
assert sink.call_args.args[1:4] == (6, (7, 90.0), "recovery_frame")
row = sink.call_args.kwargs
assert len(row["yolo_validation"]) == len(row["yolo_reacquire"]) == 12
assert row["yolo_validation_count"] == row["yolo_reacquire_count"] == 33
assert row["yolo_selected"]["bbox_px"] == selected[0]
assert row["ostrack"]["score"] is None and row["yolo_validation"][0]["confidence"] is None
assert len(row["ostrack"]["reason"]) == 160
json.dumps(row, allow_nan=False)
def test_full_queue_drops_telemetry_without_waiting(versions, tmp_path):
module = versions[1]
log_module = sys.modules[module.record_zoom_blocks.__module__]
with patch.object(log_module, "Thread"):
writer = log_module.EpisodeTelemetryLogger(tmp_path, capacity=1)
result_type = module.visual_tracker_manager.update.__globals__["VisualTrackResult"]
result = result_type(False, REJECTED, .29, 0, .8, 40, "area ratio 11.34", 17)
state = dict(mode="TRACK", tracking_active=True, track_started_at=90.0, ptz_generation=7)
with patch.object(module.config, "PREDICTIVE_ZOOM_TELEMETRY_DIR", str(tmp_path), create=True):
with patch.dict(log_module._telemetry_loggers, {str(tmp_path): writer}):
for _ in range(2):
module._record_ptz17_recovery(
6, state, 100.0, 17, 1280, 720, result, input_kind="visual_control",
outcome="geometry_no_match", trusted_bbox=TRUSTED, reference_bbox=REJECTED,
validation_targets=[], reacquire_targets=[], raw_box_count=0,
)
assert writer.queue.qsize() == 1
assert writer.dropped == 1