Record PTZ17 HOLD and reacquisition frame evidence
parent
f90c7da682
commit
c74c8fb82f
@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"base_sha256": "6cf484271ef3659028b0b4d43041addb05102f91d2758ab472d968a52d24e911",
|
"base_sha256": "6cf484271ef3659028b0b4d43041addb05102f91d2758ab472d968a52d24e911",
|
||||||
"candidate_sha256": "fc0deb0c54fbd94a916493ee0c3159b02407876b32deebbbc3d819cc58a17500",
|
"candidate_sha256": "f04d8a8d7a4105fe6812afaed00655dc42cbf7e6fb014e01d9f5efb306db13ca",
|
||||||
"patch_sha256": "e4a03e3dd4cd0f99995bc37acab29d063553cb32966fb13bdd93a46b8f7c5a9f",
|
"patch_sha256": "555d329a1c437187e254330384ec1dab60dc3fb8ac859c710c9cef517ac59dd5",
|
||||||
"installed": false
|
"installed": false
|
||||||
}
|
}
|
||||||
|
|||||||
@ -0,0 +1,231 @@
|
|||||||
|
"""Offline comparison of TRACK recovery with and without PTZ17 telemetry."""
|
||||||
|
from contextlib import ExitStack
|
||||||
|
import importlib.util
|
||||||
|
import json
|
||||||
|
import math
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from unittest.mock import Mock, patch
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import pytest
|
||||||
|
import torch
|
||||||
|
|
||||||
|
|
||||||
|
HERE = Path(__file__).resolve().parent
|
||||||
|
TRUSTED = (620, 350, 640, 364)
|
||||||
|
REJECTED = (570, 325, 680, 382)
|
||||||
|
GOOD = [622, 351, 643, 365]
|
||||||
|
WRONG = [590, 335, 670, 372]
|
||||||
|
MOVED = [760, 350, 780, 364]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="module")
|
||||||
|
def versions():
|
||||||
|
runtime = Path(os.environ["PTZ17_TEST_RUNTIME"])
|
||||||
|
package = "_ptz17_recovery_runtime"
|
||||||
|
paths = [runtime / "__init__.py", Path(os.environ["PTZ17_TEST_BASELINE"]),
|
||||||
|
Path(os.environ["PTZ17_TEST_CANDIDATE"])]
|
||||||
|
modules = []
|
||||||
|
for name, path in zip((package, package + ".baseline", package + ".candidate"), paths, strict=True):
|
||||||
|
spec = importlib.util.spec_from_file_location(
|
||||||
|
name, path, submodule_search_locations=[str(runtime)] if name == package else None,
|
||||||
|
)
|
||||||
|
assert spec and spec.loader
|
||||||
|
module = importlib.util.module_from_spec(spec)
|
||||||
|
sys.modules[name] = module
|
||||||
|
spec.loader.exec_module(module)
|
||||||
|
modules.append(module)
|
||||||
|
return modules[1:]
|
||||||
|
|
||||||
|
|
||||||
|
class Boxes:
|
||||||
|
def __init__(self, observations):
|
||||||
|
self.xyxy = torch.tensor([box for box, _, _ in observations], dtype=torch.float32)
|
||||||
|
self.conf = torch.tensor([confidence for _, confidence, _ in observations])
|
||||||
|
self.cls = torch.tensor([class_id for _, _, class_id in observations])
|
||||||
|
|
||||||
|
def __len__(self):
|
||||||
|
return len(self.conf)
|
||||||
|
|
||||||
|
|
||||||
|
def exercise(module, scenario, *, cam=6, enabled=True):
|
||||||
|
st = module.state._default_ptz_state(cam)
|
||||||
|
st.update(tracking_active=True, mode="TRACK", ptz_generation=31, track_started_at=90.0,
|
||||||
|
last_seen=99.8, last_zoom_seen=99.8, last_bbox=TRUSTED, track_source="visual",
|
||||||
|
hfov_deg=59.0, zoom_status_at=99.95, abs_zoom=1.0, identity_class=0,
|
||||||
|
visual_tracker_last_detector_at=99.9, visual_tracker_last_zoom_detector_at=99.9,
|
||||||
|
visual_tracker_last_detector_conf=0.80)
|
||||||
|
result_type = module.visual_tracker_manager.update.__globals__["VisualTrackResult"]
|
||||||
|
frame = np.zeros((720, 1280, 3), dtype=np.uint8)
|
||||||
|
operations, telemetry = [], []
|
||||||
|
|
||||||
|
def emit(_config, _cam, owner, name, at, **fields):
|
||||||
|
json.dumps(fields, allow_nan=False)
|
||||||
|
if name == "recovery_frame":
|
||||||
|
telemetry.append((owner, at, fields))
|
||||||
|
|
||||||
|
with ExitStack() as stack:
|
||||||
|
stack.enter_context(patch.dict(module.state.ptz_states, {cam: st}))
|
||||||
|
for key, value in {
|
||||||
|
"PTZ_VISUAL_CONTROL_ENABLE": True, "PREDICTIVE_ZOOM_TELEMETRY_DIR": "offline" if enabled else "",
|
||||||
|
"DETECT_CONF_BY_CLASS": {0: .65, 1: .65}, "DETECT_CONF": .65,
|
||||||
|
"DRONE_ONLY": False, "MOTION_DETECT_ENABLE": False, "BT_ENABLE": True,
|
||||||
|
"VISUAL_TRACKER_MATCH_CENTER_NORM": .10, "VISUAL_TRACKER_LOSS_REACQUIRE_CENTER_BASE": .15,
|
||||||
|
"VISUAL_TRACKER_LOSS_REACQUIRE_CONFIRM_MAX_GAP_SEC": .65,
|
||||||
|
}.items():
|
||||||
|
stack.enter_context(patch.object(module.config, key, value, create=True))
|
||||||
|
clock = stack.enter_context(patch.object(module.time, "time"))
|
||||||
|
update = stack.enter_context(patch.object(module.visual_tracker_manager, "update"))
|
||||||
|
for name in ("_publish_with_boxes", "notify_detected", "call_autofocus", "return_to_preset"):
|
||||||
|
stack.enter_context(patch.object(module, name))
|
||||||
|
for name in ("stop_ptz", "move_ptz_bounded"):
|
||||||
|
stack.enter_context(patch.object(module, name, side_effect=lambda *a, key=name, **kw:
|
||||||
|
operations.append((key, a, kw))))
|
||||||
|
stack.enter_context(patch.object(module.alarms, "is_armed", return_value=False))
|
||||||
|
stack.enter_context(patch.object(module.alarms, "ingest_detection", side_effect=lambda *a:
|
||||||
|
operations.append(("alarm", a, {}))))
|
||||||
|
stack.enter_context(patch.object(module.visual_tracker_manager, "record_detector_validation"))
|
||||||
|
stack.enter_context(patch.object(module.visual_tracker_manager, "reset"))
|
||||||
|
stack.enter_context(patch.object(module.visual_tracker_manager, "should_refresh", return_value=False))
|
||||||
|
stack.enter_context(patch.object(module, "_episode_telemetry", side_effect=emit))
|
||||||
|
zoom_module = sys.modules[module.record_zoom_blocks.__module__]
|
||||||
|
stack.enter_context(patch.object(zoom_module, "_episode_telemetry", side_effect=emit))
|
||||||
|
|
||||||
|
def start(cam_idx, _image, box, *, target_class, detector_conf, now, source_seq):
|
||||||
|
operations.append(("seed", (cam_idx, tuple(box), target_class, detector_conf, now, source_seq), {}))
|
||||||
|
ok = scenario != "seed_failed"
|
||||||
|
return result_type(ok, tuple(box) if ok else None, .8, target_class, detector_conf,
|
||||||
|
0, "initialized" if ok else "no template", source_seq)
|
||||||
|
|
||||||
|
stack.enter_context(patch.object(module.visual_tracker_manager, "start", side_effect=start))
|
||||||
|
|
||||||
|
def frame_step(seq, at, tracker_box, reason, observations=(), *, paired=False, visual_only=False):
|
||||||
|
ok = reason in ("tracked", "recovered")
|
||||||
|
result = result_type(ok, tracker_box, .80 if ok else .29, 0, .80, 40, reason, seq)
|
||||||
|
update.return_value = result
|
||||||
|
clock.return_value = at
|
||||||
|
sample = module.visual_control.VisualSample(module.visual_control.owner(st), seq, at - .010, result)
|
||||||
|
module.postprocess_control(cam, frame, SimpleNamespace(boxes=Boxes(observations)), True, seq,
|
||||||
|
visual_sample=sample if paired or visual_only else None,
|
||||||
|
visual_only=visual_only)
|
||||||
|
|
||||||
|
if scenario == "hold_then_strict":
|
||||||
|
frame_step(41, 100.0, REJECTED, "validated area ratio 11.34",
|
||||||
|
[(GOOD, .50, 0), (WRONG, .95, 0)])
|
||||||
|
frame_step(42, 100.2, REJECTED, "low score", [(GOOD, .51, 0)])
|
||||||
|
elif scenario == "wide_two_frames":
|
||||||
|
frame_step(41, 100.0, None, "low score", [(MOVED, .70, 0)])
|
||||||
|
frame_step(42, 100.3, None, "low score", [(MOVED, .71, 0)])
|
||||||
|
elif scenario == "seed_failed":
|
||||||
|
frame_step(41, 100.0, None, "low score", [(GOOD, .81, 0)])
|
||||||
|
elif scenario == "visual_recovers":
|
||||||
|
frame_step(41, 100.0, REJECTED, "validated area ratio 11.34", visual_only=True)
|
||||||
|
frame_step(42, 100.1, TRUSTED, "recovered", visual_only=True)
|
||||||
|
elif scenario == "paired_hold":
|
||||||
|
frame_step(41, 100.0, REJECTED, "validated area ratio 11.34", visual_only=True)
|
||||||
|
frame_step(42, 100.1, TRUSTED, "tracked", [(GOOD, .81, 0)], paired=True)
|
||||||
|
elif scenario == "geometry_two_frames":
|
||||||
|
frame_step(41, 100.0, TRUSTED, "center jump", [(GOOD, .81, 0)])
|
||||||
|
frame_step(42, 100.2, TRUSTED, "center jump", [(GOOD, .82, 0)])
|
||||||
|
elif scenario == "ordinary_track":
|
||||||
|
frame_step(41, 100.0, TRUSTED, "tracked", [(GOOD, .81, 0)])
|
||||||
|
else:
|
||||||
|
raise ValueError(scenario)
|
||||||
|
state_evidence = {key: st.get(key) for key in (
|
||||||
|
"track_source", "zoom_state", "zoom_rearm_epoch", "zoom_rearm_votes", "zoom_prev_cmd",
|
||||||
|
"visual_reacquire_votes", "identity_state", "alarm_gate_eligible", "last_bbox",
|
||||||
|
)}
|
||||||
|
return operations, state_evidence, [fields for _, _, fields in telemetry]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("scenario,outcomes", [
|
||||||
|
("hold_then_strict", ["geometry_trusted_area_rejected", "detector_reacquire"]),
|
||||||
|
("wide_two_frames", ["loss_confirmation_pending", "detector_reacquire"]),
|
||||||
|
("seed_failed", ["detector_fallback"]),
|
||||||
|
("visual_recovers", ["geometry_no_match", "visual_recovered"]),
|
||||||
|
("paired_hold", ["geometry_no_match", "validation_only"]),
|
||||||
|
("geometry_two_frames", ["geometry_confirmation_pending", "geometry_reinitialized"]),
|
||||||
|
("ordinary_track", []),
|
||||||
|
])
|
||||||
|
def test_command_and_alarm_parity(versions, scenario, outcomes):
|
||||||
|
base, candidate = versions
|
||||||
|
previous, previous_state, _ = exercise(base, scenario)
|
||||||
|
actual, actual_state, rows = exercise(candidate, scenario)
|
||||||
|
assert (actual, actual_state) == (previous, previous_state)
|
||||||
|
assert [row["outcome"] for row in rows] == outcomes
|
||||||
|
if scenario == "hold_then_strict":
|
||||||
|
reject, recover = rows
|
||||||
|
assert reject["trusted_bbox_px"] == list(TRUSTED)
|
||||||
|
assert reject["association_reference_bbox_px"] == list(REJECTED)
|
||||||
|
assert reject["ostrack"]["bbox_px"] == list(REJECTED)
|
||||||
|
assert reject["yolo_match"]["bbox_px"] == WRONG and reject["yolo_selected"] is None
|
||||||
|
assert {tuple(box["bbox_px"]) for box in reject["yolo_reacquire"]} == {tuple(WRONG), tuple(GOOD)}
|
||||||
|
assert recover["yolo_selected"]["bbox_px"] == GOOD
|
||||||
|
assert recover["reinitialized_ostrack"]["bbox_px"] == GOOD
|
||||||
|
if scenario == "wide_two_frames":
|
||||||
|
assert [row["reacquire_votes"] for row in rows] == [1, 2]
|
||||||
|
assert rows[0]["yolo_selected"] is None
|
||||||
|
assert rows[1]["yolo_selected"]["bbox_px"] == MOVED
|
||||||
|
if scenario == "paired_hold":
|
||||||
|
paired = rows[-1]
|
||||||
|
assert paired["frame_seq"] == paired["ostrack"]["frame_seq"] == 42
|
||||||
|
assert paired["observation_utc"] == pytest.approx(100.09)
|
||||||
|
assert paired["input_kind"] == "paired_detector"
|
||||||
|
assert paired["yolo_match"]["bbox_px"] == GOOD
|
||||||
|
if scenario == "geometry_two_frames":
|
||||||
|
assert [row["reacquire_votes"] for row in rows] == [1, 2]
|
||||||
|
assert rows[-1]["ostrack"]["ok"] is False
|
||||||
|
assert rows[-1]["reinitialized_ostrack"]["ok"] is True
|
||||||
|
assert rows[-1]["reinitialized_ostrack"]["bbox_px"] == GOOD
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("cam,enabled", [(8, True), (6, False)])
|
||||||
|
def test_scope_and_disable(versions, cam, enabled):
|
||||||
|
assert exercise(versions[1], "hold_then_strict", cam=cam, enabled=enabled)[2] == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_bounds_and_finite_json(versions):
|
||||||
|
module = versions[1]
|
||||||
|
valid = module.visual_tracker_manager.update.__globals__["VisualTrackResult"]
|
||||||
|
result = valid(False, (620, 350, 640, 364), math.nan, 0, .8, 40, "x" * 900, 17)
|
||||||
|
candidates = [([k, 4, k + 20, 17], math.inf if k == 0 else .8, 0) for k in range(33)]
|
||||||
|
selected = candidates[-1]
|
||||||
|
state = dict(mode="TRACK", tracking_active=True, track_started_at=90.0, ptz_generation=7)
|
||||||
|
sink = Mock()
|
||||||
|
with patch.object(module, "_episode_telemetry", sink):
|
||||||
|
module._record_ptz17_recovery(
|
||||||
|
6, state, 100.0, 17, 1280, 720, result, input_kind="detector_control",
|
||||||
|
outcome="geometry_hold", trusted_bbox=TRUSTED, reference_bbox=result.bbox,
|
||||||
|
validation_targets=candidates, reacquire_targets=candidates, raw_box_count=33, selected=selected,
|
||||||
|
)
|
||||||
|
assert sink.call_args.args[1:4] == (6, (7, 90.0), "recovery_frame")
|
||||||
|
row = sink.call_args.kwargs
|
||||||
|
assert len(row["yolo_validation"]) == len(row["yolo_reacquire"]) == 12
|
||||||
|
assert row["yolo_validation_count"] == row["yolo_reacquire_count"] == 33
|
||||||
|
assert row["yolo_selected"]["bbox_px"] == selected[0]
|
||||||
|
assert row["ostrack"]["score"] is None and row["yolo_validation"][0]["confidence"] is None
|
||||||
|
assert len(row["ostrack"]["reason"]) == 160
|
||||||
|
json.dumps(row, allow_nan=False)
|
||||||
|
|
||||||
|
|
||||||
|
def test_full_queue_drops_telemetry_without_waiting(versions, tmp_path):
|
||||||
|
module = versions[1]
|
||||||
|
log_module = sys.modules[module.record_zoom_blocks.__module__]
|
||||||
|
with patch.object(log_module, "Thread"):
|
||||||
|
writer = log_module.EpisodeTelemetryLogger(tmp_path, capacity=1)
|
||||||
|
result_type = module.visual_tracker_manager.update.__globals__["VisualTrackResult"]
|
||||||
|
result = result_type(False, REJECTED, .29, 0, .8, 40, "area ratio 11.34", 17)
|
||||||
|
state = dict(mode="TRACK", tracking_active=True, track_started_at=90.0, ptz_generation=7)
|
||||||
|
with patch.object(module.config, "PREDICTIVE_ZOOM_TELEMETRY_DIR", str(tmp_path), create=True):
|
||||||
|
with patch.dict(log_module._telemetry_loggers, {str(tmp_path): writer}):
|
||||||
|
for _ in range(2):
|
||||||
|
module._record_ptz17_recovery(
|
||||||
|
6, state, 100.0, 17, 1280, 720, result, input_kind="visual_control",
|
||||||
|
outcome="geometry_no_match", trusted_bbox=TRUSTED, reference_bbox=REJECTED,
|
||||||
|
validation_targets=[], reacquire_targets=[], raw_box_count=0,
|
||||||
|
)
|
||||||
|
assert writer.queue.qsize() == 1
|
||||||
|
assert writer.dropped == 1
|
||||||
Loading…
Reference in New Issue