Recover PTZ17 using the last trusted target box
parent
c74c8fb82f
commit
56a378e4c8
@ -0,0 +1,6 @@
|
||||
{
|
||||
"baseline_sha256": "f04d8a8d7a4105fe6812afaed00655dc42cbf7e6fb014e01d9f5efb306db13ca",
|
||||
"candidate_sha256": "82f54976be2ee6352981c15341e2738109afd8713b06a599cb5c7464f7363dd3",
|
||||
"patch_sha256": "65fcc671e93c8691d7363169d829b2ae27b5bd18f0a1ce6c241fdd75f7a19735",
|
||||
"installed": false
|
||||
}
|
||||
@ -0,0 +1,274 @@
|
||||
"""Offline comparison of TRACK recovery with and without PTZ17 telemetry."""
|
||||
from contextlib import ExitStack
|
||||
import importlib.util
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
TRUSTED = (620, 350, 640, 364)
|
||||
REJECTED = (570, 325, 680, 382)
|
||||
GOOD = [622, 351, 643, 365]
|
||||
WRONG = [590, 335, 670, 372]
|
||||
MOVED = [760, 350, 780, 364]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def versions():
|
||||
runtime = Path(os.environ["PTZ17_TEST_RUNTIME"])
|
||||
package = "_ptz17_recovery_runtime"
|
||||
paths = [runtime / "__init__.py", Path(os.environ["PTZ17_TEST_BASELINE"]),
|
||||
Path(os.environ["PTZ17_TEST_CANDIDATE"])]
|
||||
modules = []
|
||||
for name, path in zip((package, package + ".baseline", package + ".candidate"), paths, strict=True):
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
name, path, submodule_search_locations=[str(runtime)] if name == package else None,
|
||||
)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[name] = module
|
||||
spec.loader.exec_module(module)
|
||||
modules.append(module)
|
||||
return modules[1:]
|
||||
|
||||
|
||||
class Boxes:
|
||||
def __init__(self, observations):
|
||||
self.xyxy = torch.tensor([box for box, _, _ in observations], dtype=torch.float32)
|
||||
self.conf = torch.tensor([confidence for _, confidence, _ in observations])
|
||||
self.cls = torch.tensor([class_id for _, _, class_id in observations])
|
||||
|
||||
def __len__(self):
|
||||
return len(self.conf)
|
||||
|
||||
|
||||
def exercise(module, scenario, *, cam=6, enabled=True):
|
||||
st = module.state._default_ptz_state(cam)
|
||||
st.update(tracking_active=True, mode="TRACK", ptz_generation=31, track_started_at=90.0,
|
||||
last_seen=99.8, last_zoom_seen=99.8, last_bbox=TRUSTED, track_source="visual",
|
||||
hfov_deg=59.0, zoom_status_at=99.95, abs_zoom=1.0, identity_class=0,
|
||||
visual_tracker_last_detector_at=99.9, visual_tracker_last_zoom_detector_at=99.9,
|
||||
visual_tracker_last_detector_conf=0.80)
|
||||
result_type = module.visual_tracker_manager.update.__globals__["VisualTrackResult"]
|
||||
frame = np.zeros((720, 1280, 3), dtype=np.uint8)
|
||||
operations, telemetry = [], []
|
||||
|
||||
def emit(_config, _cam, owner, name, at, **fields):
|
||||
json.dumps(fields, allow_nan=False)
|
||||
if name == "recovery_frame":
|
||||
telemetry.append((owner, at, fields))
|
||||
|
||||
with ExitStack() as stack:
|
||||
stack.enter_context(patch.dict(module.state.ptz_states, {cam: st}))
|
||||
for key, value in {
|
||||
"PTZ_VISUAL_CONTROL_ENABLE": True, "PREDICTIVE_ZOOM_TELEMETRY_DIR": "offline" if enabled else "",
|
||||
"DETECT_CONF_BY_CLASS": {0: .65, 1: .65}, "DETECT_CONF": .65,
|
||||
"DRONE_ONLY": False, "MOTION_DETECT_ENABLE": False, "BT_ENABLE": True,
|
||||
"VISUAL_TRACKER_MATCH_CENTER_NORM": .10, "VISUAL_TRACKER_LOSS_REACQUIRE_CENTER_BASE": .15,
|
||||
"VISUAL_TRACKER_LOSS_REACQUIRE_CONFIRM_MAX_GAP_SEC": .65,
|
||||
}.items():
|
||||
stack.enter_context(patch.object(module.config, key, value, create=True))
|
||||
clock = stack.enter_context(patch.object(module.time, "time"))
|
||||
update = stack.enter_context(patch.object(module.visual_tracker_manager, "update"))
|
||||
for name in ("_publish_with_boxes", "notify_detected", "call_autofocus", "return_to_preset"):
|
||||
stack.enter_context(patch.object(module, name))
|
||||
for name in ("stop_ptz", "move_ptz_bounded"):
|
||||
stack.enter_context(patch.object(module, name, side_effect=lambda *a, key=name, **kw:
|
||||
operations.append((key, a, kw))))
|
||||
stack.enter_context(patch.object(module.alarms, "is_armed", return_value=False))
|
||||
stack.enter_context(patch.object(module.alarms, "ingest_detection", side_effect=lambda *a:
|
||||
operations.append(("alarm", a, {}))))
|
||||
stack.enter_context(patch.object(module.visual_tracker_manager, "record_detector_validation"))
|
||||
stack.enter_context(patch.object(module.visual_tracker_manager, "reset"))
|
||||
stack.enter_context(patch.object(module.visual_tracker_manager, "should_refresh", return_value=False))
|
||||
stack.enter_context(patch.object(module, "_episode_telemetry", side_effect=emit))
|
||||
zoom_module = sys.modules[module.record_zoom_blocks.__module__]
|
||||
stack.enter_context(patch.object(zoom_module, "_episode_telemetry", side_effect=emit))
|
||||
|
||||
def start(cam_idx, _image, box, *, target_class, detector_conf, now, source_seq):
|
||||
operations.append(("seed", (cam_idx, tuple(box), target_class, detector_conf, now, source_seq), {}))
|
||||
ok = scenario != "seed_failed"
|
||||
return result_type(ok, tuple(box) if ok else None, .8, target_class, detector_conf,
|
||||
0, "initialized" if ok else "no template", source_seq)
|
||||
|
||||
stack.enter_context(patch.object(module.visual_tracker_manager, "start", side_effect=start))
|
||||
|
||||
def frame_step(seq, at, tracker_box, reason, observations=(), *, paired=False, visual_only=False,
|
||||
tracker_seq=None):
|
||||
ok = reason in ("tracked", "recovered")
|
||||
result = result_type(ok, tracker_box, .80 if ok else .29, 0, .80, 40, reason,
|
||||
seq if tracker_seq is None else tracker_seq)
|
||||
update.return_value = result
|
||||
clock.return_value = at
|
||||
sample = module.visual_control.VisualSample(module.visual_control.owner(st), seq, at - .010, result)
|
||||
module.postprocess_control(cam, frame, SimpleNamespace(boxes=Boxes(observations)), True, seq,
|
||||
visual_sample=sample if paired or visual_only else None,
|
||||
visual_only=visual_only)
|
||||
|
||||
if scenario in {"trusted_pair", "duplicate", "gap", "interrupted", "mismatched_tracker", "weak"}:
|
||||
good = (GOOD, .51 if scenario != "weak" else .15, 0)
|
||||
rejected_reason = "validated area ratio 11.34"
|
||||
frame_step(41, 100.0, REJECTED, rejected_reason, [good, (WRONG, .95, 0)])
|
||||
if scenario == "interrupted":
|
||||
frame_step(42, 100.1, REJECTED, rejected_reason, [(WRONG, .95, 0)])
|
||||
if scenario == "mismatched_tracker":
|
||||
frame_step(42, 100.1, REJECTED, rejected_reason, [good], tracker_seq=43)
|
||||
frame_step(41 if scenario == "duplicate" else 43 if scenario == "interrupted" else 42,
|
||||
100.7 if scenario == "gap" else 100.2, REJECTED, rejected_reason, [good],
|
||||
tracker_seq=43 if scenario == "mismatched_tracker" else None)
|
||||
elif scenario == "hold_then_strict":
|
||||
frame_step(41, 100.0, REJECTED, "validated area ratio 11.34",
|
||||
[(GOOD, .50, 0), (WRONG, .95, 0)])
|
||||
frame_step(42, 100.2, REJECTED, "low score", [(GOOD, .51, 0)])
|
||||
elif scenario == "wide_two_frames":
|
||||
frame_step(41, 100.0, None, "low score", [(MOVED, .70, 0)])
|
||||
frame_step(42, 100.3, None, "low score", [(MOVED, .71, 0)])
|
||||
elif scenario == "seed_failed":
|
||||
frame_step(41, 100.0, None, "low score", [(GOOD, .81, 0)])
|
||||
elif scenario == "visual_recovers":
|
||||
frame_step(41, 100.0, REJECTED, "validated area ratio 11.34", visual_only=True)
|
||||
frame_step(42, 100.1, TRUSTED, "recovered", visual_only=True)
|
||||
elif scenario == "paired_hold":
|
||||
frame_step(41, 100.0, REJECTED, "validated area ratio 11.34", visual_only=True)
|
||||
frame_step(42, 100.1, TRUSTED, "tracked", [(GOOD, .81, 0)], paired=True)
|
||||
elif scenario == "geometry_two_frames":
|
||||
frame_step(41, 100.0, TRUSTED, "center jump", [(GOOD, .81, 0)])
|
||||
frame_step(42, 100.2, TRUSTED, "center jump", [(GOOD, .82, 0)])
|
||||
elif scenario == "ordinary_track":
|
||||
frame_step(41, 100.0, TRUSTED, "tracked", [(GOOD, .81, 0)])
|
||||
else:
|
||||
raise ValueError(scenario)
|
||||
state_evidence = {key: st.get(key) for key in (
|
||||
"track_source", "zoom_state", "zoom_rearm_epoch", "zoom_rearm_votes", "zoom_prev_cmd",
|
||||
"visual_reacquire_votes", "identity_state", "alarm_gate_eligible", "last_bbox",
|
||||
)}
|
||||
return operations, state_evidence, [fields for _, _, fields in telemetry]
|
||||
|
||||
|
||||
def test_trusted_geometry_recovers_on_second_detector_frame(versions):
|
||||
baseline, candidate = versions
|
||||
old_operations, _, old_rows = exercise(baseline, "trusted_pair")
|
||||
new_operations, _, new_rows = exercise(candidate, "trusted_pair")
|
||||
assert not any(action == "seed" for action, _, _ in old_operations)
|
||||
assert [row["outcome"] for row in old_rows] == ["geometry_trusted_area_rejected", "geometry_no_match"]
|
||||
assert [row["outcome"] for row in new_rows] == ["geometry_confirmation_pending", "geometry_reinitialized"]
|
||||
assert [row["reacquire_votes"] for row in new_rows] == [1, 2]
|
||||
assert new_rows[0]["association_reference_bbox_px"] == list(TRUSTED)
|
||||
assert new_rows[1]["yolo_selected"]["bbox_px"] == GOOD
|
||||
assert new_rows[1]["zoom_rearm_required"]
|
||||
assert [action for action, _, _ in new_operations].count("seed") == 1
|
||||
assert next(action for action, _, _ in new_operations if action != "alarm") == "stop_ptz"
|
||||
assert not any(action == "move_ptz_bounded" and args[3] > 0 for action, args, _ in new_operations)
|
||||
assert not any(action == "alarm" and args[-1] for action, args, _ in new_operations)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scenario", ["duplicate", "gap", "interrupted", "mismatched_tracker", "weak"])
|
||||
def test_rejected_evidence_cannot_recover(scenario, versions):
|
||||
operations, _, rows = exercise(versions[1], scenario)
|
||||
assert not any(action in {"seed", "move_ptz_bounded"} for action, _, _ in operations)
|
||||
assert all(row["reacquire_votes"] <= 1 for row in rows)
|
||||
assert any(action == "stop_ptz" for action, _, _ in operations)
|
||||
|
||||
|
||||
def test_trusted_geometry_change_is_limited_to_ptz17(versions):
|
||||
old, new = (exercise(module, "trusted_pair", cam=8) for module in versions)
|
||||
assert old == new
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scenario,outcomes", [
|
||||
("hold_then_strict", ["geometry_confirmation_pending", "detector_reacquire"]),
|
||||
("wide_two_frames", ["loss_confirmation_pending", "detector_reacquire"]),
|
||||
("seed_failed", ["detector_fallback"]),
|
||||
("visual_recovers", ["geometry_no_current_detector_frame", "visual_recovered"]),
|
||||
("paired_hold", ["geometry_no_current_detector_frame", "validation_only"]),
|
||||
("geometry_two_frames", ["geometry_confirmation_pending", "geometry_reinitialized"]),
|
||||
("ordinary_track", []),
|
||||
])
|
||||
def test_command_and_alarm_parity(versions, scenario, outcomes):
|
||||
base, candidate = versions
|
||||
previous, previous_state, _ = exercise(base, scenario)
|
||||
actual, actual_state, rows = exercise(candidate, scenario)
|
||||
assert (actual, actual_state) == (previous, previous_state)
|
||||
assert [row["outcome"] for row in rows] == outcomes
|
||||
if scenario == "hold_then_strict":
|
||||
reject, recover = rows
|
||||
assert reject["trusted_bbox_px"] == list(TRUSTED)
|
||||
assert reject["association_reference_bbox_px"] == list(TRUSTED)
|
||||
assert reject["ostrack"]["bbox_px"] == list(REJECTED)
|
||||
assert reject["yolo_match"]["bbox_px"] == GOOD and reject["yolo_selected"] is None
|
||||
assert {tuple(box["bbox_px"]) for box in reject["yolo_reacquire"]} == {tuple(WRONG), tuple(GOOD)}
|
||||
assert recover["yolo_selected"]["bbox_px"] == GOOD
|
||||
assert recover["reinitialized_ostrack"]["bbox_px"] == GOOD
|
||||
if scenario == "wide_two_frames":
|
||||
assert [row["reacquire_votes"] for row in rows] == [1, 2]
|
||||
assert rows[0]["yolo_selected"] is None
|
||||
assert rows[1]["yolo_selected"]["bbox_px"] == MOVED
|
||||
if scenario == "paired_hold":
|
||||
paired = rows[-1]
|
||||
assert paired["frame_seq"] == paired["ostrack"]["frame_seq"] == 42
|
||||
assert paired["observation_utc"] == pytest.approx(100.09)
|
||||
assert paired["input_kind"] == "paired_detector"
|
||||
assert paired["yolo_match"]["bbox_px"] == GOOD
|
||||
if scenario == "geometry_two_frames":
|
||||
assert [row["reacquire_votes"] for row in rows] == [1, 2]
|
||||
assert rows[-1]["ostrack"]["ok"] is False
|
||||
assert rows[-1]["reinitialized_ostrack"]["ok"] is True
|
||||
assert rows[-1]["reinitialized_ostrack"]["bbox_px"] == GOOD
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cam,enabled", [(8, True), (6, False)])
|
||||
def test_scope_and_disable(versions, cam, enabled):
|
||||
assert exercise(versions[1], "hold_then_strict", cam=cam, enabled=enabled)[2] == []
|
||||
|
||||
|
||||
def test_bounds_and_finite_json(versions):
|
||||
module = versions[1]
|
||||
valid = module.visual_tracker_manager.update.__globals__["VisualTrackResult"]
|
||||
result = valid(False, (620, 350, 640, 364), math.nan, 0, .8, 40, "x" * 900, 17)
|
||||
candidates = [([k, 4, k + 20, 17], math.inf if k == 0 else .8, 0) for k in range(33)]
|
||||
selected = candidates[-1]
|
||||
state = dict(mode="TRACK", tracking_active=True, track_started_at=90.0, ptz_generation=7)
|
||||
sink = Mock()
|
||||
with patch.object(module, "_episode_telemetry", sink):
|
||||
module._record_ptz17_recovery(
|
||||
6, state, 100.0, 17, 1280, 720, result, input_kind="detector_control",
|
||||
outcome="geometry_hold", trusted_bbox=TRUSTED, reference_bbox=result.bbox,
|
||||
validation_targets=candidates, reacquire_targets=candidates, raw_box_count=33, selected=selected,
|
||||
)
|
||||
assert sink.call_args.args[1:4] == (6, (7, 90.0), "recovery_frame")
|
||||
row = sink.call_args.kwargs
|
||||
assert len(row["yolo_validation"]) == len(row["yolo_reacquire"]) == 12
|
||||
assert row["yolo_validation_count"] == row["yolo_reacquire_count"] == 33
|
||||
assert row["yolo_selected"]["bbox_px"] == selected[0]
|
||||
assert row["ostrack"]["score"] is None and row["yolo_validation"][0]["confidence"] is None
|
||||
assert len(row["ostrack"]["reason"]) == 160
|
||||
json.dumps(row, allow_nan=False)
|
||||
|
||||
|
||||
def test_full_queue_drops_telemetry_without_waiting(versions, tmp_path):
|
||||
module = versions[1]
|
||||
log_module = sys.modules[module.record_zoom_blocks.__module__]
|
||||
with patch.object(log_module, "Thread"):
|
||||
writer = log_module.EpisodeTelemetryLogger(tmp_path, capacity=1)
|
||||
result_type = module.visual_tracker_manager.update.__globals__["VisualTrackResult"]
|
||||
result = result_type(False, REJECTED, .29, 0, .8, 40, "area ratio 11.34", 17)
|
||||
state = dict(mode="TRACK", tracking_active=True, track_started_at=90.0, ptz_generation=7)
|
||||
with patch.object(module.config, "PREDICTIVE_ZOOM_TELEMETRY_DIR", str(tmp_path), create=True):
|
||||
with patch.dict(log_module._telemetry_loggers, {str(tmp_path): writer}):
|
||||
for _ in range(2):
|
||||
module._record_ptz17_recovery(
|
||||
6, state, 100.0, 17, 1280, 720, result, input_kind="visual_control",
|
||||
outcome="geometry_no_match", trusted_bbox=TRUSTED, reference_bbox=REJECTED,
|
||||
validation_targets=[], reacquire_targets=[], raw_box_count=0,
|
||||
)
|
||||
assert writer.queue.qsize() == 1
|
||||
assert writer.dropped == 1
|
||||
@ -0,0 +1,68 @@
|
||||
--- a/postprocess.py
|
||||
+++ b/postprocess.py
|
||||
@@ -7904,14 +7904,31 @@
|
||||
geometry_match = validated_geometry_target = None
|
||||
seeded_geometry = None
|
||||
geometry_outcome = "geometry_no_box"
|
||||
- if (gross_visual_jump or geometric_reject) and visual_result.bbox is not None:
|
||||
+ geometry_reference = visual_result.bbox
|
||||
+ trusted_geometry_recovery = cam_idx == _PTZ17_CAM_IDX and geometric_reject
|
||||
+ same_detector_frame = False
|
||||
+ if trusted_geometry_recovery:
|
||||
+ # The rejected proposal is unavailable as a spatial reference.
|
||||
+ # Only a detector pass on this exact sequence may vote.
|
||||
+ same_detector_frame = (
|
||||
+ not visual_only and source_seq is not None and source_seq > 0
|
||||
+ and visual_result.source_seq == source_seq
|
||||
+ )
|
||||
+ geometry_reference = (
|
||||
+ previous_control_bbox if same_detector_frame
|
||||
+ and isinstance(previous_control_bbox, (tuple, list))
|
||||
+ and len(previous_control_bbox) == 4 else None
|
||||
+ )
|
||||
+ if not same_detector_frame:
|
||||
+ geometry_outcome = "geometry_no_current_detector_frame"
|
||||
+ if (gross_visual_jump or geometric_reject) and geometry_reference is not None:
|
||||
# A large motion is safe only when the *current* detector agrees
|
||||
# with that new location. A backend result that passed its own
|
||||
# geometry checks may be refreshed immediately. A rejected
|
||||
# location needs two independent detector frames; one large
|
||||
# cloud/building box must never switch the controlled target.
|
||||
validated_geometry_target = _find_matching_detector_target(
|
||||
- visual_result.bbox,
|
||||
+ geometry_reference,
|
||||
reacquire_targets,
|
||||
w,
|
||||
h,
|
||||
@@ -8005,7 +8022,7 @@
|
||||
_record_ptz17_recovery(
|
||||
cam_idx, st, now, source_seq, w, h, visual_result,
|
||||
input_kind=recovery_input_kind, outcome="geometry_reinitialized",
|
||||
- trusted_bbox=previous_control_bbox, reference_bbox=visual_result.bbox,
|
||||
+ trusted_bbox=previous_control_bbox, reference_bbox=geometry_reference,
|
||||
validation_targets=validation_targets, reacquire_targets=reacquire_targets,
|
||||
raw_box_count=raw_box_count, detector_match=geometry_match,
|
||||
selected=validated_geometry_target, seeded=seeded_geometry,
|
||||
@@ -8017,6 +8034,13 @@
|
||||
geometric_reject = False
|
||||
geometry_reinitialized = True
|
||||
_clear_loss_reacquire_candidate(st, "geometry_reinitialized")
|
||||
+ if (
|
||||
+ trusted_geometry_recovery and same_detector_frame and geometric_reject
|
||||
+ and geometry_match is None and source_seq is not None
|
||||
+ and source_seq > int(st.get("visual_reacquire_source_seq", -1))
|
||||
+ ):
|
||||
+ _clear_loss_reacquire_candidate(st, "geometry_detector_miss")
|
||||
+ st["visual_reacquire_source_seq"] = source_seq
|
||||
if geometric_reject:
|
||||
# visual_tracker.py deliberately keeps the last trusted bbox and
|
||||
# requires two consistent recovery frames. Do not destroy that
|
||||
@@ -8044,7 +8068,7 @@
|
||||
_record_ptz17_recovery(
|
||||
cam_idx, st, now, source_seq, w, h, visual_result,
|
||||
input_kind=recovery_input_kind, outcome=geometry_outcome,
|
||||
- trusted_bbox=previous_control_bbox, reference_bbox=visual_result.bbox,
|
||||
+ trusted_bbox=previous_control_bbox, reference_bbox=geometry_reference,
|
||||
validation_targets=validation_targets, reacquire_targets=reacquire_targets,
|
||||
raw_box_count=raw_box_count,
|
||||
detector_match=geometry_match, selected=validated_geometry_target,
|
||||
Loading…
Reference in New Issue