You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
Chameleon/integration/ptz17_hold_capture/hold_capture.patch

441 lines
24 KiB
Diff

--- a/postprocess.py
+++ b/postprocess.py
@@ -46,6 +46,7 @@
from .tracking_policy import should_allow_offsector_park, should_enforce_track_sector_limits
from .bt_tracker import ByteTracker
from .visual_tracker import (
+ VisualTrackResult,
bbox_center_distance_norm,
bbox_iou,
visual_tracker_manager,
@@ -776,6 +777,157 @@
}
_episode_telemetry(config, _PTZ17_CAM_IDX, (generation, started_at),
"decision_frame", at, **values)
+
+
+_PTZ17_RECOVERY_BOX_LIMIT = 12
+_PTZ17_RECOVERY_REASON_LIMIT = 160
+_PTZ17_CAPTURE_BOX_LIMIT = 64
+_PTZ17_CAPTURE_TAIL_S = 3.0
+_PTZ17_CAPTURE_STATE_KEYS = (
+ "last_bbox", "last_seen", "loss_since", "visual_reacquire_bbox", "visual_reacquire_at",
+ "visual_reacquire_votes", "visual_reacquire_source_seq", "zoom_rearm_epoch",
+ "zoom_rearm_required", "zoom_rearm_reason",
+)
+_DetectorBox = tuple[list[int], float, int]
+
+
+def _recovery_box_px(box: tuple[int, ...] | list[int] | None) -> list[int] | None:
+ """Snapshot xyxy pixels from the top left of the processing image, x right and y down."""
+ if box is None or len(box) != 4:
+ return None
+ try:
+ return [int(value) for value in box]
+ except (TypeError, ValueError, OverflowError):
+ return None
+
+
+def _recovery_detector_box(candidate: _DetectorBox | None) -> dict[str, Any] | None:
+ if candidate is None:
+ return None
+ box, score, class_id = candidate
+ confidence = float(score)
+ return dict(bbox_px=_recovery_box_px(box), confidence=confidence if math.isfinite(confidence) else None,
+ class_id=int(class_id))
+
+
+def _record_ptz17_recovery(
+ cam_idx: int, st: dict, observation_utc: float, source_seq: int | None,
+ frame_w: int, frame_h: int, visual_result: VisualTrackResult, *,
+ input_kind: str, outcome: str, trusted_bbox: tuple[int, ...] | list[int] | None,
+ reference_bbox: tuple[int, ...] | list[int] | None,
+ validation_targets: list[_DetectorBox], reacquire_targets: list[_DetectorBox],
+ raw_box_count: int, expected_track_class: int | None = None, detector_match: _DetectorBox | None = None,
+ selected: _DetectorBox | None = None, seeded: VisualTrackResult | None = None,
+) -> None:
+ """Record already available inputs and the recovery outcome off the control thread."""
+ if (cam_idx != _PTZ17_CAM_IDX or st.get("mode") != "TRACK" or not st.get("tracking_active")
+ or not getattr(config, "PREDICTIVE_ZOOM_TELEMETRY_DIR", "logs/ptz_zoom")):
+ return
+ _, generation, started_at, rearm_epoch = visual_control.owner(st)
+ if started_at <= 0.0 or not math.isfinite(started_at) or not math.isfinite(observation_utc):
+ return
+
+ def tracker_fields(result: VisualTrackResult) -> dict[str, Any]:
+ score = float(result.score)
+ return dict(bbox_px=_recovery_box_px(result.bbox), ok=bool(result.ok),
+ score=score if math.isfinite(score) else None, class_id=int(result.target_class),
+ frame_seq=int(result.source_seq) if result.source_seq is not None else None,
+ reason=str(result.reason)[:_PTZ17_RECOVERY_REASON_LIMIT])
+
+ _episode_telemetry(
+ config, cam_idx, (generation, started_at), "recovery_frame", time.time(),
+ observation_utc=observation_utc, frame_seq=int(source_seq) if source_seq is not None else None,
+ frame_size_px=[frame_w, frame_h], input_kind=input_kind, outcome=outcome,
+ expected_track_class=expected_track_class,
+ trusted_bbox_px=_recovery_box_px(trusted_bbox), association_reference_bbox_px=_recovery_box_px(reference_bbox),
+ ostrack=tracker_fields(visual_result), reinitialized_ostrack=tracker_fields(seeded) if seeded is not None else None,
+ yolo_raw_count=raw_box_count,
+ yolo_validation_count=len(validation_targets), yolo_reacquire_count=len(reacquire_targets),
+ yolo_validation=[_recovery_detector_box(box) for box in validation_targets[:_PTZ17_RECOVERY_BOX_LIMIT]],
+ yolo_reacquire=[_recovery_detector_box(box) for box in reacquire_targets[:_PTZ17_RECOVERY_BOX_LIMIT]],
+ yolo_match=_recovery_detector_box(detector_match), yolo_selected=_recovery_detector_box(selected),
+ reacquire_votes=int(st.get("visual_reacquire_votes", 0) or 0),
+ reacquire_vote_frame_seq=int(st.get("visual_reacquire_source_seq", -1)),
+ zoom_rearm_epoch=rearm_epoch, zoom_rearm_required=bool(st.get("zoom_rearm_required")),
+ zoom_rearm_votes=int(st.get("zoom_rearm_votes", 0) or 0),
+ )
+
+
+def _finish_ptz17_hold_capture(st: dict, control_utc: float, reason: str) -> None:
+ window = st.pop("ptz17_hold_capture", None)
+ if window is None:
+ return
+ owner, window_utc, count, _ = window
+ _, generation, started_at, _ = visual_control.owner(st)
+ if owner == (generation, started_at):
+ _episode_telemetry(config, _PTZ17_CAM_IDX, owner, "hold_capture_end", control_utc,
+ schema=1, window_utc=window_utc, input_count=count, close_reason=reason)
+
+
+def _capture_ptz17_hold_input(
+ st: dict, control_entry_utc: float, observation_utc: float, source_seq: int | None,
+ frame_w: int, frame_h: int, visual_result: VisualTrackResult, *,
+ input_kind: str, geometry_hold: bool, captured_utc: float | None,
+ detector_result_present: bool, raw_box_count: int, expected_track_class: int | None,
+ validation_targets: list[_DetectorBox], reacquire_targets: list[_DetectorBox],
+ before: dict | None = None,
+) -> None:
+ """Queue ordered recovery inputs. Coordinates are xyxy pixels, origin top left."""
+ if (st.get("mode") != "TRACK" or not st.get("tracking_active")
+ or not getattr(config, "PREDICTIVE_ZOOM_TELEMETRY_DIR", "logs/ptz_zoom")):
+ return
+ _, generation, started_at, _ = visual_control.owner(st)
+ if started_at <= 0.0 or not all(map(math.isfinite, (started_at, control_entry_utc, observation_utc))):
+ return
+ owner = (generation, started_at)
+ window = st.get("ptz17_hold_capture")
+ if window is not None and window[0] != owner:
+ st.pop("ptz17_hold_capture", None)
+ window = None
+ if window is not None and control_entry_utc > window[3]:
+ _finish_ptz17_hold_capture(st, control_entry_utc, "tail_elapsed")
+ window = None
+ if window is None and not geometry_hold:
+ return
+ if window is None:
+ window = (owner, control_entry_utc, 0, control_entry_utc + _PTZ17_CAPTURE_TAIL_S)
+ _, window_utc, count, deadline_utc = window
+ if geometry_hold:
+ deadline_utc = max(deadline_utc, control_entry_utc + _PTZ17_CAPTURE_TAIL_S)
+ st["ptz17_hold_capture"] = (owner, window_utc, count + 1, deadline_utc)
+ evidence = st if before is None else before
+
+ def finite(value: float | None) -> float | None:
+ return value if value is not None and math.isfinite(value) else None
+
+ _episode_telemetry(
+ config, _PTZ17_CAM_IDX, owner, "hold_capture_input", control_entry_utc,
+ schema=1, window_utc=window_utc, input_idx=count + 1, geometry_hold=geometry_hold,
+ frame_seq=source_seq, frame_size_px=[frame_w, frame_h], input_kind=input_kind,
+ observation_utc=observation_utc, captured_utc=finite(captured_utc),
+ detector_frame_seq=source_seq if detector_result_present else None,
+ detector_result_present=detector_result_present, yolo_raw_count=raw_box_count,
+ expected_track_class=expected_track_class,
+ ostrack=dict(bbox_px=_recovery_box_px(visual_result.bbox), ok=bool(visual_result.ok),
+ score=finite(visual_result.score), class_id=int(visual_result.target_class),
+ detector_conf=finite(visual_result.detector_conf), age_frames=int(visual_result.age_frames),
+ frame_seq=visual_result.source_seq,
+ reason=str(visual_result.reason)[:_PTZ17_RECOVERY_REASON_LIMIT]),
+ trusted_bbox_px=_recovery_box_px(evidence.get("last_bbox")),
+ last_seen_utc=finite(evidence.get("last_seen")),
+ loss_since_utc=finite(evidence.get("loss_since")),
+ reacquire_before=dict(bbox_px=_recovery_box_px(evidence.get("visual_reacquire_bbox")),
+ at_utc=finite(evidence.get("visual_reacquire_at")),
+ votes=int(evidence.get("visual_reacquire_votes") or 0),
+ frame_seq=int(evidence["visual_reacquire_source_seq"])
+ if evidence.get("visual_reacquire_source_seq") is not None else -1),
+ zoom_rearm_before=dict(epoch=int(evidence.get("zoom_rearm_epoch") or 0),
+ required=bool(evidence.get("zoom_rearm_required")),
+ reason=str(evidence.get("zoom_rearm_reason") or "")[:_PTZ17_RECOVERY_REASON_LIMIT]),
+ validation_count=len(validation_targets), reacquire_count=len(reacquire_targets),
+ validation=[_recovery_detector_box(box) for box in validation_targets[:_PTZ17_CAPTURE_BOX_LIMIT]],
+ reacquire=[_recovery_detector_box(box) for box in reacquire_targets[:_PTZ17_CAPTURE_BOX_LIMIT]],
+ )
def _ptz17_target_focus_score(
@@ -6782,6 +6934,8 @@
source="stop",
generation=int(st.get("ptz_generation", 0) or 0),
)
+ if cam_idx == _PTZ17_CAM_IDX:
+ _finish_ptz17_hold_capture(st, now, "track_end")
_clear_acquire_fields(st)
visual_tracker_manager.reset(cam_idx, reason=reason)
# Помечаем что парковка уже сделана — watchdog не будет дублировать
@@ -6936,11 +7090,20 @@
detector_targets: list[tuple[list[int], float, int]],
validation_targets: list[tuple[list[int], float, int]], width: int, height: int,
steering_targets: list[tuple[list[float], float, int]] | None = None,
+ *, recovery_targets: list[_DetectorBox] | None = None, raw_box_count: int = 0,
) -> None:
"""Validate the same image without rewinding live control or loss recovery."""
result = sample.result
bbox = result.bbox
if not result.ok or bbox is None:
+ if recovery_targets is not None:
+ _record_ptz17_recovery(
+ cam_idx, st, float(sample.captured_at), sample.source_seq, width, height, result,
+ input_kind="paired_detector", outcome="paired_tracker_unusable",
+ trusted_bbox=st.get("last_bbox"), reference_bbox=bbox,
+ validation_targets=validation_targets, reacquire_targets=recovery_targets,
+ raw_box_count=raw_box_count,
+ )
return
now = float(sample.captured_at)
target_class = int(result.target_class)
@@ -6962,6 +7125,14 @@
visual_tracker_manager.reset(cam_idx, reason='paired detector identity mismatch')
stop_ptz(cam_idx, source='stop', generation=int(st.get('ptz_generation', 0) or 0))
alarms.ingest_detection(cam_idx, False)
+ if recovery_targets is not None:
+ _record_ptz17_recovery(
+ cam_idx, st, now, sample.source_seq, width, height, result,
+ input_kind="paired_detector", outcome="paired_identity_stop",
+ trusted_bbox=st.get("last_bbox"), reference_bbox=bbox,
+ validation_targets=candidates, reacquire_targets=recovery_targets,
+ raw_box_count=raw_box_count, expected_track_class=target_class,
+ )
return
st['visual_tracker_mismatch_frames'] = 0
if matched is not None and now >= float(st.get('visual_tracker_last_detector_at', 0.) or 0.):
@@ -6981,6 +7152,14 @@
detector_conf=max(alarm_conf, identity_conf),
alarm_eligible=alarm_eligible, cam_idx=cam_idx)
alarms.ingest_detection(cam_idx, alarm_eligible)
+ if recovery_targets is not None:
+ _record_ptz17_recovery(
+ cam_idx, st, now, sample.source_seq, width, height, result,
+ input_kind="paired_detector", outcome="validation_only",
+ trusted_bbox=st.get("last_bbox"), reference_bbox=bbox,
+ validation_targets=candidates, reacquire_targets=recovery_targets,
+ raw_box_count=raw_box_count, expected_track_class=target_class, detector_match=matched,
+ )
@visual_control.serialized_control
@@ -6998,6 +7177,7 @@
h, w = frame_bgr.shape[:2]
w_inv, h_inv = (1.0 / w), (1.0 / h)
now = time.time()
+ control_entry_utc = now
# Intermediate control and delayed detector evidence have different jobs.
# The latter must never replay old image coordinates into the actuator.
@@ -7438,8 +7618,22 @@
)
if evidence_only:
- _consume_paired_detector_evidence(cam_idx, st, visual_sample, detector_targets,
- validation_targets, w, h, steering_targets)
+ if cam_idx == _PTZ17_CAM_IDX and "ptz17_hold_capture" in st:
+ _capture_ptz17_hold_input(
+ st, control_entry_utc, now, source_seq, w, h, visual_sample.result,
+ input_kind="paired_detector", geometry_hold=False, captured_utc=visual_sample.captured_at,
+ detector_result_present=boxes_t is not None, raw_box_count=raw_box_count,
+ expected_track_class=int(visual_sample.result.target_class),
+ validation_targets=validation_targets, reacquire_targets=reacquire_targets,
+ )
+ trace_recovery = cam_idx == _PTZ17_CAM_IDX and (
+ not visual_sample.result.ok or visual_sample.result.reason == "recovered"
+ or st.get("loss_since") is not None or st.get("zoom_rearm_reason") == "geometry_reject"
+ )
+ _consume_paired_detector_evidence(
+ cam_idx, st, visual_sample, detector_targets, validation_targets, w, h, steering_targets,
+ recovery_targets=reacquire_targets if trace_recovery else None, raw_box_count=raw_box_count,
+ )
return
if bt_active:
@@ -7759,6 +7953,7 @@
# its own gate, but a bad backend result/config override must still
# be unable to move the camera by following a remote cloud edge.
previous_control_bbox = st.get("last_bbox")
+ recovery_input_kind = "visual_control" if visual_only else "detector_control"
gross_visual_jump = False
visual_control_jump = 0.0
visual_control_limit = float(
@@ -7801,6 +7996,26 @@
("center jump", "area ratio", "validated area ratio")
)
)
+ capture_input = None
+ if cam_idx == _PTZ17_CAM_IDX and (geometric_reject or "ptz17_hold_capture" in st):
+ capture_before = {key: st.get(key) for key in _PTZ17_CAPTURE_STATE_KEYS}
+
+ def capture_input(held: bool) -> None:
+ _capture_ptz17_hold_input(
+ st, control_entry_utc, now, source_seq, w, h, visual_result,
+ input_kind=recovery_input_kind, geometry_hold=held,
+ captured_utc=visual_sample.captured_at if visual_sample is not None else None,
+ detector_result_present=not visual_only and boxes_t is not None,
+ raw_box_count=raw_box_count, expected_track_class=expected_track_class,
+ validation_targets=validation_targets, reacquire_targets=reacquire_targets,
+ before=capture_before,
+ )
+
+ if not (geometric_reject or gross_visual_jump):
+ capture_input(False)
+ geometry_match = validated_geometry_target = None
+ seeded_geometry = None
+ geometry_outcome = "geometry_no_box"
if (gross_visual_jump or geometric_reject) and visual_result.bbox is not None:
# A large motion is safe only when the *current* detector agrees
# with that new location. A backend result that passed its own
@@ -7822,6 +8037,8 @@
),
allow_target_class_compatibility=True,
)
+ geometry_match = validated_geometry_target
+ geometry_outcome = "geometry_no_match"
if (
validated_geometry_target is not None
and isinstance(previous_control_bbox, (tuple, list))
@@ -7852,6 +8069,7 @@
)
):
validated_geometry_target = None
+ geometry_outcome = "geometry_trusted_area_rejected"
if geometric_reject and validated_geometry_target is not None:
validated_geometry_target = _confirm_loss_reacquire_candidate(
st,
@@ -7861,6 +8079,7 @@
h,
source_seq=source_seq,
)
+ geometry_outcome = "geometry_confirmation_pending"
if validated_geometry_target is not None:
preserved_track_class = (
int(visual_result.target_class)
@@ -7880,6 +8099,7 @@
now=now,
source_seq=source_seq,
)
+ geometry_outcome = "geometry_seed_failed"
if seeded_geometry.ok and seeded_geometry.bbox is not None:
logger.info(
"[OSTRACK/IDENTITY] cam=%s detector validated geometry jump; reinitialized",
@@ -7893,6 +8113,18 @@
_arm_zoom_revalidation(
st, now, "geometry_reinitialized", cam_idx=cam_idx
)
+ if cam_idx == _PTZ17_CAM_IDX:
+ if capture_input is not None:
+ capture_input(geometric_reject)
+ _record_ptz17_recovery(
+ cam_idx, st, now, source_seq, w, h, visual_result,
+ input_kind=recovery_input_kind, outcome="geometry_reinitialized",
+ trusted_bbox=previous_control_bbox, reference_bbox=visual_result.bbox,
+ validation_targets=validation_targets, reacquire_targets=reacquire_targets,
+ raw_box_count=raw_box_count, detector_match=geometry_match,
+ selected=validated_geometry_target, seeded=seeded_geometry,
+ expected_track_class=expected_track_class,
+ )
visual_result = seeded_geometry
reference_bbox = seeded_geometry.bbox
gross_visual_jump = False
@@ -7922,6 +8154,18 @@
source="stop",
generation=int(st.get("ptz_generation", 0) or 0),
)
+ if cam_idx == _PTZ17_CAM_IDX:
+ if capture_input is not None:
+ capture_input(True)
+ _record_ptz17_recovery(
+ cam_idx, st, now, source_seq, w, h, visual_result,
+ input_kind=recovery_input_kind, outcome=geometry_outcome,
+ trusted_bbox=previous_control_bbox, reference_bbox=visual_result.bbox,
+ validation_targets=validation_targets, reacquire_targets=reacquire_targets,
+ raw_box_count=raw_box_count,
+ detector_match=geometry_match, selected=validated_geometry_target,
+ seeded=seeded_geometry, expected_track_class=expected_track_class,
+ )
logger.warning(
"[OSTRACK/IDENTITY] cam=%s HOLD rejected geometry reason=%s; awaiting recovery",
cam_idx,
@@ -7950,6 +8194,18 @@
source="stop",
generation=int(st.get("ptz_generation", 0) or 0),
)
+ if cam_idx == _PTZ17_CAM_IDX:
+ if capture_input is not None:
+ capture_input(False)
+ _record_ptz17_recovery(
+ cam_idx, st, now, source_seq, w, h, visual_result,
+ input_kind=recovery_input_kind, outcome="control_jump_stop",
+ trusted_bbox=previous_control_bbox, reference_bbox=visual_result.bbox,
+ validation_targets=validation_targets, reacquire_targets=reacquire_targets,
+ raw_box_count=raw_box_count, detector_match=geometry_match,
+ selected=validated_geometry_target, seeded=seeded_geometry,
+ expected_track_class=expected_track_class,
+ )
logger.warning(
"[OSTRACK/IDENTITY] cam=%s STOP rejected geometry reason=%s",
cam_idx,
@@ -7961,6 +8217,7 @@
matched_validation = None
matched_reacquire = None
+ widened_candidate = None
if reference_bbox is not None:
matched_validation = _find_visual_validation_target(
reference_bbox,
@@ -8241,6 +8498,17 @@
det_xyxy.append(tuple(visual_result.bbox))
visual_track_label_index = len(det_labels)
det_labels.append("target track")
+ if cam_idx == _PTZ17_CAM_IDX and not geometry_reinitialized and (
+ visual_result.reason == "recovered" or st.get("loss_since") is not None
+ ):
+ _record_ptz17_recovery(
+ cam_idx, st, now, source_seq, w, h, visual_result,
+ input_kind=recovery_input_kind, outcome="visual_recovered",
+ trusted_bbox=previous_control_bbox, reference_bbox=reference_bbox,
+ validation_targets=validation_targets, reacquire_targets=reacquire_targets,
+ raw_box_count=raw_box_count, detector_match=matched_validation,
+ expected_track_class=expected_track_class,
+ )
else:
ptz17_validation_timeout = bool(
visual_only and _ptz17_validation_timeout_steering(
@@ -8284,6 +8552,7 @@
_arm_zoom_revalidation(
st, now, "visual_reacquire", cam_idx=cam_idx
)
+ seeded = None
if matched_reacquire is not None:
preserved_track_class = (
int(visual_result.target_class)
@@ -8327,6 +8596,19 @@
target_source = "none"
st["visual_tracker_score"] = float(visual_result.score)
st["visual_tracker_last_reason"] = visual_result.reason
+ if cam_idx == _PTZ17_CAM_IDX:
+ _record_ptz17_recovery(
+ cam_idx, st, now, source_seq, w, h, visual_result,
+ input_kind=recovery_input_kind,
+ outcome=target_source if matched_reacquire is not None else
+ "loss_confirmation_pending" if widened_candidate is not None else "loss_no_match",
+ trusted_bbox=previous_control_bbox, reference_bbox=reference_bbox,
+ validation_targets=validation_targets, reacquire_targets=reacquire_targets,
+ raw_box_count=raw_box_count,
+ detector_match=widened_candidate if widened_candidate is not None else matched_reacquire,
+ selected=matched_reacquire, seeded=seeded,
+ expected_track_class=expected_track_class,
+ )
elif best_target is not None:
# ACQUIRE has already approved this target. Seed the appearance
# tracker on the same frame before entering full TRACK.