--- a/postprocess.py +++ b/postprocess.py @@ -46,6 +46,7 @@ from .tracking_policy import should_allow_offsector_park, should_enforce_track_sector_limits from .bt_tracker import ByteTracker from .visual_tracker import ( + VisualTrackResult, bbox_center_distance_norm, bbox_iou, visual_tracker_manager, @@ -776,6 +777,157 @@ } _episode_telemetry(config, _PTZ17_CAM_IDX, (generation, started_at), "decision_frame", at, **values) + + +_PTZ17_RECOVERY_BOX_LIMIT = 12 +_PTZ17_RECOVERY_REASON_LIMIT = 160 +_PTZ17_CAPTURE_BOX_LIMIT = 64 +_PTZ17_CAPTURE_TAIL_S = 3.0 +_PTZ17_CAPTURE_STATE_KEYS = ( + "last_bbox", "last_seen", "loss_since", "visual_reacquire_bbox", "visual_reacquire_at", + "visual_reacquire_votes", "visual_reacquire_source_seq", "zoom_rearm_epoch", + "zoom_rearm_required", "zoom_rearm_reason", +) +_DetectorBox = tuple[list[int], float, int] + + +def _recovery_box_px(box: tuple[int, ...] | list[int] | None) -> list[int] | None: + """Snapshot xyxy pixels from the top left of the processing image, x right and y down.""" + if box is None or len(box) != 4: + return None + try: + return [int(value) for value in box] + except (TypeError, ValueError, OverflowError): + return None + + +def _recovery_detector_box(candidate: _DetectorBox | None) -> dict[str, Any] | None: + if candidate is None: + return None + box, score, class_id = candidate + confidence = float(score) + return dict(bbox_px=_recovery_box_px(box), confidence=confidence if math.isfinite(confidence) else None, + class_id=int(class_id)) + + +def _record_ptz17_recovery( + cam_idx: int, st: dict, observation_utc: float, source_seq: int | None, + frame_w: int, frame_h: int, visual_result: VisualTrackResult, *, + input_kind: str, outcome: str, trusted_bbox: tuple[int, ...] | list[int] | None, + reference_bbox: tuple[int, ...] | list[int] | None, + validation_targets: list[_DetectorBox], reacquire_targets: list[_DetectorBox], + raw_box_count: int, expected_track_class: int | None = None, detector_match: _DetectorBox | None = None, + selected: _DetectorBox | None = None, seeded: VisualTrackResult | None = None, +) -> None: + """Record already available inputs and the recovery outcome off the control thread.""" + if (cam_idx != _PTZ17_CAM_IDX or st.get("mode") != "TRACK" or not st.get("tracking_active") + or not getattr(config, "PREDICTIVE_ZOOM_TELEMETRY_DIR", "logs/ptz_zoom")): + return + _, generation, started_at, rearm_epoch = visual_control.owner(st) + if started_at <= 0.0 or not math.isfinite(started_at) or not math.isfinite(observation_utc): + return + + def tracker_fields(result: VisualTrackResult) -> dict[str, Any]: + score = float(result.score) + return dict(bbox_px=_recovery_box_px(result.bbox), ok=bool(result.ok), + score=score if math.isfinite(score) else None, class_id=int(result.target_class), + frame_seq=int(result.source_seq) if result.source_seq is not None else None, + reason=str(result.reason)[:_PTZ17_RECOVERY_REASON_LIMIT]) + + _episode_telemetry( + config, cam_idx, (generation, started_at), "recovery_frame", time.time(), + observation_utc=observation_utc, frame_seq=int(source_seq) if source_seq is not None else None, + frame_size_px=[frame_w, frame_h], input_kind=input_kind, outcome=outcome, + expected_track_class=expected_track_class, + trusted_bbox_px=_recovery_box_px(trusted_bbox), association_reference_bbox_px=_recovery_box_px(reference_bbox), + ostrack=tracker_fields(visual_result), reinitialized_ostrack=tracker_fields(seeded) if seeded is not None else None, + yolo_raw_count=raw_box_count, + yolo_validation_count=len(validation_targets), yolo_reacquire_count=len(reacquire_targets), + yolo_validation=[_recovery_detector_box(box) for box in validation_targets[:_PTZ17_RECOVERY_BOX_LIMIT]], + yolo_reacquire=[_recovery_detector_box(box) for box in reacquire_targets[:_PTZ17_RECOVERY_BOX_LIMIT]], + yolo_match=_recovery_detector_box(detector_match), yolo_selected=_recovery_detector_box(selected), + reacquire_votes=int(st.get("visual_reacquire_votes", 0) or 0), + reacquire_vote_frame_seq=int(st.get("visual_reacquire_source_seq", -1)), + zoom_rearm_epoch=rearm_epoch, zoom_rearm_required=bool(st.get("zoom_rearm_required")), + zoom_rearm_votes=int(st.get("zoom_rearm_votes", 0) or 0), + ) + + +def _finish_ptz17_hold_capture(st: dict, control_utc: float, reason: str) -> None: + window = st.pop("ptz17_hold_capture", None) + if window is None: + return + owner, window_utc, count, _ = window + _, generation, started_at, _ = visual_control.owner(st) + if owner == (generation, started_at): + _episode_telemetry(config, _PTZ17_CAM_IDX, owner, "hold_capture_end", control_utc, + schema=1, window_utc=window_utc, input_count=count, close_reason=reason) + + +def _capture_ptz17_hold_input( + st: dict, control_entry_utc: float, observation_utc: float, source_seq: int | None, + frame_w: int, frame_h: int, visual_result: VisualTrackResult, *, + input_kind: str, geometry_hold: bool, captured_utc: float | None, + detector_result_present: bool, raw_box_count: int, expected_track_class: int | None, + validation_targets: list[_DetectorBox], reacquire_targets: list[_DetectorBox], + before: dict | None = None, +) -> None: + """Queue ordered recovery inputs. Coordinates are xyxy pixels, origin top left.""" + if (st.get("mode") != "TRACK" or not st.get("tracking_active") + or not getattr(config, "PREDICTIVE_ZOOM_TELEMETRY_DIR", "logs/ptz_zoom")): + return + _, generation, started_at, _ = visual_control.owner(st) + if started_at <= 0.0 or not all(map(math.isfinite, (started_at, control_entry_utc, observation_utc))): + return + owner = (generation, started_at) + window = st.get("ptz17_hold_capture") + if window is not None and window[0] != owner: + st.pop("ptz17_hold_capture", None) + window = None + if window is not None and control_entry_utc > window[3]: + _finish_ptz17_hold_capture(st, control_entry_utc, "tail_elapsed") + window = None + if window is None and not geometry_hold: + return + if window is None: + window = (owner, control_entry_utc, 0, control_entry_utc + _PTZ17_CAPTURE_TAIL_S) + _, window_utc, count, deadline_utc = window + if geometry_hold: + deadline_utc = max(deadline_utc, control_entry_utc + _PTZ17_CAPTURE_TAIL_S) + st["ptz17_hold_capture"] = (owner, window_utc, count + 1, deadline_utc) + evidence = st if before is None else before + + def finite(value: float | None) -> float | None: + return value if value is not None and math.isfinite(value) else None + + _episode_telemetry( + config, _PTZ17_CAM_IDX, owner, "hold_capture_input", control_entry_utc, + schema=1, window_utc=window_utc, input_idx=count + 1, geometry_hold=geometry_hold, + frame_seq=source_seq, frame_size_px=[frame_w, frame_h], input_kind=input_kind, + observation_utc=observation_utc, captured_utc=finite(captured_utc), + detector_frame_seq=source_seq if detector_result_present else None, + detector_result_present=detector_result_present, yolo_raw_count=raw_box_count, + expected_track_class=expected_track_class, + ostrack=dict(bbox_px=_recovery_box_px(visual_result.bbox), ok=bool(visual_result.ok), + score=finite(visual_result.score), class_id=int(visual_result.target_class), + detector_conf=finite(visual_result.detector_conf), age_frames=int(visual_result.age_frames), + frame_seq=visual_result.source_seq, + reason=str(visual_result.reason)[:_PTZ17_RECOVERY_REASON_LIMIT]), + trusted_bbox_px=_recovery_box_px(evidence.get("last_bbox")), + last_seen_utc=finite(evidence.get("last_seen")), + loss_since_utc=finite(evidence.get("loss_since")), + reacquire_before=dict(bbox_px=_recovery_box_px(evidence.get("visual_reacquire_bbox")), + at_utc=finite(evidence.get("visual_reacquire_at")), + votes=int(evidence.get("visual_reacquire_votes") or 0), + frame_seq=int(evidence["visual_reacquire_source_seq"]) + if evidence.get("visual_reacquire_source_seq") is not None else -1), + zoom_rearm_before=dict(epoch=int(evidence.get("zoom_rearm_epoch") or 0), + required=bool(evidence.get("zoom_rearm_required")), + reason=str(evidence.get("zoom_rearm_reason") or "")[:_PTZ17_RECOVERY_REASON_LIMIT]), + validation_count=len(validation_targets), reacquire_count=len(reacquire_targets), + validation=[_recovery_detector_box(box) for box in validation_targets[:_PTZ17_CAPTURE_BOX_LIMIT]], + reacquire=[_recovery_detector_box(box) for box in reacquire_targets[:_PTZ17_CAPTURE_BOX_LIMIT]], + ) def _ptz17_target_focus_score( @@ -6782,6 +6934,8 @@ source="stop", generation=int(st.get("ptz_generation", 0) or 0), ) + if cam_idx == _PTZ17_CAM_IDX: + _finish_ptz17_hold_capture(st, now, "track_end") _clear_acquire_fields(st) visual_tracker_manager.reset(cam_idx, reason=reason) # Помечаем что парковка уже сделана — watchdog не будет дублировать @@ -6936,11 +7090,20 @@ detector_targets: list[tuple[list[int], float, int]], validation_targets: list[tuple[list[int], float, int]], width: int, height: int, steering_targets: list[tuple[list[float], float, int]] | None = None, + *, recovery_targets: list[_DetectorBox] | None = None, raw_box_count: int = 0, ) -> None: """Validate the same image without rewinding live control or loss recovery.""" result = sample.result bbox = result.bbox if not result.ok or bbox is None: + if recovery_targets is not None: + _record_ptz17_recovery( + cam_idx, st, float(sample.captured_at), sample.source_seq, width, height, result, + input_kind="paired_detector", outcome="paired_tracker_unusable", + trusted_bbox=st.get("last_bbox"), reference_bbox=bbox, + validation_targets=validation_targets, reacquire_targets=recovery_targets, + raw_box_count=raw_box_count, + ) return now = float(sample.captured_at) target_class = int(result.target_class) @@ -6962,6 +7125,14 @@ visual_tracker_manager.reset(cam_idx, reason='paired detector identity mismatch') stop_ptz(cam_idx, source='stop', generation=int(st.get('ptz_generation', 0) or 0)) alarms.ingest_detection(cam_idx, False) + if recovery_targets is not None: + _record_ptz17_recovery( + cam_idx, st, now, sample.source_seq, width, height, result, + input_kind="paired_detector", outcome="paired_identity_stop", + trusted_bbox=st.get("last_bbox"), reference_bbox=bbox, + validation_targets=candidates, reacquire_targets=recovery_targets, + raw_box_count=raw_box_count, expected_track_class=target_class, + ) return st['visual_tracker_mismatch_frames'] = 0 if matched is not None and now >= float(st.get('visual_tracker_last_detector_at', 0.) or 0.): @@ -6981,6 +7152,14 @@ detector_conf=max(alarm_conf, identity_conf), alarm_eligible=alarm_eligible, cam_idx=cam_idx) alarms.ingest_detection(cam_idx, alarm_eligible) + if recovery_targets is not None: + _record_ptz17_recovery( + cam_idx, st, now, sample.source_seq, width, height, result, + input_kind="paired_detector", outcome="validation_only", + trusted_bbox=st.get("last_bbox"), reference_bbox=bbox, + validation_targets=candidates, reacquire_targets=recovery_targets, + raw_box_count=raw_box_count, expected_track_class=target_class, detector_match=matched, + ) @visual_control.serialized_control @@ -6998,6 +7177,7 @@ h, w = frame_bgr.shape[:2] w_inv, h_inv = (1.0 / w), (1.0 / h) now = time.time() + control_entry_utc = now # Intermediate control and delayed detector evidence have different jobs. # The latter must never replay old image coordinates into the actuator. @@ -7438,8 +7618,22 @@ ) if evidence_only: - _consume_paired_detector_evidence(cam_idx, st, visual_sample, detector_targets, - validation_targets, w, h, steering_targets) + if cam_idx == _PTZ17_CAM_IDX and "ptz17_hold_capture" in st: + _capture_ptz17_hold_input( + st, control_entry_utc, now, source_seq, w, h, visual_sample.result, + input_kind="paired_detector", geometry_hold=False, captured_utc=visual_sample.captured_at, + detector_result_present=boxes_t is not None, raw_box_count=raw_box_count, + expected_track_class=int(visual_sample.result.target_class), + validation_targets=validation_targets, reacquire_targets=reacquire_targets, + ) + trace_recovery = cam_idx == _PTZ17_CAM_IDX and ( + not visual_sample.result.ok or visual_sample.result.reason == "recovered" + or st.get("loss_since") is not None or st.get("zoom_rearm_reason") == "geometry_reject" + ) + _consume_paired_detector_evidence( + cam_idx, st, visual_sample, detector_targets, validation_targets, w, h, steering_targets, + recovery_targets=reacquire_targets if trace_recovery else None, raw_box_count=raw_box_count, + ) return if bt_active: @@ -7759,6 +7953,7 @@ # its own gate, but a bad backend result/config override must still # be unable to move the camera by following a remote cloud edge. previous_control_bbox = st.get("last_bbox") + recovery_input_kind = "visual_control" if visual_only else "detector_control" gross_visual_jump = False visual_control_jump = 0.0 visual_control_limit = float( @@ -7801,6 +7996,26 @@ ("center jump", "area ratio", "validated area ratio") ) ) + capture_input = None + if cam_idx == _PTZ17_CAM_IDX and (geometric_reject or "ptz17_hold_capture" in st): + capture_before = {key: st.get(key) for key in _PTZ17_CAPTURE_STATE_KEYS} + + def capture_input(held: bool) -> None: + _capture_ptz17_hold_input( + st, control_entry_utc, now, source_seq, w, h, visual_result, + input_kind=recovery_input_kind, geometry_hold=held, + captured_utc=visual_sample.captured_at if visual_sample is not None else None, + detector_result_present=not visual_only and boxes_t is not None, + raw_box_count=raw_box_count, expected_track_class=expected_track_class, + validation_targets=validation_targets, reacquire_targets=reacquire_targets, + before=capture_before, + ) + + if not (geometric_reject or gross_visual_jump): + capture_input(False) + geometry_match = validated_geometry_target = None + seeded_geometry = None + geometry_outcome = "geometry_no_box" if (gross_visual_jump or geometric_reject) and visual_result.bbox is not None: # A large motion is safe only when the *current* detector agrees # with that new location. A backend result that passed its own @@ -7822,6 +8037,8 @@ ), allow_target_class_compatibility=True, ) + geometry_match = validated_geometry_target + geometry_outcome = "geometry_no_match" if ( validated_geometry_target is not None and isinstance(previous_control_bbox, (tuple, list)) @@ -7852,6 +8069,7 @@ ) ): validated_geometry_target = None + geometry_outcome = "geometry_trusted_area_rejected" if geometric_reject and validated_geometry_target is not None: validated_geometry_target = _confirm_loss_reacquire_candidate( st, @@ -7861,6 +8079,7 @@ h, source_seq=source_seq, ) + geometry_outcome = "geometry_confirmation_pending" if validated_geometry_target is not None: preserved_track_class = ( int(visual_result.target_class) @@ -7880,6 +8099,7 @@ now=now, source_seq=source_seq, ) + geometry_outcome = "geometry_seed_failed" if seeded_geometry.ok and seeded_geometry.bbox is not None: logger.info( "[OSTRACK/IDENTITY] cam=%s detector validated geometry jump; reinitialized", @@ -7893,6 +8113,18 @@ _arm_zoom_revalidation( st, now, "geometry_reinitialized", cam_idx=cam_idx ) + if cam_idx == _PTZ17_CAM_IDX: + if capture_input is not None: + capture_input(geometric_reject) + _record_ptz17_recovery( + cam_idx, st, now, source_seq, w, h, visual_result, + input_kind=recovery_input_kind, outcome="geometry_reinitialized", + trusted_bbox=previous_control_bbox, reference_bbox=visual_result.bbox, + validation_targets=validation_targets, reacquire_targets=reacquire_targets, + raw_box_count=raw_box_count, detector_match=geometry_match, + selected=validated_geometry_target, seeded=seeded_geometry, + expected_track_class=expected_track_class, + ) visual_result = seeded_geometry reference_bbox = seeded_geometry.bbox gross_visual_jump = False @@ -7922,6 +8154,18 @@ source="stop", generation=int(st.get("ptz_generation", 0) or 0), ) + if cam_idx == _PTZ17_CAM_IDX: + if capture_input is not None: + capture_input(True) + _record_ptz17_recovery( + cam_idx, st, now, source_seq, w, h, visual_result, + input_kind=recovery_input_kind, outcome=geometry_outcome, + trusted_bbox=previous_control_bbox, reference_bbox=visual_result.bbox, + validation_targets=validation_targets, reacquire_targets=reacquire_targets, + raw_box_count=raw_box_count, + detector_match=geometry_match, selected=validated_geometry_target, + seeded=seeded_geometry, expected_track_class=expected_track_class, + ) logger.warning( "[OSTRACK/IDENTITY] cam=%s HOLD rejected geometry reason=%s; awaiting recovery", cam_idx, @@ -7950,6 +8194,18 @@ source="stop", generation=int(st.get("ptz_generation", 0) or 0), ) + if cam_idx == _PTZ17_CAM_IDX: + if capture_input is not None: + capture_input(False) + _record_ptz17_recovery( + cam_idx, st, now, source_seq, w, h, visual_result, + input_kind=recovery_input_kind, outcome="control_jump_stop", + trusted_bbox=previous_control_bbox, reference_bbox=visual_result.bbox, + validation_targets=validation_targets, reacquire_targets=reacquire_targets, + raw_box_count=raw_box_count, detector_match=geometry_match, + selected=validated_geometry_target, seeded=seeded_geometry, + expected_track_class=expected_track_class, + ) logger.warning( "[OSTRACK/IDENTITY] cam=%s STOP rejected geometry reason=%s", cam_idx, @@ -7961,6 +8217,7 @@ matched_validation = None matched_reacquire = None + widened_candidate = None if reference_bbox is not None: matched_validation = _find_visual_validation_target( reference_bbox, @@ -8241,6 +8498,17 @@ det_xyxy.append(tuple(visual_result.bbox)) visual_track_label_index = len(det_labels) det_labels.append("target track") + if cam_idx == _PTZ17_CAM_IDX and not geometry_reinitialized and ( + visual_result.reason == "recovered" or st.get("loss_since") is not None + ): + _record_ptz17_recovery( + cam_idx, st, now, source_seq, w, h, visual_result, + input_kind=recovery_input_kind, outcome="visual_recovered", + trusted_bbox=previous_control_bbox, reference_bbox=reference_bbox, + validation_targets=validation_targets, reacquire_targets=reacquire_targets, + raw_box_count=raw_box_count, detector_match=matched_validation, + expected_track_class=expected_track_class, + ) else: ptz17_validation_timeout = bool( visual_only and _ptz17_validation_timeout_steering( @@ -8284,6 +8552,7 @@ _arm_zoom_revalidation( st, now, "visual_reacquire", cam_idx=cam_idx ) + seeded = None if matched_reacquire is not None: preserved_track_class = ( int(visual_result.target_class) @@ -8327,6 +8596,19 @@ target_source = "none" st["visual_tracker_score"] = float(visual_result.score) st["visual_tracker_last_reason"] = visual_result.reason + if cam_idx == _PTZ17_CAM_IDX: + _record_ptz17_recovery( + cam_idx, st, now, source_seq, w, h, visual_result, + input_kind=recovery_input_kind, + outcome=target_source if matched_reacquire is not None else + "loss_confirmation_pending" if widened_candidate is not None else "loss_no_match", + trusted_bbox=previous_control_bbox, reference_bbox=reference_bbox, + validation_targets=validation_targets, reacquire_targets=reacquire_targets, + raw_box_count=raw_box_count, + detector_match=widened_candidate if widened_candidate is not None else matched_reacquire, + selected=matched_reacquire, seeded=seeded, + expected_track_class=expected_track_class, + ) elif best_target is not None: # ACQUIRE has already approved this target. Seed the appearance # tracker on the same frame before entering full TRACK.