WIP: PTZ — привязка кадров для текущей модели #2

Draft
YaroslavKartashov wants to merge 1 commits from feature/legacy-ptz-frame-handoff into main

1
.gitattributes vendored

@ -0,0 +1 @@
/integration/legacy_ptz_frame_handoff/server.patch text eol=lf -whitespace

@ -0,0 +1,23 @@
# PTZ frame handoff для текущей модели
Отдельный кандидат из работы над UNKNOWN R3. Два изменения на фактическом серверном снимке:
- Остаточный возраст промежуточного кадра измеряется от декодирования. Привязка к подключению и владельцу проверяется до и после OSTrack.
- Детектор получает тот же кадр, что и парное наблюдение OSTrack. Результат парной обработки отбрасывается по истечении 650 мс или при смене подключения либо владельца.
Непарный инференс, обработка обзорных камер и пороги текущего детектора остаются в штатном пути. Патч не содержит изменений моделей, правил тревоги, управления линзой и ограничений движения. Время декодирования локальное, поэтому тесты не измеряют задержку от экспозиции камеры.
## Офлайн-проверка
В новом локальном каталоге разместите под `ptz_tracker_modular/` два исходных файла, совпадающих с `baseline` в `pins.json`. `streaming.py` берётся из фактического Docker bind. Из нового каталога:
```sh
git -c core.autocrlf=false apply --check /path/to/server.patch
git -c core.autocrlf=false apply /path/to/server.patch
python -B test_legacy_frame_handoff.py
git -c core.autocrlf=false apply -R /path/to/server.patch
```
Расположите файл теста в этом же каталоге. Сверьте SHA-256 до применения, после применения и после отката. Восемь проверок выполняются с подменой внешних соединений, трекера и управления камерой. Оба файла после обратного патча побайтно совпадают со снимком.
Применение на объекте требует отдельно проверенной процедуры замены обоих фактических источников и восстановления при частичном сбое. Локальный обратный патч сам по себе проверяет только восстановление содержимого файлов.

@ -0,0 +1,24 @@
{
"changed": {
"streaming.py": {
"baseline": "313828be8be66a5ebbf2365ec91de965128f06da16c54ff2332733a0e2fc701c",
"candidate": "a6c41eeba90c95f03bb78b7a5dad9e52595eb8310b4171efc5c523a17c509704"
},
"visual_control_probe.py": {
"baseline": "a5d6608c0fefa316a48384685ddf9b35bcd7dc1a5f7b1f8f1b348490d448832f",
"candidate": "6d81759060646a4af08d5d0de49ebad7ac417e1a789e37eff2403fff6d2162c9"
}
},
"patch_sha256": "5c472d70613cd91caed7af43872300c399e3c27c6bcdc8334f096066819a581e",
"source_captured_at": "2026-11-09T04:35:25.030505+00:00",
"preserved_source_sha256": {
"postprocess.py": "5326c9d3093b7c863310b9c0a7a85010a1a83ff014631a054ac9a16cc31c8fc5",
"ptz_io.py": "dacc7e95a2abc58deee0e3f6a4614aeb529acbee33aab5bd692ef16c39e33da7",
"model.py": "c2b1d73c634fa3724f302ff83c07c990d09ff4c6521eb7699fadaae9e3974e92",
"alarms.py": "f71d75f34664f48ffc230d54f3ee10168ad1d7f7942fbe815bc319d2764701ca",
"events.py": "c6606f18fdb17df59fbe74f38a500c0a0c0a9e7c8989bab9cf13685a726c8c53",
"main.py": "d1da4c165a89ff86713be8e940fd86cef84276250bb9f08ef42cfb8c8038618a",
"visual_control.py": "46cfa956f2fb567a3eae60aba5f96734dc3241761928e21eec2b9d56303322a6"
},
"exact_forward_and_reverse_roundtrip": true
}

@ -0,0 +1,179 @@
--- a/ptz_tracker_modular/streaming.py
+++ b/ptz_tracker_modular/streaming.py
@@ -3,3 +3,3 @@
from . import visual_control
-_visual_feed_captured_at: dict[int, float] = {}
+from typing import NamedTuple
@@ -60,3 +60,11 @@
_cpp_stream_map_missing_logged: str = ""
-_visual_feed_pending: Dict[int, Tuple[np.ndarray, int, float]] = {}
+class _VisualReceipt(NamedTuple):
+ source: object
+ owner: tuple
+ utc: float
+ monotonic: float
+
+
+_visual_source = None
+_visual_feed_pending: Dict[int, Tuple[np.ndarray, int, float, _VisualReceipt]] = {}
_visual_feed_last_served: Dict[int, float] = {}
@@ -228,3 +236,3 @@
waiting = []
- for cam_id, (_, _, queued_at) in _visual_feed_pending.items():
+ for cam_id, (_, _, queued_at, _) in _visual_feed_pending.items():
st = state.ptz_states.get(int(cam_id), {})
@@ -256,4 +264,3 @@
break
- frame_bgr, source_seq, queued_at = _visual_feed_pending.pop(cam_id)
- captured_at = _visual_feed_captured_at.pop(cam_id, time.time())
+ frame_bgr, source_seq, queued_at, receipt = _visual_feed_pending.pop(cam_id)
st = state.ptz_states.get(cam_id)
@@ -273,3 +280,4 @@
source_seq=int(source_seq),
- captured_at=captured_at,
+ captured_at=receipt.utc,
+ receipt=receipt,
),
@@ -299,2 +307,3 @@
source_seq: int,
+ *, receipt: Optional[_VisualReceipt],
) -> None:
@@ -302,6 +311,5 @@
global _visual_feed_worker_task
- if not _visual_feed_is_active(cam_id, st):
+ if receipt is None or not _visual_feed_is_active(cam_id, st):
_visual_feed_pending.pop(cam_id, None)
_visual_feed_last_served.pop(cam_id, None)
- _visual_feed_captured_at.pop(cam_id, None)
return
@@ -315,4 +323,3 @@
queued_at = float(previous[2]) if previous is not None else time.perf_counter()
- _visual_feed_pending[int(cam_id)] = (frame_bgr, int(source_seq), queued_at)
- _visual_feed_captured_at[int(cam_id)] = time.time()
+ _visual_feed_pending[int(cam_id)] = (frame_bgr, int(source_seq), queued_at, receipt)
if _visual_feed_worker_task is None or _visual_feed_worker_task.done():
@@ -2092,2 +2099,3 @@
async def cpp_ws_loop() -> None:
+ global _visual_source
frames_total = 0
@@ -2119,2 +2127,3 @@
logger.info("[CPP WS] connected]")
+ source = _visual_source = object()
_cpp_stream_map_logged.clear()
@@ -2208,5 +2217,14 @@
img = None
+ receipt = None
+
+ def decode():
+ owner = visual_control.owner(pst) if idx in config.PTZ_CAM_IDS else None
+ pixels = _decode_image(encoded)
+ if pixels is None or owner is None:
+ return pixels, None
+ return pixels, _VisualReceipt(source, owner, time.time(), time.monotonic())
+
frontend_due = can_publish and _frontend_raw_preview_due(idx, pst, now_pc)
if frontend_due:
- img = _decode_image(encoded)
+ img, receipt = decode()
if img is not None:
@@ -2224,5 +2242,5 @@
if getattr(config, "DROP_DECODE_WHEN_BUSY", False) and pst["proc_busy"]:
- img_busy = img if img is not None else _decode_image(encoded)
+ img_busy, receipt = (img, receipt) if img is not None else decode()
if img_busy is not None:
- _feed_visual_tracker_frame(idx, img_busy, pst, source_seq)
+ _feed_visual_tracker_frame(idx, img_busy, pst, source_seq, receipt=receipt)
pst["pending_frame"] = img_busy
@@ -2235,3 +2253,3 @@
if img is None:
- img = _decode_image(encoded)
+ img, receipt = decode()
if img is None:
@@ -2284,3 +2302,3 @@
if pst["proc_busy"]:
- _feed_visual_tracker_frame(idx, img, pst, source_seq)
+ _feed_visual_tracker_frame(idx, img, pst, source_seq, receipt=receipt)
skipped_busy_total += 1
@@ -2294,3 +2312,3 @@
if (now_pc - last_submit) < infer_min_dt:
- _feed_visual_tracker_frame(idx, img, pst, source_seq)
+ _feed_visual_tracker_frame(idx, img, pst, source_seq, receipt=receipt)
if publish_raw_allowed:
@@ -2345,3 +2363,5 @@
logger.error("[CPP WS] error: %s", exc)
- await asyncio.sleep(0.5)
+ finally:
+ _visual_source = None
+ await asyncio.sleep(0.5)
@@ -2563,12 +2583,23 @@
+def _visual_receipt_current(st, receipt, max_age):
+ return (receipt is not None and _visual_source is not None
+ and receipt.source is _visual_source
+ and receipt.owner == visual_control.owner(st)
+ and 0 <= time.monotonic() - receipt.monotonic <= max_age
+ and 0 <= time.time() - receipt.utc <= max_age)
+
+
@visual_control_probe.trace_feed_update
def _update_visual_feed(cam_id: int, frame_bgr: np.ndarray, source_seq: int,
- captured_at: Optional[float] = None) -> None:
+ captured_at: Optional[float] = None, *, receipt: _VisualReceipt) -> None:
with visual_control.control_lock(cam_id):
st = state.ptz_states.get(cam_id)
- if st is None:
+ if (st is None or not _visual_receipt_current(st, receipt, .25)
+ or not _visual_feed_is_active(cam_id, st)):
return
- captured_at = time.time() if captured_at is None else float(captured_at)
- token = visual_control.owner(st)
+ captured_at = receipt.utc
+ token = receipt.owner
result = visual_control_probe.call_tracker_update(visual_tracker_manager.update, cam_id, frame_bgr, source_seq=source_seq)
+ if not _visual_receipt_current(st, receipt, .25):
+ return
_store_visual_feed_result(cam_id, source_seq, result)
@@ -2589,2 +2620,3 @@
visual_control.retain(cam_id, frame_bgr, sample)
+ st['visual_paired_receipt'] = (sample, receipt)
@@ -2605,3 +2637,7 @@
paired_frame, sample = saved
- future.visual_sample = sample
+ provenance = st.get('visual_paired_receipt')
+ if (provenance is None or provenance[0] is not sample
+ or not _visual_receipt_current(st, provenance[1], .25)):
+ return item
+ future.visual_sample, future.visual_receipt = provenance
return cam_id, paired_frame, is_ptz, sample.source_seq, future, queued_at
@@ -2612,3 +2648,7 @@
return postprocess_control(cam_id, frame, res, is_ptz, source_seq)
- return postprocess_control(cam_id, frame, res, is_ptz, source_seq,
- visual_sample=sample)
+ with visual_control.control_lock(cam_id):
+ st = state.ptz_states.get(cam_id)
+ if st is None or not _visual_receipt_current(st, future.visual_receipt, .65):
+ return
+ return postprocess_control(cam_id, frame, res, is_ptz, source_seq,
+ visual_sample=sample)
--- a/ptz_tracker_modular/visual_control_probe.py
+++ b/ptz_tracker_modular/visual_control_probe.py
@@ -87,4 +87,4 @@
@wraps(function)
- def wrapped(camera, frame, st, source_seq):
- result = function(camera, frame, st, source_seq)
+ def wrapped(camera, frame, st, source_seq, **kwargs):
+ result = function(camera, frame, st, source_seq, **kwargs)
try:
@@ -103,3 +103,3 @@
@wraps(function)
- def wrapped(camera, frame, source_seq, captured_at=None):
+ def wrapped(camera, frame, source_seq, captured_at=None, **kwargs):
start = _perf()
@@ -117,3 +117,3 @@
try:
- return function(camera, frame, source_seq, captured_at=captured_at)
+ return function(camera, frame, source_seq, captured_at=captured_at, **kwargs)
finally:

@ -0,0 +1,193 @@
"""Offline tests for the legacy PTZ frame handoff using the actual server functions."""
from __future__ import annotations
import ast
from contextlib import nullcontext
from functools import wraps
from pathlib import Path
from types import SimpleNamespace
from typing import NamedTuple
import unittest
ROOT = Path(__file__).resolve().parent / 'ptz_tracker_modular'
def definitions(file, names, env):
path = ROOT / file
tree = ast.parse(path.read_text(encoding='utf-8'))
definitions = [node for node in tree.body if getattr(node, 'name', None) in names]
future = tree.body[0] if isinstance(tree.body[0], ast.ImportFrom) and tree.body[0].module == '__future__' else None
exec(compile(ast.Module(body=([future] if future else []) + definitions, type_ignores=[]), str(path), 'exec'), env)
class FrameHandoffTests(unittest.TestCase):
def setUp(self):
self.source = object()
self.frame = object()
self.utc, self.mono = 100., 50.
self.owner = (0, 3, 99., 1)
self.st = {'mode': 'TRACK', 'tracking_active': True}
self.st.update(owner=self.owner)
self.ptz = []
self.tracker = []
self.saved = None
self.env = dict(
NamedTuple=NamedTuple, _visual_source=self.source, state=SimpleNamespace(ptz_states={0: self.st}),
time=SimpleNamespace(time=lambda: self.utc, monotonic=lambda: self.mono),
visual_control=SimpleNamespace(control_lock=lambda cam: nullcontext(),
owner=lambda st: st['owner'], enabled=lambda: True,
VisualSample=lambda owner, seq, at, result: SimpleNamespace(owner=owner, source_seq=seq, captured_at=at, result=result),
retain=self.retain, latest=lambda *args: self.saved),
visual_control_probe=SimpleNamespace(trace_feed_update=lambda f:f,
call_tracker_update=lambda update,*args,**kwargs: update(*args,**kwargs)),
visual_tracker_manager=SimpleNamespace(update=self.update_tracker),
config=SimpleNamespace(), _visual_feed_is_active=lambda *args: True,
_store_visual_feed_result=lambda *args: None,
postprocess_control=lambda *args,**kw:self.ptz.append((args,kw)),
)
definitions('streaming.py', ('_VisualReceipt','_visual_receipt_current','_update_visual_feed',
'_prepare_visual_inference_item','_postprocess_inference_item'), self.env)
def retain(self, camera, frame, sample):
self.saved = frame, sample
def update_tracker(self, camera, frame, source_seq):
self.tracker.append((camera, frame, source_seq))
return SimpleNamespace(ok=True)
def visual(self, **overrides):
fields = dict(source=self.source, owner=self.owner, utc=100., monotonic=50.)
fields.update(overrides)
receipt = self.env['_VisualReceipt'](**fields)
self.env['_update_visual_feed'](0, self.frame, 1, receipt=receipt)
def test_recent_frame_centers_with_original_decode_time(self):
self.utc += .11
self.mono += .11
self.visual()
self.assertEqual(self.tracker, [(0,self.frame,1)])
self.assertTrue(self.ptz[0][1]['visual_only'])
self.assertEqual(self.ptz[0][1]['visual_sample'].captured_at, 100.)
def test_stale_foreign_and_future_frames_do_not_reach_tracker(self):
for override in ({'monotonic':49.7}, {'utc':100.2}, {'source':object()},
{'owner':(0,2,99.,1)}):
with self.subTest(override=override):
self.visual(**override)
self.assertEqual(self.tracker, [])
self.assertEqual(self.ptz, [])
def test_frame_aging_during_tracker_update_is_not_sent_to_ptz(self):
def update(*args, **kw):
self.mono += .3
return self.update_tracker(*args, **kw)
self.env['visual_tracker_manager'].update = update
self.visual()
self.assertEqual(len(self.tracker), 1)
self.assertEqual(self.ptz, [])
def test_paired_detection_keeps_image_and_rejects_late_result(self):
self.visual()
queued = (0, object(), True, 2, SimpleNamespace(), 51.)
cam, frame, ptz, seq, future, _ = self.env['_prepare_visual_inference_item'](queued)
self.assertIs(frame, self.frame)
self.assertEqual(seq, 1)
self.utc += .4
self.mono += .4
self.env['_postprocess_inference_item'](cam, frame, 'legacy results', ptz, seq, future)
self.assertEqual(self.ptz[-1][0][2], 'legacy results')
self.assertIs(self.ptz[-1][1]['visual_sample'], self.saved[1])
self.mono += .3
self.env['_postprocess_inference_item'](cam, frame, 'late results', ptz, seq, future)
self.assertEqual(len(self.ptz), 2)
def test_paired_detection_discarded_after_reconnect(self):
self.visual()
cam, frame, ptz, seq, future, _ = self.env['_prepare_visual_inference_item'](
(0, object(), True, 2, SimpleNamespace(), 51.))
self.env['_visual_source'] = object()
self.env['_postprocess_inference_item'](cam, frame, 'legacy results', ptz, seq, future)
self.assertEqual(len(self.ptz), 1)
def test_unpaired_detector_results_keep_legacy_entry_point(self):
post = self.env['_postprocess_inference_item']
for ptz in (True,False):
post(0, self.frame, 'legacy results', ptz, 1, SimpleNamespace())
self.assertEqual(self.ptz[-1], ((0, self.frame, 'legacy results', ptz, 1), {}))
def test_decoder_busy_branch_uses_original_receipt_on_both_preview_paths(self):
class StopProbe(Exception):
pass
async def stop_after_connection(delay):
raise StopProbe
class Socket:
async def __aenter__(self):
return self
async def __aexit__(self, *exc):
pass
async def __aiter__(self):
yield b'\x01\x00\x00\x17'
for preview in (False, True):
with self.subTest(preview=preview):
seen, errors, decoded = [], [], []
self.utc, self.mono = 100., 50.
self.st.update(proc_busy=True, last_source_seq=0)
def image_decode(buffer, mode):
decoded.append(1)
return self.frame
def publish(*args, **kwargs):
self.utc, self.mono = 100.2, 50.2
self.env.update(
websockets=SimpleNamespace(connect=lambda *args,**kwargs:Socket()),
asyncio=SimpleNamespace(sleep=stop_after_connection),
logger=SimpleNamespace(info=lambda *args:None, error=lambda *args:errors.append(args)),
config=SimpleNamespace(CAMERA_CONFIG={0:{}}, PTZ_CAM_IDS=(0,),
DROP_DECODE_WHEN_BUSY=True, CPP_WS_URI='offline'),
cv2=SimpleNamespace(imdecode=image_decode,IMREAD_COLOR=1),
np=SimpleNamespace(frombuffer=lambda data,dtype:data,uint8=int),
_cpp_stream_map_logged=set(), _resolve_cpp_cam_id=int,
ensure_runtime_state_for_camera=lambda *args:None,
_has_recent_video_client=lambda *args:True,
_frontend_raw_preview_due=lambda *args:preview,
_publish_frontend_raw_preview=publish,
_target_predecode_fps=lambda *args:0,
_feed_visual_tracker_frame=lambda *args,**kw:seen.append((args,kw['receipt'],self.env['_visual_source'])),
)
self.env['time'].perf_counter=lambda:50.
self.env['state'].detected_cameras={0}
definitions('streaming.py', ('cpp_ws_loop',), self.env)
with self.assertRaises(StopProbe):
self.env['cpp_ws_loop']().send(None)
self.assertFalse(errors)
self.assertEqual(len(decoded),1)
self.assertEqual(len(seen),1)
args, receipt, source = seen[0]
self.assertEqual(args, (0,self.frame,self.st,1))
self.assertIs(receipt.source, source)
self.assertEqual((receipt.utc, receipt.monotonic), (100.,50.))
self.assertIsNone(self.env['_visual_source'])
def test_probe_decorators_forward_receipt(self):
got = []
env = dict(wraps=wraps, _perf=lambda:0., _wall=lambda:100.,
_context=SimpleNamespace(), _safe_record=lambda *args,**kwargs:None)
definitions('visual_control_probe.py', ('trace_enqueue','trace_feed_update'), env)
def enqueue(camera, frame, st, source_seq, *, receipt):
got.append(receipt)
def update(camera, frame, source_seq, captured_at=None, *, receipt):
got.append(receipt)
env['trace_enqueue'](enqueue)(0, self.frame, {}, 1, receipt=self.source)
env['trace_feed_update'](update)(0, self.frame, 1, 100., receipt=self.source)
self.assertEqual(got, [self.source,self.source])
if __name__ == '__main__':
unittest.main(verbosity=2)
Loading…
Cancel
Save