You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
Chameleon/tests/test_unknown_inspection_run...

212 lines
10 KiB
Python

"""New V2 crop/provenance checks. Synthetic pixels, no models or network."""
import importlib.util
import json
from pathlib import Path
import sys
import threading
import time
import types
import unittest
from dataclasses import replace
import numpy as np
sys.dont_write_bytecode = True
HERE = Path(__file__).resolve().parents[1]
package = types.ModuleType('ptz_tracker_modular')
package.__path__ = [str(HERE/'ptz_tracker_modular')]
sys.modules['ptz_tracker_modular'] = package
from ptz_tracker_modular.frame_time import FrameClock
from ptz_tracker_modular.inspection_runtime import CropContext, CropWorker, check_stamp, current_detector, provenance
from ptz_tracker_modular.inspection_input import adapt_result
from ptz_tracker_modular.inspection_input import InspectionVisual
BOX = (8., 8., 48., 48.)
OWNER = (123, 7, 99.125, 0)
SOURCE = ('decoder', 'epoch1', 0)
SCORES = (('multirotor_uav', .94), ('fixed_wing_uav', .02), ('bird', .02), ('manned_aircraft', .02))
def stamp(seq=1, at=50., camera=0, epoch='epoch1'):
return FrameClock(camera, 'decoder', epoch).observe(source_seq=seq,
mono_before_ns=round(at*1e9), mono_after_ns=round(at*1e9), utc_ns=round((at+50)*1e9))
class RuntimeTests(unittest.TestCase):
def setUp(self):
self.worker = CropWorker((0, 2, 4, 6, 8, 10))
self.ctx = CropContext(0, 1, OWNER, SOURCE, 'b'*64)
self.worker.bind(self.ctx)
rng = np.random.default_rng(1)
self.frame = rng.integers(0, 256, (64, 64, 3), dtype=np.uint8)
def submit(self, *, ctx=None, seq=1, at=50., occluded=False, bbox=BOX):
ctx = ctx or self.ctx
return self.worker.submit(ctx, stamp(seq, at, ctx.camera, ctx.source[1]), self.frame,
bbox, now=at, occluded=occluded)
def test_real_crop_copies_exact_pixels_and_original_stamp(self):
self.assertTrue(self.submit())
original = self.frame[8:48, 8:48].copy()
self.frame[:] = 0
def classifier(pixels):
np.testing.assert_array_equal(pixels, original)
self.assertFalse(pixels.flags.writeable)
return SCORES
self.assertTrue(self.worker.run_one(classifier, clock=lambda:50.1))
st, rec = self.worker.take_stamped(self.ctx, now=50.2)
self.assertEqual((rec.source_seq, rec.observed_at, rec.shape), (1, 100., (64, 64)))
self.assertEqual(st.decode_mono_ns, 50_000_000_000)
self.assertFalse(rec.occluded)
self.assertGreater(rec.sharpness, 0)
def test_unknown_occlusion_quality_remains_unusable(self):
self.assertTrue(self.submit(occluded=None))
self.worker.run_one(lambda p:SCORES, clock=lambda:50.1)
self.assertTrue(self.worker.take(self.ctx, now=50.2).occluded)
def test_low_score_callback_cannot_hide_source_clipping(self):
self.assertTrue(self.submit(bbox=(0., 8., 40., 48.)))
self.worker.run_one(lambda p:SCORES, clock=lambda:50.1)
self.assertTrue(self.worker.take(self.ctx, now=50.2).clipped)
def test_slow_classifier_does_not_lock_submit_or_poll(self):
self.submit()
entered, release = threading.Event(), threading.Event()
def classify(pixels):
entered.set()
self.assertTrue(release.wait(2))
return SCORES
t = threading.Thread(target=lambda:self.worker.run_one(classify, clock=lambda:50.1))
t.start()
try:
self.assertTrue(entered.wait(1))
self.assertTrue(self.submit(seq=2, at=50.05))
self.assertIsNone(self.worker.take(self.ctx, now=50.1))
self.assertFalse(self.worker.run_one(lambda p:SCORES, clock=lambda:50.1))
self.assertEqual(self.worker.counts(), (1, 0))
finally:
release.set(); t.join(2)
self.assertFalse(t.is_alive())
def test_six_camera_queue_is_bounded_and_fair(self):
contexts = [replace(self.ctx, camera=c) for c in (0, 2, 4, 6, 8, 10)]
for c in contexts:
self.worker.bind(c)
self.assertTrue(self.submit(ctx=c))
for seq in range(2, 100):
self.assertTrue(self.submit(seq=seq, at=50.+seq*.001))
order = []
for _ in range(6):
self.assertTrue(self.worker.run_one(lambda p:SCORES, clock=lambda:50.2))
# Metadata results can be taken without retaining image buffers.
for c in contexts:
r = self.worker.take(c, now=50.2)
if r: order.append((r.camera, r.source_seq))
self.assertEqual(order, [(0,99),(2,1),(4,1),(6,1),(8,1),(10,1)])
self.assertEqual(self.worker.counts(), (0,0))
def test_old_inflight_answer_cannot_survive_revoke_rebind_aba(self):
self.submit()
def classify(p):
self.worker.revoke(0); self.worker.bind(self.ctx)
return SCORES
self.assertFalse(self.worker.run_one(classify, clock=lambda:50.1))
self.assertIsNone(self.worker.take(self.ctx, now=50.1))
def test_old_context_poll_cannot_consume_new_owner_result(self):
new=replace(self.ctx,inspection_id=2,owner=(124,8,100.,0))
self.worker.bind(new)
self.submit(ctx=new)
self.assertTrue(self.worker.run_one(lambda p:SCORES,clock=lambda:50.1))
self.assertIsNone(self.worker.take(self.ctx,now=50.2))
self.assertIsNotNone(self.worker.take(new,now=50.2))
def test_owner_source_model_or_session_change_drops_old_completion(self):
for changed in (replace(self.ctx, owner=(124,8,100.,0)),
replace(self.ctx, source=('decoder','epoch2',0)),
replace(self.ctx, classifier_sha='c'*64), replace(self.ctx, inspection_id=2)):
self.worker.bind(self.ctx); self.submit()
self.assertFalse(self.worker.run_one(lambda p:(self.worker.bind(changed) or SCORES), clock=lambda:50.1))
self.assertIsNone(self.worker.take(changed, now=50.1))
def test_stale_start_does_not_call_model(self):
self.submit()
self.assertFalse(self.worker.run_one(lambda p:self.fail('model called'), clock=lambda:50.651))
def test_completion_and_take_recheck_original_monotonic_age(self):
self.submit()
clock = [50.1]
def classify(p): clock[0]=50.651; return SCORES
self.assertFalse(self.worker.run_one(classify, clock=lambda:clock[0]))
self.submit(seq=2, at=51.)
self.assertTrue(self.worker.run_one(lambda p:SCORES, clock=lambda:51.1))
self.assertIsNone(self.worker.take(self.ctx, now=51.651))
def test_exception_retains_unknown_and_next_job_can_run(self):
self.submit()
def bad(p): raise RuntimeError('synthetic model error')
self.assertFalse(self.worker.run_one(bad, clock=lambda:50.1))
self.assertIsNone(self.worker.take(self.ctx, now=50.1))
self.submit(seq=2, at=50.2)
self.assertTrue(self.worker.run_one(lambda p:SCORES, clock=lambda:50.3))
def test_duplicate_reverse_time_wrong_epoch_and_excess_crop_refused(self):
self.assertTrue(self.submit())
self.assertFalse(self.submit())
self.assertFalse(self.submit(seq=2, at=49.9))
wrong = replace(self.ctx, source=('decoder','epoch2',0))
self.assertFalse(self.submit(ctx=wrong, seq=2, at=50.1))
small = CropWorker((0,), max_pixels=10); small.bind(self.ctx)
self.assertFalse(small.submit(self.ctx, stamp(), self.frame, BOX, now=50.))
def test_binding_cannot_be_learned_from_untrusted_result(self):
result = types.SimpleNamespace(inspection_stamp=stamp(), inspection_owner=OWNER)
state = {}
self.assertNotEqual(provenance(state,result,None,camera=0,seq=1,owner=OWNER,mono=50.,visual_only=False),'')
self.assertNotIn('inspection_source_binding', state)
def test_detector_monotonic_deadline_duplicate_and_clock_epoch(self):
s = stamp(); result = types.SimpleNamespace(inspection_stamp=s,inspection_owner=OWNER)
state = {'inspection_source_binding':SOURCE}
self.assertEqual(provenance(state,result,None,camera=0,seq=1,owner=OWNER,mono=50.1,visual_only=False),'')
self.assertTrue(current_detector(state,50.65))
self.assertFalse(current_detector(state,50.651))
self.assertNotEqual(provenance(state,result,None,camera=0,seq=1,owner=OWNER,mono=50.2,visual_only=False),'')
result.inspection_stamp = replace(stamp(2,50.2),clock_epoch=1)
self.assertNotEqual(provenance(state,result,None,camera=0,seq=2,owner=OWNER,mono=50.2,visual_only=False),'')
def test_visual_stamp_required_and_age_independent_of_wall(self):
state = {'inspection_source_binding':SOURCE}
old = types.SimpleNamespace(inspection_stamp=stamp(), inspection_owner=OWNER, captured_at=999.)
self.assertNotEqual(provenance(state,None,old,camera=0,seq=1,owner=OWNER,mono=50.251,visual_only=True),'')
self.assertEqual(provenance(state,None,old,camera=0,seq=1,owner=OWNER,mono=50.2,visual_only=True),'')
def test_stamp_future_or_foreign_owner_rejected(self):
for kw in ({'now':49.9}, {'seq':2}, {'camera':2}, {'owner':(1,)}, {'binding':('other','epoch1',0)}):
args=dict(camera=0,seq=1,owner=OWNER,expected_owner=OWNER,binding=SOURCE,now=50.,age=.65)
args.update(kw)
self.assertFalse(check_stamp(stamp(),**args))
def test_visual_adapter_cannot_restamp_sequence_owner_or_wall_time(self):
s=stamp(); original=types.SimpleNamespace(source_seq=1,captured_at=100.,owner=OWNER,result=object())
self.assertIs(InspectionVisual(original,s,OWNER).result,original.result)
for bad in (replace(s,source_seq=2),replace(s,decode_utc_ns=101_000_000_000)):
with self.assertRaises(ValueError): InspectionVisual(original,bad,OWNER)
with self.assertRaises(ValueError): InspectionVisual(original,s,(1,))
def test_malformed_or_unbounded_score_map_is_bounded_and_refused(self):
invalid = ((),(('drone',1.),),tuple((n,float('nan')) for n,v in SCORES),
tuple((n,1.) for n,v in SCORES),('oops',)*4)
for i,scores in enumerate(invalid):
self.assertTrue(self.submit(seq=i+1,at=50.+i*.01))
self.assertFalse(self.worker.run_one(lambda p:scores,clock=lambda:50.1))
self.submit(seq=10,at=50.2)
def unbounded(p):
while True: yield ('bird',.25)
self.assertFalse(self.worker.run_one(unbounded,clock=lambda:50.3))
if __name__ == '__main__':
unittest.main(verbosity=2)