From 94856d0fb8dc674a2a8c1b3ed194ed73bd823d50 Mon Sep 17 00:00:00 2001 From: Sergey Revyakin Date: Tue, 5 May 2026 14:23:17 +0700 Subject: [PATCH] =?UTF-8?q?=D1=81=D0=BA=D1=80=D0=B8=D0=BF=D1=82=20=D0=B4?= =?UTF-8?q?=D0=BB=D1=8F=20=D0=BF=D0=B0=D1=80=D1=81=D0=B8=D0=BD=D0=B3=D0=B0?= =?UTF-8?q?=20=D0=BB=D0=BE=D0=B3=D0=BE=D0=B2=20=D1=81=20nn=5Fserver?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/capture_nn_results.py | 223 ++++++++++++++++++++++++++++++++++ 1 file changed, 223 insertions(+) create mode 100644 scripts/capture_nn_results.py diff --git a/scripts/capture_nn_results.py b/scripts/capture_nn_results.py new file mode 100644 index 0000000..22b2d3e --- /dev/null +++ b/scripts/capture_nn_results.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +import argparse +import csv +import json +import os +import re +import subprocess +import sys +import time +from datetime import datetime, timezone + + +RESULT_RE = re.compile( + r"^(?P\S+)\s+RESULT Модель (?P\d+) с типом (?P.+?): " + r"(?P\S+) \(probability=(?P[-+]?\d+(?:\.\d+)?)\)\s*$" +) +FREQ_RE = re.compile(r"(\d{3,5})") + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Capture NN inference RESULT logs into CSV or JSONL until time or size limit." + ) + parser.add_argument( + "--output", + required=True, + help="Output file path (.csv or .jsonl recommended).", + ) + parser.add_argument( + "--format", + choices=("csv", "jsonl"), + default="csv", + help="Output format.", + ) + parser.add_argument( + "--minutes", + type=float, + default=0.0, + help="Stop after this many minutes. 0 means no time limit.", + ) + parser.add_argument( + "--max-bytes", + type=int, + default=3 * 1024 * 1024 * 1024, + help="Stop when output file reaches this size in bytes. Default: 3 GiB.", + ) + parser.add_argument( + "--since", + default=None, + help="Optional docker logs --since value, e.g. 20m, 2h, 2026-05-04T12:00:00.", + ) + parser.add_argument( + "--tail", + type=int, + default=0, + help="How many previous log lines to include before following. Default: 0.", + ) + parser.add_argument( + "--compose-file", + default="deploy/docker/docker-compose.yml", + help="Path to docker compose file.", + ) + parser.add_argument( + "--service", + default="dronedetector-nn-server", + help="Docker compose service name.", + ) + parser.add_argument( + "--follow", + action="store_true", + help="Follow logs live. By default the script captures a finite history snapshot.", + ) + return parser.parse_args() + + +def extract_freq(model_type): + matches = FREQ_RE.findall(model_type) + if not matches: + return "" + known_freqs = {"433", "750", "868", "915", "1200", "1500", "2400", "3300", "4500", "5200", "5800"} + for value in matches: + if value in known_freqs: + return value + return matches[0] + + +def parse_docker_timestamp(value): + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return datetime.now(timezone.utc) + + +def open_output(path, fmt): + os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True) + fh = open(path, "a", encoding="utf-8", newline="") + writer = None + if fmt == "csv": + writer = csv.writer(fh) + if fh.tell() == 0: + writer.writerow( + [ + "docker_timestamp", + "event_time_iso", + "event_time_epoch", + "freq", + "model_id", + "model_type", + "prediction", + "probability", + ] + ) + fh.flush() + return fh, writer + + +def write_record(fh, writer, fmt, record): + if fmt == "csv": + writer.writerow( + [ + record["docker_timestamp"], + record["event_time_iso"], + record["event_time_epoch"], + record["freq"], + record["model_id"], + record["model_type"], + record["prediction"], + record["probability"], + ] + ) + else: + fh.write(json.dumps(record, ensure_ascii=False) + "\n") + fh.flush() + + +def build_command(args): + cmd = [ + "docker", + "compose", + "-f", + args.compose_file, + "logs", + "--timestamps", + "--no-log-prefix", + args.service, + ] + if args.since: + cmd[5:5] = ["--since", args.since] + if int(args.tail) > 0: + cmd[-1:-1] = ["--tail", str(args.tail)] + if args.follow: + cmd.insert(-1, "-f") + return cmd + + +def main(): + args = parse_args() + deadline = time.time() + (args.minutes * 60.0) if args.minutes > 0 else None + + fh, writer = open_output(args.output, args.format) + cmd = build_command(args) + + print("Running:", " ".join(cmd), file=sys.stderr) + proc = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + encoding="utf-8", + errors="replace", + bufsize=1, + ) + + captured = 0 + try: + assert proc.stdout is not None + for line in proc.stdout: + if deadline is not None and time.time() >= deadline: + print("Stopping: time limit reached", file=sys.stderr) + break + + match = RESULT_RE.match(line.rstrip("\n")) + if not match: + continue + + docker_dt = parse_docker_timestamp(match.group("docker_ts")) + event_dt = docker_dt.astimezone() + model_type = match.group("model_type") + record = { + "docker_timestamp": match.group("docker_ts"), + "event_time_iso": event_dt.isoformat(timespec="seconds"), + "event_time_epoch": round(docker_dt.timestamp(), 3), + "freq": extract_freq(model_type), + "model_id": int(match.group("model_id")), + "model_type": model_type, + "prediction": match.group("prediction"), + "probability": float(match.group("probability")), + } + write_record(fh, writer, args.format, record) + captured += 1 + + if fh.tell() >= args.max_bytes: + print("Stopping: file size limit reached", file=sys.stderr) + break + finally: + try: + proc.terminate() + except Exception: + pass + try: + proc.wait(timeout=5) + except Exception: + try: + proc.kill() + except Exception: + pass + fh.close() + + print(f"Captured {captured} inference results into {args.output}", file=sys.stderr) + + +if __name__ == "__main__": + main()