diff --git a/NN_server/Model.py b/NN_server/Model.py index c8e31c3..9acdd00 100644 --- a/NN_server/Model.py +++ b/NN_server/Model.py @@ -154,18 +154,22 @@ class Model(object): except Exception as exc: print(str(exc)) - def _prepare_data(self, data=None): + def _prepare_data(self, data=None, ind_inference=None): try: + if ind_inference is None: + ind_inference = Model.get_ind_inference() print('Подготовка данных' + self._shablon) - self._data = self._pre_func(data, src=self._src_result, ind_inference=Model.get_ind_inference()) + self._data = self._pre_func(data, src=self._src_result, ind_inference=ind_inference) except Exception as exc: print(str(exc)) - def _post_data(self, prediction=None): + def _post_data(self, prediction=None, ind_inference=None): + if ind_inference is None: + ind_inference = Model.get_ind_inference() print('Постобработка данных' + self._shablon) self._ind_inference += 1 self._post_func(src=self._src_result, data=self._data, model_id=self._model_id, model_type=self._type_model, - ind_inference=Model.get_ind_inference(), prediction=prediction) + ind_inference=ind_inference, prediction=prediction) def get_test_inference(self): try: @@ -248,23 +252,25 @@ class Model(object): except Exception as exc: print(str(exc)) - def get_inference(self, data=None): + def get_inference(self, data=None, ind_inference=None): try: - return self._inference(data=data) + return self._inference(data=data, ind_inference=ind_inference) except Exception as exc: print(str(exc)) return None - def _inference(self, data=None): + def _inference(self, data=None, ind_inference=None): try: - Model._add_in_result_list(type_model=self._type_model, ind_inference=self.get_ind_inference(), list_to_add=[]) - self._prepare_data(data=data) + if ind_inference is None: + ind_inference = Model.get_ind_inference() + Model._add_in_result_list(type_model=self._type_model, ind_inference=ind_inference, list_to_add=[]) + self._prepare_data(data=data, ind_inference=ind_inference) print('Инференс' + self._shablon) prediction, probability = self._inference_func(data=self._data, model=self._model, mapping=self._classes, shablon=self._shablon) print('RESULT' + self._shablon + ': ' + str(prediction) + ' (probability=' + str(probability) + ')') - Model._add_in_result_list(type_model=self._type_model, ind_inference=self.get_ind_inference(), list_to_add=[prediction, probability]) - self._post_data(prediction=prediction) + Model._add_in_result_list(type_model=self._type_model, ind_inference=ind_inference, list_to_add=[prediction, probability]) + self._post_data(prediction=prediction, ind_inference=ind_inference) gc.collect() return prediction, probability diff --git a/NN_server/Models/ensemble_1200_v44.py b/NN_server/Models/ensemble_1200_v44.py index d805a4f..11a7239 100644 --- a/NN_server/Models/ensemble_1200_v44.py +++ b/NN_server/Models/ensemble_1200_v44.py @@ -6,6 +6,8 @@ import torch import cv2 import gc import io +import os +import re def _render_plot(values, figsize=(16, 16), dpi=16): @@ -36,6 +38,42 @@ def _render_plot(values, figsize=(16, 16), dpi=16): return np.asarray(cv2.split(img), dtype=np.float32) +def _prune_old_inference_images(src, model_type, model_id, keep_last=200): + try: + keep_last = int(os.getenv("INFERENCE_IMAGE_KEEP_LAST", str(keep_last))) + except ValueError: + keep_last = keep_last + + if keep_last <= 0 or not src or not os.path.isdir(src): + return + + pattern = re.compile( + r"_inference_(\d+)_.*_" + + re.escape(str(model_id)) + + "_" + + re.escape(str(model_type)) + + r"\.png$" + ) + grouped = {} + for name in os.listdir(src): + match = pattern.match(name) + if match is None: + continue + grouped.setdefault(int(match.group(1)), []).append(name) + + if len(grouped) <= keep_last: + return + + for old_result_id in sorted(grouped)[: len(grouped) - keep_last]: + for name in grouped[old_result_id]: + try: + os.remove(os.path.join(src, name)) + except FileNotFoundError: + pass + except OSError as exc: + print(f"failed to remove old inference image {name}: {exc}") + + def pre_func_ensemble(data=None, src="", ind_inference=0): try: import matplotlib.pyplot as plt @@ -164,7 +202,7 @@ def post_func_ensemble(src="", model_type="", prediction="", model_id=0, ind_inf matplotlib.use("Agg") plt.ioff() - if int(ind_inference) <= 100 and isinstance(data, (list, tuple)) and len(data) >= 2: + if isinstance(data, (list, tuple)) and len(data) >= 2: fig, ax = plt.subplots() ax.imshow(np.moveaxis(data[0], 0, -1)) plt.savefig(src + "_inference_" + str(ind_inference) + "_" + prediction + "_real_" + str(model_id) + "_" + model_type + ".png") @@ -183,6 +221,8 @@ def post_func_ensemble(src="", model_type="", prediction="", model_id=0, ind_inf cv2.destroyAllWindows() gc.collect() + _prune_old_inference_images(src, model_type, model_id) + plt.clf() plt.cla() plt.close() diff --git a/NN_server/Models/ensemble_2400_v44.py b/NN_server/Models/ensemble_2400_v44.py index d805a4f..11a7239 100644 --- a/NN_server/Models/ensemble_2400_v44.py +++ b/NN_server/Models/ensemble_2400_v44.py @@ -6,6 +6,8 @@ import torch import cv2 import gc import io +import os +import re def _render_plot(values, figsize=(16, 16), dpi=16): @@ -36,6 +38,42 @@ def _render_plot(values, figsize=(16, 16), dpi=16): return np.asarray(cv2.split(img), dtype=np.float32) +def _prune_old_inference_images(src, model_type, model_id, keep_last=200): + try: + keep_last = int(os.getenv("INFERENCE_IMAGE_KEEP_LAST", str(keep_last))) + except ValueError: + keep_last = keep_last + + if keep_last <= 0 or not src or not os.path.isdir(src): + return + + pattern = re.compile( + r"_inference_(\d+)_.*_" + + re.escape(str(model_id)) + + "_" + + re.escape(str(model_type)) + + r"\.png$" + ) + grouped = {} + for name in os.listdir(src): + match = pattern.match(name) + if match is None: + continue + grouped.setdefault(int(match.group(1)), []).append(name) + + if len(grouped) <= keep_last: + return + + for old_result_id in sorted(grouped)[: len(grouped) - keep_last]: + for name in grouped[old_result_id]: + try: + os.remove(os.path.join(src, name)) + except FileNotFoundError: + pass + except OSError as exc: + print(f"failed to remove old inference image {name}: {exc}") + + def pre_func_ensemble(data=None, src="", ind_inference=0): try: import matplotlib.pyplot as plt @@ -164,7 +202,7 @@ def post_func_ensemble(src="", model_type="", prediction="", model_id=0, ind_inf matplotlib.use("Agg") plt.ioff() - if int(ind_inference) <= 100 and isinstance(data, (list, tuple)) and len(data) >= 2: + if isinstance(data, (list, tuple)) and len(data) >= 2: fig, ax = plt.subplots() ax.imshow(np.moveaxis(data[0], 0, -1)) plt.savefig(src + "_inference_" + str(ind_inference) + "_" + prediction + "_real_" + str(model_id) + "_" + model_type + ".png") @@ -183,6 +221,8 @@ def post_func_ensemble(src="", model_type="", prediction="", model_id=0, ind_inf cv2.destroyAllWindows() gc.collect() + _prune_old_inference_images(src, model_type, model_id) + plt.clf() plt.cla() plt.close() diff --git a/NN_server/server.py b/NN_server/server.py index 36c751b..d811ef3 100644 --- a/NN_server/server.py +++ b/NN_server/server.py @@ -30,6 +30,7 @@ loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) queue = asyncio.Queue() semaphore = asyncio.Semaphore(3) +receive_data_lock = threading.Lock() prediction_list = [] result_msg = {} @@ -97,11 +98,7 @@ def collect_inference_images(result_id, model_name=''): if exact_images: return result_id, exact_images - if not grouped_images: - return result_id, [] - - latest_result_id = max(grouped_images) - return latest_result_id, grouped_images[latest_result_id] + return result_id, [] def send_inference_result(payload): @@ -180,6 +177,11 @@ def run_example(): @app.route('/receive_data', methods=['POST']) def receive_data(): + with receive_data_lock: + return _receive_data_locked() + + +def _receive_data_locked(): try: print() data = request.json @@ -203,7 +205,7 @@ def receive_data(): print('-' * 100) print(str(model)) result_msg[str(model.get_model_name())] = {'freq': freq} - prediction, probability = model.get_inference([np.asarray(data['data_real'], dtype=np.float32), np.asarray(data['data_imag'], dtype=np.float32)]) + prediction, probability = model.get_inference([np.asarray(data['data_real'], dtype=np.float32), np.asarray(data['data_imag'], dtype=np.float32)], ind_inference=result_id) result_msg[str(model.get_model_name())]['prediction'] = prediction result_msg[str(model.get_model_name())]['probability'] = str(probability) prediction_list.append(prediction) diff --git a/capture_hourly.sh b/capture_hourly.sh index 34d1166..49f1cf5 100755 --- a/capture_hourly.sh +++ b/capture_hourly.sh @@ -1,99 +1,59 @@ #!/usr/bin/env bash set -Eeuo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SCRIPT_OWNER="${CAPTURE_USER:-$(stat -c %U "$SCRIPT_DIR")}" +SCRIPT_OWNER_HOME="$(getent passwd "$SCRIPT_OWNER" | cut -d: -f6 || true)" +cd "$SCRIPT_DIR" + ############################ # НАСТРОЙКИ ############################ -# Точка сохранения, по умолчанию nvme/data. -BASE_DIR="${BASE_DIR:-/mnt/data/noise}" +BASE_DIR="/mnt/data/dataset_6_5_26" -# Путь к python из venv -PYTHON_BIN="${PYTHON_BIN:-$PWD/.venv-sdr/bin/python}" +PYTHON_BIN="${PYTHON_BIN:-$SCRIPT_DIR/.venv-sdr/bin/python}" +SCRIPT_PATH="${SCRIPT_PATH:-$SCRIPT_DIR/scripts_nn/data_saver_headless.py}" +ENV_FILE="${ENV_FILE:-$SCRIPT_DIR/.env}" -# Путь к headless скрипту -SCRIPT_PATH="${SCRIPT_PATH:-$PWD/scripts_nn/data_saver_headless.py}" +RUN_ONCE="${RUN_ONCE:-0}" +CAPTURE_LOG_FILE="${CAPTURE_LOG_FILE:-$BASE_DIR/capture_hourly.log}" -# Env-файл читается точечно: source .env небезопасен из-за сложных NN_* строк. -ENV_FILE="${ENV_FILE:-$PWD/.env}" +SYSTEMCTL_BIN=(systemctl) +CURRENT_CAPTURE_PID="" +CURRENT_MOCK_PID="" +CURRENT_SERVICE_UNIT="" +STOPPED_SERVICE_UNIT="" # Лимиты -PER_FREQ_LIMIT_BYTES=$((3 * 1024 * 1024 * 1024)) # 3 GiB на частоту за запуск -TOTAL_LIMIT_BYTES=$((128 * 1024 * 1024 * 1024)) # общий лимит 512 GiB -CYCLE_SECONDS="${CYCLE_SECONDS:-36}" # один цикл в час +PER_FREQ_LIMIT_GIB="${PER_FREQ_LIMIT_GIB:-12}" +TOTAL_LIMIT_GIB="${TOTAL_LIMIT_GIB:-266}" + +PER_FREQ_LIMIT_BYTES=$((PER_FREQ_LIMIT_GIB * 1024 * 1024 * 1024)) +TOTAL_LIMIT_BYTES=$((TOTAL_LIMIT_GIB * 1024 * 1024 * 1024)) + +# Для обычного hourly можно поставить 3600. +# Для RUN_ONCE это почти не важно. +CYCLE_SECONDS="${CYCLE_SECONDS:-1}" # Параметры SDR -SAMP_RATE="20e6" -SPLIT_SIZE="400000" -DELAY="0.1" -RF_GAIN="12" -IF_GAIN="30" -BB_GAIN="36" - -# Моки для server-to-master на время capture. +SAMP_RATE="${SAMP_RATE:-20e6}" +SPLIT_SIZE="${SPLIT_SIZE:-400000}" +DELAY="${DELAY:-0.25}" +RF_GAIN="${RF_GAIN:-12}" +IF_GAIN="${IF_GAIN:-12}" +BB_GAIN="${BB_GAIN:-0}" + +############################ +# MOCK SENDER +############################ + CAPTURE_MOCK_SEND_ENABLED="${CAPTURE_MOCK_SEND_ENABLED:-1}" CAPTURE_MOCK_HOST="${CAPTURE_MOCK_HOST:-127.0.0.1}" CAPTURE_MOCK_INTERVAL_SECONDS="${CAPTURE_MOCK_INTERVAL_SECONDS:-1}" CAPTURE_MOCK_TIMEOUT_SECONDS="${CAPTURE_MOCK_TIMEOUT_SECONDS:-0.3}" CAPTURE_MOCK_LOG_SUCCESS="${CAPTURE_MOCK_LOG_SUCCESS:-0}" -MOCK_SENDER_PID="" -ACTIVE_CAPTURE_PID="" -ACTIVE_MOCK_PID="" -ACTIVE_SERVICE_UNIT="" -SYSTEMCTL_BIN=(systemctl) - -############################ -# ЧАСТОТЫ И SERIAL ИЗ ENV -############################ - -#ORDER=(433 750 915 1200 2400 3300 4500 5200 5800) - -ORDER=(1200) -declare -A SERIAL -declare -A FREQ_HZ -declare -A SERVICE_UNIT - -#SERIAL[433]="$hack_433" -FREQ_HZ[433]="433000000" - -#SERIAL[750]="$hack_750" -FREQ_HZ[750]="750000000" - -#SERIAL[868]="$hack_868" -FREQ_HZ[868]="868000000" - -#SERIAL[915]="$hack_915" -FREQ_HZ[915]="915000000" - -SERIAL[1200]="0000000000000000114864dc38638a1b" -FREQ_HZ[1200]="1200000000" - -#SERIAL[2400]="0000000000000000600463dc29789bc7" -FREQ_HZ[2400]="1200000000" - -#SERIAL[3300]="$hack_3300" -FREQ_HZ[3300]="3300000000" - -#SERIAL[4500]="$hack_4500" -FREQ_HZ[4500]="4500000000" - -#SERIAL[5200]="$hack_5200" -FREQ_HZ[5200]="5200000000" - -SERIAL[5800]="0000000000000000518864dc33743883" -FREQ_HZ[5800]="5800000000" - -SERVICE_UNIT[433]="dronedetector-sdr-433.service" -SERVICE_UNIT[750]="dronedetector-sdr-750.service" -SERVICE_UNIT[868]="dronedetector-sdr-868.service" -SERVICE_UNIT[915]="dronedetector-sdr-915.service" -SERVICE_UNIT[1200]="dronedetector-sdr-1200.service" -SERVICE_UNIT[2400]="dronedetector-sdr-2400.service" -SERVICE_UNIT[3300]="dronedetector-sdr-3300.service" -SERVICE_UNIT[4500]="dronedetector-sdr-4500.service" -SERVICE_UNIT[5200]="dronedetector-sdr-5200.service" -SERVICE_UNIT[5800]="dronedetector-sdr-5800.service" ############################ # ВСПОМОГАТЕЛЬНОЕ @@ -107,54 +67,6 @@ systemctl_run() { "${SYSTEMCTL_BIN[@]}" "$@" } -service_exists() { - local unit="$1" - systemctl_run show "$unit" >/dev/null 2>&1 -} - -service_is_active() { - local unit="$1" - systemctl_run is-active --quiet "$unit" >/dev/null 2>&1 -} - -stop_band_service() { - local band="$1" - local unit="${SERVICE_UNIT[$band]:-}" - - ACTIVE_SERVICE_UNIT="" - if [[ -z "$unit" ]]; then - log "Для band=$band не найден service unit" - return 0 - fi - - if ! service_exists "$unit"; then - log "Service unit $unit не установлен, пропускаю stop/start" - return 0 - fi - - if ! service_is_active "$unit"; then - log "Service $unit не активен, stop/start не нужен" - return 0 - fi - - log "Останавливаю service $unit перед записью band=$band" - if ! systemctl_run stop "$unit"; then - log "Не удалось остановить service $unit" - return 1 - fi - ACTIVE_SERVICE_UNIT="$unit" -} - -start_active_service() { - if [[ -z "$ACTIVE_SERVICE_UNIT" ]]; then - return 0 - fi - - log "Запускаю service $ACTIVE_SERVICE_UNIT после записи" - systemctl_run start "$ACTIVE_SERVICE_UNIT" || log "Не удалось запустить service $ACTIVE_SERVICE_UNIT" - ACTIVE_SERVICE_UNIT="" -} - env_get() { local key="$1" local default="${2:-}" @@ -166,19 +78,24 @@ env_get() { awk -v key="$key" -v default="$default" ' BEGIN { found = 0 } + $0 ~ "^[[:space:]]*" key "=" { value = $0 + sub("^[[:space:]]*" key "=", "", value) sub("[[:space:]]+#.*$", "", value) gsub("^[[:space:]]+|[[:space:]]+$", "", value) + if ((substr(value, 1, 1) == "\"" && substr(value, length(value), 1) == "\"") || (substr(value, 1, 1) == "'"'"'" && substr(value, length(value), 1) == "'"'"'")) { value = substr(value, 2, length(value) - 2) } + print value found = 1 exit } + END { if (!found) { print default @@ -196,32 +113,145 @@ mock_bool_enabled() { mock_url() { local port endpoint + port="${CAPTURE_MOCK_PORT:-$(env_get locport "$(env_get GENERAL_SERVER_PORT 5010)")}" endpoint="${CAPTURE_MOCK_ENDPOINT:-$(env_get freq_endpoint process_data)}" + endpoint="${endpoint#/}" + printf 'http://%s:%s/%s' "$CAPTURE_MOCK_HOST" "$port" "$endpoint" } +service_exists() { + local unit="$1" + systemctl_run show "$unit" >/dev/null 2>&1 +} + +service_is_active() { + local unit="$1" + systemctl_run is-active --quiet "$unit" >/dev/null 2>&1 +} + +dir_size_bytes() { + local path="$1" + + if [[ -e "$path" ]]; then + du -sb "$path" 2>/dev/null | awk '{print $1}' + else + echo 0 + fi +} + +total_size_bytes() { + dir_size_bytes "$BASE_DIR" +} + +terminate_pid() { + local pid="${1:-}" + + if [[ -z "$pid" ]]; then + return 0 + fi + + if ! kill -0 "$pid" 2>/dev/null; then + return 0 + fi + + kill -TERM "$pid" 2>/dev/null || true + + for _ in {1..10}; do + if ! kill -0 "$pid" 2>/dev/null; then + break + fi + sleep 1 + done + + if kill -0 "$pid" 2>/dev/null; then + log "PID=$pid не завершился по TERM, отправляю KILL" + kill -KILL "$pid" 2>/dev/null || true + fi + + wait "$pid" 2>/dev/null || true +} + +stop_band_service() { + local band="$1" + local unit="${SERVICE_UNIT[$band]:-}" + + STOPPED_SERVICE_UNIT="" + CURRENT_SERVICE_UNIT="" + + if [[ -z "$unit" ]]; then + log "Для band=$band не найден service unit" + return 0 + fi + + if ! service_exists "$unit"; then + log "Service unit $unit не установлен, пропускаю stop/start" + return 0 + fi + + log "Останавливаю service $unit перед записью band=$band" + + if systemctl_run stop "$unit"; then + log "Service $unit остановлен или уже был остановлен" + else + log "Не удалось остановить service $unit" + fi + + STOPPED_SERVICE_UNIT="$unit" + CURRENT_SERVICE_UNIT="$unit" + + return 0 +} + +start_stopped_service() { + local unit="${STOPPED_SERVICE_UNIT:-}" + + if [[ -z "$unit" ]]; then + return 0 + fi + + log "Запускаю service $unit после записи" + + if systemctl_run start "$unit"; then + log "Service $unit запущен" + else + log "Не удалось запустить service $unit" + systemctl_run status "$unit" --no-pager || true + fi + + STOPPED_SERVICE_UNIT="" + CURRENT_SERVICE_UNIT="" +} + start_mock_sender() { local band="$1" local log_file="$2" local url - MOCK_SENDER_PID="" + CURRENT_MOCK_PID="" + if ! mock_bool_enabled "$CAPTURE_MOCK_SEND_ENABLED"; then log "Mock sender отключен CAPTURE_MOCK_SEND_ENABLED=$CAPTURE_MOCK_SEND_ENABLED" return 0 fi url="$(mock_url)" + log "Старт mock sender band=$band url=$url amplitude=0 interval=${CAPTURE_MOCK_INTERVAL_SECONDS}s" - "$PYTHON_BIN" - "$url" "$band" "$CAPTURE_MOCK_INTERVAL_SECONDS" "$CAPTURE_MOCK_TIMEOUT_SECONDS" <<'PY' >>"$log_file" 2>&1 & + "$PYTHON_BIN" - \ + "$url" \ + "$band" \ + "$CAPTURE_MOCK_INTERVAL_SECONDS" \ + "$CAPTURE_MOCK_TIMEOUT_SECONDS" \ + "$CAPTURE_MOCK_LOG_SUCCESS" \ + >>"$log_file" 2>&1 <<'PY' & + import json -import os import sys import time -import urllib.error import urllib.request @@ -229,61 +259,133 @@ url = sys.argv[1] freq = str(sys.argv[2]) interval = float(sys.argv[3]) timeout = float(sys.argv[4]) +log_success = sys.argv[5].lower() in {"1", "true", "yes", "on"} + payload = json.dumps({"freq": freq, "amplitude": 0}).encode("utf-8") headers = {"Content-Type": "application/json"} -log_success = os.getenv("CAPTURE_MOCK_LOG_SUCCESS", "0").lower() in {"1", "true", "yes", "on"} -print(f"[capture-mock] started url={url} freq={freq} amplitude=0 interval={interval}", flush=True) +print( + f"[capture-mock] started url={url} freq={freq} amplitude=0 interval={interval}", + flush=True, +) + while True: try: - req = urllib.request.Request(url, data=payload, headers=headers, method="POST") + req = urllib.request.Request( + url, + data=payload, + headers=headers, + method="POST", + ) + with urllib.request.urlopen(req, timeout=timeout) as response: if log_success: print(f"[capture-mock] sent status={response.status}", flush=True) + except Exception as exc: print(f"[capture-mock] send failed: {exc}", flush=True) + time.sleep(interval) PY - MOCK_SENDER_PID=$! - log "Mock sender PID=$MOCK_SENDER_PID" + + CURRENT_MOCK_PID=$! + log "Mock sender PID=$CURRENT_MOCK_PID" } stop_mock_sender() { local pid="${1:-}" + if [[ -z "$pid" ]]; then return 0 fi + if kill -0 "$pid" 2>/dev/null; then log "Останавливаю mock sender PID=$pid" - kill -TERM "$pid" 2>/dev/null || true - wait "$pid" 2>/dev/null || true + terminate_pid "$pid" fi } -cleanup_active_jobs() { +cleanup_capture() { local rc=$? - if [[ -n "${ACTIVE_CAPTURE_PID:-}" ]] && kill -0 "$ACTIVE_CAPTURE_PID" 2>/dev/null; then - log "Останавливаю active capture PID=$ACTIVE_CAPTURE_PID" - kill -TERM "$ACTIVE_CAPTURE_PID" 2>/dev/null || true - wait "$ACTIVE_CAPTURE_PID" 2>/dev/null || true + + if [[ -n "${CURRENT_CAPTURE_PID:-}" ]] && kill -0 "$CURRENT_CAPTURE_PID" 2>/dev/null; then + log "Останавливаю текущий capture PID=$CURRENT_CAPTURE_PID" + terminate_pid "$CURRENT_CAPTURE_PID" fi - stop_mock_sender "${ACTIVE_MOCK_PID:-}" - start_active_service - exit "$rc" -} + CURRENT_CAPTURE_PID="" -dir_size_bytes() { - local path="$1" - if [[ -e "$path" ]]; then - du -sb "$path" 2>/dev/null | awk '{print $1}' - else - echo 0 + if [[ -n "${CURRENT_MOCK_PID:-}" ]] && kill -0 "$CURRENT_MOCK_PID" 2>/dev/null; then + log "Останавливаю текущий mock sender PID=$CURRENT_MOCK_PID" + terminate_pid "$CURRENT_MOCK_PID" fi + CURRENT_MOCK_PID="" + + start_stopped_service + + exit "$rc" } -total_size_bytes() { - dir_size_bytes "$BASE_DIR" +on_signal() { + local sig="$1" + log "Получен сигнал $sig, завершаю работу" + exit 130 } + +############################ +# ЧАСТОТЫ И SERIAL +############################ + +ORDER=(2400) + +declare -A SERIAL +declare -A FREQ_HZ +declare -A SERVICE_UNIT + +SERIAL[433]="$(env_get hack_433)" +FREQ_HZ[433]="433000000" + +SERIAL[750]="$(env_get hack_750)" +FREQ_HZ[750]="750000000" + +SERIAL[868]="$(env_get hack_868)" +FREQ_HZ[868]="868000000" + +SERIAL[915]="$(env_get hack_915)" +FREQ_HZ[915]="915000000" + +SERIAL[1200]="$(env_get hack_1200)" +FREQ_HZ[1200]="1200000000" + +SERIAL[2400]="$(env_get hack_2400)" +FREQ_HZ[2400]="2400000000" + +SERIAL[3300]="$(env_get hack_3300)" +FREQ_HZ[3300]="3300000000" + +SERIAL[4500]="$(env_get hack_4500)" +FREQ_HZ[4500]="4500000000" + +SERIAL[5200]="$(env_get hack_5200)" +FREQ_HZ[5200]="5200000000" + +SERIAL[5800]="$(env_get hack_5800)" +FREQ_HZ[5800]="5800000000" + +SERVICE_UNIT[433]="dronedetector-sdr-433.service" +SERVICE_UNIT[750]="dronedetector-sdr-750.service" +SERVICE_UNIT[868]="dronedetector-sdr-868.service" +SERVICE_UNIT[915]="dronedetector-sdr-915.service" +SERVICE_UNIT[1200]="dronedetector-sdr-1200.service" +SERVICE_UNIT[2400]="dronedetector-sdr-2400.service" +SERVICE_UNIT[3300]="dronedetector-sdr-3300.service" +SERVICE_UNIT[4500]="dronedetector-sdr-4500.service" +SERVICE_UNIT[5200]="dronedetector-sdr-5200.service" +SERVICE_UNIT[5800]="dronedetector-sdr-5800.service" + +############################ +# ОСНОВНАЯ ЛОГИКА +############################ + ensure_requirements() { if [[ ! -x "$PYTHON_BIN" ]]; then echo "Не найден python: $PYTHON_BIN" >&2 @@ -310,35 +412,63 @@ ensure_requirements() { fi mkdir -p "$BASE_DIR" + mkdir -p "$(dirname "$CAPTURE_LOG_FILE")" - local dev - dev="$(df -P "$BASE_DIR" | awk 'NR==2 {print $1}')" + if [[ -n "${CAPTURE_ORDER:-}" ]]; then + ORDER=() + local band + for band in ${CAPTURE_ORDER//,/ }; do + ORDER+=("$band") + done + fi + if [[ "${#ORDER[@]}" -eq 0 ]]; then + echo "ORDER пустой. Укажите частоты в ORDER или задайте CAPTURE_ORDER." >&2 + exit 1 + fi + + log "Рабочая директория: $SCRIPT_DIR" + log "BASE_DIR: $BASE_DIR" + log "ORDER: ${ORDER[*]}" + log "RUN_ONCE: $RUN_ONCE" + log "PER_FREQ_LIMIT_GIB: $PER_FREQ_LIMIT_GIB" + log "TOTAL_LIMIT_GIB: $TOTAL_LIMIT_GIB" + log "CYCLE_SECONDS: $CYCLE_SECONDS" + log "CAPTURE_MOCK_SEND_ENABLED: $CAPTURE_MOCK_SEND_ENABLED" + log "PYTHON_BIN: $PYTHON_BIN" + log "SCRIPT_PATH: $SCRIPT_PATH" } run_one_freq() { local band="$1" - local serial="${SERIAL[$band]}" - local freq="${FREQ_HZ[$band]}" + local serial="${SERIAL[$band]:-}" + local freq="${FREQ_HZ[$band]:-}" if [[ -z "$serial" ]]; then log "Для band=$band пустой serial, пропускаю" return 0 fi + if [[ -z "$freq" ]]; then + log "Для band=$band не задана частота, пропускаю" + return 0 + fi + local ts out_dir log_file ts="$(date '+%F_%H-%M-%S')" out_dir="$BASE_DIR/$band/$ts" log_file="$out_dir/run.log" - mkdir -p "$out_dir" - log "Старт band=$band serial=$serial freq=$freq dir=$out_dir" - if ! stop_band_service "$band"; then + + if ! mkdir -p "$out_dir"; then + log "Не удалось создать каталог $out_dir" return 1 fi + stop_band_service "$band" + "$PYTHON_BIN" "$SCRIPT_PATH" \ --serial "$serial" \ --freq "$freq" \ @@ -353,77 +483,72 @@ run_one_freq() { >"$log_file" 2>&1 & local pid=$! - local mock_pid="" - log "PID=$pid" - ACTIVE_CAPTURE_PID="$pid" + CURRENT_CAPTURE_PID="$pid" + log "Capture PID=$pid" + start_mock_sender "$band" "$log_file" - mock_pid="$MOCK_SENDER_PID" - ACTIVE_MOCK_PID="$mock_pid" + local mock_pid="$CURRENT_MOCK_PID" while kill -0 "$pid" 2>/dev/null; do local cur_dir_size cur_total_size + cur_dir_size="$(dir_size_bytes "$out_dir")" cur_total_size="$(total_size_bytes)" if (( cur_total_size >= TOTAL_LIMIT_BYTES )); then - log "Достигнут общий лимит 512 GiB. Останавливаю PID=$pid" - kill -TERM "$pid" 2>/dev/null || true - wait "$pid" 2>/dev/null || true + log "Достигнут общий лимит ${TOTAL_LIMIT_GIB} GiB. Останавливаю PID=$pid" + + terminate_pid "$pid" + CURRENT_CAPTURE_PID="" + stop_mock_sender "$mock_pid" - start_active_service - ACTIVE_CAPTURE_PID="" - ACTIVE_MOCK_PID="" + CURRENT_MOCK_PID="" + + start_stopped_service + return 2 fi if (( cur_dir_size >= PER_FREQ_LIMIT_BYTES )); then - log "Для band=$band достигнут лимит 3 GiB. Останавливаю PID=$pid" - kill -TERM "$pid" 2>/dev/null || true + log "Для band=$band достигнут лимит ${PER_FREQ_LIMIT_GIB} GiB. Останавливаю PID=$pid" - for _ in {1..10}; do - if ! kill -0 "$pid" 2>/dev/null; then - break - fi - sleep 1 - done - - if kill -0 "$pid" 2>/dev/null; then - log "PID=$pid не завершился по TERM, отправляю KILL" - kill -KILL "$pid" 2>/dev/null || true - fi + terminate_pid "$pid" + CURRENT_CAPTURE_PID="" - wait "$pid" 2>/dev/null || true - stop_mock_sender "$mock_pid" - start_active_service - ACTIVE_CAPTURE_PID="" - ACTIVE_MOCK_PID="" break fi sleep 1 done + wait "$pid" 2>/dev/null || true + CURRENT_CAPTURE_PID="" + stop_mock_sender "$mock_pid" - start_active_service - ACTIVE_CAPTURE_PID="" - ACTIVE_MOCK_PID="" + CURRENT_MOCK_PID="" + + start_stopped_service + log "Завершен band=$band, размер=$(du -sh "$out_dir" | awk '{print $1}')" + return 0 } main_loop() { while true; do local total_before cycle_start elapsed sleep_left + total_before="$(total_size_bytes)" if (( total_before >= TOTAL_LIMIT_BYTES )); then - log "Общий размер уже >= 512 GiB, выхожу" + log "Общий размер уже >= ${TOTAL_LIMIT_GIB} GiB, выхожу" break fi cycle_start="$(date +%s)" log "Новый цикл" + local band for band in "${ORDER[@]}"; do if (( $(total_size_bytes) >= TOTAL_LIMIT_BYTES )); then log "Общий лимит достигнут внутри цикла, выхожу" @@ -431,19 +556,28 @@ main_loop() { fi run_one_freq "$band" || { - rc=$? - if [[ $rc -eq 2 ]]; then + local rc=$? + + if [[ "$rc" -eq 2 ]]; then log "Остановка по общему лимиту" return 0 fi + + log "Ошибка записи band=$band rc=$rc" + return "$rc" } done + if [[ "$RUN_ONCE" == "1" ]]; then + log "RUN_ONCE=1, завершаю работу после одного цикла" + break + fi + elapsed=$(( $(date +%s) - cycle_start )) sleep_left=$(( CYCLE_SECONDS - elapsed )) if (( sleep_left > 0 )); then - log "Цикл занял ${elapsed} сек, жду ${sleep_left} сек до следующего часа" + log "Цикл занял ${elapsed} сек, жду ${sleep_left} сек" sleep "$sleep_left" else log "Цикл занял ${elapsed} сек, паузы нет" @@ -451,8 +585,9 @@ main_loop() { done } -if [[ "${CAPTURE_HOURLY_SOURCE_ONLY:-0}" != "1" ]]; then - trap cleanup_active_jobs INT TERM - ensure_requirements - main_loop -fi +trap cleanup_capture EXIT +trap 'on_signal INT' INT +trap 'on_signal TERM' TERM + +ensure_requirements +main_loop \ No newline at end of file diff --git a/orange_scripts/main_2400.py b/orange_scripts/main_2400.py index c15c661..9dabeb0 100644 --- a/orange_scripts/main_2400.py +++ b/orange_scripts/main_2400.py @@ -67,8 +67,8 @@ class get_center_freq(gr.top_block): self.rtlsdr_source_0.set_sample_rate(samp_rate) self.rtlsdr_source_0.set_center_freq(center_freq, 0) self.rtlsdr_source_0.set_freq_corr(0, 0) - self.rtlsdr_source_0.set_gain(16, 0) - self.rtlsdr_source_0.set_if_gain(16, 0) + self.rtlsdr_source_0.set_gain(0, 0) + self.rtlsdr_source_0.set_if_gain(0, 0) self.rtlsdr_source_0.set_bb_gain(0, 0) self.rtlsdr_source_0.set_antenna('', 0) self.rtlsdr_source_0.set_bandwidth(0, 0) diff --git a/src/main_2400.py b/src/main_2400.py index 8790050..a036c1b 100644 --- a/src/main_2400.py +++ b/src/main_2400.py @@ -40,8 +40,8 @@ class get_center_freq(gr.top_block): self.rtlsdr_source_0.set_sample_rate(self.samp_rate) self.rtlsdr_source_0.set_center_freq(self.center_freq, 0) self.rtlsdr_source_0.set_freq_corr(0, 0) - self.rtlsdr_source_0.set_gain(24, 0) - self.rtlsdr_source_0.set_if_gain(24, 0) + self.rtlsdr_source_0.set_gain(0, 0) + self.rtlsdr_source_0.set_if_gain(0, 0) self.rtlsdr_source_0.set_bb_gain(100, 0) self.rtlsdr_source_0.set_antenna('', 0) self.rtlsdr_source_0.set_bandwidth(0, 0) diff --git a/telemetry/telemetry_server.py b/telemetry/telemetry_server.py index 83d7113..99298a7 100644 --- a/telemetry/telemetry_server.py +++ b/telemetry/telemetry_server.py @@ -144,9 +144,10 @@ def _resolve_latest_images_for_model(payload: Dict[str, Any]) -> Dict[str, Any]: return payload current_id = int(payload.get('result_id', 0) or 0) - resolved_id = current_id if current_id in grouped else max(grouped) - payload['result_id'] = resolved_id - payload['images'] = sorted(grouped[resolved_id]) + if current_id in grouped: + payload['images'] = sorted(grouped[current_id]) + else: + payload['images'] = _sanitize_image_names(payload.get('images', [])) return payload