DroneDetector/capture_hourly.sh

#!/usr/bin/env bash
set -Eeuo pipefail

############################
# НАСТРОЙКИ
############################

# Точка сохранения, по умолчанию nvme/data.
BASE_DIR="${BASE_DIR:-/mnt/data/noise}"

# Путь к python из venv
PYTHON_BIN="${PYTHON_BIN:-$PWD/.venv-sdr/bin/python}"

# Путь к headless скрипту
SCRIPT_PATH="${SCRIPT_PATH:-$PWD/scripts_nn/data_saver_headless.py}"

# Env-файл читается точечно: source .env небезопасен из-за сложных NN_* строк.
ENV_FILE="${ENV_FILE:-$PWD/.env}"


# Лимиты
PER_FREQ_LIMIT_BYTES=$((3 * 1024 * 1024 * 1024))     # 3 GiB на частоту за запуск
TOTAL_LIMIT_BYTES=$((128 * 1024 * 1024 * 1024))      # общий лимит 512 GiB
CYCLE_SECONDS="${CYCLE_SECONDS:-36}"            # один цикл в час

# Параметры SDR
SAMP_RATE="20e6"
SPLIT_SIZE="400000"
DELAY="0.1"
RF_GAIN="12"
IF_GAIN="30"
BB_GAIN="36"

# Моки для server-to-master на время capture.
CAPTURE_MOCK_SEND_ENABLED="${CAPTURE_MOCK_SEND_ENABLED:-1}"
CAPTURE_MOCK_HOST="${CAPTURE_MOCK_HOST:-127.0.0.1}"
CAPTURE_MOCK_INTERVAL_SECONDS="${CAPTURE_MOCK_INTERVAL_SECONDS:-1}"
CAPTURE_MOCK_TIMEOUT_SECONDS="${CAPTURE_MOCK_TIMEOUT_SECONDS:-0.3}"
CAPTURE_MOCK_LOG_SUCCESS="${CAPTURE_MOCK_LOG_SUCCESS:-0}"
MOCK_SENDER_PID=""
ACTIVE_CAPTURE_PID=""
ACTIVE_MOCK_PID=""
ACTIVE_SERVICE_UNIT=""
SYSTEMCTL_BIN=(systemctl)

############################
# ЧАСТОТЫ И SERIAL ИЗ ENV
############################

#ORDER=(433 750 915 1200 2400 3300 4500 5200 5800)

ORDER=(1200)
declare -A SERIAL
declare -A FREQ_HZ
declare -A SERVICE_UNIT

#SERIAL[433]="$hack_433"
FREQ_HZ[433]="433000000"

#SERIAL[750]="$hack_750"
FREQ_HZ[750]="750000000"

#SERIAL[868]="$hack_868"
FREQ_HZ[868]="868000000"

#SERIAL[915]="$hack_915"
FREQ_HZ[915]="915000000"

SERIAL[1200]="0000000000000000114864dc38638a1b"
FREQ_HZ[1200]="1200000000"

#SERIAL[2400]="0000000000000000600463dc29789bc7"
FREQ_HZ[2400]="1200000000"

#SERIAL[3300]="$hack_3300"
FREQ_HZ[3300]="3300000000"

#SERIAL[4500]="$hack_4500"
FREQ_HZ[4500]="4500000000"

#SERIAL[5200]="$hack_5200"
FREQ_HZ[5200]="5200000000"

SERIAL[5800]="0000000000000000518864dc33743883"
FREQ_HZ[5800]="5800000000"

SERVICE_UNIT[433]="dronedetector-sdr-433.service"
SERVICE_UNIT[750]="dronedetector-sdr-750.service"
SERVICE_UNIT[868]="dronedetector-sdr-868.service"
SERVICE_UNIT[915]="dronedetector-sdr-915.service"
SERVICE_UNIT[1200]="dronedetector-sdr-1200.service"
SERVICE_UNIT[2400]="dronedetector-sdr-2400.service"
SERVICE_UNIT[3300]="dronedetector-sdr-3300.service"
SERVICE_UNIT[4500]="dronedetector-sdr-4500.service"
SERVICE_UNIT[5200]="dronedetector-sdr-5200.service"
SERVICE_UNIT[5800]="dronedetector-sdr-5800.service"

############################
# ВСПОМОГАТЕЛЬНОЕ
############################

log() {
  printf '[%s] %s\n' "$(date '+%F %T')" "$*"
}

systemctl_run() {
  "${SYSTEMCTL_BIN[@]}" "$@"
}

service_exists() {
  local unit="$1"
  systemctl_run show "$unit" >/dev/null 2>&1
}

service_is_active() {
  local unit="$1"
  systemctl_run is-active --quiet "$unit" >/dev/null 2>&1
}

stop_band_service() {
  local band="$1"
  local unit="${SERVICE_UNIT[$band]:-}"

  ACTIVE_SERVICE_UNIT=""
  if [[ -z "$unit" ]]; then
    log "Для band=$band не найден service unit"
    return 0
  fi

  if ! service_exists "$unit"; then
    log "Service unit $unit не установлен, пропускаю stop/start"
    return 0
  fi

  if ! service_is_active "$unit"; then
    log "Service $unit не активен, stop/start не нужен"
    return 0
  fi

  log "Останавливаю service $unit перед записью band=$band"
  if ! systemctl_run stop "$unit"; then
    log "Не удалось остановить service $unit"
    return 1
  fi
  ACTIVE_SERVICE_UNIT="$unit"
}

start_active_service() {
  if [[ -z "$ACTIVE_SERVICE_UNIT" ]]; then
    return 0
  fi

  log "Запускаю service $ACTIVE_SERVICE_UNIT после записи"
  systemctl_run start "$ACTIVE_SERVICE_UNIT" || log "Не удалось запустить service $ACTIVE_SERVICE_UNIT"
  ACTIVE_SERVICE_UNIT=""
}

env_get() {
  local key="$1"
  local default="${2:-}"

  if [[ ! -f "$ENV_FILE" ]]; then
    printf '%s\n' "$default"
    return 0
  fi

  awk -v key="$key" -v default="$default" '
    BEGIN { found = 0 }
    $0 ~ "^[[:space:]]*" key "=" {
      value = $0
      sub("^[[:space:]]*" key "=", "", value)
      sub("[[:space:]]+#.*$", "", value)
      gsub("^[[:space:]]+|[[:space:]]+$", "", value)
      if ((substr(value, 1, 1) == "\"" && substr(value, length(value), 1) == "\"") ||
          (substr(value, 1, 1) == "'"'"'" && substr(value, length(value), 1) == "'"'"'")) {
        value = substr(value, 2, length(value) - 2)
      }
      print value
      found = 1
      exit
    }
    END {
      if (!found) {
        print default
      }
    }
  ' "$ENV_FILE"
}

mock_bool_enabled() {
  case "${1,,}" in
    1|true|yes|on) return 0 ;;
    *) return 1 ;;
  esac
}

mock_url() {
  local port endpoint
  port="${CAPTURE_MOCK_PORT:-$(env_get locport "$(env_get GENERAL_SERVER_PORT 5010)")}"
  endpoint="${CAPTURE_MOCK_ENDPOINT:-$(env_get freq_endpoint process_data)}"
  endpoint="${endpoint#/}"
  printf 'http://%s:%s/%s' "$CAPTURE_MOCK_HOST" "$port" "$endpoint"
}

start_mock_sender() {
  local band="$1"
  local log_file="$2"
  local url

  MOCK_SENDER_PID=""
  if ! mock_bool_enabled "$CAPTURE_MOCK_SEND_ENABLED"; then
    log "Mock sender отключен CAPTURE_MOCK_SEND_ENABLED=$CAPTURE_MOCK_SEND_ENABLED"
    return 0
  fi

  url="$(mock_url)"
  log "Старт mock sender band=$band url=$url amplitude=0 interval=${CAPTURE_MOCK_INTERVAL_SECONDS}s"

  "$PYTHON_BIN" - "$url" "$band" "$CAPTURE_MOCK_INTERVAL_SECONDS" "$CAPTURE_MOCK_TIMEOUT_SECONDS" <<'PY' >>"$log_file" 2>&1 &
import json
import os
import sys
import time
import urllib.error
import urllib.request


url = sys.argv[1]
freq = str(sys.argv[2])
interval = float(sys.argv[3])
timeout = float(sys.argv[4])
payload = json.dumps({"freq": freq, "amplitude": 0}).encode("utf-8")
headers = {"Content-Type": "application/json"}
log_success = os.getenv("CAPTURE_MOCK_LOG_SUCCESS", "0").lower() in {"1", "true", "yes", "on"}

print(f"[capture-mock] started url={url} freq={freq} amplitude=0 interval={interval}", flush=True)
while True:
    try:
        req = urllib.request.Request(url, data=payload, headers=headers, method="POST")
        with urllib.request.urlopen(req, timeout=timeout) as response:
            if log_success:
                print(f"[capture-mock] sent status={response.status}", flush=True)
    except Exception as exc:
        print(f"[capture-mock] send failed: {exc}", flush=True)
    time.sleep(interval)
PY
  MOCK_SENDER_PID=$!
  log "Mock sender PID=$MOCK_SENDER_PID"
}

stop_mock_sender() {
  local pid="${1:-}"
  if [[ -z "$pid" ]]; then
    return 0
  fi
  if kill -0 "$pid" 2>/dev/null; then
    log "Останавливаю mock sender PID=$pid"
    kill -TERM "$pid" 2>/dev/null || true
    wait "$pid" 2>/dev/null || true
  fi
}

cleanup_active_jobs() {
  local rc=$?
  if [[ -n "${ACTIVE_CAPTURE_PID:-}" ]] && kill -0 "$ACTIVE_CAPTURE_PID" 2>/dev/null; then
    log "Останавливаю active capture PID=$ACTIVE_CAPTURE_PID"
    kill -TERM "$ACTIVE_CAPTURE_PID" 2>/dev/null || true
    wait "$ACTIVE_CAPTURE_PID" 2>/dev/null || true
  fi
  stop_mock_sender "${ACTIVE_MOCK_PID:-}"
  start_active_service
  exit "$rc"
}

dir_size_bytes() {
  local path="$1"
  if [[ -e "$path" ]]; then
    du -sb "$path" 2>/dev/null | awk '{print $1}'
  else
    echo 0
  fi
}

total_size_bytes() {
  dir_size_bytes "$BASE_DIR"
}
ensure_requirements() {
  if [[ ! -x "$PYTHON_BIN" ]]; then
    echo "Не найден python: $PYTHON_BIN" >&2
    exit 1
  fi

  if [[ ! -f "$SCRIPT_PATH" ]]; then
    echo "Не найден script: $SCRIPT_PATH" >&2
    exit 1
  fi

  if ! command -v systemctl >/dev/null 2>&1; then
    echo "Не найден systemctl" >&2
    exit 1
  fi

  if [[ "${EUID}" -ne 0 ]]; then
    if command -v sudo >/dev/null 2>&1; then
      SYSTEMCTL_BIN=(sudo systemctl)
    else
      echo "Для управления service нужен root или sudo" >&2
      exit 1
    fi
  fi

  mkdir -p "$BASE_DIR"

  local dev
  dev="$(df -P "$BASE_DIR" | awk 'NR==2 {print $1}')"


}

run_one_freq() {
  local band="$1"
  local serial="${SERIAL[$band]}"
  local freq="${FREQ_HZ[$band]}"

  if [[ -z "$serial" ]]; then
    log "Для band=$band пустой serial, пропускаю"
    return 0
  fi

  local ts out_dir log_file
  ts="$(date '+%F_%H-%M-%S')"
  out_dir="$BASE_DIR/$band/$ts"
  log_file="$out_dir/run.log"

  mkdir -p "$out_dir"

  log "Старт band=$band serial=$serial freq=$freq dir=$out_dir"
  if ! stop_band_service "$band"; then
    return 1
  fi

  "$PYTHON_BIN" "$SCRIPT_PATH" \
    --serial "$serial" \
    --freq "$freq" \
    --save-dir "$out_dir" \
    --file-tag "${band}_" \
    --samp-rate "$SAMP_RATE" \
    --split-size "$SPLIT_SIZE" \
    --delay "$DELAY" \
    --rf-gain "$RF_GAIN" \
    --if-gain "$IF_GAIN" \
    --bb-gain "$BB_GAIN" \
    >"$log_file" 2>&1 &

  local pid=$!
  local mock_pid=""
  log "PID=$pid"
  ACTIVE_CAPTURE_PID="$pid"
  start_mock_sender "$band" "$log_file"
  mock_pid="$MOCK_SENDER_PID"
  ACTIVE_MOCK_PID="$mock_pid"

  while kill -0 "$pid" 2>/dev/null; do
    local cur_dir_size cur_total_size
    cur_dir_size="$(dir_size_bytes "$out_dir")"
    cur_total_size="$(total_size_bytes)"

    if (( cur_total_size >= TOTAL_LIMIT_BYTES )); then
      log "Достигнут общий лимит 512 GiB. Останавливаю PID=$pid"
      kill -TERM "$pid" 2>/dev/null || true
      wait "$pid" 2>/dev/null || true
      stop_mock_sender "$mock_pid"
      start_active_service
      ACTIVE_CAPTURE_PID=""
      ACTIVE_MOCK_PID=""
      return 2
    fi

    if (( cur_dir_size >= PER_FREQ_LIMIT_BYTES )); then
      log "Для band=$band достигнут лимит 3 GiB. Останавливаю PID=$pid"
      kill -TERM "$pid" 2>/dev/null || true

      for _ in {1..10}; do
        if ! kill -0 "$pid" 2>/dev/null; then
          break
        fi
        sleep 1
      done

      if kill -0 "$pid" 2>/dev/null; then
        log "PID=$pid не завершился по TERM, отправляю KILL"
        kill -KILL "$pid" 2>/dev/null || true
      fi

      wait "$pid" 2>/dev/null || true
      stop_mock_sender "$mock_pid"
      start_active_service
      ACTIVE_CAPTURE_PID=""
      ACTIVE_MOCK_PID=""
      break
    fi

    sleep 1
  done

  stop_mock_sender "$mock_pid"
  start_active_service
  ACTIVE_CAPTURE_PID=""
  ACTIVE_MOCK_PID=""
  log "Завершен band=$band, размер=$(du -sh "$out_dir" | awk '{print $1}')"
  return 0
}

main_loop() {
  while true; do
    local total_before cycle_start elapsed sleep_left
    total_before="$(total_size_bytes)"

    if (( total_before >= TOTAL_LIMIT_BYTES )); then
      log "Общий размер уже >= 512 GiB, выхожу"
      break
    fi

    cycle_start="$(date +%s)"
    log "Новый цикл"

    for band in "${ORDER[@]}"; do
      if (( $(total_size_bytes) >= TOTAL_LIMIT_BYTES )); then
        log "Общий лимит достигнут внутри цикла, выхожу"
        return 0
      fi

      run_one_freq "$band" || {
        rc=$?
        if [[ $rc -eq 2 ]]; then
          log "Остановка по общему лимиту"
          return 0
        fi
      }
    done

    elapsed=$(( $(date +%s) - cycle_start ))
    sleep_left=$(( CYCLE_SECONDS - elapsed ))

    if (( sleep_left > 0 )); then
      log "Цикл занял ${elapsed} сек, жду ${sleep_left} сек до следующего часа"
      sleep "$sleep_left"
    else
      log "Цикл занял ${elapsed} сек, паузы нет"
    fi
  done
}

if [[ "${CAPTURE_HOURLY_SOURCE_ONLY:-0}" != "1" ]]; then
  trap cleanup_active_jobs INT TERM
  ensure_requirements
  main_loop
fi