From 06dc5fa71e2c989822976630e5a2a5767b00920b Mon Sep 17 00:00:00 2001 From: Sergey Revyakin Date: Mon, 27 Apr 2026 19:45:52 +0700 Subject: [PATCH] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8=D0=BB=20?= =?UTF-8?q?=D1=81=D0=B1=D0=BE=D1=80=20=D0=B4=D0=B0=D0=BD=D0=BD=D1=8B=D1=85?= =?UTF-8?q?=20=D0=B4=D0=BB=D1=8F=20neptune?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 15 +- capture_hourly.sh | 411 ++++++++++++++++++++++-------- install_all.sh | 22 ++ scripts_nn/data_saver_headless.py | 342 ++++++++++++++++++++----- 4 files changed, 616 insertions(+), 174 deletions(-) mode change 100644 => 100755 scripts_nn/data_saver_headless.py diff --git a/README.md b/README.md index b3859a8..1d8b1b4 100644 --- a/README.md +++ b/README.md @@ -232,4 +232,17 @@ curl -X POST 'http://127.0.0.1:5010/process_data' \ ### Read_energy Neptune ``` bash ./.venv-sdr/bin/python read_energy.py --backend iio --uri ip:192.168.2.1 --only 2400 --refresh 0.5 --interval 0.2 --vec-len 1024 -``` \ No newline at end of file +``` + +### Capture_hourly Neptune +``` bash +CAPTURE_ORDER=2400 \ +CAPTURE_BACKEND_2400=iio \ +CAPTURE_IIO_URI_2400=ip:192.168.2.1 \ +RUN_ONCE=1 \ +./capture_hourly.sh +``` + +Дополнительные параметры `Neptune` для `capture_hourly` читаются из `.env` или окружения в формате +`CAPTURE_IIO__2400`, например `CAPTURE_IIO_DEVICE_2400`, `CAPTURE_IIO_PHY_DEVICE_2400`, +`CAPTURE_IIO_GAIN_MODE_2400`, `CAPTURE_IIO_HARDWAREGAIN_2400`, `CAPTURE_IIO_BANDWIDTH_2400`. diff --git a/capture_hourly.sh b/capture_hourly.sh index 51fd8f2..f0e1cdc 100755 --- a/capture_hourly.sh +++ b/capture_hourly.sh @@ -1,5 +1,3 @@ -# echo 'RUN_ONCE=1 CAPTURE_ORDER=433 /home/sergei/work/DroneDetector/DroneDetector/capture_hourly.sh >> /home/sergei/capture_hourly_night.log 2>&1' | sudo at 15:00 2026-07-22 - #!/usr/bin/env bash set -Eeuo pipefail @@ -8,91 +6,114 @@ SCRIPT_OWNER="${CAPTURE_USER:-$(stat -c %U "$SCRIPT_DIR")}" SCRIPT_OWNER_HOME="$(getent passwd "$SCRIPT_OWNER" | cut -d: -f6)" cd "$SCRIPT_DIR" -source "$SCRIPT_DIR/.env" -############################ -# НАСТРОЙКИ -############################ +ENV_FILE="$SCRIPT_DIR/.env" +declare -A DOTENV_VALUES -BASE_DIR="${CAPTURE_BASE_DIR:-${SCRIPT_OWNER_HOME}/dataset/noise}" +trim_whitespace() { + local value="$1" + value="${value#"${value%%[![:space:]]*}"}" + value="${value%"${value##*[![:space:]]}"}" + printf '%s' "$value" +} -# Путь к python из venv -PYTHON_BIN="${PYTHON_BIN:-$SCRIPT_DIR/.venv-sdr/bin/python}" +load_env_file() { + local line key value quote_char -# Путь к headless скрипту -SCRIPT_PATH="${SCRIPT_PATH:-$SCRIPT_DIR/scripts_nn/data_saver_headless.py}" + if [[ ! -f "$ENV_FILE" ]]; then + echo "Не найден .env: $ENV_FILE" >&2 + exit 1 + fi -RUN_ONCE="${RUN_ONCE:-0}" -CAPTURE_LOG_FILE="${CAPTURE_LOG_FILE:-$BASE_DIR/capture_hourly.log}" + while IFS= read -r line || [[ -n "$line" ]]; do + line="${line%$'\r'}" + line="$(trim_whitespace "$line")" -SYSTEMCTL_BIN=(systemctl) -CURRENT_CAPTURE_PID="" -CURRENT_SERVICE_UNIT="" + [[ -z "$line" || "$line" == \#* ]] && continue + if [[ "$line" == export\ * ]]; then + line="${line#export }" + fi + [[ "$line" == *=* ]] || continue -# Лимиты + key="$(trim_whitespace "${line%%=*}")" + value="$(trim_whitespace "${line#*=}")" -lim_all=10 -PER_FREQ_LIMIT_BYTES=$((1 * 1024 * 1024 * 1024)) # GiB на частоту за запуск -TOTAL_LIMIT_BYTES=$((lim_all * 1024 * 1024 * 1024)) # общий лимит GiB -CYCLE_SECONDS=1 # один цикл в час - -# Параметры SDR -SAMP_RATE="20e6" -SPLIT_SIZE="400000" -DELAY="0.01" -RF_GAIN="12" -IF_GAIN="30" -BB_GAIN="36" + if [[ ${#value} -ge 2 ]]; then + quote_char="${value:0:1}" + if [[ ( "$quote_char" == "'" || "$quote_char" == '"' ) && "${value: -1}" == "$quote_char" ]]; then + value="${value:1:${#value}-2}" + fi + fi -############################ -# ЧАСТОТЫ И SERIAL ИЗ ENV -############################ + DOTENV_VALUES["$key"]="$value" + done < "$ENV_FILE" +} -ORDER=(433) +get_env_setting() { + local key="$1" + local default="${2-}" + if [[ -n "${!key+x}" ]]; then + printf '%s' "${!key}" + elif [[ -n "${DOTENV_VALUES[$key]+x}" ]]; then + printf '%s' "${DOTENV_VALUES[$key]}" + else + printf '%s' "$default" + fi +} -declare -A SERIAL -declare -A FREQ_HZ -declare -A SERVICE_UNIT +get_band_setting() { + local prefix="$1" + local band="$2" + local default="${3-}" -SERIAL[433]="$hack_433" -FREQ_HZ[433]="433000000" + get_env_setting "${prefix}_${band}" "$(get_env_setting "$prefix" "$default")" +} -SERIAL[750]="$hack_750" -FREQ_HZ[750]="750000000" +load_env_file +############################ +# НАСТРОЙКИ +############################ -SERIAL[915]="$hack_915" -FREQ_HZ[915]="915000000" +BASE_DIR="$(get_env_setting CAPTURE_BASE_DIR "${SCRIPT_OWNER_HOME}/dataset/noise")" +PYTHON_BIN="$(get_env_setting PYTHON_BIN "$SCRIPT_DIR/.venv-sdr/bin/python")" +SCRIPT_PATH="$(get_env_setting SCRIPT_PATH "$SCRIPT_DIR/scripts_nn/data_saver_headless.py")" -SERIAL[1200]="$hack_1200" -FREQ_HZ[1200]="1200000000" +RUN_ONCE="$(get_env_setting RUN_ONCE "0")" +CAPTURE_LOG_FILE="$(get_env_setting CAPTURE_LOG_FILE "$BASE_DIR/capture_hourly.log")" -SERIAL[2400]="$hack_2400" -FREQ_HZ[2400]="2400000000" +SYSTEMCTL_BIN=(systemctl) +CURRENT_CAPTURE_PID="" +CURRENT_SERVICE_UNITS=() -SERIAL[3300]="$hack_3300" -FREQ_HZ[3300]="3300000000" +lim_all=10 +PER_FREQ_LIMIT_BYTES=$((1 * 1024 * 1024 * 1024)) +TOTAL_LIMIT_BYTES=$((lim_all * 1024 * 1024 * 1024)) +CYCLE_SECONDS="$(get_env_setting CAPTURE_CYCLE_SECONDS "1")" -SERIAL[4500]="$hack_4500" -FREQ_HZ[4500]="4500000000" +DEFAULT_SAMP_RATE="$(get_env_setting CAPTURE_SAMP_RATE "20e6")" +DEFAULT_SPLIT_SIZE="$(get_env_setting CAPTURE_SPLIT_SIZE "400000")" +DEFAULT_DELAY="$(get_env_setting CAPTURE_DELAY "0.01")" +DEFAULT_RF_GAIN="$(get_env_setting CAPTURE_RF_GAIN "12")" +DEFAULT_IF_GAIN="$(get_env_setting CAPTURE_IF_GAIN "30")" +DEFAULT_BB_GAIN="$(get_env_setting CAPTURE_BB_GAIN "36")" -SERIAL[5200]="$hack_5200" -FREQ_HZ[5200]="5200000000" +ORDER=(433) +SUPPORTED_BANDS=(433 750 915 1200 1500 2400 3300 4500 5200 5800) -SERIAL[5800]="$hack_5800" -FREQ_HZ[5800]="5800000000" +declare -A SERIAL +declare -A FREQ_HZ +declare -A SERVICE_UNIT +declare -A BACKEND -SERVICE_UNIT[433]="dronedetector-sdr-433.service" -SERVICE_UNIT[750]="dronedetector-sdr-750.service" -SERVICE_UNIT[915]="dronedetector-sdr-915.service" -SERVICE_UNIT[1200]="dronedetector-sdr-1200.service" -SERVICE_UNIT[2400]="dronedetector-sdr-2400.service" -SERVICE_UNIT[3300]="dronedetector-sdr-3300.service" -SERVICE_UNIT[4500]="dronedetector-sdr-4500.service" -SERVICE_UNIT[5200]="dronedetector-sdr-5200.service" -SERVICE_UNIT[5800]="dronedetector-sdr-5800.service" +for band in "${SUPPORTED_BANDS[@]}"; do + SERIAL["$band"]="$(get_env_setting "hack_${band}" "$(get_env_setting "HACKID_${band}")")" + FREQ_HZ["$band"]="$(get_env_setting "c_freq_${band}" "$band")e6" + SERVICE_UNIT["$band"]="dronedetector-sdr-${band}.service" + BACKEND["$band"]="$(tr '[:upper:]' '[:lower:]' <<<"$(get_band_setting CAPTURE_BACKEND "$band" "hackrf")")" +done ############################ # ВСПОМОГАТЕЛЬНОЕ @@ -111,36 +132,117 @@ service_exists() { systemctl_run show "$unit" >/dev/null 2>&1 } -stop_band_service() { +service_is_active() { + local unit="$1" + systemctl_run is-active --quiet "$unit" +} + +remember_service_unit() { + local unit="$1" + local existing + + for existing in "${CURRENT_SERVICE_UNITS[@]}"; do + if [[ "$existing" == "$unit" ]]; then + return 0 + fi + done + + CURRENT_SERVICE_UNITS+=("$unit") +} + +stop_iio_band_service() { local band="$1" local unit="${SERVICE_UNIT[$band]:-}" + CURRENT_SERVICE_UNITS=() + if [[ -z "$unit" ]]; then log "Для band=$band не найден service unit" return 0 fi if ! service_exists "$unit"; then - log "Service unit $unit не установлен, пропускаю stop/start" + log "Service unit $unit не установлен, пропускаю" return 0 fi - log "Останавливаю service $unit перед записью band=$band" + if service_is_active "$unit"; then + remember_service_unit "$unit" + fi + + log "Останавливаю service $unit перед записью band=$band backend=iio" systemctl_run stop "$unit" - CURRENT_SERVICE_UNIT="$unit" +} + +stop_hackrf_band_services() { + local band="$1" + local serial="${SERIAL[$band]:-}" + local other_band unit other_backend other_serial + + if [[ -z "$serial" ]]; then + log "Для band=$band пустой serial, пропускаю stop/start сервисов" + return 0 + fi + + CURRENT_SERVICE_UNITS=() + for other_band in "${!SERVICE_UNIT[@]}"; do + unit="${SERVICE_UNIT[$other_band]:-}" + other_backend="${BACKEND[$other_band]:-hackrf}" + other_serial="${SERIAL[$other_band]:-}" + + if [[ "$other_backend" != "hackrf" || -z "$unit" || -z "$other_serial" || "$other_serial" != "$serial" ]]; then + continue + fi + + if ! service_exists "$unit"; then + log "Service unit $unit не установлен, пропускаю" + continue + fi + + if service_is_active "$unit"; then + remember_service_unit "$unit" + fi + + log "Останавливаю service $unit перед записью band=$band" + systemctl_run stop "$unit" + done +} + +stop_band_service() { + local band="$1" + local backend="${BACKEND[$band]:-hackrf}" + + case "$backend" in + hackrf) + stop_hackrf_band_services "$band" + ;; + iio) + stop_iio_band_service "$band" + ;; + *) + log "Неизвестный backend=$backend для band=$band" + return 1 + ;; + esac } start_current_service() { - if [[ -z "$CURRENT_SERVICE_UNIT" ]]; then + local unit + + if [[ "${#CURRENT_SERVICE_UNITS[@]}" -eq 0 ]]; then return 0 fi - log "Запускаю service $CURRENT_SERVICE_UNIT после записи" - systemctl_run start "$CURRENT_SERVICE_UNIT" - CURRENT_SERVICE_UNIT="" + for unit in "${CURRENT_SERVICE_UNITS[@]}"; do + log "Запускаю service $unit после записи" + systemctl_run start "$unit" + done + CURRENT_SERVICE_UNITS=() } cleanup_capture() { + local unit + if [[ -n "$CURRENT_CAPTURE_PID" ]] && kill -0 "$CURRENT_CAPTURE_PID" 2>/dev/null; then log "Останавливаю текущий PID=$CURRENT_CAPTURE_PID при завершении скрипта" kill -TERM "$CURRENT_CAPTURE_PID" 2>/dev/null || true @@ -148,10 +250,12 @@ cleanup_capture() { fi CURRENT_CAPTURE_PID="" - if [[ -n "$CURRENT_SERVICE_UNIT" ]]; then - log "Восстанавливаю service $CURRENT_SERVICE_UNIT при завершении скрипта" - systemctl_run start "$CURRENT_SERVICE_UNIT" || true - CURRENT_SERVICE_UNIT="" + if [[ "${#CURRENT_SERVICE_UNITS[@]}" -gt 0 ]]; then + for unit in "${CURRENT_SERVICE_UNITS[@]}"; do + log "Восстанавливаю service $unit при завершении скрипта" + systemctl_run start "$unit" || true + done + CURRENT_SERVICE_UNITS=() fi } @@ -175,6 +279,8 @@ total_size_bytes() { } ensure_requirements() { + local capture_order band backend + if [[ ! -x "$PYTHON_BIN" ]]; then echo "Не найден python: $PYTHON_BIN" >&2 exit 1 @@ -202,10 +308,10 @@ ensure_requirements() { mkdir -p "$BASE_DIR" mkdir -p "$(dirname "$CAPTURE_LOG_FILE")" - if [[ -n "${CAPTURE_ORDER:-}" ]]; then + capture_order="$(get_env_setting CAPTURE_ORDER)" + if [[ -n "$capture_order" ]]; then ORDER=() - local band - for band in ${CAPTURE_ORDER//,/ }; do + for band in ${capture_order//,/ }; do ORDER+=("$band") done fi @@ -215,29 +321,132 @@ ensure_requirements() { exit 1 fi + for band in "${ORDER[@]}"; do + backend="${BACKEND[$band]:-}" + if [[ -z "$backend" ]]; then + echo "Band $band не поддерживается capture_hourly." >&2 + exit 1 + fi + + case "$backend" in + hackrf|iio) + ;; + *) + echo "Неизвестный backend=$backend для band=$band" >&2 + exit 1 + ;; + esac + + if [[ "$backend" == "iio" ]]; then + if ! command -v iio_attr >/dev/null 2>&1; then + echo "Не найден iio_attr для band=$band" >&2 + exit 1 + fi + if ! command -v iio_readdev >/dev/null 2>&1; then + echo "Не найден iio_readdev для band=$band" >&2 + exit 1 + fi + fi + done + log "Рабочая директория: $SCRIPT_DIR" log "BASE_DIR: $BASE_DIR" log "ORDER: ${ORDER[*]}" log "RUN_ONCE: $RUN_ONCE" - } run_one_freq() { local band="$1" + local backend="${BACKEND[$band]}" local serial="${SERIAL[$band]}" local freq="${FREQ_HZ[$band]}" - - if [[ -z "$serial" ]]; then - log "Для band=$band пустой serial, пропускаю" + local ts out_dir log_file pid + local samp_rate split_size delay + local rf_gain if_gain bb_gain + local iio_uri iio_device iio_phy_device iio_i_channel iio_q_channel + local iio_lo_channel iio_port_select iio_gain_mode iio_hardwaregain + local iio_timeout_ms iio_settle iio_bandwidth iio_samples_per_read + local backend_desc + local -a cmd + + if [[ "$backend" == "hackrf" && -z "$serial" ]]; then + log "Для band=$band backend=hackrf, но serial пустой, пропускаю" return 0 fi - local ts out_dir log_file ts="$(date '+%F_%H-%M-%S')" out_dir="$BASE_DIR/$band/$ts" log_file="$out_dir/run.log" - log "Старт band=$band serial=$serial freq=$freq dir=$out_dir" + samp_rate="$(get_band_setting CAPTURE_SAMP_RATE "$band" "$DEFAULT_SAMP_RATE")" + split_size="$(get_band_setting CAPTURE_SPLIT_SIZE "$band" "$DEFAULT_SPLIT_SIZE")" + delay="$(get_band_setting CAPTURE_DELAY "$band" "$DEFAULT_DELAY")" + + cmd=( + "$PYTHON_BIN" + "$SCRIPT_PATH" + "--backend" "$backend" + "--freq" "$freq" + "--save-dir" "$out_dir" + "--file-tag" "${band}_" + "--samp-rate" "$samp_rate" + "--split-size" "$split_size" + "--delay" "$delay" + ) + + if [[ "$backend" == "hackrf" ]]; then + rf_gain="$(get_band_setting CAPTURE_RF_GAIN "$band" "$DEFAULT_RF_GAIN")" + if_gain="$(get_band_setting CAPTURE_IF_GAIN "$band" "$DEFAULT_IF_GAIN")" + bb_gain="$(get_band_setting CAPTURE_BB_GAIN "$band" "$DEFAULT_BB_GAIN")" + cmd+=( + "--serial" "$serial" + "--rf-gain" "$rf_gain" + "--if-gain" "$if_gain" + "--bb-gain" "$bb_gain" + ) + backend_desc="serial=$serial" + else + iio_uri="$(get_band_setting CAPTURE_IIO_URI "$band" "ip:192.168.2.1")" + iio_device="$(get_band_setting CAPTURE_IIO_DEVICE "$band" "cf-ad9361-lpc")" + iio_phy_device="$(get_band_setting CAPTURE_IIO_PHY_DEVICE "$band" "ad9361-phy")" + iio_i_channel="$(get_band_setting CAPTURE_IIO_I_CHANNEL "$band" "voltage0")" + iio_q_channel="$(get_band_setting CAPTURE_IIO_Q_CHANNEL "$band" "voltage1")" + iio_lo_channel="$(get_band_setting CAPTURE_IIO_LO_CHANNEL "$band" "altvoltage0")" + iio_port_select="$(get_band_setting CAPTURE_IIO_PORT_SELECT "$band" "A_BALANCED")" + iio_gain_mode="$(get_band_setting CAPTURE_IIO_GAIN_MODE "$band" "slow_attack")" + iio_hardwaregain="$(get_band_setting CAPTURE_IIO_HARDWAREGAIN "$band")" + iio_timeout_ms="$(get_band_setting CAPTURE_IIO_TIMEOUT_MS "$band" "4000")" + iio_settle="$(get_band_setting CAPTURE_SETTLE "$band" "0.12")" + iio_bandwidth="$(get_band_setting CAPTURE_IIO_BANDWIDTH "$band")" + iio_samples_per_read="$(get_band_setting CAPTURE_IIO_SAMPLES_PER_READ "$band")" + + cmd+=( + "--iio-uri" "$iio_uri" + "--iio-device" "$iio_device" + "--iio-phy-device" "$iio_phy_device" + "--iio-i-channel" "$iio_i_channel" + "--iio-q-channel" "$iio_q_channel" + "--iio-lo-channel" "$iio_lo_channel" + "--iio-port-select" "$iio_port_select" + "--iio-gain-mode" "$iio_gain_mode" + "--timeout-ms" "$iio_timeout_ms" + "--settle" "$iio_settle" + ) + + if [[ -n "$iio_hardwaregain" ]]; then + cmd+=("--iio-hardwaregain" "$iio_hardwaregain") + fi + if [[ -n "$iio_bandwidth" ]]; then + cmd+=("--bandwidth" "$iio_bandwidth") + fi + if [[ -n "$iio_samples_per_read" ]]; then + cmd+=("--samples-per-read" "$iio_samples_per_read") + fi + + backend_desc="uri=$iio_uri" + fi + + log "Старт band=$band backend=$backend $backend_desc freq=$freq dir=$out_dir" if ! mkdir -p "$out_dir"; then log "Не удалось создать каталог $out_dir" return 1 @@ -245,20 +454,8 @@ run_one_freq() { stop_band_service "$band" - "$PYTHON_BIN" "$SCRIPT_PATH" \ - --serial "$serial" \ - --freq "$freq" \ - --save-dir "$out_dir" \ - --file-tag "${band}_" \ - --samp-rate "$SAMP_RATE" \ - --split-size "$SPLIT_SIZE" \ - --delay "$DELAY" \ - --rf-gain "$RF_GAIN" \ - --if-gain "$IF_GAIN" \ - --bb-gain "$BB_GAIN" \ - >"$log_file" 2>&1 & - - local pid=$! + "${cmd[@]}" >"$log_file" 2>&1 & + pid=$! CURRENT_CAPTURE_PID="$pid" log "PID=$pid" @@ -268,7 +465,7 @@ run_one_freq() { cur_total_size="$(total_size_bytes)" if (( cur_total_size >= TOTAL_LIMIT_BYTES )); then - log "Достигнут общий лимит $lim_all GiB. Останавливаю PID=$pid" + log "Достигнут общий лимит ${lim_all} GiB. Останавливаю PID=$pid" kill -TERM "$pid" 2>/dev/null || true wait "$pid" 2>/dev/null || true CURRENT_CAPTURE_PID="" @@ -277,7 +474,7 @@ run_one_freq() { fi if (( cur_dir_size >= PER_FREQ_LIMIT_BYTES )); then - log "Для band=$band достигнут лимит 3 GiB. Останавливаю PID=$pid" + log "Для band=$band достигнут лимит 1 GiB. Останавливаю PID=$pid" kill -TERM "$pid" 2>/dev/null || true for _ in {1..10}; do @@ -301,17 +498,18 @@ run_one_freq() { CURRENT_CAPTURE_PID="" start_current_service - log "Завершен band=$band, размер=$(du -sh "$out_dir" | awk '{print $1}')" + log "Завершен band=$band backend=$backend, размер=$(du -sh "$out_dir" | awk '{print $1}')" return 0 } main_loop() { + local total_before cycle_start elapsed sleep_left rc + while true; do - local total_before cycle_start elapsed sleep_left total_before="$(total_size_bytes)" if (( total_before >= TOTAL_LIMIT_BYTES )); then - log "Общий размер уже >= GiB, выхожу" + log "Общий размер уже >= ${lim_all} GiB, выхожу" break fi @@ -329,10 +527,9 @@ main_loop() { if [[ $rc -eq 2 ]]; then log "Остановка по общему лимиту" return 0 - else - log "Ошибка записи band=$band rc=$rc" - return "$rc" fi + log "Ошибка записи band=$band rc=$rc" + return "$rc" } done @@ -345,7 +542,7 @@ main_loop() { sleep_left=$(( CYCLE_SECONDS - elapsed )) if (( sleep_left > 0 )); then - log "Цикл занял ${elapsed} сек, жду ${sleep_left} сек до следующего часа" + log "Цикл занял ${elapsed} сек, жду ${sleep_left} сек до следующего запуска" sleep "$sleep_left" else log "Цикл занял ${elapsed} сек, паузы нет" diff --git a/install_all.sh b/install_all.sh index 1cb5ae2..e0f0955 100755 --- a/install_all.sh +++ b/install_all.sh @@ -25,6 +25,11 @@ SDR_UNITS=( ) CONFIGURED_SDR_UNITS=() +IIO_TOOL_BINS=( + /usr/bin/iio_attr + /usr/bin/iio_readdev + /usr/bin/iio_info +) log() { printf '[install_all] %s\n' "$*" @@ -177,10 +182,26 @@ install_host_non_python_deps() { libusb-1.0-0 \ libusb-1.0-0-dev \ hackrf \ + libiio-utils \ gnuradio \ gr-osmosdr } +link_host_sdr_tools_into_venv() { + local venv_path="$1" + local tool_path tool_name + + for tool_path in "${IIO_TOOL_BINS[@]}"; do + if [[ ! -x "$tool_path" ]]; then + log "Skipping missing SDR host tool: ${tool_path}" + continue + fi + + tool_name="$(basename "$tool_path")" + ln -sf "$tool_path" "$venv_path/bin/$tool_name" + done +} + setup_sdr_python_env() { log "Setting up SDR python environment" local venv_path="${PROJECT_ROOT}/.venv-sdr" @@ -191,6 +212,7 @@ setup_sdr_python_env() { "$venv_path/bin/pip" install --upgrade pip "$venv_path/bin/pip" install -r "${PROJECT_ROOT}/deploy/requirements/sdr_host.txt" + link_host_sdr_tools_into_venv "$venv_path" chown -R "${RUN_USER}:${RUN_GROUP}" "$venv_path" } diff --git a/scripts_nn/data_saver_headless.py b/scripts_nn/data_saver_headless.py old mode 100644 new mode 100755 index c002959..87437a5 --- a/scripts_nn/data_saver_headless.py +++ b/scripts_nn/data_saver_headless.py @@ -1,27 +1,23 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +import argparse import os -import sys -import time import signal -import argparse +import subprocess +import sys import threading +import time import numpy as np from gnuradio import gr import osmosdr +IIO_MIN_SAMPLE_RATE = 2_083_333 -class SimsiSink(gr.sync_block): - def __init__(self, save_dir="./signal", file_tag="fragment_", split_size=1000000, delay=0.0): - gr.sync_block.__init__( - self, - name="Simsi_Sink", - in_sig=[np.complex64], - out_sig=None, - ) +class RotatingIqWriter: + def __init__(self, save_dir="./signal", file_tag="fragment_", split_size=1_000_000, delay=0.0): self.save_dir = str(save_dir) self.file_tag = str(file_tag) self.split_size = int(split_size) @@ -86,35 +82,56 @@ class SimsiSink(gr.sync_block): self.file_index += 1 self._open_next_file() - def work(self, input_items, output_items): - data = input_items[0] + def write_samples(self, samples): + data = np.asarray(samples, dtype=np.complex64).reshape(-1) offset = 0 - total = len(data) + total = int(data.size) while offset < total: remaining = self.split_size - self.current_len chunk = min(remaining, total - offset) - - self.current_fd.write(data[offset:offset + chunk].copy()) + block = np.ascontiguousarray(data[offset:offset + chunk], dtype=np.complex64) + self.current_fd.write(block.tobytes()) self.current_len += chunk offset += chunk if self.current_len >= self.split_size: self._rotate_file() - return len(data) - - def stop(self): + def close(self): try: if self.current_fd is not None: self.current_fd.close() self.current_fd = None finally: self._remove_in_progress() + + +class SimsiSink(gr.sync_block): + def __init__(self, save_dir="./signal", file_tag="fragment_", split_size=1_000_000, delay=0.0): + gr.sync_block.__init__( + self, + name="Simsi_Sink", + in_sig=[np.complex64], + out_sig=None, + ) + self.writer = RotatingIqWriter( + save_dir=save_dir, + file_tag=file_tag, + split_size=split_size, + delay=delay, + ) + + def work(self, input_items, output_items): + self.writer.write_samples(input_items[0]) + return len(input_items[0]) + + def stop(self): + self.writer.close() return True -class DataSaver(gr.top_block): +class HackRfDataSaver(gr.top_block): def __init__( self, serial: str, @@ -130,47 +147,204 @@ class DataSaver(gr.top_block): ): super().__init__("data_saver_headless", catch_exceptions=True) - self.serial = serial - self.freq = float(freq) - self.save_dir = save_dir - self.file_tag = file_tag - self.samp_rate = float(samp_rate) - self.split_size = int(split_size) - self.delay = float(delay) - self.rf_gain = float(rf_gain) - self.if_gain = float(if_gain) - self.bb_gain = float(bb_gain) - - dev_args = f"numchan=1 hackrf={self.serial}" + dev_args = f"numchan=1 hackrf={serial}" self.source = osmosdr.source(args=dev_args) self.source.set_time_unknown_pps(osmosdr.time_spec_t()) - self.source.set_sample_rate(self.samp_rate) - self.source.set_center_freq(self.freq, 0) + self.source.set_sample_rate(float(samp_rate)) + self.source.set_center_freq(float(freq), 0) self.source.set_freq_corr(0, 0) self.source.set_dc_offset_mode(0, 0) self.source.set_iq_balance_mode(0, 0) self.source.set_gain_mode(False, 0) - self.source.set_gain(self.rf_gain, 0) - self.source.set_if_gain(self.if_gain, 0) - self.source.set_bb_gain(self.bb_gain, 0) + self.source.set_gain(float(rf_gain), 0) + self.source.set_if_gain(float(if_gain), 0) + self.source.set_bb_gain(float(bb_gain), 0) self.source.set_antenna("", 0) self.source.set_bandwidth(0, 0) self.sink = SimsiSink( - save_dir=self.save_dir, - file_tag=self.file_tag, - split_size=self.split_size, - delay=self.delay, + save_dir=save_dir, + file_tag=file_tag, + split_size=split_size, + delay=delay, ) self.connect((self.source, 0), (self.sink, 0)) +class IioCapture: + def __init__(self, args: argparse.Namespace): + self.args = args + self.writer = RotatingIqWriter( + save_dir=args.save_dir, + file_tag=args.file_tag, + split_size=args.split_size, + delay=args.delay, + ) + self._static_signature = None + self._last_freq_hz = None + self._input_channels = [args.iio_i_channel] + if args.iio_q_channel not in self._input_channels: + self._input_channels.append(args.iio_q_channel) + + def _run(self, cmd): + proc = subprocess.run(list(cmd), capture_output=True, text=True) + if proc.returncode != 0: + details = (proc.stderr or "").strip() or (proc.stdout or "").strip() or f"exit code {proc.returncode}" + raise RuntimeError(details) + return proc.stdout + + def _run_binary(self, cmd, stop_event: threading.Event): + proc = subprocess.Popen(list(cmd), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + try: + while True: + try: + stdout, stderr = proc.communicate(timeout=0.2) + break + except subprocess.TimeoutExpired: + if stop_event.is_set(): + proc.terminate() + try: + proc.communicate(timeout=1.0) + except subprocess.TimeoutExpired: + proc.kill() + proc.communicate() + raise InterruptedError("stop requested") + if proc.returncode != 0: + details = stderr.decode("utf-8", errors="ignore").strip() or stdout.decode("utf-8", errors="ignore").strip() + raise RuntimeError(details or f"exit code {proc.returncode}") + return stdout + finally: + if proc.poll() is None: + proc.kill() + proc.communicate() + + def _set_input_attr(self, channel: str, attr: str, value: str): + self._run( + [ + "iio_attr", + "-u", + self.args.iio_uri, + "-q", + "-i", + "-c", + self.args.iio_phy_device, + channel, + attr, + value, + ] + ) + + def _set_output_attr(self, channel: str, attr: str, value: str): + self._run( + [ + "iio_attr", + "-u", + self.args.iio_uri, + "-q", + "-o", + "-c", + self.args.iio_phy_device, + channel, + attr, + value, + ] + ) + + def ensure_configured(self): + sample_rate = int(round(max(float(self.args.samp_rate), IIO_MIN_SAMPLE_RATE))) + bandwidth = None + if self.args.bandwidth is not None: + bandwidth = int(round(max(float(self.args.bandwidth), 200_000.0))) + + signature = ( + float(sample_rate), + None if bandwidth is None else float(bandwidth), + self.args.iio_gain_mode, + self.args.iio_hardwaregain, + self.args.iio_port_select, + ) + if signature == self._static_signature: + return + + for channel in self._input_channels: + self._set_input_attr(channel, "sampling_frequency", str(sample_rate)) + if bandwidth is not None: + self._set_input_attr(channel, "rf_bandwidth", str(bandwidth)) + if self.args.iio_port_select: + self._set_input_attr(channel, "rf_port_select", self.args.iio_port_select) + if self.args.iio_gain_mode: + self._set_input_attr(channel, "gain_control_mode", self.args.iio_gain_mode) + if self.args.iio_gain_mode == "manual" and self.args.iio_hardwaregain is not None: + self._set_input_attr(channel, "hardwaregain", f"{self.args.iio_hardwaregain:.6f}") + + self._static_signature = signature + + def tune(self): + target = int(round(float(self.args.freq))) + if self._last_freq_hz == target: + return + self._set_output_attr(self.args.iio_lo_channel, "frequency", str(target)) + self._last_freq_hz = target + + def read_chunk(self, stop_event: threading.Event): + samples_per_read = self.args.samples_per_read + if samples_per_read is None: + samples_per_read = max(4096, min(int(self.args.split_size), 262_144)) + + raw = self._run_binary( + [ + "iio_readdev", + "-u", + self.args.iio_uri, + "-T", + str(int(self.args.timeout_ms)), + "-b", + str(int(samples_per_read)), + "-s", + str(int(samples_per_read)), + self.args.iio_device, + self.args.iio_i_channel, + self.args.iio_q_channel, + ], + stop_event, + ) + + values = np.frombuffer(raw, dtype=" 0: + time.sleep(self.args.settle) + + while not stop_event.is_set(): + try: + samples = self.read_chunk(stop_event) + except InterruptedError: + break + self.writer.write_samples(samples) + + def close(self): + self.writer.close() + + def parse_args(): - parser = argparse.ArgumentParser(description="Headless GNU Radio IQ saver for HackRF") + parser = argparse.ArgumentParser(description="Headless IQ saver for HackRF or IIO SDR backends") - parser.add_argument("--serial", required=True, help="HackRF serial number") + parser.add_argument("--backend", choices=("hackrf", "iio"), default="hackrf", help="SDR backend") + parser.add_argument("--serial", help="HackRF serial number") parser.add_argument("--freq", type=float, required=True, help="Center frequency in Hz") parser.add_argument("--save-dir", required=True, help="Directory for output IQ files") parser.add_argument("--file-tag", default="fragment_", help="Prefix for output files") @@ -183,55 +357,91 @@ def parse_args(): parser.add_argument("--if-gain", type=float, default=30, help="HackRF IF gain") parser.add_argument("--bb-gain", type=float, default=36, help="HackRF BB gain") + parser.add_argument("--bandwidth", type=float, default=None, help="Optional IIO RF bandwidth in Hz") + parser.add_argument("--iio-uri", default="ip:192.168.2.1", help="IIO URI") + parser.add_argument("--iio-device", default="cf-ad9361-lpc", help="IIO RX buffer device") + parser.add_argument("--iio-phy-device", default="ad9361-phy", help="IIO PHY device") + parser.add_argument("--iio-i-channel", default="voltage0", help="IIO I channel") + parser.add_argument("--iio-q-channel", default="voltage1", help="IIO Q channel") + parser.add_argument("--iio-lo-channel", default="altvoltage0", help="IIO LO channel for RX frequency") + parser.add_argument("--iio-port-select", default="A_BALANCED", help="IIO rf_port_select value") + parser.add_argument("--iio-gain-mode", default="slow_attack", help="IIO gain_control_mode value") + parser.add_argument("--iio-hardwaregain", type=float, default=None, help="IIO hardware gain in dB for manual mode") + parser.add_argument("--timeout-ms", type=int, default=4000, help="IIO read timeout in milliseconds") + parser.add_argument("--settle", type=float, default=0.12, help="Delay after tuning for IIO backend") + parser.add_argument("--samples-per-read", type=int, default=None, help="IIO complex samples per read call") + return parser.parse_args() def main(): args = parse_args() + stop_event = threading.Event() - tb = DataSaver( - serial=args.serial, - freq=args.freq, - save_dir=args.save_dir, - file_tag=args.file_tag, - samp_rate=args.samp_rate, - split_size=args.split_size, - delay=args.delay, - rf_gain=args.rf_gain, - if_gain=args.if_gain, - bb_gain=args.bb_gain, - ) + if args.backend == "hackrf" and not args.serial: + print("--serial is required for --backend hackrf", file=sys.stderr) + sys.exit(2) - stop_event = threading.Event() + capture = None + tb = None def handle_signal(sig, frame): print(f"Received signal {sig}, stopping...", flush=True) stop_event.set() - tb.stop() - tb.wait() + if tb is not None: + tb.stop() + tb.wait() signal.signal(signal.SIGINT, handle_signal) signal.signal(signal.SIGTERM, handle_signal) - print("Starting flowgraph...", flush=True) - print(f" serial: {args.serial}", flush=True) + print("Starting capture...", flush=True) + print(f" backend: {args.backend}", flush=True) print(f" freq: {args.freq}", flush=True) print(f" save_dir: {args.save_dir}", flush=True) print(f" file_tag: {args.file_tag}", flush=True) print(f" samp_rate: {args.samp_rate}", flush=True) print(f" split_size: {args.split_size}", flush=True) - print(f" gains: rf={args.rf_gain} if={args.if_gain} bb={args.bb_gain}", flush=True) - - tb.start() try: - while not stop_event.is_set(): - time.sleep(0.5) + if args.backend == "hackrf": + print(f" serial: {args.serial}", flush=True) + print(f" gains: rf={args.rf_gain} if={args.if_gain} bb={args.bb_gain}", flush=True) + tb = HackRfDataSaver( + serial=args.serial, + freq=args.freq, + save_dir=args.save_dir, + file_tag=args.file_tag, + samp_rate=args.samp_rate, + split_size=args.split_size, + delay=args.delay, + rf_gain=args.rf_gain, + if_gain=args.if_gain, + bb_gain=args.bb_gain, + ) + tb.start() + while not stop_event.is_set(): + time.sleep(0.5) + else: + print(f" uri: {args.iio_uri}", flush=True) + print( + f" iio: device={args.iio_device} phy={args.iio_phy_device} " + f"port={args.iio_port_select} gain_mode={args.iio_gain_mode}", + flush=True, + ) + capture = IioCapture(args) + capture.run(stop_event) except KeyboardInterrupt: handle_signal(signal.SIGINT, None) + finally: + if capture is not None: + capture.close() + if tb is not None and not stop_event.is_set(): + tb.stop() + tb.wait() print("Stopped.", flush=True) if __name__ == "__main__": - main() \ No newline at end of file + main()