Исправление бага с размерностью при обучении

.gitignore

@@ -188,4 +188,6 @@ runtime/
 
 /.venv-*/*
 
-/train_scripts/models
+/train_scripts/models
+
+logs/nn_results_*.csv

NN_server/Model.py

@@ -262,6 +262,7 @@ class Model(object):
             print('Инференс' + self._shablon)
             prediction, probability = self._inference_func(data=self._data, model=self._model, mapping=self._classes,
                                                            shablon=self._shablon)
+            print('RESULT' + self._shablon + ': ' + str(prediction) + ' (probability=' + str(probability) + ')')
             Model._add_in_result_list(type_model=self._type_model, ind_inference=self.get_ind_inference(), list_to_add=[prediction, probability])
             self._post_data(prediction=prediction)
 

NN_server/Models/ensemble_1200_v44.py

@@ -0,0 +1,197 @@
+from torchvision import models
+import torch.nn as nn
+import matplotlib
+import numpy as np
+import torch
+import cv2
+import gc
+import io
+
+
+def _render_plot(values, figsize=(16, 16), dpi=16):
+    import matplotlib.pyplot as plt
+
+    fig = plt.figure(figsize=figsize)
+    plt.axes(ylim=(-1, 1))
+    plt.plot(values, color="black")
+    plt.gca().set_axis_off()
+    plt.subplots_adjust(top=1, bottom=0, right=1, left=0, hspace=0, wspace=0)
+    plt.margins(0, 0)
+
+    buf = io.BytesIO()
+    fig.savefig(buf, format="png", dpi=dpi)
+    buf.seek(0)
+    img_arr = np.frombuffer(buf.getvalue(), dtype=np.uint8)
+    buf.close()
+
+    img = cv2.imdecode(img_arr, 1)
+    if img is None:
+        raise RuntimeError("failed to decode plot image")
+
+    plt.clf()
+    plt.cla()
+    plt.close()
+    plt.close(fig)
+
+    return np.asarray(cv2.split(img), dtype=np.float32)
+
+
+def pre_func_ensemble(data=None, src="", ind_inference=0):
+    try:
+        import matplotlib.pyplot as plt
+
+        matplotlib.use("Agg")
+        plt.ioff()
+
+        real = np.asarray(data[0], dtype=np.float32)
+        imag = np.asarray(data[1], dtype=np.float32)
+        signal = real + 1j * imag
+
+        img_real = _render_plot(signal.real)
+        img_mag = _render_plot(np.abs(signal))
+
+        cv2.destroyAllWindows()
+        gc.collect()
+
+        print("Подготовка данных завершена")
+        print()
+        return [img_real, img_mag]
+
+    except Exception as exc:
+        print(str(exc))
+        return None
+
+
+def build_func_ensemble(file_model="", file_config="", num_classes=None):
+    try:
+        import matplotlib.pyplot as plt
+
+        matplotlib.use("Agg")
+        plt.ioff()
+        torch.cuda.empty_cache()
+
+        num_classes = 2
+        model1 = models.resnet18(pretrained=False)
+        model2 = models.resnet50(pretrained=False)
+
+        model1.fc = nn.Linear(model1.fc.in_features, num_classes)
+        model2.fc = nn.Linear(model2.fc.in_features, num_classes)
+
+        class Ensemble(nn.Module):
+            def __init__(self, model1, model2):
+                super().__init__()
+                self.model1 = model1
+                self.model2 = model2
+                self.fc = nn.Linear(2 * num_classes, num_classes)
+
+            def forward(self, x):
+                if isinstance(x, (list, tuple)):
+                    x1 = x[0]
+                    x2 = x[1] if len(x) > 1 else x[0]
+                else:
+                    x1 = x
+                    x2 = x
+                y1 = self.model1(x1)
+                y2 = self.model2(x2)
+                y = torch.cat((y1, y2), dim=1)
+                return self.fc(y)
+
+        model = Ensemble(model1, model2)
+
+        device = "cuda" if torch.cuda.is_available() else "cpu"
+        if device != "cpu":
+            model = model.to(device)
+        model.load_state_dict(torch.load(file_model, map_location=device))
+        model.eval()
+
+        cv2.destroyAllWindows()
+        gc.collect()
+
+        print("Инициализация модели завершена")
+        print()
+        return model
+
+    except Exception as exc:
+        print(str(exc))
+        return None
+
+
+def inference_func_ensemble(data=None, model=None, mapping=None, shablon=""):
+    try:
+        cv2.destroyAllWindows()
+        gc.collect()
+        torch.cuda.empty_cache()
+
+        device = "cuda" if torch.cuda.is_available() else "cpu"
+        if isinstance(data, (list, tuple)) and len(data) >= 2:
+            inputs = [
+                torch.unsqueeze(torch.tensor(data[0]).cpu(), 0).to(device).float(),
+                torch.unsqueeze(torch.tensor(data[1]).cpu(), 0).to(device).float(),
+            ]
+        else:
+            tensor = torch.unsqueeze(torch.tensor(data).cpu(), 0).to(device).float()
+            inputs = [tensor, tensor]
+
+        with torch.no_grad():
+            output = model(inputs)
+            _, predict = torch.max(output.data, 1)
+
+        prediction = mapping[int(np.asarray(predict.cpu())[0])]
+        print("PREDICTION" + shablon + ": " + prediction)
+
+        output = output.cpu()
+        label = np.asarray(np.argmax(output, axis=1))[0]
+        output = np.asarray(torch.squeeze(output, 0))
+        expon = np.exp(output - np.max(output))
+        probability = round((expon / expon.sum())[label], 2)
+
+        cv2.destroyAllWindows()
+        gc.collect()
+        print("Уверенность" + shablon + " в предсказании: " + str(probability))
+        print("Инференс завершен")
+        print()
+        return [prediction, probability]
+
+    except Exception as exc:
+        print(str(exc))
+        return None
+
+
+def post_func_ensemble(src="", model_type="", prediction="", model_id=0, ind_inference=0, data=None):
+    try:
+        import matplotlib.pyplot as plt
+
+        matplotlib.use("Agg")
+        plt.ioff()
+
+        if int(ind_inference) <= 100 and isinstance(data, (list, tuple)) and len(data) >= 2:
+            fig, ax = plt.subplots()
+            ax.imshow(np.moveaxis(data[0], 0, -1))
+            plt.savefig(src + "_inference_" + str(ind_inference) + "_" + prediction + "_real_" + str(model_id) + "_" + model_type + ".png")
+            plt.clf()
+            plt.cla()
+            plt.close(fig)
+            cv2.destroyAllWindows()
+            gc.collect()
+
+            fig, ax = plt.subplots()
+            ax.imshow(np.moveaxis(data[1], 0, -1))
+            plt.savefig(src + "_inference_" + str(ind_inference) + "_" + prediction + "_mod_" + str(model_id) + "_" + model_type + ".png")
+            plt.clf()
+            plt.cla()
+            plt.close(fig)
+            cv2.destroyAllWindows()
+            gc.collect()
+
+        plt.clf()
+        plt.cla()
+        plt.close()
+        cv2.destroyAllWindows()
+        gc.collect()
+
+        print("Постобработка завершена")
+        print()
+
+    except Exception as exc:
+        print(str(exc))
+        return None

NN_server/Models/ensemble_2400_v44.py

@@ -0,0 +1,197 @@
+from torchvision import models
+import torch.nn as nn
+import matplotlib
+import numpy as np
+import torch
+import cv2
+import gc
+import io
+
+
+def _render_plot(values, figsize=(16, 16), dpi=16):
+    import matplotlib.pyplot as plt
+
+    fig = plt.figure(figsize=figsize)
+    plt.axes(ylim=(-1, 1))
+    plt.plot(values, color="black")
+    plt.gca().set_axis_off()
+    plt.subplots_adjust(top=1, bottom=0, right=1, left=0, hspace=0, wspace=0)
+    plt.margins(0, 0)
+
+    buf = io.BytesIO()
+    fig.savefig(buf, format="png", dpi=dpi)
+    buf.seek(0)
+    img_arr = np.frombuffer(buf.getvalue(), dtype=np.uint8)
+    buf.close()
+
+    img = cv2.imdecode(img_arr, 1)
+    if img is None:
+        raise RuntimeError("failed to decode plot image")
+
+    plt.clf()
+    plt.cla()
+    plt.close()
+    plt.close(fig)
+
+    return np.asarray(cv2.split(img), dtype=np.float32)
+
+
+def pre_func_ensemble(data=None, src="", ind_inference=0):
+    try:
+        import matplotlib.pyplot as plt
+
+        matplotlib.use("Agg")
+        plt.ioff()
+
+        real = np.asarray(data[0], dtype=np.float32)
+        imag = np.asarray(data[1], dtype=np.float32)
+        signal = real + 1j * imag
+
+        img_real = _render_plot(signal.real)
+        img_mag = _render_plot(np.abs(signal))
+
+        cv2.destroyAllWindows()
+        gc.collect()
+
+        print("Подготовка данных завершена")
+        print()
+        return [img_real, img_mag]
+
+    except Exception as exc:
+        print(str(exc))
+        return None
+
+
+def build_func_ensemble(file_model="", file_config="", num_classes=None):
+    try:
+        import matplotlib.pyplot as plt
+
+        matplotlib.use("Agg")
+        plt.ioff()
+        torch.cuda.empty_cache()
+
+        num_classes = 2
+        model1 = models.resnet18(pretrained=False)
+        model2 = models.resnet50(pretrained=False)
+
+        model1.fc = nn.Linear(model1.fc.in_features, num_classes)
+        model2.fc = nn.Linear(model2.fc.in_features, num_classes)
+
+        class Ensemble(nn.Module):
+            def __init__(self, model1, model2):
+                super().__init__()
+                self.model1 = model1
+                self.model2 = model2
+                self.fc = nn.Linear(2 * num_classes, num_classes)
+
+            def forward(self, x):
+                if isinstance(x, (list, tuple)):
+                    x1 = x[0]
+                    x2 = x[1] if len(x) > 1 else x[0]
+                else:
+                    x1 = x
+                    x2 = x
+                y1 = self.model1(x1)
+                y2 = self.model2(x2)
+                y = torch.cat((y1, y2), dim=1)
+                return self.fc(y)
+
+        model = Ensemble(model1, model2)
+
+        device = "cuda" if torch.cuda.is_available() else "cpu"
+        if device != "cpu":
+            model = model.to(device)
+        model.load_state_dict(torch.load(file_model, map_location=device))
+        model.eval()
+
+        cv2.destroyAllWindows()
+        gc.collect()
+
+        print("Инициализация модели завершена")
+        print()
+        return model
+
+    except Exception as exc:
+        print(str(exc))
+        return None
+
+
+def inference_func_ensemble(data=None, model=None, mapping=None, shablon=""):
+    try:
+        cv2.destroyAllWindows()
+        gc.collect()
+        torch.cuda.empty_cache()
+
+        device = "cuda" if torch.cuda.is_available() else "cpu"
+        if isinstance(data, (list, tuple)) and len(data) >= 2:
+            inputs = [
+                torch.unsqueeze(torch.tensor(data[0]).cpu(), 0).to(device).float(),
+                torch.unsqueeze(torch.tensor(data[1]).cpu(), 0).to(device).float(),
+            ]
+        else:
+            tensor = torch.unsqueeze(torch.tensor(data).cpu(), 0).to(device).float()
+            inputs = [tensor, tensor]
+
+        with torch.no_grad():
+            output = model(inputs)
+            _, predict = torch.max(output.data, 1)
+
+        prediction = mapping[int(np.asarray(predict.cpu())[0])]
+        print("PREDICTION" + shablon + ": " + prediction)
+
+        output = output.cpu()
+        label = np.asarray(np.argmax(output, axis=1))[0]
+        output = np.asarray(torch.squeeze(output, 0))
+        expon = np.exp(output - np.max(output))
+        probability = round((expon / expon.sum())[label], 2)
+
+        cv2.destroyAllWindows()
+        gc.collect()
+        print("Уверенность" + shablon + " в предсказании: " + str(probability))
+        print("Инференс завершен")
+        print()
+        return [prediction, probability]
+
+    except Exception as exc:
+        print(str(exc))
+        return None
+
+
+def post_func_ensemble(src="", model_type="", prediction="", model_id=0, ind_inference=0, data=None):
+    try:
+        import matplotlib.pyplot as plt
+
+        matplotlib.use("Agg")
+        plt.ioff()
+
+        if int(ind_inference) <= 100 and isinstance(data, (list, tuple)) and len(data) >= 2:
+            fig, ax = plt.subplots()
+            ax.imshow(np.moveaxis(data[0], 0, -1))
+            plt.savefig(src + "_inference_" + str(ind_inference) + "_" + prediction + "_real_" + str(model_id) + "_" + model_type + ".png")
+            plt.clf()
+            plt.cla()
+            plt.close(fig)
+            cv2.destroyAllWindows()
+            gc.collect()
+
+            fig, ax = plt.subplots()
+            ax.imshow(np.moveaxis(data[1], 0, -1))
+            plt.savefig(src + "_inference_" + str(ind_inference) + "_" + prediction + "_mod_" + str(model_id) + "_" + model_type + ".png")
+            plt.clf()
+            plt.cla()
+            plt.close(fig)
+            cv2.destroyAllWindows()
+            gc.collect()
+
+        plt.clf()
+        plt.cla()
+        plt.close()
+        cv2.destroyAllWindows()
+        gc.collect()
+
+        print("Постобработка завершена")
+        print()
+
+    except Exception as exc:
+        print(str(exc))
+        return None

NN_server/server.py

@@ -137,19 +137,29 @@ def receive_data():
                 print()
                 
                 try:
+                                    
                     result = 0
-                    if (int(freq) == 2400 and (prediction_list[0] in ['drone', 'drone_noise'] or (prediction_list[0] == 'wifi' and float(probability) >= 0.95))) or (int(freq) == 1200 and (prediction_list[0] in ['drone'] and float(probability) >= 0.95)):
-                        result += 0
-                    if int(freq) in [915]:
-                        result = 0
-                    if int(freq) in []:
+                    freq_int = int(freq)
+                    prediction = prediction_list[0]
+                    prob = float(probability)
+
+                    if freq_int == 2400:
+                        if prediction in ["drone", "drone_noise"]:
+                            result += 0
+                        elif prediction == "wifi" and prob >= 0.95:
+                            result += 0
+
+                    elif freq_int == 1200:
+                        if prediction == "drone" and prob >= 0.95:
+                            result += 8
+
+                    elif freq_int == 915:
                         result = 0
-                    data_to_send={
-                        'freq': str(freq),
-                        'amplitude': result
-                        #'triggered': False if result < 7 else True,
-                        #'light_len': result
-                        }
+
+                    data_to_send = {
+                        "freq": str(freq),
+                        "amplitude": result,
+                    }
                     response = requests.post("http://{0}:{1}/process_data".format(gen_server_ip, gen_server_port), json=data_to_send)
                     if response.status_code == 200:
                         print("Данные успешно отправлены!")

deploy/docker/docker-compose.yml

@@ -20,6 +20,7 @@ services:
       - ../../runtime:/app/runtime
       - ../../src:/app/src
       - ../../common:/app/common
+      - ../../train_scripts:/app/train_scripts:ro
     networks:
       - dronedetector-net
     extra_hosts:
@@ -50,6 +51,7 @@ services:
       - ../../.env:/app/.env:ro
       - ../../NN_server:/app/NN_server
       - ../../common:/app/common
+      - ../../train_scripts:/app/train_scripts:ro
     gpus: all
     networks:
       - dronedetector-net

logs/analysis.ipynb

@@ -0,0 +1,673 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "215a6c3a",
+   "metadata": {},
+   "source": [
+    "# NN inference analysis\n",
+    "\n",
+    "Анализ CSV с результатами инференса: доля класса `drone`, частоты срабатываний, уверенность модели и интервалы между `drone`-классификациями."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "id": "4e8cff32",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CSV path: /home/sibscience-4/from_ssh/DroneDetector/logs/nn_results_live_6gb.csv\n",
+      "Rows: 27258\n",
+      "Time range: 2026-05-04 17:35:21.019627763+07:00 -> 2026-05-05 12:17:19.369858371+07:00\n",
+      "Missing freq rows: 0\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>docker_timestamp</th>\n",
+       "      <th>event_time_iso</th>\n",
+       "      <th>event_time_epoch</th>\n",
+       "      <th>freq</th>\n",
+       "      <th>model_id</th>\n",
+       "      <th>model_type</th>\n",
+       "      <th>prediction</th>\n",
+       "      <th>probability</th>\n",
+       "      <th>ts</th>\n",
+       "      <th>local_time</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>2026-05-04T10:35:21.019627763Z</td>\n",
+       "      <td>2026-05-04T17:35:21+07:00</td>\n",
+       "      <td>1.777891e+09</td>\n",
+       "      <td>2400</td>\n",
+       "      <td>2</td>\n",
+       "      <td>ensemble_2400_v44</td>\n",
+       "      <td>drone</td>\n",
+       "      <td>0.99</td>\n",
+       "      <td>2026-05-04 10:35:21.019627763+00:00</td>\n",
+       "      <td>2026-05-04 17:35:21.019627763+07:00</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>2026-05-04T10:35:21.019631281Z</td>\n",
+       "      <td>2026-05-04T17:35:21+07:00</td>\n",
+       "      <td>1.777891e+09</td>\n",
+       "      <td>1200</td>\n",
+       "      <td>1</td>\n",
+       "      <td>ensemble_1200_v44</td>\n",
+       "      <td>noise</td>\n",
+       "      <td>1.00</td>\n",
+       "      <td>2026-05-04 10:35:21.019631281+00:00</td>\n",
+       "      <td>2026-05-04 17:35:21.019631281+07:00</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>2026-05-04T10:35:27.048188525Z</td>\n",
+       "      <td>2026-05-04T17:35:27+07:00</td>\n",
+       "      <td>1.777891e+09</td>\n",
+       "      <td>2400</td>\n",
+       "      <td>2</td>\n",
+       "      <td>ensemble_2400_v44</td>\n",
+       "      <td>drone</td>\n",
+       "      <td>0.99</td>\n",
+       "      <td>2026-05-04 10:35:27.048188525+00:00</td>\n",
+       "      <td>2026-05-04 17:35:27.048188525+07:00</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>2026-05-04T10:35:29.238925690Z</td>\n",
+       "      <td>2026-05-04T17:35:29+07:00</td>\n",
+       "      <td>1.777891e+09</td>\n",
+       "      <td>1200</td>\n",
+       "      <td>1</td>\n",
+       "      <td>ensemble_1200_v44</td>\n",
+       "      <td>noise</td>\n",
+       "      <td>1.00</td>\n",
+       "      <td>2026-05-04 10:35:29.238925690+00:00</td>\n",
+       "      <td>2026-05-04 17:35:29.238925690+07:00</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>2026-05-04T10:35:32.842234116Z</td>\n",
+       "      <td>2026-05-04T17:35:32+07:00</td>\n",
+       "      <td>1.777891e+09</td>\n",
+       "      <td>2400</td>\n",
+       "      <td>2</td>\n",
+       "      <td>ensemble_2400_v44</td>\n",
+       "      <td>drone</td>\n",
+       "      <td>0.92</td>\n",
+       "      <td>2026-05-04 10:35:32.842234116+00:00</td>\n",
+       "      <td>2026-05-04 17:35:32.842234116+07:00</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                 docker_timestamp             event_time_iso  \\\n",
+       "0  2026-05-04T10:35:21.019627763Z  2026-05-04T17:35:21+07:00   \n",
+       "1  2026-05-04T10:35:21.019631281Z  2026-05-04T17:35:21+07:00   \n",
+       "2  2026-05-04T10:35:27.048188525Z  2026-05-04T17:35:27+07:00   \n",
+       "3  2026-05-04T10:35:29.238925690Z  2026-05-04T17:35:29+07:00   \n",
+       "4  2026-05-04T10:35:32.842234116Z  2026-05-04T17:35:32+07:00   \n",
+       "\n",
+       "   event_time_epoch  freq  model_id         model_type prediction  \\\n",
+       "0      1.777891e+09  2400         2  ensemble_2400_v44      drone   \n",
+       "1      1.777891e+09  1200         1  ensemble_1200_v44      noise   \n",
+       "2      1.777891e+09  2400         2  ensemble_2400_v44      drone   \n",
+       "3      1.777891e+09  1200         1  ensemble_1200_v44      noise   \n",
+       "4      1.777891e+09  2400         2  ensemble_2400_v44      drone   \n",
+       "\n",
+       "   probability                                  ts  \\\n",
+       "0         0.99 2026-05-04 10:35:21.019627763+00:00   \n",
+       "1         1.00 2026-05-04 10:35:21.019631281+00:00   \n",
+       "2         0.99 2026-05-04 10:35:27.048188525+00:00   \n",
+       "3         1.00 2026-05-04 10:35:29.238925690+00:00   \n",
+       "4         0.92 2026-05-04 10:35:32.842234116+00:00   \n",
+       "\n",
+       "                           local_time  \n",
+       "0 2026-05-04 17:35:21.019627763+07:00  \n",
+       "1 2026-05-04 17:35:21.019631281+07:00  \n",
+       "2 2026-05-04 17:35:27.048188525+07:00  \n",
+       "3 2026-05-04 17:35:29.238925690+07:00  \n",
+       "4 2026-05-04 17:35:32.842234116+07:00  "
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "from pathlib import Path\n",
+    "import pandas as pd\n",
+    "\n",
+    "csv_path = Path('/home/sibscience-4/from_ssh/DroneDetector/logs/nn_results_live_6gb.csv')\n",
+    "df = pd.read_csv(csv_path)\n",
+    "\n",
+    "df['ts'] = pd.to_datetime(df['docker_timestamp'], utc=True)\n",
+    "df['local_time'] = df['ts'].dt.tz_convert('Asia/Novosibirsk')\n",
+    "df['freq'] = pd.to_numeric(df['freq'], errors='coerce').astype('Int64')\n",
+    "df['probability'] = pd.to_numeric(df['probability'], errors='coerce')\n",
+    "df = df.sort_values('ts').reset_index(drop=True)\n",
+    "\n",
+    "print(f'CSV path: {csv_path}')\n",
+    "print(f'Rows: {len(df)}')\n",
+    "print(f'Time range: {df[\"local_time\"].min()} -> {df[\"local_time\"].max()}')\n",
+    "print(f'Missing freq rows: {df[\"freq\"].isna().sum()}')\n",
+    "display(df.head())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0fcc6dd6",
+   "metadata": {},
+   "source": [
+    "## Общая сводка по частотам и классам"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "id": "7bf0fc3f",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>freq</th>\n",
+       "      <th>prediction</th>\n",
+       "      <th>count</th>\n",
+       "      <th>avg_probability</th>\n",
+       "      <th>min_probability</th>\n",
+       "      <th>max_probability</th>\n",
+       "      <th>freq_total</th>\n",
+       "      <th>class_rate</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>1200</td>\n",
+       "      <td>drone</td>\n",
+       "      <td>3</td>\n",
+       "      <td>0.840000</td>\n",
+       "      <td>0.78</td>\n",
+       "      <td>0.91</td>\n",
+       "      <td>13632</td>\n",
+       "      <td>0.000220</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>1200</td>\n",
+       "      <td>noise</td>\n",
+       "      <td>13629</td>\n",
+       "      <td>0.997436</td>\n",
+       "      <td>0.91</td>\n",
+       "      <td>1.00</td>\n",
+       "      <td>13632</td>\n",
+       "      <td>0.999780</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>2400</td>\n",
+       "      <td>drone</td>\n",
+       "      <td>11921</td>\n",
+       "      <td>0.868013</td>\n",
+       "      <td>0.50</td>\n",
+       "      <td>1.00</td>\n",
+       "      <td>13626</td>\n",
+       "      <td>0.874872</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>2400</td>\n",
+       "      <td>noise</td>\n",
+       "      <td>1705</td>\n",
+       "      <td>0.649185</td>\n",
+       "      <td>0.50</td>\n",
+       "      <td>1.00</td>\n",
+       "      <td>13626</td>\n",
+       "      <td>0.125128</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   freq prediction  count  avg_probability  min_probability  max_probability  \\\n",
+       "0  1200      drone      3         0.840000             0.78             0.91   \n",
+       "1  1200      noise  13629         0.997436             0.91             1.00   \n",
+       "2  2400      drone  11921         0.868013             0.50             1.00   \n",
+       "3  2400      noise   1705         0.649185             0.50             1.00   \n",
+       "\n",
+       "   freq_total  class_rate  \n",
+       "0       13632    0.000220  \n",
+       "1       13632    0.999780  \n",
+       "2       13626    0.874872  \n",
+       "3       13626    0.125128  "
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "class_summary = (\n",
+    "    df.groupby(['freq', 'prediction'], dropna=False)\n",
+    "      .agg(\n",
+    "          count=('prediction', 'size'),\n",
+    "          avg_probability=('probability', 'mean'),\n",
+    "          min_probability=('probability', 'min'),\n",
+    "          max_probability=('probability', 'max'),\n",
+    "      )\n",
+    "      .reset_index()\n",
+    ")\n",
+    "\n",
+    "freq_total = df.groupby('freq', dropna=False).size().rename('freq_total').reset_index()\n",
+    "class_summary = class_summary.merge(freq_total, on='freq', how='left')\n",
+    "class_summary['class_rate'] = class_summary['count'] / class_summary['freq_total']\n",
+    "class_summary = class_summary.sort_values(['freq', 'prediction']).reset_index(drop=True)\n",
+    "\n",
+    "display(class_summary)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "039fcfa6",
+   "metadata": {},
+   "source": [
+    "## Статистика по классу drone"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "id": "c1ed2bc4",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>freq</th>\n",
+       "      <th>total_inferences</th>\n",
+       "      <th>drone_count</th>\n",
+       "      <th>avg_drone_probability</th>\n",
+       "      <th>median_drone_probability</th>\n",
+       "      <th>min_drone_probability</th>\n",
+       "      <th>max_drone_probability</th>\n",
+       "      <th>first_drone_time</th>\n",
+       "      <th>last_drone_time</th>\n",
+       "      <th>drone_rate</th>\n",
+       "      <th>dataset_duration_min</th>\n",
+       "      <th>drone_per_min</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>1200</td>\n",
+       "      <td>13632</td>\n",
+       "      <td>3</td>\n",
+       "      <td>0.840000</td>\n",
+       "      <td>0.83</td>\n",
+       "      <td>0.78</td>\n",
+       "      <td>0.91</td>\n",
+       "      <td>2026-05-05 03:18:55.374394280+07:00</td>\n",
+       "      <td>2026-05-05 08:39:23.676669045+07:00</td>\n",
+       "      <td>0.000220</td>\n",
+       "      <td>1121.972504</td>\n",
+       "      <td>0.002674</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>2400</td>\n",
+       "      <td>13626</td>\n",
+       "      <td>11921</td>\n",
+       "      <td>0.868013</td>\n",
+       "      <td>0.93</td>\n",
+       "      <td>0.50</td>\n",
+       "      <td>1.00</td>\n",
+       "      <td>2026-05-04 17:35:21.019627763+07:00</td>\n",
+       "      <td>2026-05-05 12:17:19.369858371+07:00</td>\n",
+       "      <td>0.874872</td>\n",
+       "      <td>1121.972504</td>\n",
+       "      <td>10.625038</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   freq  total_inferences  drone_count  avg_drone_probability  \\\n",
+       "0  1200             13632            3               0.840000   \n",
+       "1  2400             13626        11921               0.868013   \n",
+       "\n",
+       "   median_drone_probability  min_drone_probability  max_drone_probability  \\\n",
+       "0                      0.83                   0.78                   0.91   \n",
+       "1                      0.93                   0.50                   1.00   \n",
+       "\n",
+       "                     first_drone_time                     last_drone_time  \\\n",
+       "0 2026-05-05 03:18:55.374394280+07:00 2026-05-05 08:39:23.676669045+07:00   \n",
+       "1 2026-05-04 17:35:21.019627763+07:00 2026-05-05 12:17:19.369858371+07:00   \n",
+       "\n",
+       "   drone_rate  dataset_duration_min  drone_per_min  \n",
+       "0    0.000220           1121.972504       0.002674  \n",
+       "1    0.874872           1121.972504      10.625038  "
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "drone = df[df['prediction'].eq('drone')].copy()\n",
+    "\n",
+    "total_by_freq = df.groupby('freq', dropna=False).size().rename('total_inferences')\n",
+    "drone_by_freq = drone.groupby('freq', dropna=False).agg(\n",
+    "    drone_count=('prediction', 'size'),\n",
+    "    avg_drone_probability=('probability', 'mean'),\n",
+    "    median_drone_probability=('probability', 'median'),\n",
+    "    min_drone_probability=('probability', 'min'),\n",
+    "    max_drone_probability=('probability', 'max'),\n",
+    "    first_drone_time=('local_time', 'min'),\n",
+    "    last_drone_time=('local_time', 'max'),\n",
+    ")\n",
+    "\n",
+    "drone_stats = total_by_freq.to_frame().join(drone_by_freq, how='left').fillna({'drone_count': 0})\n",
+    "drone_stats['drone_count'] = drone_stats['drone_count'].astype(int)\n",
+    "drone_stats['drone_rate'] = drone_stats['drone_count'] / drone_stats['total_inferences']\n",
+    "\n",
+    "if len(df) > 1:\n",
+    "    duration_min = (df['ts'].max() - df['ts'].min()).total_seconds() / 60\n",
+    "else:\n",
+    "    duration_min = 0\n",
+    "\n",
+    "drone_stats['dataset_duration_min'] = duration_min\n",
+    "drone_stats['drone_per_min'] = drone_stats['drone_count'] / duration_min if duration_min > 0 else 0\n",
+    "drone_stats = drone_stats.reset_index().sort_values('freq').reset_index(drop=True)\n",
+    "\n",
+    "display(drone_stats)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "56ae5e2b",
+   "metadata": {},
+   "source": [
+    "## Интервалы между drone-классификациями"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "id": "0c43eb07",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>freq</th>\n",
+       "      <th>interval_count</th>\n",
+       "      <th>avg_interval_sec</th>\n",
+       "      <th>median_interval_sec</th>\n",
+       "      <th>min_interval_sec</th>\n",
+       "      <th>max_interval_sec</th>\n",
+       "      <th>p90_interval_sec</th>\n",
+       "      <th>p95_interval_sec</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>1200</td>\n",
+       "      <td>2</td>\n",
+       "      <td>9614.151137</td>\n",
+       "      <td>9614.151137</td>\n",
+       "      <td>619.196112</td>\n",
+       "      <td>18609.106163</td>\n",
+       "      <td>16810.115158</td>\n",
+       "      <td>17709.610661</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>2400</td>\n",
+       "      <td>11920</td>\n",
+       "      <td>5.647513</td>\n",
+       "      <td>4.974157</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>1210.107950</td>\n",
+       "      <td>9.203728</td>\n",
+       "      <td>10.079097</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   freq  interval_count  avg_interval_sec  median_interval_sec  \\\n",
+       "0  1200               2       9614.151137          9614.151137   \n",
+       "1  2400           11920          5.647513             4.974157   \n",
+       "\n",
+       "   min_interval_sec  max_interval_sec  p90_interval_sec  p95_interval_sec  \n",
+       "0        619.196112      18609.106163      16810.115158      17709.610661  \n",
+       "1          0.000000       1210.107950          9.203728         10.079097  "
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "if drone.empty:\n",
+    "    print('No drone predictions found')\n",
+    "    drone_interval_stats = pd.DataFrame()\n",
+    "else:\n",
+    "    drone = drone.sort_values(['freq', 'ts']).copy()\n",
+    "    drone['dt_drone_freq_sec'] = drone.groupby('freq')['ts'].diff().dt.total_seconds()\n",
+    "    drone_interval_stats = (\n",
+    "        drone.groupby('freq', dropna=False)['dt_drone_freq_sec']\n",
+    "             .agg(\n",
+    "                 interval_count='count',\n",
+    "                 avg_interval_sec='mean',\n",
+    "                 median_interval_sec='median',\n",
+    "                 min_interval_sec='min',\n",
+    "                 max_interval_sec='max',\n",
+    "                 p90_interval_sec=lambda s: s.quantile(0.90),\n",
+    "                 p95_interval_sec=lambda s: s.quantile(0.95),\n",
+    "             )\n",
+    "             .reset_index()\n",
+    "    )\n",
+    "    display(drone_interval_stats)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2ecf96f0",
+   "metadata": {},
+   "source": [
+    "## Drone-события"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ff14a339",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "drone_events = drone[[\n",
+    "    'local_time',\n",
+    "    'docker_timestamp',\n",
+    "    'freq',\n",
+    "    'model_id',\n",
+    "    'model_type',\n",
+    "    'prediction',\n",
+    "    'probability',\n",
+    "]].sort_values('local_time').reset_index(drop=True)\n",
+    "\n",
+    "display(drone_events.head(50))\n",
+    "display(drone_events.tail(50))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "376b84d0",
+   "metadata": {},
+   "source": [
+    "## Частота drone-классификаций по минутам"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5f82f5c1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if drone.empty:\n",
+    "    print('No drone predictions found')\n",
+    "else:\n",
+    "    drone_per_minute = (\n",
+    "        drone.set_index('local_time')\n",
+    "             .groupby('freq')\n",
+    "             .resample('1min')\n",
+    "             .size()\n",
+    "             .rename('drone_count')\n",
+    "             .reset_index()\n",
+    "    )\n",
+    "    display(drone_per_minute.tail(100))\n",
+    "\n",
+    "    pivot = drone_per_minute.pivot_table(\n",
+    "        index='local_time',\n",
+    "        columns='freq',\n",
+    "        values='drone_count',\n",
+    "        fill_value=0,\n",
+    "    )\n",
+    "    display(pivot.tail(50))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ad570d05",
+   "metadata": {},
+   "source": [
+    "## Быстрые графики"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "895550a1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import matplotlib.pyplot as plt\n",
+    "\n",
+    "if not drone.empty:\n",
+    "    ax = drone_stats.set_index('freq')['drone_rate'].plot(kind='bar', figsize=(8, 4), title='Drone rate by frequency')\n",
+    "    ax.set_ylabel('drone_count / total_inferences')\n",
+    "    plt.show()\n",
+    "\n",
+    "    ax = drone.boxplot(column='probability', by='freq', figsize=(8, 4))\n",
+    "    ax.set_title('Drone probability by frequency')\n",
+    "    ax.set_xlabel('freq')\n",
+    "    ax.set_ylabel('probability')\n",
+    "    plt.suptitle('')\n",
+    "    plt.show()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "pygments_lexer": "ipython3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

scripts/capture_nn_results.py

@@ -0,0 +1,223 @@
+#!/usr/bin/env python3
+import argparse
+import csv
+import json
+import os
+import re
+import subprocess
+import sys
+import time
+from datetime import datetime, timezone
+
+
+RESULT_RE = re.compile(
+    r"^(?P<docker_ts>\S+)\s+RESULT Модель (?P<model_id>\d+) с типом (?P<model_type>.+?): "
+    r"(?P<prediction>\S+) \(probability=(?P<probability>[-+]?\d+(?:\.\d+)?)\)\s*$"
+)
+FREQ_RE = re.compile(r"(\d{3,5})")
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description="Capture NN inference RESULT logs into CSV or JSONL until time or size limit."
+    )
+    parser.add_argument(
+        "--output",
+        required=True,
+        help="Output file path (.csv or .jsonl recommended).",
+    )
+    parser.add_argument(
+        "--format",
+        choices=("csv", "jsonl"),
+        default="csv",
+        help="Output format.",
+    )
+    parser.add_argument(
+        "--minutes",
+        type=float,
+        default=0.0,
+        help="Stop after this many minutes. 0 means no time limit.",
+    )
+    parser.add_argument(
+        "--max-bytes",
+        type=int,
+        default=3 * 1024 * 1024 * 1024,
+        help="Stop when output file reaches this size in bytes. Default: 3 GiB.",
+    )
+    parser.add_argument(
+        "--since",
+        default=None,
+        help="Optional docker logs --since value, e.g. 20m, 2h, 2026-05-04T12:00:00.",
+    )
+    parser.add_argument(
+        "--tail",
+        type=int,
+        default=0,
+        help="How many previous log lines to include before following. Default: 0.",
+    )
+    parser.add_argument(
+        "--compose-file",
+        default="deploy/docker/docker-compose.yml",
+        help="Path to docker compose file.",
+    )
+    parser.add_argument(
+        "--service",
+        default="dronedetector-nn-server",
+        help="Docker compose service name.",
+    )
+    parser.add_argument(
+        "--follow",
+        action="store_true",
+        help="Follow logs live. By default the script captures a finite history snapshot.",
+    )
+    return parser.parse_args()
+
+
+def extract_freq(model_type):
+    matches = FREQ_RE.findall(model_type)
+    if not matches:
+        return ""
+    known_freqs = {"433", "750", "868", "915", "1200", "1500", "2400", "3300", "4500", "5200", "5800"}
+    for value in matches:
+        if value in known_freqs:
+            return value
+    return matches[0]
+
+
+def parse_docker_timestamp(value):
+    try:
+        return datetime.fromisoformat(value.replace("Z", "+00:00"))
+    except ValueError:
+        return datetime.now(timezone.utc)
+
+
+def open_output(path, fmt):
+    os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True)
+    fh = open(path, "a", encoding="utf-8", newline="")
+    writer = None
+    if fmt == "csv":
+        writer = csv.writer(fh)
+        if fh.tell() == 0:
+            writer.writerow(
+                [
+                    "docker_timestamp",
+                    "event_time_iso",
+                    "event_time_epoch",
+                    "freq",
+                    "model_id",
+                    "model_type",
+                    "prediction",
+                    "probability",
+                ]
+            )
+            fh.flush()
+    return fh, writer
+
+
+def write_record(fh, writer, fmt, record):
+    if fmt == "csv":
+        writer.writerow(
+            [
+                record["docker_timestamp"],
+                record["event_time_iso"],
+                record["event_time_epoch"],
+                record["freq"],
+                record["model_id"],
+                record["model_type"],
+                record["prediction"],
+                record["probability"],
+            ]
+        )
+    else:
+        fh.write(json.dumps(record, ensure_ascii=False) + "\n")
+    fh.flush()
+
+
+def build_command(args):
+    cmd = [
+        "docker",
+        "compose",
+        "-f",
+        args.compose_file,
+        "logs",
+        "--timestamps",
+        "--no-log-prefix",
+        args.service,
+    ]
+    if args.since:
+        cmd[5:5] = ["--since", args.since]
+    if int(args.tail) > 0:
+        cmd[-1:-1] = ["--tail", str(args.tail)]
+    if args.follow:
+        cmd.insert(-1, "-f")
+    return cmd
+
+
+def main():
+    args = parse_args()
+    deadline = time.time() + (args.minutes * 60.0) if args.minutes > 0 else None
+
+    fh, writer = open_output(args.output, args.format)
+    cmd = build_command(args)
+
+    print("Running:", " ".join(cmd), file=sys.stderr)
+    proc = subprocess.Popen(
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        encoding="utf-8",
+        errors="replace",
+        bufsize=1,
+    )
+
+    captured = 0
+    try:
+        assert proc.stdout is not None
+        for line in proc.stdout:
+            if deadline is not None and time.time() >= deadline:
+                print("Stopping: time limit reached", file=sys.stderr)
+                break
+
+            match = RESULT_RE.match(line.rstrip("\n"))
+            if not match:
+                continue
+
+            docker_dt = parse_docker_timestamp(match.group("docker_ts"))
+            event_dt = docker_dt.astimezone()
+            model_type = match.group("model_type")
+            record = {
+                "docker_timestamp": match.group("docker_ts"),
+                "event_time_iso": event_dt.isoformat(timespec="seconds"),
+                "event_time_epoch": round(docker_dt.timestamp(), 3),
+                "freq": extract_freq(model_type),
+                "model_id": int(match.group("model_id")),
+                "model_type": model_type,
+                "prediction": match.group("prediction"),
+                "probability": float(match.group("probability")),
+            }
+            write_record(fh, writer, args.format, record)
+            captured += 1
+
+            if fh.tell() >= args.max_bytes:
+                print("Stopping: file size limit reached", file=sys.stderr)
+                break
+    finally:
+        try:
+            proc.terminate()
+        except Exception:
+            pass
+        try:
+            proc.wait(timeout=5)
+        except Exception:
+            try:
+                proc.kill()
+            except Exception:
+                pass
+        fh.close()
+
+    print(f"Captured {captured} inference results into {args.output}", file=sys.stderr)
+
+
+if __name__ == "__main__":
+    main()

train_scripts/Training_models2pic_val_loss.ipynb

@@ -0,0 +1,523 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "e1db882b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/sibscience-4/from_ssh/DroneDetector/.venv-train/lib/python3.12/site-packages/matplotlib/projections/__init__.py:63: UserWarning: Unable to import Axes3D. This may be due to multiple versions of Matplotlib being installed (e.g. as a system package and as a pip package). As a result, the 3D projection is not available.\n",
+      "  warnings.warn(\"Unable to import Axes3D. This may be due to multiple versions of \"\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "cuda\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "220"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from sklearn.model_selection import train_test_split\n",
+    "from torch.utils.data import Dataset, DataLoader\n",
+    "from torch import default_generator, randperm\n",
+    "from torch.utils.data.dataset import Subset\n",
+    "import torchvision.transforms as transforms\n",
+    "from torchvision.io import read_image\n",
+    "from importlib import import_module\n",
+    "import matplotlib.pyplot as plt\n",
+    "from torchvision import models\n",
+    "import torch, torchvision\n",
+    "from pathlib import Path\n",
+    "from PIL import Image\n",
+    "import torch.nn as nn\n",
+    "from tqdm import tqdm\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "import matplotlib\n",
+    "import os, shutil\n",
+    "import mlconfig\n",
+    "import random\n",
+    "import shutil\n",
+    "import timeit\n",
+    "import copy\n",
+    "import time\n",
+    "import cv2\n",
+    "import csv\n",
+    "import sys\n",
+    "import io\n",
+    "import gc\n",
+    "\n",
+    "plt.rcParams[\"savefig.bbox\"] = 'tight'\n",
+    "torch.manual_seed(1)\n",
+    "#matplotlib.use('Agg')\n",
+    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
+    "print(device)\n",
+    "torch.cuda.empty_cache()\n",
+    "cv2.destroyAllWindows()\n",
+    "gc.collect()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "8e009995",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def prepare_and_learning_detection(num_classes, num_samples, path_dataset, model_name, config_name, model,selected_freq):\n",
+    "    num_samples_per_class = num_samples // num_classes\n",
+    "\n",
+    "    #----------Создаём папку для сохранения результатов обучения--------------\n",
+    "    os.makedirs(\"models\", exist_ok=True)\n",
+    "    ind = 1\n",
+    "    while True:\n",
+    "        if os.path.exists(\"models/\" + model_name + str(ind)):\n",
+    "            ind += 1\n",
+    "        else:\n",
+    "            os.mkdir(\"models/\" + model_name + str(ind))\n",
+    "            path_res = \"models/\" + model_name + str(ind) + '/'\n",
+    "            break\n",
+    "    \n",
+    "    #----------Создаём файл dataset.csv для обучения--------------\n",
+    "    \n",
+    "    pd_columns = ['file_name']\n",
+    "    df = pd.DataFrame(columns=pd_columns)\n",
+    "    \n",
+    "    subdirs = os.listdir(path_dataset)\n",
+    "    \n",
+    "    for subdir in subdirs:\n",
+    "        freq_dir = os.path.join(path_dataset, subdir, str(selected_freq)+\"_jpg\")\n",
+    "        if not os.path.isdir(freq_dir):\n",
+    "            print(\"Error1\")\n",
+    "            continue\n",
+    "        \n",
+    "        files_k=[f for f in os.listdir(freq_dir)]\n",
+    "        print(len(files_k))\n",
+    "        \n",
+    "        files = [\n",
+    "                f for f in os.listdir(freq_dir)\n",
+    "                if os.path.isfile(os.path.join(freq_dir, f)) and f.endswith('imag.png')\n",
+    "            ]\n",
+    "        num_samples_per_class = min(num_samples_per_class, len(files))\n",
+    "        print(f\"num_samples per class {subdir} is {num_samples_per_class}\")\n",
+    "\n",
+    "    for subdir in subdirs:\n",
+    "        freq_dir = os.path.join(path_dataset, subdir, str(selected_freq)+\"_jpg\")\n",
+    "        if not os.path.isdir(freq_dir):\n",
+    "            print(\"Error1\")\n",
+    "            continue\n",
+    "\n",
+    "        files = [\n",
+    "            f for f in os.listdir(freq_dir)\n",
+    "            if os.path.isfile(os.path.join(freq_dir, f)) and f.endswith('imag.png')\n",
+    "        ]\n",
+    "        random.shuffle(files)\n",
+    "        files_to_process = files[:num_samples_per_class]\n",
+    "\n",
+    "        for file in files_to_process:\n",
+    "            row = pd.DataFrame({\n",
+    "                pd_columns[0]: [str(os.path.join(freq_dir, file))]\n",
+    "            })\n",
+    "            df = pd.concat([df, row], ignore_index=True)\n",
+    "\n",
+    "    dataset_csv_path = os.path.join(path_res, 'dataset.csv')\n",
+    "    df.to_csv(dataset_csv_path, index=False)\n",
+    "\n",
+    "    if not os.path.exists(dataset_csv_path):\n",
+    "        raise RuntimeError(f'dataset.csv was not created: {dataset_csv_path}')\n",
+    "    #----------Импортируем параметры для обучения--------------\n",
+    "    \n",
+    "    def load_function(attr):\n",
+    "        module_, func = attr.rsplit('.', maxsplit=1)\n",
+    "        return getattr(import_module(module_), func)\n",
+    "    \n",
+    "    config = mlconfig.load('config_' + config_name + '.yaml')\n",
+    "    \n",
+    "    #----------Создаём класс датасета--------------\n",
+    "    \n",
+    "    class MyDataset(Dataset):\n",
+    "        def __init__(self, path_dataset, csv_file):\n",
+    "            data=[]\n",
+    "            with open(os.path.join(path_dataset, csv_file), newline='') as csvfile:\n",
+    "                reader = csv.reader(csvfile, delimiter=' ', quotechar='|')\n",
+    "                for row in list(reader)[1:]:\n",
+    "                    row = str(row)\n",
+    "                    data.append(row[2: len(row)-2])\n",
+    "            self.path_dataset = path_dataset\n",
+    "            self.target_shape = None\n",
+    "            self.target_hw = None\n",
+    "            self.sig_filenames = self._validate_files(data)\n",
+    "\n",
+    "        def _pair_paths(self, filename):\n",
+    "            base = os.path.splitext(filename)[0]\n",
+    "            if base.endswith(\"real\"):\n",
+    "                return base + \".png\", base[:-4] + \"imag.png\"\n",
+    "            if base.endswith(\"imag\"):\n",
+    "                return base[:-4] + \"real.png\", base + \".png\"\n",
+    "            return None, None\n",
+    "\n",
+    "        @staticmethod\n",
+    "        def _read_shape(path):\n",
+    "            img = cv2.imread(path)\n",
+    "            if img is None:\n",
+    "                return None\n",
+    "            return img.shape\n",
+    "\n",
+    "        def _validate_files(self, filenames):\n",
+    "            from collections import Counter\n",
+    "\n",
+    "            candidates = []\n",
+    "            dropped = []\n",
+    "            shape_counter = Counter()\n",
+    "\n",
+    "            for filename in filenames:\n",
+    "                real_path, imag_path = self._pair_paths(filename)\n",
+    "                if real_path is None or imag_path is None:\n",
+    "                    dropped.append((filename, \"bad file suffix\"))\n",
+    "                    continue\n",
+    "\n",
+    "                real_shape = self._read_shape(real_path)\n",
+    "                imag_shape = self._read_shape(imag_path)\n",
+    "                if real_shape is None or imag_shape is None:\n",
+    "                    dropped.append((filename, f\"read failed real={real_shape} imag={imag_shape}\"))\n",
+    "                    continue\n",
+    "                if real_shape != imag_shape:\n",
+    "                    dropped.append((filename, f\"pair shape mismatch real={real_shape} imag={imag_shape}\"))\n",
+    "                    continue\n",
+    "\n",
+    "                shape_counter[real_shape] += 1\n",
+    "                candidates.append((filename, real_shape))\n",
+    "\n",
+    "            if not candidates:\n",
+    "                raise RuntimeError(\"No valid image pairs left after shape validation\")\n",
+    "\n",
+    "            preferred_shape = (1600, 1600, 3)\n",
+    "            if shape_counter.get(preferred_shape, 0) > 0:\n",
+    "                target_shape = preferred_shape\n",
+    "            else:\n",
+    "                target_shape = shape_counter.most_common(1)[0][0]\n",
+    "\n",
+    "            self.target_shape = target_shape\n",
+    "            self.target_hw = target_shape[:2]\n",
+    "            valid = [filename for filename, _shape in candidates]\n",
+    "            resized_count = sum(1 for _filename, shape in candidates if shape != target_shape)\n",
+    "\n",
+    "            print(f\"[dataset-shape-filter] shape_distribution={dict(shape_counter)}\")\n",
+    "            print(\n",
+    "                f\"[dataset-shape-filter] target_shape={target_shape} \"\n",
+    "                f\"kept={len(valid)} dropped={len(dropped)} will_resize={resized_count}\"\n",
+    "            )\n",
+    "            for filename, reason in dropped[:20]:\n",
+    "                print(f\"[dataset-shape-filter] drop {filename}: {reason}\")\n",
+    "            if len(dropped) > 20:\n",
+    "                print(f\"[dataset-shape-filter] ... {len(dropped) - 20} more dropped\")\n",
+    "\n",
+    "            return valid\n",
+    "\n",
+    "        def __len__(self):\n",
+    "            return len(self.sig_filenames)\n",
+    "\n",
+    "        def _read_image_chw(self, path):\n",
+    "            img = cv2.imread(path)\n",
+    "            if img is None:\n",
+    "                raise RuntimeError(f\"failed to read image: {path}\")\n",
+    "            if img.shape[:2] != self.target_hw:\n",
+    "                img = cv2.resize(\n",
+    "                    img,\n",
+    "                    (self.target_hw[1], self.target_hw[0]),\n",
+    "                    interpolation=cv2.INTER_AREA,\n",
+    "                )\n",
+    "            return np.asarray(cv2.split(img), dtype=np.float32)\n",
+    "\n",
+    "        def __getitem__(self, idx):\n",
+    "            real_path, imag_path = self._pair_paths(self.sig_filenames[idx])\n",
+    "            image_real = self._read_image_chw(real_path)\n",
+    "            image_imag = self._read_image_chw(imag_path)\n",
+    "\n",
+    "            path_parts = set(self.sig_filenames[idx].split('/'))\n",
+    "            if 'drone' in path_parts:\n",
+    "                label = torch.tensor(0)\n",
+    "            elif 'noise' in path_parts:\n",
+    "                label = torch.tensor(1)\n",
+    "            else:\n",
+    "                raise RuntimeError(f\"cannot infer label from path: {self.sig_filenames[idx]}\")\n",
+    "\n",
+    "            return image_real, image_imag, label\n",
+    "\n",
+    "    #----------Создаём датасет--------------\n",
+    "    \n",
+    "    dataset = MyDataset(path_dataset=path_res, csv_file='dataset.csv')\n",
+    "    train_set, valid_set = torch.utils.data.random_split(dataset, [0.7, 0.3], generator=torch.Generator().manual_seed(42))\n",
+    "    batch_size = config.batch_size\n",
+    "    train_dataloader = torch.utils.data.DataLoader(train_set, batch_size=batch_size, shuffle=True, drop_last=True)\n",
+    "    valid_dataloader = torch.utils.data.DataLoader(valid_set, batch_size=batch_size, shuffle=False, drop_last=False)\n",
+    "    \n",
+    "    dataloaders = {}\n",
+    "    dataloaders['train'] = train_dataloader\n",
+    "    dataloaders['val'] = valid_dataloader\n",
+    "    dataset_sizes = {}\n",
+    "    dataset_sizes['train'] = len(train_set)\n",
+    "    dataset_sizes['val'] = len(valid_set)\n",
+    "\n",
+    "    #----------Обучаем модель--------------\n",
+    "\n",
+    "    val_loss = []\n",
+    "    val_acc = []\n",
+    "    train_loss = []\n",
+    "    train_acc = []\n",
+    "    epochs = config.epoch\n",
+    "    min_delta = 1e-4\n",
+    "    \n",
+    "    best_val_loss = float('inf')\n",
+    "    best_model = copy.deepcopy(model.state_dict())\n",
+    "    limit = config.limit\n",
+    "    ind_limit = 0\n",
+    "    epoch_limit = epochs\n",
+    "    \n",
+    "    start = timeit.default_timer()\n",
+    "    for epoch in range(1, epochs+1):\n",
+    "        print(f\"Epoch : {epoch}\\n\")\n",
+    "        \n",
+    "        for phase in ['train', 'val']:\n",
+    "            if phase == 'train':\n",
+    "                model.train()\n",
+    "            else:\n",
+    "                model.eval()\n",
+    "\n",
+    "            running_loss = 0.0\n",
+    "            running_corrects = 0\n",
+    "    \n",
+    "            for (img1, img2, label) in tqdm(dataloaders[phase]):\n",
+    "                img1, img2, label = img1.to(device), img2.to(device), label.to(device)\n",
+    "                optimizer.zero_grad()\n",
+    "    \n",
+    "                with torch.set_grad_enabled(phase == 'train'):\n",
+    "                    output = model([img1, img2])\n",
+    "                    _, pred = torch.max(output.data, 1)\n",
+    "                    loss = criterion(output, label)\n",
+    "                    if phase == 'train':\n",
+    "                        loss.backward()\n",
+    "                        optimizer.step()\n",
+    "                \n",
+    "                running_loss += loss.item() * img1.size(0)\n",
+    "                running_corrects += torch.sum(pred == label.data)\n",
+    "    \n",
+    "            epoch_loss = running_loss / dataset_sizes[phase]\n",
+    "            epoch_acc = running_corrects.double() / dataset_sizes[phase]\n",
+    "                \n",
+    "            print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))\n",
+    "                    \n",
+    "            if phase == 'train':\n",
+    "                train_loss.append(epoch_loss)\n",
+    "                train_acc.append(epoch_acc)\n",
+    "            else:\n",
+    "                val_loss.append(epoch_loss)\n",
+    "                val_acc.append(epoch_acc)\n",
+    "                scheduler.step(epoch_loss)\n",
+    "\n",
+    "                current_lr = optimizer.param_groups[0]['lr']\n",
+    "                print(f'val lr: {current_lr:.8f}')\n",
+    "\n",
+    "                if epoch_loss < (best_val_loss - min_delta):\n",
+    "                    ind_limit = 0\n",
+    "                    best_val_loss = epoch_loss\n",
+    "                    best_model = copy.deepcopy(model.state_dict())\n",
+    "                    torch.save(best_model, path_res + model_name + '.pth')\n",
+    "                    print(f'saved best model with val_loss={best_val_loss:.4f}')\n",
+    "                else:\n",
+    "                    ind_limit += 1\n",
+    "                    print(f'early stopping patience: {ind_limit}/{limit}')\n",
+    "                    \n",
+    "                if ind_limit >= limit:\n",
+    "                    break\n",
+    "                \n",
+    "        if ind_limit >= limit:\n",
+    "            epoch_limit = epoch\n",
+    "            break\n",
+    "            \n",
+    "        print()\n",
+    "    \n",
+    "    end = timeit.default_timer()\n",
+    "    print(f\"Total time elapsed = {end - start} seconds\")\n",
+    "    epoch_limit += 1\n",
+    "    \n",
+    "    #----------Вывод графиков и сохранение результатов обучения--------------\n",
+    "    \n",
+    "    train_acc = np.asarray(list(map(lambda x: x.item(), train_acc)))\n",
+    "    val_acc = np.asarray(list(map(lambda x: x.item(), val_acc)))\n",
+    "    \n",
+    "    np.save(path_res+'train_acc.npy', train_acc)\n",
+    "    np.save(path_res+'val_acc.npy', val_acc)\n",
+    "    np.save(path_res+'train_loss.npy', train_loss)\n",
+    "    np.save(path_res+'val_loss.npy', val_loss)\n",
+    "    \n",
+    "    plt.figure()\n",
+    "    plt.plot(range(1,epoch_limit), train_loss, color='blue')\n",
+    "    plt.plot(range(1,epoch_limit), val_loss, color='red')\n",
+    "    plt.xlabel('Epoch')\n",
+    "    plt.ylabel('Loss')  \n",
+    "             \n",
+    "    plt.title('Loss Curve')\n",
+    "    plt.legend(['Train Loss', 'Validation Loss'])\n",
+    "    plt.show()\n",
+    "    plt.clf()\n",
+    "    plt.cla()\n",
+    "    plt.close()\n",
+    "    \n",
+    "    plt.figure()\n",
+    "    plt.plot(range(1,epoch_limit), train_acc, color='blue')\n",
+    "    plt.plot(range(1,epoch_limit), val_acc, color='red')\n",
+    "    plt.xlabel('Epoch')\n",
+    "    plt.ylabel('Accuracy')\n",
+    "    plt.title('Accuracy Curve')\n",
+    "    plt.legend(['Train Accuracy', 'Validation Accuracy'])\n",
+    "    plt.show()\n",
+    "    \n",
+    "    plt.clf()\n",
+    "    plt.cla()\n",
+    "    plt.close()\n",
+    "    torch.cuda.empty_cache()\n",
+    "    cv2.destroyAllWindows()\n",
+    "    del model\n",
+    "    gc.collect()\n",
+    "\n",
+    "    return path_res, model_name"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "bbbe7fea",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/sibscience-4/from_ssh/DroneDetector/.venv-train/lib/python3.12/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
+      "  warnings.warn(\n",
+      "/home/sibscience-4/from_ssh/DroneDetector/.venv-train/lib/python3.12/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=None`.\n",
+      "  warnings.warn(msg)\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Error1\n",
+      "Error1\n",
+      "Error1\n",
+      "Error1\n",
+      "Error1\n",
+      "Error1\n"
+     ]
+    },
+    {
+     "ename": "RuntimeError",
+     "evalue": "No valid image pairs left after shape validation",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
+      "\u001b[31mRuntimeError\u001b[39m                              Traceback (most recent call last)",
+      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[4]\u001b[39m\u001b[32m, line 45\u001b[39m\n\u001b[32m     41\u001b[39m     model = model.to(device)\n\u001b[32m     42\u001b[39m \n\u001b[32m     43\u001b[39m \u001b[38;5;66;03m#----------Создания датасета и обучение модели--------------\u001b[39;00m\n\u001b[32m     44\u001b[39m \n\u001b[32m---> \u001b[39m\u001b[32m45\u001b[39m path_res, model_name = prepare_and_learning_detection(num_classes = num_classes, num_samples = 10000, path_dataset = \"/mnt/data/Dataset_overlay\", \n\u001b[32m     46\u001b[39m                      selected_freq=\u001b[32m1200\u001b[39m,model_name = config_name+\u001b[33m\"1200_\"\u001b[39m, config_name = config_name, model=model)\n\u001b[32m     47\u001b[39m \n\u001b[32m     48\u001b[39m \n",
+      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[2]\u001b[39m\u001b[32m, line 183\u001b[39m, in \u001b[36mprepare_and_learning_detection\u001b[39m\u001b[34m(num_classes, num_samples, path_dataset, model_name, config_name, model, selected_freq)\u001b[39m\n\u001b[32m    179\u001b[39m             \u001b[38;5;28;01mreturn\u001b[39;00m image_real, image_imag, label\n\u001b[32m    180\u001b[39m \n\u001b[32m    181\u001b[39m     \u001b[38;5;66;03m#----------Создаём датасет--------------\u001b[39;00m\n\u001b[32m    182\u001b[39m \n\u001b[32m--> \u001b[39m\u001b[32m183\u001b[39m     dataset = MyDataset(path_dataset=path_res, csv_file=\u001b[33m'dataset.csv'\u001b[39m)\n\u001b[32m    184\u001b[39m     train_set, valid_set = torch.utils.data.random_split(dataset, [\u001b[32m0.7\u001b[39m, \u001b[32m0.3\u001b[39m], generator=torch.Generator().manual_seed(\u001b[32m42\u001b[39m))\n\u001b[32m    185\u001b[39m     batch_size = config.batch_size\n\u001b[32m    186\u001b[39m     train_dataloader = torch.utils.data.DataLoader(train_set, batch_size=batch_size, shuffle=\u001b[38;5;28;01mTrue\u001b[39;00m, drop_last=\u001b[38;5;28;01mTrue\u001b[39;00m)\n",
+      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[2]\u001b[39m\u001b[32m, line 83\u001b[39m, in \u001b[36mprepare_and_learning_detection.<locals>.MyDataset.__init__\u001b[39m\u001b[34m(self, path_dataset, csv_file)\u001b[39m\n\u001b[32m     79\u001b[39m                     data.append(row[\u001b[32m2\u001b[39m: len(row)-\u001b[32m2\u001b[39m])\n\u001b[32m     80\u001b[39m             self.path_dataset = path_dataset\n\u001b[32m     81\u001b[39m             self.target_shape = \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[32m     82\u001b[39m             self.target_hw = \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[32m---> \u001b[39m\u001b[32m83\u001b[39m             self.sig_filenames = self._validate_files(data)\n",
+      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[2]\u001b[39m\u001b[32m, line 126\u001b[39m, in \u001b[36mprepare_and_learning_detection.<locals>.MyDataset._validate_files\u001b[39m\u001b[34m(self, filenames)\u001b[39m\n\u001b[32m    122\u001b[39m                 shape_counter[real_shape] += \u001b[32m1\u001b[39m\n\u001b[32m    123\u001b[39m                 candidates.append((filename, real_shape))\n\u001b[32m    124\u001b[39m \n\u001b[32m    125\u001b[39m             \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28;01mnot\u001b[39;00m candidates:\n\u001b[32m--> \u001b[39m\u001b[32m126\u001b[39m                 \u001b[38;5;28;01mraise\u001b[39;00m RuntimeError(\u001b[33m\"No valid image pairs left after shape validation\"\u001b[39m)\n\u001b[32m    127\u001b[39m \n\u001b[32m    128\u001b[39m             preferred_shape = (\u001b[32m1600\u001b[39m, \u001b[32m1600\u001b[39m, \u001b[32m3\u001b[39m)\n\u001b[32m    129\u001b[39m             \u001b[38;5;28;01mif\u001b[39;00m shape_counter.get(preferred_shape, \u001b[32m0\u001b[39m) > \u001b[32m0\u001b[39m:\n",
+      "\u001b[31mRuntimeError\u001b[39m: No valid image pairs left after shape validation"
+     ]
+    }
+   ],
+   "source": [
+    "torch.cuda.empty_cache()\n",
+    "cv2.destroyAllWindows()\n",
+    "gc.collect()\n",
+    "\n",
+    "config_name = \"ensemble\"\n",
+    "    \n",
+    "def load_function(attr):\n",
+    "    module_, func = attr.rsplit('.', maxsplit=1)\n",
+    "    return getattr(import_module(module_), func)\n",
+    "    \n",
+    "config = mlconfig.load('config_' + config_name + '.yaml')\n",
+    "\n",
+    "model1 = models.resnet18(pretrained=False)\n",
+    "model2 = models.resnet50(pretrained=False)\n",
+    "\n",
+    "num_classes = 2\n",
+    "\n",
+    "model1.fc = nn.Linear(model1.fc.in_features, num_classes)\n",
+    "model2.fc = nn.Linear(model2.fc.in_features, num_classes)\n",
+    "\n",
+    "class Ensemble(nn.Module):\n",
+    "    def __init__(self, model1, model2):\n",
+    "        super(Ensemble, self).__init__()\n",
+    "        self.model1 = model1\n",
+    "        self.model2 = model2\n",
+    "        self.fc = nn.Linear(2 * num_classes, num_classes)\n",
+    "\n",
+    "    def forward(self, x):\n",
+    "        x1 = self.model1(x[0])\n",
+    "        x2 = self.model2(x[1])\n",
+    "        x = torch.cat((x1, x2), dim=1)\n",
+    "        x = self.fc(x)\n",
+    "        return x\n",
+    "model = Ensemble(model1, model2)\n",
+    "\n",
+    "optimizer = load_function(config.optimizer.name)(model.parameters(), lr=config.optimizer.lr)\n",
+    "criterion = load_function(config.loss_function.name)()\n",
+    "scheduler = load_function(config.scheduler.name)(optimizer, step_size=config.scheduler.step_size, gamma=config.scheduler.gamma)\n",
+    "\n",
+    "if device != 'cpu':\n",
+    "    model = model.to(device)\n",
+    "\n",
+    "#----------Создания датасета и обучение модели--------------\n",
+    "\n",
+    "path_res, model_name = prepare_and_learning_detection(num_classes = num_classes, num_samples = 10000, path_dataset = \"/mnt/data/Dataset_overlay\", \n",
+    "                     selected_freq=2400,model_name = config_name+\"2400_\", config_name = config_name, model=model)\n",
+    "\n",
+    "\n",
+    "torch.cuda.empty_cache()\n",
+    "cv2.destroyAllWindows()\n",
+    "del model\n",
+    "gc.collect()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "usr",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

train_scripts/create_dataset_overlay.py

@@ -0,0 +1,217 @@
+#!/usr/bin/env python3
+import argparse
+import csv
+import os
+import random
+import shutil
+from pathlib import Path
+
+import cv2
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+import numpy as np
+from tqdm import tqdm
+
+
+DEFAULT_FREQS = (1200, 2400)
+PNG_SUFFIXES = ("_real.png", "_imag.png", "_spec.png")
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description=(
+            "Build a two-class image dataset with drone signatures overlaid on noise images. "
+            "The output is ready for Training_models2pic_val_loss.ipynb."
+        )
+    )
+    parser.add_argument("--drone-root", default="/mnt/data/Dataset/drone")
+    parser.add_argument("--noise-img-root", default="/mnt/data/Dataset_img/noise")
+    parser.add_argument("--output-root", default="/mnt/data/Dataset_overlay")
+    parser.add_argument("--freqs", default=",".join(str(v) for v in DEFAULT_FREQS))
+    parser.add_argument("--alpha", type=float, default=1.0, help="Overlay strength: 1.0 keeps the darkest drone/noise pixels.")
+    parser.add_argument("--limit-per-freq", type=int, default=0, help="0 means use all available noise images per frequency.")
+    parser.add_argument("--seed", type=int, default=42)
+    parser.add_argument("--copy-noise", action="store_true", help="Copy noise files instead of hardlinking them.")
+    parser.add_argument("--overwrite", action="store_true")
+    parser.add_argument("--dry-run", action="store_true")
+    parser.add_argument("--no-progress", action="store_true", help="Disable tqdm progress bars.")
+    return parser.parse_args()
+
+
+def parse_freqs(value):
+    return [int(item.strip()) for item in value.split(",") if item.strip()]
+
+
+def collect_drone_npys(root, freq):
+    root = Path(root)
+    candidates = []
+    candidates.extend((root / str(freq)).rglob("*.npy"))
+    candidates.extend((root / f"{freq}_jpg").glob("*.npy"))
+    return sorted({p for p in candidates if p.is_file()})
+
+
+def collect_noise_npys(root, freq):
+    root = Path(root)
+    candidates = []
+    candidates.extend((root / f"{freq}_jpg").glob("*.npy"))
+    candidates.extend((root / str(freq)).rglob("*.npy"))
+    return sorted({p for p in candidates if p.is_file()})
+
+
+def load_image_tensor(path):
+    arr = np.load(path)
+    if arr.ndim == 2:
+        arr = np.stack([arr, arr, arr], axis=0)
+    if arr.ndim == 3 and arr.shape[-1] in (1, 3) and arr.shape[0] not in (1, 3):
+        arr = np.moveaxis(arr, -1, 0)
+    if arr.ndim != 3:
+        raise ValueError(f"expected 3D image tensor, got shape={arr.shape} path={path}")
+    if arr.shape[0] == 1:
+        arr = np.repeat(arr, 3, axis=0)
+    if arr.shape[0] < 3:
+        raise ValueError(f"expected at least 3 channels, got shape={arr.shape} path={path}")
+    return arr[:3].astype(np.float32, copy=False)
+
+
+def resize_like(arr, shape):
+    if arr.shape == shape:
+        return arr
+    channels, height, width = shape
+    resized = []
+    for channel in arr[:channels]:
+        resized.append(cv2.resize(channel, (width, height), interpolation=cv2.INTER_LINEAR))
+    return np.asarray(resized, dtype=np.float32)
+
+
+def overlay_tensors(noise, drone, alpha):
+    drone = resize_like(drone, noise.shape)
+    bright_overlay = np.minimum(noise, drone)
+    mixed = (1.0 - alpha) * noise + alpha * bright_overlay
+    return np.clip(mixed, 0, 255).astype(np.float32)
+
+
+def to_uint8(channel):
+    arr = np.asarray(channel, dtype=np.float32)
+    finite = arr[np.isfinite(arr)]
+    if finite.size == 0:
+        return np.zeros(arr.shape, dtype=np.uint8)
+    min_v = float(finite.min())
+    max_v = float(finite.max())
+    if min_v >= 0.0 and max_v <= 255.0:
+        return np.clip(arr, 0, 255).astype(np.uint8)
+    if max_v == min_v:
+        return np.zeros(arr.shape, dtype=np.uint8)
+    norm = (arr - min_v) / (max_v - min_v)
+    return np.clip(norm * 255.0, 0, 255).astype(np.uint8)
+
+
+def save_notebook_style_png(path, channel):
+    fig = plt.figure(figsize=(16, 16))
+    plt.imshow(channel)
+    plt.savefig(path)
+    plt.clf()
+    plt.cla()
+    plt.close()
+    plt.close(fig)
+
+
+def save_sample(base_path, tensor):
+    np.save(str(base_path) + ".npy", tensor.astype(np.float32))
+    for idx, suffix in enumerate(PNG_SUFFIXES):
+        save_notebook_style_png(str(base_path) + suffix, tensor[idx])
+
+
+def link_or_copy(src, dst, copy_file):
+    dst.parent.mkdir(parents=True, exist_ok=True)
+    if dst.exists():
+        return
+    if copy_file:
+        shutil.copy2(src, dst)
+        return
+    try:
+        os.link(src, dst)
+    except OSError:
+        shutil.copy2(src, dst)
+
+
+def copy_noise_family(noise_npy, out_dir, copy_file):
+    base_name = noise_npy.name[:-4] if noise_npy.name.endswith(".npy") else noise_npy.name
+    link_or_copy(noise_npy, out_dir / noise_npy.name, copy_file)
+    for suffix in PNG_SUFFIXES:
+        sidecar = noise_npy.with_name(base_name + suffix)
+        if sidecar.exists():
+            link_or_copy(sidecar, out_dir / sidecar.name, copy_file)
+
+
+def main():
+    args = parse_args()
+    random.seed(args.seed)
+    freqs = parse_freqs(args.freqs)
+    output_root = Path(args.output_root)
+    manifest_rows = []
+
+    if args.overwrite and output_root.exists() and not args.dry_run:
+        shutil.rmtree(output_root)
+
+    for freq in freqs:
+        drone_files = collect_drone_npys(args.drone_root, freq)
+        noise_files = collect_noise_npys(args.noise_img_root, freq)
+        if not drone_files:
+            raise RuntimeError(f"no drone npy files found for freq={freq} under {args.drone_root}")
+        if not noise_files:
+            raise RuntimeError(f"no noise image npy files found for freq={freq} under {args.noise_img_root}")
+
+        count = len(noise_files) if args.limit_per_freq <= 0 else min(args.limit_per_freq, len(noise_files))
+        selected_noise = noise_files[:]
+        random.shuffle(selected_noise)
+        selected_noise = selected_noise[:count]
+
+        out_drone_dir = output_root / "drone" / f"{freq}_jpg"
+        out_noise_dir = output_root / "noise" / f"{freq}_jpg"
+
+        print(f"freq={freq}: drone_source={len(drone_files)} noise_source={len(noise_files)} output_per_class={count}")
+        if args.dry_run:
+            continue
+
+        out_drone_dir.mkdir(parents=True, exist_ok=True)
+        out_noise_dir.mkdir(parents=True, exist_ok=True)
+
+        iterator = tqdm(
+            enumerate(selected_noise),
+            total=count,
+            desc=f"overlay freq={freq}",
+            unit="sample",
+            disable=args.no_progress,
+        )
+        for idx, noise_path in iterator:
+            drone_path = drone_files[idx % len(drone_files)]
+            noise_tensor = load_image_tensor(noise_path)
+            drone_tensor = load_image_tensor(drone_path)
+            mixed = overlay_tensors(noise_tensor, drone_tensor, args.alpha)
+
+            out_base = out_drone_dir / f"overlay_{freq}_{idx:06d}"
+            save_sample(out_base, mixed)
+            copy_noise_family(noise_path, out_noise_dir, args.copy_noise)
+
+            manifest_rows.append({
+                "freq": freq,
+                "output": str(out_base) + ".npy",
+                "noise_source": str(noise_path),
+                "drone_source": str(drone_path),
+                "alpha": args.alpha,
+            })
+
+    if not args.dry_run:
+        manifest_path = output_root / "overlay_manifest.csv"
+        with manifest_path.open("w", newline="", encoding="utf-8") as fh:
+            writer = csv.DictWriter(fh, fieldnames=["freq", "output", "noise_source", "drone_source", "alpha"])
+            writer.writeheader()
+            writer.writerows(manifest_rows)
+        print(f"wrote {len(manifest_rows)} overlay samples")
+        print(f"manifest: {manifest_path}")
+        print(f"dataset: {output_root}")
+
+
+if __name__ == "__main__":
+    main()