SergeyRevyakin
/
DroneDetector
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
Automatica_1_v2
Automatica-3
Ufa
neptune
main
automatica-1
main_2
new_read_pipe
fft
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from '5624b3065f'
${ noResults }
DroneDetector/train_scripts/ImageDatasetCreate_spec_ima...

565 lines
15 KiB
Plaintext
Raw Blame History

{
"cells": [
{
"cell_type": "code",
"execution_count": 2,
"id": "4fdb98fc-65bb-467e-be0c-168fee9b0fca",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"cuda:0\n"
]
},
{
"data": {
"text/plain": [
"<contextlib.ExitStack at 0x7d183e4bd220>"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"import time\n",
"import io\n",
"import cv2\n",
"import copy\n",
"import os\n",
"from tqdm import tqdm\n",
"import torch.nn as nn\n",
"import torch\n",
"import torchvision\n",
"from torch.utils.data import Dataset\n",
"from torch import default_generator, randperm\n",
"from PIL import Image\n",
"#from torch._utils import _accumulate\n",
"import csv\n",
"from torch.utils.data.dataset import Subset\n",
"from scipy import ndimage\n",
"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
"print(device)\n",
"batch_size = 16\n",
"momentum=0.9\n",
"lr = 1e-3\n",
"import random\n",
"sub_sample = 0.5\n",
"import matplotlib\n",
"import gc\n",
"import torchsig.utils as u\n",
"import torchsig.transforms.transforms as T\n",
"from torchsig.transforms import functional as F\n",
"matplotlib.use('Agg')\n",
"import matplotlib as mpl\n",
"mpl.rcParams['agg.path.chunksize'] = 256*256\n",
"plt.ioff()"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "4848b066-2e09-4c1c-b8fa-8e3fa84d907a",
"metadata": {},
"outputs": [],
"source": [
"s = T.Spectrogram(nperseg=1024)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "9267fbe1",
"metadata": {},
"outputs": [],
"source": [
"def sig2pic_with_spec(path_to_data, filename, specT=None,figsize=(16,16), dpi=16, resize = None):\n",
" try:\n",
" if specT is None:\n",
" specT = T.Spectrogram(nperseg=1024)\n",
" with open(path_to_data + filename, 'rb') as file:\n",
" tmp = np.frombuffer(file.read(), dtype=np.complex64)\n",
" signal = tmp\n",
"\n",
" #rint(\"vSE ok\")\n",
"\n",
" spectr = np.array(F.spectrogram(signal, fft_size=specT.fft_size, fft_stride=specT.fft_stride)[:, :figsize[0] * dpi])\n",
" #print(\"VSE OK\")\n",
" mag = np.abs(signal)\n",
" real = signal.real\n",
"\n",
" fig2 = plt.figure(figsize = figsize)\n",
" plt.axes(ylim=(-1, 1))\n",
"\n",
" plt.plot(real, color='black')\n",
" plt.gca().set_axis_off()\n",
" plt.subplots_adjust(top = 1, bottom = 0, right = 1, left = 0, hspace = 0, wspace = 0)\n",
" plt.margins(0,0)\n",
" buf2 = io.BytesIO()\n",
" fig2.savefig(buf2, format=\"png\", dpi=dpi)\n",
" buf2.seek(0)\n",
" img_arr2 = np.frombuffer(buf2.getvalue(), dtype=np.uint8)\n",
" buf2.close()\n",
" img2 = cv2.imdecode(img_arr2, 1)\n",
" img2 = cv2.cvtColor(img2, cv2.COLOR_BGR2GRAY)\n",
" plt.clf()\n",
" plt.cla()\n",
" plt.close()\n",
" plt.close(fig2)\n",
"\n",
" fig3 = plt.figure(figsize = figsize)\n",
" plt.axes(ylim=(-1, 1))\n",
"\n",
" plt.plot(mag, color='black')\n",
" plt.gca().set_axis_off()\n",
" plt.subplots_adjust(top = 1, bottom = 0, right = 1, left = 0, hspace = 0, wspace = 0)\n",
" plt.margins(0,0)\n",
" buf3 = io.BytesIO()\n",
" fig3.savefig(buf3, format=\"png\", dpi=dpi)\n",
" buf3.seek(0)\n",
" img_arr3 = np.frombuffer(buf3.getvalue(), dtype=np.uint8)\n",
" buf3.close()\n",
" img3 = cv2.imdecode(img_arr3, 1)\n",
" img3 = cv2.cvtColor(img3, cv2.COLOR_BGR2GRAY)\n",
" plt.clf()\n",
" plt.cla()\n",
" plt.close()\n",
" plt.close(fig3)\n",
"\n",
" if resize != None:\n",
" resized_real = cv2.resize(img2, resize)\n",
" resized_mag = cv2.resize(img3, resize)\n",
" resized_spectr = cv2.resize(spectr, resize)\n",
" img = np.asarray([resized_real, resized_mag, resized_spectr], dtype=np.float32)\n",
" return img\n",
" img = np.asarray([img2, img3, spectr], dtype=np.float32)\n",
" return img\n",
" except Exception as e:\n",
" print(str(e))\n",
" return None"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "448da74a-e0ae-44d8-9877-8dd1f257a24f",
"metadata": {},
"outputs": [],
"source": [
"selected_freq=750\n",
"\n",
"path_to_binaries = f'/mnt/nvme1/dataset/{selected_freq}'\n",
"path_to_pictures = f'/mnt/nvme1/dataset_img/noise/{selected_freq}_jpg'"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "ac4945a8-29c4-4da4-945f-08658953e3e5",
"metadata": {},
"outputs": [],
"source": [
"from tqdm import tqdm"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "6f226f86-5d72-4573-8af6-750128b70263",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"2026-03-25_16-59-25: 0%| | 0/40 [00:00<?, ?it/s]"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2026-03-25_16-59-25: 100%|██████████| 40/40 [00:47<00:00, 1.18s/it]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Dir: 2026-03-25_16-59-25 finished!\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2026-03-26_02-59-25: 100%|██████████| 40/40 [00:47<00:00, 1.18s/it]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Dir: 2026-03-26_02-59-25 finished!\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2026-03-25_14-59-25: 100%|██████████| 40/40 [00:35<00:00, 1.11it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Dir: 2026-03-25_14-59-25 finished!\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2026-03-26_04-59-26: 100%|██████████| 40/40 [00:49<00:00, 1.23s/it]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Dir: 2026-03-26_04-59-26 finished!\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2026-03-25_17-59-26: 100%|██████████| 40/40 [00:44<00:00, 1.12s/it]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Dir: 2026-03-25_17-59-26 finished!\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2026-03-25_20-59-26: 100%|██████████| 40/40 [00:47<00:00, 1.20s/it]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Dir: 2026-03-25_20-59-26 finished!\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2026-03-25_22-59-25: 100%|██████████| 40/40 [00:45<00:00, 1.13s/it]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Dir: 2026-03-25_22-59-25 finished!\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2026-03-25_14-56-12: 100%|██████████| 25/25 [00:22<00:00, 1.12it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Dir: 2026-03-25_14-56-12 finished!\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2026-03-26_00-59-27: 100%|██████████| 40/40 [00:46<00:00, 1.16s/it]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Dir: 2026-03-26_00-59-27 finished!\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2026-03-25_19-59-25: 100%|██████████| 40/40 [00:44<00:00, 1.11s/it]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Dir: 2026-03-25_19-59-25 finished!\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2026-03-26_05-59-25: 100%|██████████| 40/40 [00:48<00:00, 1.20s/it]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Dir: 2026-03-26_05-59-25 finished!\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2026-03-26_08-59-25: 100%|██████████| 40/40 [00:47<00:00, 1.18s/it]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Dir: 2026-03-26_08-59-25 finished!\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2026-03-26_07-59-25: 100%|██████████| 40/40 [00:46<00:00, 1.16s/it]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Dir: 2026-03-26_07-59-25 finished!\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2026-03-26_06-59-26: 100%|██████████| 40/40 [00:48<00:00, 1.20s/it]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Dir: 2026-03-26_06-59-26 finished!\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2026-03-26_01-59-26: 100%|██████████| 40/40 [00:47<00:00, 1.20s/it]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Dir: 2026-03-26_01-59-26 finished!\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2026-03-25_15-59-26: 100%|██████████| 40/40 [00:45<00:00, 1.14s/it]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Dir: 2026-03-25_15-59-26 finished!\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2026-03-25_23-59-26: 100%|██████████| 40/40 [00:48<00:00, 1.20s/it]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Dir: 2026-03-25_23-59-26 finished!\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2026-03-25_21-59-26: 100%|██████████| 40/40 [00:46<00:00, 1.17s/it]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Dir: 2026-03-25_21-59-26 finished!\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2026-03-25_18-59-25: 100%|██████████| 40/40 [00:47<00:00, 1.19s/it]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Dir: 2026-03-25_18-59-25 finished!\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2026-03-26_03-59-26: 100%|██████████| 40/40 [00:47<00:00, 1.19s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Dir: 2026-03-26_03-59-26 finished!\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n"
]
}
],
"source": [
"size = (256,256)\n",
"\n",
"if not os.path.exists(path_to_pictures):\n",
" os.mkdir(path_to_pictures)\n",
"\n",
"for subdir in os.listdir(path_to_binaries):\n",
" filepath = path_to_binaries + '/' + subdir + '/'\n",
"\n",
" if not os.path.isdir(filepath):\n",
" continue\n",
"\n",
" files = os.listdir(filepath)\n",
" k = max(1, int(len(files) * 0.04))\n",
" files = random.sample(files, k)\n",
" for file in tqdm(files, desc=subdir):\n",
" full_input_path = filepath + file\n",
"\n",
" if not os.path.isfile(full_input_path):\n",
" continue\n",
"\n",
" if file in ('run.log', 'reading_in_progress'):\n",
" continue\n",
"\n",
" save_base = subdir + '__' + file\n",
"\n",
" savepath = path_to_pictures + '/' + save_base + '.npy'\n",
" savepath_real_png = path_to_pictures + '/' + save_base + '_real' + '.png'\n",
" savepath_imag_png = path_to_pictures + '/' + save_base + '_imag' + '.png'\n",
" savepath_spec_png = path_to_pictures + '/' + save_base + '_spec' + '.png'\n",
"\n",
" if not os.path.exists(savepath):\n",
" img = sig2pic_with_spec(path_to_data=filepath, filename=file, specT=s, resize=size)\n",
" gc.collect()\n",
"\n",
" try:\n",
" plt.imshow(img[0])\n",
" plt.savefig(savepath_real_png)\n",
" plt.clf()\n",
" plt.cla()\n",
" plt.close()\n",
"\n",
" plt.imshow(img[1])\n",
" plt.savefig(savepath_imag_png)\n",
" plt.clf()\n",
" plt.cla()\n",
" plt.close()\n",
"\n",
" plt.imshow(img[2])\n",
" plt.savefig(savepath_spec_png)\n",
" plt.clf()\n",
" plt.cla()\n",
" plt.close()\n",
"\n",
" np.save(savepath, img)\n",
"\n",
" except Exception:\n",
" continue\n",
"\n",
" print('Dir: ', subdir , ' finished!')"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "58ff5fbd",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": ".venv-train (3.12.3)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.3"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Reference in New Issue View Git Blame Copy Permalink