Compare commits
19 Commits
main
...
Automatica
| Author | SHA1 | Date |
|---|---|---|
|
|
9d7a0b4fdc | 13 hours ago |
|
|
fed635a0ac | 13 hours ago |
|
|
1b97c0756a | 13 hours ago |
|
|
7ca6c8b26a | 1 day ago |
|
|
1fb308c6fa | 2 days ago |
|
|
d0f2a26cdc | 2 days ago |
|
|
98f6fbdbdc | 2 days ago |
|
|
3bf93aab3f | 2 days ago |
|
|
7ad17bb4c4 | 2 days ago |
|
|
0b65c2980d | 2 days ago |
|
|
c70a25cb8f | 2 days ago |
|
|
94856d0fb8 | 2 days ago |
|
|
783fb40eb0 | 2 days ago |
|
|
a1c99ebf9f | 2 days ago |
|
|
0fad5d6404 | 2 days ago |
|
|
c0ccecc270 | 2 days ago |
|
|
6a492a036b | 2 days ago |
|
|
8801da18c4 | 3 days ago |
|
|
e0b258c911 | 3 days ago |
@ -1,17 +1,50 @@
|
||||
.git
|
||||
.vscode
|
||||
.venv-sdr
|
||||
# Default-deny: include only files required by Docker images.
|
||||
*
|
||||
|
||||
!.dockerignore
|
||||
!deploy/
|
||||
!deploy/docker/
|
||||
!deploy/docker/**
|
||||
!deploy/requirements/
|
||||
!deploy/requirements/**
|
||||
!src/
|
||||
!src/**
|
||||
!common/
|
||||
!common/**
|
||||
!NN_server/
|
||||
!NN_server/**
|
||||
!telemetry/
|
||||
!telemetry/**
|
||||
!torchsig/
|
||||
!torchsig/**
|
||||
|
||||
# Drop heavy or host-only artifacts even if their parent directory is included.
|
||||
.git/
|
||||
.vscode/
|
||||
.venv*/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
*.pyo
|
||||
|
||||
# Heavy host-only SDR sources
|
||||
*.tar
|
||||
*.tar.gz
|
||||
*.tgz
|
||||
*.zip
|
||||
*.iq
|
||||
logs/
|
||||
runtime/
|
||||
train_scripts/
|
||||
scripts_nn/
|
||||
orange_scripts/
|
||||
signal/
|
||||
gnuradio/
|
||||
gr-osmosdr/
|
||||
gr-osmosdr-0.2.6/
|
||||
|
||||
# Local runtime artifacts
|
||||
NN_server/result/
|
||||
|
||||
# Legacy install artifacts not needed in docker image
|
||||
install_scripts/
|
||||
NN_server/NN/
|
||||
NN_server/result/
|
||||
NN_server/__pycache__/
|
||||
src/__pycache__/
|
||||
common/__pycache__/
|
||||
telemetry/__pycache__/
|
||||
torchsig/__pycache__/
|
||||
torchsig/docs/
|
||||
|
||||
@ -1,272 +0,0 @@
|
||||
#################
|
||||
# GENERAL
|
||||
#################
|
||||
module_name=dronedetector
|
||||
freqs=433,750,868,915,1200,1500,2400,3300,4500,5200,5800
|
||||
|
||||
signal_threshold=0.02
|
||||
signal_threshold_433=0.0195
|
||||
signal_threshold_700=0.01
|
||||
signal_threshold_868=0.01
|
||||
signal_threshold_5200=0.01
|
||||
signal_threshold_5800=0.01
|
||||
|
||||
# Runtime flags
|
||||
# 1/0, true/false, yes/no, on/off
|
||||
debug_flag=0
|
||||
debug_freq_flag=0
|
||||
debug_module_flag=0
|
||||
send_to_module_flag=1
|
||||
save_data_flag=0
|
||||
freq_endpoint=process_data
|
||||
send_to_master_flag=1
|
||||
send_to_jammer_flag=0
|
||||
jammer_timeout=2
|
||||
master_timeout=1
|
||||
amount_connection_attempts=3
|
||||
mad_k_on=5.0
|
||||
mad_eps=0.05
|
||||
# Per-frequency MAD sensitivity (scanner frequencies)
|
||||
|
||||
path_to_save_medians=/tmp/dronedetector/noises_medians/
|
||||
path_to_save_alarms=/tmp/dronedetector/alarms/
|
||||
elems_to_save=signal,abs_signal
|
||||
file_types_to_save=npy,npy
|
||||
smb_host=127.0.0.1
|
||||
smb_port=139
|
||||
smb_user=guest
|
||||
smb_pass=guest
|
||||
shared_folder=shared
|
||||
the_pc_name=dronedetector-host
|
||||
remote_pc_name=dronedetector-storage
|
||||
smb_domain=WORKGROUP
|
||||
|
||||
#################
|
||||
# 433
|
||||
#################
|
||||
hack_433=
|
||||
c_freq_433=433
|
||||
f_step_433=-20e6
|
||||
f_bases_433=0.480e9
|
||||
f_roofs_433=0.400e9
|
||||
signal_length_433=1000000
|
||||
buffer_columns_size_433=11
|
||||
num_of_thinning_iter_433=32
|
||||
multiply_factor_433=1.3
|
||||
num_for_alarm_433=4
|
||||
mad_k_on_433=0
|
||||
dbfs_linear_offset_db_433=0
|
||||
dbfs_linear_abs_median_scale_433=0
|
||||
|
||||
#################
|
||||
# 750
|
||||
#################
|
||||
hack_750=
|
||||
c_freq_750=750
|
||||
f_step_750=-20e6
|
||||
f_bases_750=0.755e9
|
||||
f_roofs_750=0.695e9
|
||||
signal_length_750=1000000
|
||||
buffer_columns_size_750=11
|
||||
num_of_thinning_iter_750=32
|
||||
multiply_factor_750=1.3
|
||||
num_for_alarm_750=4
|
||||
mad_k_on_750=0
|
||||
dbfs_linear_offset_db_750=0
|
||||
dbfs_linear_abs_median_scale_750=0
|
||||
|
||||
|
||||
#################
|
||||
# 915
|
||||
#################
|
||||
hack_915=
|
||||
c_freq_915=915
|
||||
f_step_915=-20e6
|
||||
f_bases_915=0.98e9
|
||||
f_roofs_915=0.840e9
|
||||
signal_length_915=1000000
|
||||
buffer_columns_size_915=11
|
||||
num_of_thinning_iter_915=32
|
||||
multiply_factor_915=1.3
|
||||
num_for_alarm_915=4
|
||||
mad_k_on_915=0
|
||||
dbfs_linear_offset_db_915=0
|
||||
dbfs_linear_abs_median_scale_915=0
|
||||
|
||||
#################
|
||||
# 1200
|
||||
#################
|
||||
hack_1200=
|
||||
c_freq_1200=1200
|
||||
f_step_1200=-20e6
|
||||
f_bases_1200=1.1e9
|
||||
f_roofs_1200=1.3e9
|
||||
signal_length_1200=1000000
|
||||
buffer_columns_size_1200=11
|
||||
num_of_thinning_iter_1200=32
|
||||
multiply_factor_1200=1.3
|
||||
num_for_alarm_1200=4
|
||||
mad_k_on_1200=0
|
||||
dbfs_linear_offset_db_1200=0
|
||||
dbfs_linear_abs_median_scale_1200=0
|
||||
|
||||
|
||||
#################
|
||||
# 1500
|
||||
#################
|
||||
hack_1500=
|
||||
c_freq_1500=1500
|
||||
f_step_1500=-20e6
|
||||
f_bases_1500=1.605e9
|
||||
f_roofs_1500=1.405e9
|
||||
signal_length_1500=1000000
|
||||
buffer_columns_size_1500=11
|
||||
num_of_thinning_iter_1500=32
|
||||
multiply_factor_1500=1.3
|
||||
num_for_alarm_1500=4
|
||||
mad_k_on_1500=0
|
||||
dbfs_linear_offset_db_1500=0
|
||||
dbfs_linear_abs_median_scale_1500=0
|
||||
|
||||
|
||||
#################
|
||||
# 2400
|
||||
#################
|
||||
hack_2400=
|
||||
c_freq_2400=2400
|
||||
f_step_2400=-20e6
|
||||
f_bases_2400=2.3e9
|
||||
f_roofs_2400=2.5e9
|
||||
signal_length_2400=1000000
|
||||
buffer_columns_size_2400=11
|
||||
num_of_thinning_iter_2400=32
|
||||
multiply_factor_2400=1.3
|
||||
num_for_alarm_2400=4
|
||||
mad_k_on_2400=0
|
||||
dbfs_linear_offset_db_2400=0
|
||||
dbfs_linear_abs_median_scale_2400=0
|
||||
|
||||
#################
|
||||
# 3300
|
||||
#################
|
||||
hack_3300=
|
||||
c_freq_3300=3300
|
||||
f_step_3300=-20e6
|
||||
f_bases_3300=3.5e9
|
||||
f_roofs_3300=3.3e9
|
||||
signal_length_3300=1000000
|
||||
buffer_columns_size_3300=11
|
||||
num_of_thinning_iter_3300=32
|
||||
multiply_factor_3300=1.3
|
||||
num_for_alarm_3300=3
|
||||
mad_k_on_3300=0
|
||||
dbfs_linear_offset_db_3300=0
|
||||
dbfs_linear_abs_median_scale_3300=0
|
||||
|
||||
#################
|
||||
# 4500
|
||||
#################
|
||||
hack_4500=
|
||||
c_freq_4500=4500
|
||||
f_step_4500=-20e6
|
||||
f_bases_4500=4.7e9
|
||||
f_roofs_4500=4.5e9
|
||||
signal_length_4500=1000000
|
||||
buffer_columns_size_4500=11
|
||||
num_of_thinning_iter_4500=32
|
||||
multiply_factor_4500=1.3
|
||||
num_for_alarm_4500=2
|
||||
mad_k_on_4500=0
|
||||
dbfs_linear_offset_db_4500=0
|
||||
dbfs_linear_abs_median_scale_4500=0
|
||||
|
||||
|
||||
#################
|
||||
# 5200
|
||||
#################
|
||||
hack_5200=
|
||||
c_freq_5200=5200
|
||||
f_step_5200=-20e6
|
||||
f_bases_5200=5.3e9
|
||||
f_roofs_5200=5.1e9
|
||||
signal_length_5200=1000000
|
||||
buffer_columns_size_5200=11
|
||||
num_of_thinning_iter_5200=32
|
||||
multiply_factor_5200=1.08
|
||||
num_for_alarm_5200=3
|
||||
mad_k_on_5200=0
|
||||
dbfs_linear_offset_db_5200=0
|
||||
dbfs_linear_abs_median_scale_4500=0
|
||||
|
||||
#################
|
||||
# 5800
|
||||
#################
|
||||
hack_5800=
|
||||
c_freq_5800=5800
|
||||
f_step_5800=-20e6
|
||||
f_bases_5800=5.8e9
|
||||
f_roofs_5800=5.5e9
|
||||
signal_length_5800=1000000
|
||||
buffer_columns_size_5800=11
|
||||
num_of_thinning_iter_5800=32
|
||||
multiply_factor_5800=1.08
|
||||
num_for_alarm_5800=3
|
||||
mad_k_on_5800=0
|
||||
dbfs_linear_offset_db_5200=0
|
||||
dbfs_linear_abs_median_scale_5800=0
|
||||
|
||||
|
||||
#################
|
||||
# MODULE SERVER (server_to_master)
|
||||
#################
|
||||
lochost=0.0.0.0
|
||||
locport=5010
|
||||
master_server_ip=192.168.11.223
|
||||
master_server_port=3000
|
||||
module_mac=bc:fc:e7:ca:87:2b
|
||||
module_ip=192.168.11.223
|
||||
threshold_to_alarm=5
|
||||
num_of_clear_packs=5
|
||||
time_to_jam=30
|
||||
time_to_fresh=5
|
||||
active_interval_to_send=1
|
||||
passive_interval_to_send=60
|
||||
gpsport=/dev/null
|
||||
latitude=1
|
||||
longitude=1
|
||||
jamhost=127.0.0.1
|
||||
jamport=3000
|
||||
|
||||
#################
|
||||
# ORANGE SDR (915/1200/2400 -> NN_server)
|
||||
#################
|
||||
POROG_2400=0.01
|
||||
HACKID_2400=
|
||||
POROG_1200=0.01
|
||||
HACKID_1200=
|
||||
POROG_915=0.01
|
||||
HACKID_915=
|
||||
SERVER_IP_1=127.0.0.1
|
||||
SERVER_PORT_1=8080
|
||||
SERVER_IP_2=127.0.0.1
|
||||
SERVER_PORT_2=8080
|
||||
|
||||
#################
|
||||
# NN_SERVER
|
||||
#################
|
||||
FREQS=915,1200,2400
|
||||
PATH_TO_NN=/app/NN_server/NN/
|
||||
SRC_RESULT=/app/NN_server/result/
|
||||
SRC_EXAMPLE=${PATH_TO_NN}example/
|
||||
|
||||
NN_1='${PATH_TO_NN}resnet18_1.pth && ${PATH_TO_NN}config_resnet18.yaml && ${SRC_EXAMPLE} && ${SRC_RESULT} && Resnet18_1_2400 && build_func_resnet18 && pre_func_resnet18 && inference_func_resnet18 && post_func_resnet18 && [drone,noise,wifi] && 10 && 1 && /app/NN_server/datasets/full_dataset_pic/'
|
||||
NN_21='${PATH_TO_NN}ensemble_1.2.pth && ${PATH_TO_NN}config_ensemble.yaml && ${SRC_EXAMPLE} && ${SRC_RESULT} && ensemble_1200 && build_func_ensemble && pre_func_ensemble && inference_func_ensemble && post_func_ensemble && [drone,noise] && 10 && 1 && /app/NN_server/datasets/full_dataset/'
|
||||
NN_22='${PATH_TO_NN}ensemble_915.pth && ${PATH_TO_NN}config_ensemble.yaml && ${SRC_EXAMPLE} && ${SRC_RESULT} && ensemble_915 && build_func_ensemble && pre_func_ensemble && inference_func_ensemble && post_func_ensemble && [drone,noise] && 10 && 1 && /app/NN_server/datasets/full_dataset/'
|
||||
|
||||
GENERAL_SERVER_IP=dronedetector-server-to-master
|
||||
GENERAL_SERVER_PORT=5010
|
||||
SERVER_IP=0.0.0.0
|
||||
SERVER_PORT=8080
|
||||
|
||||
NN_HOT_RELOAD=1
|
||||
|
||||
@ -0,0 +1,247 @@
|
||||
<#
|
||||
.Synopsis
|
||||
Activate a Python virtual environment for the current PowerShell session.
|
||||
|
||||
.Description
|
||||
Pushes the python executable for a virtual environment to the front of the
|
||||
$Env:PATH environment variable and sets the prompt to signify that you are
|
||||
in a Python virtual environment. Makes use of the command line switches as
|
||||
well as the `pyvenv.cfg` file values present in the virtual environment.
|
||||
|
||||
.Parameter VenvDir
|
||||
Path to the directory that contains the virtual environment to activate. The
|
||||
default value for this is the parent of the directory that the Activate.ps1
|
||||
script is located within.
|
||||
|
||||
.Parameter Prompt
|
||||
The prompt prefix to display when this virtual environment is activated. By
|
||||
default, this prompt is the name of the virtual environment folder (VenvDir)
|
||||
surrounded by parentheses and followed by a single space (ie. '(.venv) ').
|
||||
|
||||
.Example
|
||||
Activate.ps1
|
||||
Activates the Python virtual environment that contains the Activate.ps1 script.
|
||||
|
||||
.Example
|
||||
Activate.ps1 -Verbose
|
||||
Activates the Python virtual environment that contains the Activate.ps1 script,
|
||||
and shows extra information about the activation as it executes.
|
||||
|
||||
.Example
|
||||
Activate.ps1 -VenvDir C:\Users\MyUser\Common\.venv
|
||||
Activates the Python virtual environment located in the specified location.
|
||||
|
||||
.Example
|
||||
Activate.ps1 -Prompt "MyPython"
|
||||
Activates the Python virtual environment that contains the Activate.ps1 script,
|
||||
and prefixes the current prompt with the specified string (surrounded in
|
||||
parentheses) while the virtual environment is active.
|
||||
|
||||
.Notes
|
||||
On Windows, it may be required to enable this Activate.ps1 script by setting the
|
||||
execution policy for the user. You can do this by issuing the following PowerShell
|
||||
command:
|
||||
|
||||
PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser
|
||||
|
||||
For more information on Execution Policies:
|
||||
https://go.microsoft.com/fwlink/?LinkID=135170
|
||||
|
||||
#>
|
||||
Param(
|
||||
[Parameter(Mandatory = $false)]
|
||||
[String]
|
||||
$VenvDir,
|
||||
[Parameter(Mandatory = $false)]
|
||||
[String]
|
||||
$Prompt
|
||||
)
|
||||
|
||||
<# Function declarations --------------------------------------------------- #>
|
||||
|
||||
<#
|
||||
.Synopsis
|
||||
Remove all shell session elements added by the Activate script, including the
|
||||
addition of the virtual environment's Python executable from the beginning of
|
||||
the PATH variable.
|
||||
|
||||
.Parameter NonDestructive
|
||||
If present, do not remove this function from the global namespace for the
|
||||
session.
|
||||
|
||||
#>
|
||||
function global:deactivate ([switch]$NonDestructive) {
|
||||
# Revert to original values
|
||||
|
||||
# The prior prompt:
|
||||
if (Test-Path -Path Function:_OLD_VIRTUAL_PROMPT) {
|
||||
Copy-Item -Path Function:_OLD_VIRTUAL_PROMPT -Destination Function:prompt
|
||||
Remove-Item -Path Function:_OLD_VIRTUAL_PROMPT
|
||||
}
|
||||
|
||||
# The prior PYTHONHOME:
|
||||
if (Test-Path -Path Env:_OLD_VIRTUAL_PYTHONHOME) {
|
||||
Copy-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME -Destination Env:PYTHONHOME
|
||||
Remove-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME
|
||||
}
|
||||
|
||||
# The prior PATH:
|
||||
if (Test-Path -Path Env:_OLD_VIRTUAL_PATH) {
|
||||
Copy-Item -Path Env:_OLD_VIRTUAL_PATH -Destination Env:PATH
|
||||
Remove-Item -Path Env:_OLD_VIRTUAL_PATH
|
||||
}
|
||||
|
||||
# Just remove the VIRTUAL_ENV altogether:
|
||||
if (Test-Path -Path Env:VIRTUAL_ENV) {
|
||||
Remove-Item -Path env:VIRTUAL_ENV
|
||||
}
|
||||
|
||||
# Just remove VIRTUAL_ENV_PROMPT altogether.
|
||||
if (Test-Path -Path Env:VIRTUAL_ENV_PROMPT) {
|
||||
Remove-Item -Path env:VIRTUAL_ENV_PROMPT
|
||||
}
|
||||
|
||||
# Just remove the _PYTHON_VENV_PROMPT_PREFIX altogether:
|
||||
if (Get-Variable -Name "_PYTHON_VENV_PROMPT_PREFIX" -ErrorAction SilentlyContinue) {
|
||||
Remove-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Scope Global -Force
|
||||
}
|
||||
|
||||
# Leave deactivate function in the global namespace if requested:
|
||||
if (-not $NonDestructive) {
|
||||
Remove-Item -Path function:deactivate
|
||||
}
|
||||
}
|
||||
|
||||
<#
|
||||
.Description
|
||||
Get-PyVenvConfig parses the values from the pyvenv.cfg file located in the
|
||||
given folder, and returns them in a map.
|
||||
|
||||
For each line in the pyvenv.cfg file, if that line can be parsed into exactly
|
||||
two strings separated by `=` (with any amount of whitespace surrounding the =)
|
||||
then it is considered a `key = value` line. The left hand string is the key,
|
||||
the right hand is the value.
|
||||
|
||||
If the value starts with a `'` or a `"` then the first and last character is
|
||||
stripped from the value before being captured.
|
||||
|
||||
.Parameter ConfigDir
|
||||
Path to the directory that contains the `pyvenv.cfg` file.
|
||||
#>
|
||||
function Get-PyVenvConfig(
|
||||
[String]
|
||||
$ConfigDir
|
||||
) {
|
||||
Write-Verbose "Given ConfigDir=$ConfigDir, obtain values in pyvenv.cfg"
|
||||
|
||||
# Ensure the file exists, and issue a warning if it doesn't (but still allow the function to continue).
|
||||
$pyvenvConfigPath = Join-Path -Resolve -Path $ConfigDir -ChildPath 'pyvenv.cfg' -ErrorAction Continue
|
||||
|
||||
# An empty map will be returned if no config file is found.
|
||||
$pyvenvConfig = @{ }
|
||||
|
||||
if ($pyvenvConfigPath) {
|
||||
|
||||
Write-Verbose "File exists, parse `key = value` lines"
|
||||
$pyvenvConfigContent = Get-Content -Path $pyvenvConfigPath
|
||||
|
||||
$pyvenvConfigContent | ForEach-Object {
|
||||
$keyval = $PSItem -split "\s*=\s*", 2
|
||||
if ($keyval[0] -and $keyval[1]) {
|
||||
$val = $keyval[1]
|
||||
|
||||
# Remove extraneous quotations around a string value.
|
||||
if ("'""".Contains($val.Substring(0, 1))) {
|
||||
$val = $val.Substring(1, $val.Length - 2)
|
||||
}
|
||||
|
||||
$pyvenvConfig[$keyval[0]] = $val
|
||||
Write-Verbose "Adding Key: '$($keyval[0])'='$val'"
|
||||
}
|
||||
}
|
||||
}
|
||||
return $pyvenvConfig
|
||||
}
|
||||
|
||||
|
||||
<# Begin Activate script --------------------------------------------------- #>
|
||||
|
||||
# Determine the containing directory of this script
|
||||
$VenvExecPath = Split-Path -Parent $MyInvocation.MyCommand.Definition
|
||||
$VenvExecDir = Get-Item -Path $VenvExecPath
|
||||
|
||||
Write-Verbose "Activation script is located in path: '$VenvExecPath'"
|
||||
Write-Verbose "VenvExecDir Fullname: '$($VenvExecDir.FullName)"
|
||||
Write-Verbose "VenvExecDir Name: '$($VenvExecDir.Name)"
|
||||
|
||||
# Set values required in priority: CmdLine, ConfigFile, Default
|
||||
# First, get the location of the virtual environment, it might not be
|
||||
# VenvExecDir if specified on the command line.
|
||||
if ($VenvDir) {
|
||||
Write-Verbose "VenvDir given as parameter, using '$VenvDir' to determine values"
|
||||
}
|
||||
else {
|
||||
Write-Verbose "VenvDir not given as a parameter, using parent directory name as VenvDir."
|
||||
$VenvDir = $VenvExecDir.Parent.FullName.TrimEnd("\\/")
|
||||
Write-Verbose "VenvDir=$VenvDir"
|
||||
}
|
||||
|
||||
# Next, read the `pyvenv.cfg` file to determine any required value such
|
||||
# as `prompt`.
|
||||
$pyvenvCfg = Get-PyVenvConfig -ConfigDir $VenvDir
|
||||
|
||||
# Next, set the prompt from the command line, or the config file, or
|
||||
# just use the name of the virtual environment folder.
|
||||
if ($Prompt) {
|
||||
Write-Verbose "Prompt specified as argument, using '$Prompt'"
|
||||
}
|
||||
else {
|
||||
Write-Verbose "Prompt not specified as argument to script, checking pyvenv.cfg value"
|
||||
if ($pyvenvCfg -and $pyvenvCfg['prompt']) {
|
||||
Write-Verbose " Setting based on value in pyvenv.cfg='$($pyvenvCfg['prompt'])'"
|
||||
$Prompt = $pyvenvCfg['prompt'];
|
||||
}
|
||||
else {
|
||||
Write-Verbose " Setting prompt based on parent's directory's name. (Is the directory name passed to venv module when creating the virtual environment)"
|
||||
Write-Verbose " Got leaf-name of $VenvDir='$(Split-Path -Path $venvDir -Leaf)'"
|
||||
$Prompt = Split-Path -Path $venvDir -Leaf
|
||||
}
|
||||
}
|
||||
|
||||
Write-Verbose "Prompt = '$Prompt'"
|
||||
Write-Verbose "VenvDir='$VenvDir'"
|
||||
|
||||
# Deactivate any currently active virtual environment, but leave the
|
||||
# deactivate function in place.
|
||||
deactivate -nondestructive
|
||||
|
||||
# Now set the environment variable VIRTUAL_ENV, used by many tools to determine
|
||||
# that there is an activated venv.
|
||||
$env:VIRTUAL_ENV = $VenvDir
|
||||
|
||||
if (-not $Env:VIRTUAL_ENV_DISABLE_PROMPT) {
|
||||
|
||||
Write-Verbose "Setting prompt to '$Prompt'"
|
||||
|
||||
# Set the prompt to include the env name
|
||||
# Make sure _OLD_VIRTUAL_PROMPT is global
|
||||
function global:_OLD_VIRTUAL_PROMPT { "" }
|
||||
Copy-Item -Path function:prompt -Destination function:_OLD_VIRTUAL_PROMPT
|
||||
New-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Description "Python virtual environment prompt prefix" -Scope Global -Option ReadOnly -Visibility Public -Value $Prompt
|
||||
|
||||
function global:prompt {
|
||||
Write-Host -NoNewline -ForegroundColor Green "($_PYTHON_VENV_PROMPT_PREFIX) "
|
||||
_OLD_VIRTUAL_PROMPT
|
||||
}
|
||||
$env:VIRTUAL_ENV_PROMPT = $Prompt
|
||||
}
|
||||
|
||||
# Clear PYTHONHOME
|
||||
if (Test-Path -Path Env:PYTHONHOME) {
|
||||
Copy-Item -Path Env:PYTHONHOME -Destination Env:_OLD_VIRTUAL_PYTHONHOME
|
||||
Remove-Item -Path Env:PYTHONHOME
|
||||
}
|
||||
|
||||
# Add the venv to the PATH
|
||||
Copy-Item -Path Env:PATH -Destination Env:_OLD_VIRTUAL_PATH
|
||||
$Env:PATH = "$VenvExecDir$([System.IO.Path]::PathSeparator)$Env:PATH"
|
||||
@ -0,0 +1,70 @@
|
||||
# This file must be used with "source bin/activate" *from bash*
|
||||
# You cannot run it directly
|
||||
|
||||
deactivate () {
|
||||
# reset old environment variables
|
||||
if [ -n "${_OLD_VIRTUAL_PATH:-}" ] ; then
|
||||
PATH="${_OLD_VIRTUAL_PATH:-}"
|
||||
export PATH
|
||||
unset _OLD_VIRTUAL_PATH
|
||||
fi
|
||||
if [ -n "${_OLD_VIRTUAL_PYTHONHOME:-}" ] ; then
|
||||
PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}"
|
||||
export PYTHONHOME
|
||||
unset _OLD_VIRTUAL_PYTHONHOME
|
||||
fi
|
||||
|
||||
# Call hash to forget past commands. Without forgetting
|
||||
# past commands the $PATH changes we made may not be respected
|
||||
hash -r 2> /dev/null
|
||||
|
||||
if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then
|
||||
PS1="${_OLD_VIRTUAL_PS1:-}"
|
||||
export PS1
|
||||
unset _OLD_VIRTUAL_PS1
|
||||
fi
|
||||
|
||||
unset VIRTUAL_ENV
|
||||
unset VIRTUAL_ENV_PROMPT
|
||||
if [ ! "${1:-}" = "nondestructive" ] ; then
|
||||
# Self destruct!
|
||||
unset -f deactivate
|
||||
fi
|
||||
}
|
||||
|
||||
# unset irrelevant variables
|
||||
deactivate nondestructive
|
||||
|
||||
# on Windows, a path can contain colons and backslashes and has to be converted:
|
||||
if [ "${OSTYPE:-}" = "cygwin" ] || [ "${OSTYPE:-}" = "msys" ] ; then
|
||||
# transform D:\path\to\venv to /d/path/to/venv on MSYS
|
||||
# and to /cygdrive/d/path/to/venv on Cygwin
|
||||
export VIRTUAL_ENV=$(cygpath /home/sibscience-4/from_ssh/DroneDetector/.venv-sdr)
|
||||
else
|
||||
# use the path as-is
|
||||
export VIRTUAL_ENV=/home/sibscience-4/from_ssh/DroneDetector/.venv-sdr
|
||||
fi
|
||||
|
||||
_OLD_VIRTUAL_PATH="$PATH"
|
||||
PATH="$VIRTUAL_ENV/"bin":$PATH"
|
||||
export PATH
|
||||
|
||||
# unset PYTHONHOME if set
|
||||
# this will fail if PYTHONHOME is set to the empty string (which is bad anyway)
|
||||
# could use `if (set -u; : $PYTHONHOME) ;` in bash
|
||||
if [ -n "${PYTHONHOME:-}" ] ; then
|
||||
_OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}"
|
||||
unset PYTHONHOME
|
||||
fi
|
||||
|
||||
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then
|
||||
_OLD_VIRTUAL_PS1="${PS1:-}"
|
||||
PS1='(.venv-sdr) '"${PS1:-}"
|
||||
export PS1
|
||||
VIRTUAL_ENV_PROMPT='(.venv-sdr) '
|
||||
export VIRTUAL_ENV_PROMPT
|
||||
fi
|
||||
|
||||
# Call hash to forget past commands. Without forgetting
|
||||
# past commands the $PATH changes we made may not be respected
|
||||
hash -r 2> /dev/null
|
||||
@ -0,0 +1,27 @@
|
||||
# This file must be used with "source bin/activate.csh" *from csh*.
|
||||
# You cannot run it directly.
|
||||
|
||||
# Created by Davide Di Blasi <davidedb@gmail.com>.
|
||||
# Ported to Python 3.3 venv by Andrew Svetlov <andrew.svetlov@gmail.com>
|
||||
|
||||
alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; unsetenv VIRTUAL_ENV_PROMPT; test "\!:*" != "nondestructive" && unalias deactivate'
|
||||
|
||||
# Unset irrelevant variables.
|
||||
deactivate nondestructive
|
||||
|
||||
setenv VIRTUAL_ENV /home/sibscience-4/from_ssh/DroneDetector/.venv-sdr
|
||||
|
||||
set _OLD_VIRTUAL_PATH="$PATH"
|
||||
setenv PATH "$VIRTUAL_ENV/"bin":$PATH"
|
||||
|
||||
|
||||
set _OLD_VIRTUAL_PROMPT="$prompt"
|
||||
|
||||
if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then
|
||||
set prompt = '(.venv-sdr) '"$prompt"
|
||||
setenv VIRTUAL_ENV_PROMPT '(.venv-sdr) '
|
||||
endif
|
||||
|
||||
alias pydoc python -m pydoc
|
||||
|
||||
rehash
|
||||
@ -0,0 +1,69 @@
|
||||
# This file must be used with "source <venv>/bin/activate.fish" *from fish*
|
||||
# (https://fishshell.com/). You cannot run it directly.
|
||||
|
||||
function deactivate -d "Exit virtual environment and return to normal shell environment"
|
||||
# reset old environment variables
|
||||
if test -n "$_OLD_VIRTUAL_PATH"
|
||||
set -gx PATH $_OLD_VIRTUAL_PATH
|
||||
set -e _OLD_VIRTUAL_PATH
|
||||
end
|
||||
if test -n "$_OLD_VIRTUAL_PYTHONHOME"
|
||||
set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME
|
||||
set -e _OLD_VIRTUAL_PYTHONHOME
|
||||
end
|
||||
|
||||
if test -n "$_OLD_FISH_PROMPT_OVERRIDE"
|
||||
set -e _OLD_FISH_PROMPT_OVERRIDE
|
||||
# prevents error when using nested fish instances (Issue #93858)
|
||||
if functions -q _old_fish_prompt
|
||||
functions -e fish_prompt
|
||||
functions -c _old_fish_prompt fish_prompt
|
||||
functions -e _old_fish_prompt
|
||||
end
|
||||
end
|
||||
|
||||
set -e VIRTUAL_ENV
|
||||
set -e VIRTUAL_ENV_PROMPT
|
||||
if test "$argv[1]" != "nondestructive"
|
||||
# Self-destruct!
|
||||
functions -e deactivate
|
||||
end
|
||||
end
|
||||
|
||||
# Unset irrelevant variables.
|
||||
deactivate nondestructive
|
||||
|
||||
set -gx VIRTUAL_ENV /home/sibscience-4/from_ssh/DroneDetector/.venv-sdr
|
||||
|
||||
set -gx _OLD_VIRTUAL_PATH $PATH
|
||||
set -gx PATH "$VIRTUAL_ENV/"bin $PATH
|
||||
|
||||
# Unset PYTHONHOME if set.
|
||||
if set -q PYTHONHOME
|
||||
set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME
|
||||
set -e PYTHONHOME
|
||||
end
|
||||
|
||||
if test -z "$VIRTUAL_ENV_DISABLE_PROMPT"
|
||||
# fish uses a function instead of an env var to generate the prompt.
|
||||
|
||||
# Save the current fish_prompt function as the function _old_fish_prompt.
|
||||
functions -c fish_prompt _old_fish_prompt
|
||||
|
||||
# With the original prompt function renamed, we can override with our own.
|
||||
function fish_prompt
|
||||
# Save the return status of the last command.
|
||||
set -l old_status $status
|
||||
|
||||
# Output the venv prompt; color taken from the blue of the Python logo.
|
||||
printf "%s%s%s" (set_color 4B8BBE) '(.venv-sdr) ' (set_color normal)
|
||||
|
||||
# Restore the return status of the previous command.
|
||||
echo "exit $old_status" | .
|
||||
# Output the original/"old" prompt.
|
||||
_old_fish_prompt
|
||||
end
|
||||
|
||||
set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV"
|
||||
set -gx VIRTUAL_ENV_PROMPT '(.venv-sdr) '
|
||||
end
|
||||
@ -0,0 +1,6 @@
|
||||
#!/home/sibscience-4/from_ssh/DroneDetector/.venv-sdr/bin/python3
|
||||
import sys
|
||||
from numpy.f2py.f2py2e import main
|
||||
if __name__ == '__main__':
|
||||
sys.argv[0] = sys.argv[0].removesuffix('.exe')
|
||||
sys.exit(main())
|
||||
@ -0,0 +1,6 @@
|
||||
#!/home/sibscience-4/from_ssh/DroneDetector/.venv-sdr/bin/python3
|
||||
import sys
|
||||
from charset_normalizer.cli import cli_detect
|
||||
if __name__ == '__main__':
|
||||
sys.argv[0] = sys.argv[0].removesuffix('.exe')
|
||||
sys.exit(cli_detect())
|
||||
@ -0,0 +1,8 @@
|
||||
#!/home/sibscience-4/from_ssh/DroneDetector/.venv-sdr/bin/python3
|
||||
# -*- coding: utf-8 -*-
|
||||
import re
|
||||
import sys
|
||||
from pip._internal.cli.main import main
|
||||
if __name__ == '__main__':
|
||||
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
|
||||
sys.exit(main())
|
||||
@ -0,0 +1,8 @@
|
||||
#!/home/sibscience-4/from_ssh/DroneDetector/.venv-sdr/bin/python3
|
||||
# -*- coding: utf-8 -*-
|
||||
import re
|
||||
import sys
|
||||
from pip._internal.cli.main import main
|
||||
if __name__ == '__main__':
|
||||
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
|
||||
sys.exit(main())
|
||||
@ -0,0 +1,8 @@
|
||||
#!/home/sibscience-4/from_ssh/DroneDetector/.venv-sdr/bin/python3
|
||||
# -*- coding: utf-8 -*-
|
||||
import re
|
||||
import sys
|
||||
from pip._internal.cli.main import main
|
||||
if __name__ == '__main__':
|
||||
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
|
||||
sys.exit(main())
|
||||
@ -0,0 +1 @@
|
||||
python3
|
||||
@ -0,0 +1 @@
|
||||
/usr/bin/python3
|
||||
@ -0,0 +1 @@
|
||||
python3
|
||||
@ -0,0 +1 @@
|
||||
lib
|
||||
@ -0,0 +1,5 @@
|
||||
home = /usr/bin
|
||||
include-system-site-packages = true
|
||||
version = 3.12.3
|
||||
executable = /usr/bin/python3.12
|
||||
command = /usr/bin/python3 -m venv --system-site-packages /home/sibscience-4/from_ssh/DroneDetector/.venv-sdr
|
||||
@ -1,66 +0,0 @@
|
||||
|
||||
# CLAUDE.md
|
||||
|
||||
Behavioral guidelines to reduce common LLM coding mistakes. Merge with project-specific instructions as needed.
|
||||
|
||||
**Tradeoff:** These guidelines bias toward caution over speed. For trivial tasks, use judgment.
|
||||
|
||||
## 1. Think Before Coding
|
||||
|
||||
**Don't assume. Don't hide confusion. Surface tradeoffs.**
|
||||
|
||||
Before implementing:
|
||||
- State your assumptions explicitly. If uncertain, ask.
|
||||
- If multiple interpretations exist, present them - don't pick silently.
|
||||
- If a simpler approach exists, say so. Push back when warranted.
|
||||
- If something is unclear, stop. Name what's confusing. Ask.
|
||||
|
||||
## 2. Simplicity First
|
||||
|
||||
**Minimum code that solves the problem. Nothing speculative.**
|
||||
|
||||
- No features beyond what was asked.
|
||||
- No abstractions for single-use code.
|
||||
- No "flexibility" or "configurability" that wasn't requested.
|
||||
- No error handling for impossible scenarios.
|
||||
- If you write 200 lines and it could be 50, rewrite it.
|
||||
|
||||
Ask yourself: "Would a senior engineer say this is overcomplicated?" If yes, simplify.
|
||||
|
||||
## 3. Surgical Changes
|
||||
|
||||
**Touch only what you must. Clean up only your own mess.**
|
||||
|
||||
When editing existing code:
|
||||
- Don't "improve" adjacent code, comments, or formatting.
|
||||
- Don't refactor things that aren't broken.
|
||||
- Match existing style, even if you'd do it differently.
|
||||
- If you notice unrelated dead code, mention it - don't delete it.
|
||||
|
||||
When your changes create orphans:
|
||||
- Remove imports/variables/functions that YOUR changes made unused.
|
||||
- Don't remove pre-existing dead code unless asked.
|
||||
|
||||
The test: Every changed line should trace directly to the user's request.
|
||||
|
||||
## 4. Goal-Driven Execution
|
||||
|
||||
**Define success criteria. Loop until verified.**
|
||||
|
||||
Transform tasks into verifiable goals:
|
||||
- "Add validation" → "Write tests for invalid inputs, then make them pass"
|
||||
- "Fix the bug" → "Write a test that reproduces it, then make it pass"
|
||||
- "Refactor X" → "Ensure tests pass before and after"
|
||||
|
||||
For multi-step tasks, state a brief plan:
|
||||
```
|
||||
1. [Step] → verify: [check]
|
||||
2. [Step] → verify: [check]
|
||||
3. [Step] → verify: [check]
|
||||
```
|
||||
|
||||
Strong success criteria let you loop independently. Weak criteria ("make it work") require constant clarification.
|
||||
|
||||
---
|
||||
|
||||
**These guidelines are working if:** fewer unnecessary changes in diffs, fewer rewrites due to overcomplication, and clarifying questions come before implementation rather than after mistakes.
|
||||
@ -1,197 +0,0 @@
|
||||
from torchvision import models
|
||||
import torch.nn as nn
|
||||
import matplotlib
|
||||
import numpy as np
|
||||
import torch
|
||||
import cv2
|
||||
import gc
|
||||
import io
|
||||
|
||||
|
||||
def _render_plot(values, figsize=(16, 16), dpi=16):
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
fig = plt.figure(figsize=figsize)
|
||||
plt.axes(ylim=(-1, 1))
|
||||
plt.plot(values, color="black")
|
||||
plt.gca().set_axis_off()
|
||||
plt.subplots_adjust(top=1, bottom=0, right=1, left=0, hspace=0, wspace=0)
|
||||
plt.margins(0, 0)
|
||||
|
||||
buf = io.BytesIO()
|
||||
fig.savefig(buf, format="png", dpi=dpi)
|
||||
buf.seek(0)
|
||||
img_arr = np.frombuffer(buf.getvalue(), dtype=np.uint8)
|
||||
buf.close()
|
||||
|
||||
img = cv2.imdecode(img_arr, 1)
|
||||
if img is None:
|
||||
raise RuntimeError("failed to decode plot image")
|
||||
|
||||
plt.clf()
|
||||
plt.cla()
|
||||
plt.close()
|
||||
plt.close(fig)
|
||||
|
||||
return np.asarray(cv2.split(img), dtype=np.float32)
|
||||
|
||||
|
||||
def pre_func_ensemble(data=None, src="", ind_inference=0):
|
||||
try:
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
matplotlib.use("Agg")
|
||||
plt.ioff()
|
||||
|
||||
real = np.asarray(data[0], dtype=np.float32)
|
||||
imag = np.asarray(data[1], dtype=np.float32)
|
||||
signal = real + 1j * imag
|
||||
|
||||
img_real = _render_plot(signal.real)
|
||||
img_mag = _render_plot(np.abs(signal))
|
||||
|
||||
cv2.destroyAllWindows()
|
||||
gc.collect()
|
||||
|
||||
print("Подготовка данных завершена")
|
||||
print()
|
||||
return [img_real, img_mag]
|
||||
|
||||
except Exception as exc:
|
||||
print(str(exc))
|
||||
return None
|
||||
|
||||
|
||||
def build_func_ensemble(file_model="", file_config="", num_classes=None):
|
||||
try:
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
matplotlib.use("Agg")
|
||||
plt.ioff()
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
num_classes = 2
|
||||
model1 = models.resnet18(pretrained=False)
|
||||
model2 = models.resnet50(pretrained=False)
|
||||
|
||||
model1.fc = nn.Linear(model1.fc.in_features, num_classes)
|
||||
model2.fc = nn.Linear(model2.fc.in_features, num_classes)
|
||||
|
||||
class Ensemble(nn.Module):
|
||||
def __init__(self, model1, model2):
|
||||
super().__init__()
|
||||
self.model1 = model1
|
||||
self.model2 = model2
|
||||
self.fc = nn.Linear(2 * num_classes, num_classes)
|
||||
|
||||
def forward(self, x):
|
||||
if isinstance(x, (list, tuple)):
|
||||
x1 = x[0]
|
||||
x2 = x[1] if len(x) > 1 else x[0]
|
||||
else:
|
||||
x1 = x
|
||||
x2 = x
|
||||
y1 = self.model1(x1)
|
||||
y2 = self.model2(x2)
|
||||
y = torch.cat((y1, y2), dim=1)
|
||||
return self.fc(y)
|
||||
|
||||
model = Ensemble(model1, model2)
|
||||
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
if device != "cpu":
|
||||
model = model.to(device)
|
||||
model.load_state_dict(torch.load(file_model, map_location=device))
|
||||
model.eval()
|
||||
|
||||
cv2.destroyAllWindows()
|
||||
gc.collect()
|
||||
|
||||
print("Инициализация модели завершена")
|
||||
print()
|
||||
return model
|
||||
|
||||
except Exception as exc:
|
||||
print(str(exc))
|
||||
return None
|
||||
|
||||
|
||||
def inference_func_ensemble(data=None, model=None, mapping=None, shablon=""):
|
||||
try:
|
||||
cv2.destroyAllWindows()
|
||||
gc.collect()
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
if isinstance(data, (list, tuple)) and len(data) >= 2:
|
||||
inputs = [
|
||||
torch.unsqueeze(torch.tensor(data[0]).cpu(), 0).to(device).float(),
|
||||
torch.unsqueeze(torch.tensor(data[1]).cpu(), 0).to(device).float(),
|
||||
]
|
||||
else:
|
||||
tensor = torch.unsqueeze(torch.tensor(data).cpu(), 0).to(device).float()
|
||||
inputs = [tensor, tensor]
|
||||
|
||||
with torch.no_grad():
|
||||
output = model(inputs)
|
||||
_, predict = torch.max(output.data, 1)
|
||||
|
||||
prediction = mapping[int(np.asarray(predict.cpu())[0])]
|
||||
print("PREDICTION" + shablon + ": " + prediction)
|
||||
|
||||
output = output.cpu()
|
||||
label = np.asarray(np.argmax(output, axis=1))[0]
|
||||
output = np.asarray(torch.squeeze(output, 0))
|
||||
expon = np.exp(output - np.max(output))
|
||||
probability = round((expon / expon.sum())[label], 2)
|
||||
|
||||
cv2.destroyAllWindows()
|
||||
gc.collect()
|
||||
print("Уверенность" + shablon + " в предсказании: " + str(probability))
|
||||
print("Инференс завершен")
|
||||
print()
|
||||
return [prediction, probability]
|
||||
|
||||
except Exception as exc:
|
||||
print(str(exc))
|
||||
return None
|
||||
|
||||
|
||||
def post_func_ensemble(src="", model_type="", prediction="", model_id=0, ind_inference=0, data=None):
|
||||
try:
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
matplotlib.use("Agg")
|
||||
plt.ioff()
|
||||
|
||||
if int(ind_inference) <= 100 and isinstance(data, (list, tuple)) and len(data) >= 2:
|
||||
fig, ax = plt.subplots()
|
||||
ax.imshow(np.moveaxis(data[0], 0, -1))
|
||||
plt.savefig(src + "_inference_" + str(ind_inference) + "_" + prediction + "_real_" + str(model_id) + "_" + model_type + ".png")
|
||||
plt.clf()
|
||||
plt.cla()
|
||||
plt.close(fig)
|
||||
cv2.destroyAllWindows()
|
||||
gc.collect()
|
||||
|
||||
fig, ax = plt.subplots()
|
||||
ax.imshow(np.moveaxis(data[1], 0, -1))
|
||||
plt.savefig(src + "_inference_" + str(ind_inference) + "_" + prediction + "_mod_" + str(model_id) + "_" + model_type + ".png")
|
||||
plt.clf()
|
||||
plt.cla()
|
||||
plt.close(fig)
|
||||
cv2.destroyAllWindows()
|
||||
gc.collect()
|
||||
|
||||
plt.clf()
|
||||
plt.cla()
|
||||
plt.close()
|
||||
cv2.destroyAllWindows()
|
||||
gc.collect()
|
||||
|
||||
print("Постобработка завершена")
|
||||
print()
|
||||
|
||||
except Exception as exc:
|
||||
print(str(exc))
|
||||
return None
|
||||
@ -0,0 +1,19 @@
|
||||
[Unit]
|
||||
Description=DroneDetector SDR Router 868/915 Shared HackRF
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=__RUN_USER__
|
||||
Group=__RUN_GROUP__
|
||||
WorkingDirectory=__PROJECT_ROOT__
|
||||
EnvironmentFile=__PROJECT_ROOT__/.env
|
||||
Environment=PYTHONPATH=__PROJECT_ROOT__
|
||||
ExecStartPre=/usr/local/bin/dronedetector-precheck-sdr.sh
|
||||
ExecStart=__PROJECT_ROOT__/.venv-sdr/bin/python src/main_868_915_router.py
|
||||
Restart=always
|
||||
RestartSec=3
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@ -0,0 +1,20 @@
|
||||
[Unit]
|
||||
Description=DroneDetector SDR Scanner 868 MHz
|
||||
After=network-online.target dronedetector-sdr-868-915.service
|
||||
Wants=network-online.target
|
||||
Requires=dronedetector-sdr-868-915.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=__RUN_USER__
|
||||
Group=__RUN_GROUP__
|
||||
WorkingDirectory=__PROJECT_ROOT__
|
||||
EnvironmentFile=__PROJECT_ROOT__/.env
|
||||
Environment=PYTHONPATH=__PROJECT_ROOT__
|
||||
ExecStartPre=/usr/local/bin/dronedetector-precheck-sdr.sh
|
||||
ExecStart=__PROJECT_ROOT__/.venv-sdr/bin/python src/main_868.py
|
||||
Restart=always
|
||||
RestartSec=3
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@ -0,0 +1,22 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if ! command -v hackrf_info >/dev/null 2>&1; then
|
||||
echo "[dronedetector-precheck] hackrf_info not found. Install hackrf-tools/hackrf package." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v gnuradio-config-info >/dev/null 2>&1; then
|
||||
echo "[dronedetector-precheck] gnuradio-config-info not found. Install gnuradio." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! python3 -c "import osmosdr" >/dev/null 2>&1; then
|
||||
echo "[dronedetector-precheck] Python module osmosdr not importable." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! hackrf_info 2>/dev/null | grep -q "Found HackRF"; then
|
||||
echo "[dronedetector-precheck] HackRF device was not detected by hackrf_info." >&2
|
||||
exit 1
|
||||
fi
|
||||
@ -1,115 +0,0 @@
|
||||
|
||||
from common.runtime import load_root_env, validate_env, as_float, as_int, as_str
|
||||
import numpy as np
|
||||
import requests
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
|
||||
|
||||
load_root_env(__file__)
|
||||
validate_env("orange_scripts/compose_send_data_5800.py", {
|
||||
"POROG_5800": as_float,
|
||||
"SERVER_IP_2": as_str,
|
||||
"SERVER_PORT_2": as_int,
|
||||
})
|
||||
porog = float(os.getenv('POROG_5800'))
|
||||
server_ip_1 = os.getenv('SERVER_IP_1')
|
||||
server_port_1 = os.getenv('SERVER_PORT_1')
|
||||
server_ip_2 = os.getenv('SERVER_IP_2')
|
||||
server_port_2 = os.getenv('SERVER_PORT_2')
|
||||
PARAMS = {'split_size': 400_000, 'point_amount': 100_000}
|
||||
PARAMS['show_amount'] = 0.8 * PARAMS['point_amount']
|
||||
token = 0
|
||||
channel = 1
|
||||
flag = 0
|
||||
|
||||
##############################
|
||||
# HYPERPARAMETERS
|
||||
##############################
|
||||
f_base = 5.9e9
|
||||
f_step = 20e6
|
||||
f_roof = 5.7e9
|
||||
##############################
|
||||
# Variables
|
||||
##############################
|
||||
f = f_base
|
||||
EOCF = 0
|
||||
signal_arr = []
|
||||
|
||||
|
||||
class NumpyArrayEncoder(json.JSONEncoder):
|
||||
def default(self, obj):
|
||||
if isinstance(obj, np.integer):
|
||||
return int(obj)
|
||||
elif isinstance(obj, np.ndarray):
|
||||
return obj.tolist()
|
||||
else:
|
||||
return super(NumpyArrayEncoder, self).default(obj)
|
||||
|
||||
|
||||
def send_data(sig):
|
||||
try:
|
||||
global token
|
||||
print('#' * 10)
|
||||
print('\nОтправка пакета ' + str(token+1))
|
||||
data_to_send = {
|
||||
"freq": 5800,
|
||||
"channel": int(channel),
|
||||
"token": int(token+1),
|
||||
"data_real": np.asarray(np.array(sig, dtype=np.complex64).real, dtype=np.float32),
|
||||
"data_imag": np.asarray(np.array(sig, dtype=np.complex64).imag, dtype=np.float32)
|
||||
}
|
||||
mod_data_to_send = json.dumps(data_to_send, cls=NumpyArrayEncoder)
|
||||
response = requests.post("http://{0}:{1}/receive_data".format(server_ip_2, server_port_2), json=mod_data_to_send)
|
||||
if response.status_code == 200:
|
||||
token += 1
|
||||
print(response.text)
|
||||
print('#' * 10)
|
||||
else:
|
||||
print("Ошибка при отправке данных: ", response.status_code)
|
||||
print('#' * 10)
|
||||
except Exception as exc:
|
||||
print(str(exc))
|
||||
|
||||
|
||||
def median(sig):
|
||||
global flag
|
||||
median = abs(float(np.median(sorted(np.asarray(np.abs(np.array(sig, dtype=np.complex64)), dtype=np.float32))[int(PARAMS['show_amount']):])))
|
||||
flag = 0 if porog > median else 1
|
||||
print(channel, median, flag)
|
||||
|
||||
|
||||
def work(lvl):
|
||||
global flag
|
||||
global channel
|
||||
global f_base
|
||||
global f_step
|
||||
global f_roof
|
||||
global f
|
||||
global EOCF
|
||||
global signal_arr
|
||||
|
||||
y = np.array(lvl).ravel()
|
||||
signal_arr = np.concatenate((signal_arr, y), axis=None)
|
||||
|
||||
if f >= f_roof:
|
||||
f = f_base
|
||||
signal_arr = []
|
||||
channel = 1
|
||||
return f, EOCF
|
||||
else:
|
||||
if flag == 0 and len(signal_arr) >= PARAMS['point_amount']:
|
||||
median(signal_arr[:PARAMS['point_amount']])
|
||||
signal_arr = []
|
||||
if flag == 0:
|
||||
f += f_step
|
||||
channel += 1
|
||||
if len(signal_arr) >= PARAMS['split_size']:
|
||||
send_data(signal_arr[:PARAMS['split_size']])
|
||||
flag = 0
|
||||
signal_arr = []
|
||||
channel += 1
|
||||
f += f_step
|
||||
return f, EOCF
|
||||
@ -1,807 +0,0 @@
|
||||
"""
|
||||
./.venv-sdr/bin/python read_energy_wide.py \
|
||||
--serial 0000000000000000a18c63dc2a83b813 \
|
||||
--sample-rate 20000000 \
|
||||
--base 6000 \
|
||||
--roof 5700 \
|
||||
--step 20
|
||||
|
||||
"""
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import math
|
||||
import re
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
try:
|
||||
import numpy as np
|
||||
except Exception as exc:
|
||||
print(f"numpy import failed: {exc}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
from gnuradio import blocks, gr
|
||||
import osmosdr
|
||||
except Exception as exc:
|
||||
print(f"gnuradio/osmosdr import failed: {exc}", file=sys.stderr)
|
||||
print("Run with the SDR venv, e.g. .venv-sdr/bin/python read_energy_wide.py", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
EPS = 1e-20
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScanWindow:
|
||||
seq: int
|
||||
start_mhz: float
|
||||
end_mhz: float
|
||||
low_mhz: float
|
||||
high_mhz: float
|
||||
center_mhz: float
|
||||
status: str = "INIT"
|
||||
rms: Optional[float] = None
|
||||
power_lin: Optional[float] = None
|
||||
dbfs: Optional[float] = None
|
||||
samples: int = 0
|
||||
updated_at: float = 0.0
|
||||
error: str = ""
|
||||
pass_no: int = 0
|
||||
|
||||
|
||||
class WideProbeTop(gr.top_block):
|
||||
def __init__(
|
||||
self,
|
||||
index: int,
|
||||
center_freq_hz: float,
|
||||
sample_rate: float,
|
||||
vec_len: int,
|
||||
gain: float,
|
||||
if_gain: float,
|
||||
bb_gain: float,
|
||||
):
|
||||
super().__init__("hackrf_energy_wide_probe")
|
||||
self.probe = blocks.probe_signal_vc(vec_len)
|
||||
self.stream_to_vec = blocks.stream_to_vector(gr.sizeof_gr_complex * 1, vec_len)
|
||||
self.src = osmosdr.source(args=f"numchan=1 hackrf={index}")
|
||||
self.src.set_time_unknown_pps(osmosdr.time_spec_t())
|
||||
self.src.set_sample_rate(sample_rate)
|
||||
self.src.set_center_freq(center_freq_hz, 0)
|
||||
try:
|
||||
self.src.set_freq_corr(0, 0)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
self.src.set_gain_mode(False, 0)
|
||||
except Exception:
|
||||
pass
|
||||
for fn, val in (("set_gain", gain), ("set_if_gain", if_gain), ("set_bb_gain", bb_gain)):
|
||||
try:
|
||||
getattr(self.src, fn)(val, 0)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
self.src.set_bandwidth(0, 0)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
self.src.set_antenna("", 0)
|
||||
except Exception:
|
||||
pass
|
||||
self.connect((self.src, 0), (self.stream_to_vec, 0))
|
||||
self.connect((self.stream_to_vec, 0), (self.probe, 0))
|
||||
|
||||
def tune(self, freq_hz: float) -> None:
|
||||
self.src.set_center_freq(freq_hz, 0)
|
||||
|
||||
def read_metrics(self) -> Tuple[float, float, float, int]:
|
||||
arr = np.asarray(self.probe.level(), dtype=np.complex64)
|
||||
if arr.size == 0:
|
||||
raise RuntimeError("no samples")
|
||||
power_lin = float(np.mean(arr.real * arr.real + arr.imag * arr.imag))
|
||||
rms = math.sqrt(max(power_lin, 0.0))
|
||||
dbfs = 10.0 * math.log10(max(power_lin, EPS))
|
||||
return rms, power_lin, dbfs, int(arr.size)
|
||||
|
||||
def read_window(self, settle: float, avg_reads: int, pause_between_reads: float) -> Tuple[float, float, float, int]:
|
||||
if settle > 0:
|
||||
time.sleep(settle)
|
||||
|
||||
read_count = max(1, avg_reads)
|
||||
powers: List[float] = []
|
||||
sample_sizes: List[int] = []
|
||||
last_error: Optional[Exception] = None
|
||||
|
||||
for idx in range(read_count):
|
||||
deadline = time.time() + 1.0
|
||||
while True:
|
||||
try:
|
||||
_, power_lin, _, samples = self.read_metrics()
|
||||
powers.append(power_lin)
|
||||
sample_sizes.append(samples)
|
||||
break
|
||||
except Exception as exc:
|
||||
last_error = exc
|
||||
if time.time() >= deadline:
|
||||
raise RuntimeError(str(last_error) if last_error else "no samples")
|
||||
time.sleep(0.02)
|
||||
if idx + 1 < read_count and pause_between_reads > 0:
|
||||
time.sleep(pause_between_reads)
|
||||
|
||||
power_lin = float(sum(powers) / len(powers))
|
||||
rms = math.sqrt(max(power_lin, 0.0))
|
||||
dbfs = 10.0 * math.log10(max(power_lin, EPS))
|
||||
samples = int(sum(sample_sizes) / len(sample_sizes))
|
||||
return rms, power_lin, dbfs, samples
|
||||
|
||||
|
||||
def parse_hackrf_info() -> Dict[str, int]:
|
||||
try:
|
||||
proc = subprocess.run(["hackrf_info"], capture_output=True, text=True, timeout=15)
|
||||
except FileNotFoundError:
|
||||
raise RuntimeError("hackrf_info not found")
|
||||
except subprocess.TimeoutExpired:
|
||||
raise RuntimeError("hackrf_info timeout")
|
||||
text = (proc.stdout or "") + "\n" + (proc.stderr or "")
|
||||
out: Dict[str, int] = {}
|
||||
cur_idx: Optional[int] = None
|
||||
for line in text.splitlines():
|
||||
m = re.search(r"^Index:\s*(\d+)", line)
|
||||
if m:
|
||||
cur_idx = int(m.group(1))
|
||||
continue
|
||||
m = re.search(r"^Serial number:\s*([0-9a-fA-F]+)", line)
|
||||
if m and cur_idx is not None:
|
||||
out[m.group(1).lower()] = cur_idx
|
||||
if not out:
|
||||
raise RuntimeError("no devices parsed from hackrf_info")
|
||||
return out
|
||||
|
||||
|
||||
def fmt(value: Optional[float], spec: str) -> str:
|
||||
return "-" if value is None else format(value, spec)
|
||||
|
||||
|
||||
def build_windows(base_mhz: float, roof_mhz: float, step_mhz: float) -> List[ScanWindow]:
|
||||
if step_mhz <= 0:
|
||||
raise ValueError("step must be > 0")
|
||||
if base_mhz == roof_mhz:
|
||||
raise ValueError("base and roof must be different")
|
||||
|
||||
direction = -1.0 if roof_mhz < base_mhz else 1.0
|
||||
edge = base_mhz
|
||||
seq = 1
|
||||
windows: List[ScanWindow] = []
|
||||
|
||||
while True:
|
||||
next_edge = edge + direction * step_mhz
|
||||
if direction < 0 and next_edge < roof_mhz:
|
||||
next_edge = roof_mhz
|
||||
if direction > 0 and next_edge > roof_mhz:
|
||||
next_edge = roof_mhz
|
||||
|
||||
low_mhz = min(edge, next_edge)
|
||||
high_mhz = max(edge, next_edge)
|
||||
center_mhz = (low_mhz + high_mhz) / 2.0
|
||||
windows.append(
|
||||
ScanWindow(
|
||||
seq=seq,
|
||||
start_mhz=edge,
|
||||
end_mhz=next_edge,
|
||||
low_mhz=low_mhz,
|
||||
high_mhz=high_mhz,
|
||||
center_mhz=center_mhz,
|
||||
)
|
||||
)
|
||||
|
||||
if next_edge == roof_mhz:
|
||||
break
|
||||
edge = next_edge
|
||||
seq += 1
|
||||
|
||||
return windows
|
||||
|
||||
|
||||
def render(
|
||||
windows: List[ScanWindow],
|
||||
serial: str,
|
||||
index: int,
|
||||
sample_rate: float,
|
||||
base_mhz: float,
|
||||
roof_mhz: float,
|
||||
step_mhz: float,
|
||||
started_at: float,
|
||||
pass_no: int,
|
||||
current_seq: int,
|
||||
) -> None:
|
||||
now = time.time()
|
||||
capture_bw_mhz = sample_rate / 1e6
|
||||
current_row = next((row for row in windows if row.seq == current_seq), None)
|
||||
best_row = max(
|
||||
(row for row in windows if row.status == "OK" and row.dbfs is not None),
|
||||
key=lambda row: row.dbfs if row.dbfs is not None else float("-inf"),
|
||||
default=None,
|
||||
)
|
||||
print("\x1b[2J\x1b[H", end="")
|
||||
print("HackRF Wide Energy Monitor (relative power: RMS / linear / dBFS)")
|
||||
print(
|
||||
f"serial: {serial} | idx: {index} | sample-rate: {capture_bw_mhz:.3f} MHz | "
|
||||
f"scan: {base_mhz:.3f}->{roof_mhz:.3f} MHz step {step_mhz:.3f} MHz | "
|
||||
f"pass: {pass_no} | uptime: {int(now-started_at)}s | {time.strftime('%Y-%m-%d %H:%M:%S')}"
|
||||
)
|
||||
print()
|
||||
header = (
|
||||
f"{'cur':>3} {'seq':>3} {'window MHz':>23} {'center':>9} {'status':>8} "
|
||||
f"{'dBFS':>9} {'rms':>10} {'power':>12} {'N':>5} {'age':>5} error"
|
||||
)
|
||||
print(header)
|
||||
print("-" * len(header))
|
||||
for row in windows:
|
||||
age = "-" if row.updated_at <= 0 else f"{(now-row.updated_at):.1f}"
|
||||
err = row.error
|
||||
if len(err) > 50:
|
||||
err = err[:47] + "..."
|
||||
marker = ">>>" if row.seq == current_seq else ""
|
||||
print(
|
||||
f"{marker:>3} {row.seq:>3} "
|
||||
f"{f'{row.high_mhz:.3f}-{row.low_mhz:.3f}':>23} {row.center_mhz:>9.3f} {row.status:>8} "
|
||||
f"{fmt(row.dbfs, '.2f'):>9} {fmt(row.rms, '.6f'):>10} {fmt(row.power_lin, '.8f'):>12} "
|
||||
f"{row.samples:>5} {age:>5} {err}"
|
||||
)
|
||||
print()
|
||||
if best_row is not None:
|
||||
best_age = "-" if best_row.updated_at <= 0 else f"{(now-best_row.updated_at):.1f}"
|
||||
print(
|
||||
f"{'':>3} {'MAX':>3} "
|
||||
f"{f'{best_row.high_mhz:.3f}-{best_row.low_mhz:.3f}':>23} {best_row.center_mhz:>9.3f} {best_row.status:>8} "
|
||||
f"{fmt(best_row.dbfs, '.2f'):>9} {fmt(best_row.rms, '.6f'):>10} {fmt(best_row.power_lin, '.8f'):>12} "
|
||||
f"{best_row.samples:>5} {best_age:>5} pass={best_row.pass_no}"
|
||||
)
|
||||
elif current_row is not None:
|
||||
current_age = "-" if current_row.updated_at <= 0 else f"{(now-current_row.updated_at):.1f}"
|
||||
print(
|
||||
f"{'':>3} {'MAX':>3} "
|
||||
f"{f'{current_row.high_mhz:.3f}-{current_row.low_mhz:.3f}':>23} {current_row.center_mhz:>9.3f} {'INIT':>8} "
|
||||
f"{fmt(None, '.2f'):>9} {fmt(None, '.6f'):>10} {fmt(None, '.8f'):>12} "
|
||||
f"{0:>5} {current_age:>5} no successful windows yet"
|
||||
)
|
||||
print("Ctrl+C to stop. Window width equals step; sample-rate must be >= step to cover each window.")
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="Retune one HackRF across a wide frequency range and measure energy")
|
||||
parser.add_argument("--serial", required=True, help="HackRF serial number from hackrf_info")
|
||||
parser.add_argument("--sample-rate", type=float, required=True, help="Sample rate in Hz")
|
||||
parser.add_argument("--base", type=float, required=True, help="Scan start edge in MHz")
|
||||
parser.add_argument("--roof", type=float, required=True, help="Scan end edge in MHz")
|
||||
parser.add_argument("--step", type=float, required=True, help="Window width / retune step in MHz")
|
||||
parser.add_argument("--vec-len", type=int, default=4096, help="Probe vector length")
|
||||
parser.add_argument("--settle", type=float, default=0.12, help="Wait time after retune before reading (s)")
|
||||
parser.add_argument("--avg-reads", type=int, default=3, help="How many probe reads to average per window")
|
||||
parser.add_argument("--pause-between-reads", type=float, default=0.02, help="Pause between averaged reads (s)")
|
||||
parser.add_argument("--passes", type=int, default=0, help="Number of sweep passes, 0 means infinite")
|
||||
parser.add_argument("--gain", type=float, default=16.0, help="General gain")
|
||||
parser.add_argument("--if-gain", type=float, default=16.0, help="IF gain")
|
||||
parser.add_argument("--bb-gain", type=float, default=16.0, help="BB gain")
|
||||
return parser
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = build_parser().parse_args()
|
||||
serial = args.serial.lower()
|
||||
|
||||
try:
|
||||
windows = build_windows(args.base, args.roof, args.step)
|
||||
except ValueError as exc:
|
||||
print(f"invalid scan range: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
step_hz = args.step * 1e6
|
||||
if args.sample_rate < step_hz:
|
||||
print(
|
||||
f"sample-rate {args.sample_rate:.0f} Hz is smaller than step window {step_hz:.0f} Hz; "
|
||||
"this would leave gaps in the scan",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 2
|
||||
|
||||
try:
|
||||
serial_to_index = parse_hackrf_info()
|
||||
except Exception as exc:
|
||||
print(f"hackrf discovery failed: {exc}", file=sys.stderr)
|
||||
return 3
|
||||
|
||||
index = serial_to_index.get(serial)
|
||||
if index is None:
|
||||
print(f"serial {serial} not found in hackrf_info", file=sys.stderr)
|
||||
print("available serials:", file=sys.stderr)
|
||||
for item_serial, item_index in sorted(serial_to_index.items(), key=lambda item: item[1]):
|
||||
print(f" idx={item_index} serial={item_serial}", file=sys.stderr)
|
||||
return 4
|
||||
|
||||
stop_requested = False
|
||||
|
||||
def on_signal(signum, frame):
|
||||
nonlocal stop_requested
|
||||
stop_requested = True
|
||||
|
||||
signal.signal(signal.SIGINT, on_signal)
|
||||
signal.signal(signal.SIGTERM, on_signal)
|
||||
|
||||
probe: Optional[WideProbeTop] = None
|
||||
started_at = time.time()
|
||||
pass_no = 0
|
||||
current_seq = windows[0].seq
|
||||
|
||||
try:
|
||||
probe = WideProbeTop(
|
||||
index=index,
|
||||
center_freq_hz=windows[0].center_mhz * 1e6,
|
||||
sample_rate=args.sample_rate,
|
||||
vec_len=args.vec_len,
|
||||
gain=args.gain,
|
||||
if_gain=args.if_gain,
|
||||
bb_gain=args.bb_gain,
|
||||
)
|
||||
probe.start()
|
||||
time.sleep(max(args.settle, 0.12))
|
||||
|
||||
while not stop_requested:
|
||||
pass_no += 1
|
||||
for row in windows:
|
||||
if stop_requested:
|
||||
break
|
||||
current_seq = row.seq
|
||||
try:
|
||||
probe.tune(row.center_mhz * 1e6)
|
||||
rms, power_lin, dbfs, samples = probe.read_window(
|
||||
settle=args.settle,
|
||||
avg_reads=args.avg_reads,
|
||||
pause_between_reads=args.pause_between_reads,
|
||||
)
|
||||
row.status = "OK"
|
||||
row.rms = rms
|
||||
row.power_lin = power_lin
|
||||
row.dbfs = dbfs
|
||||
row.samples = samples
|
||||
row.error = ""
|
||||
row.updated_at = time.time()
|
||||
row.pass_no = pass_no
|
||||
except Exception as exc:
|
||||
row.status = "ERR"
|
||||
row.error = str(exc)
|
||||
row.updated_at = time.time()
|
||||
render(
|
||||
windows=windows,
|
||||
serial=serial,
|
||||
index=index,
|
||||
sample_rate=args.sample_rate,
|
||||
base_mhz=args.base,
|
||||
roof_mhz=args.roof,
|
||||
step_mhz=args.step,
|
||||
started_at=started_at,
|
||||
pass_no=pass_no,
|
||||
current_seq=current_seq,
|
||||
)
|
||||
if args.passes > 0 and pass_no >= args.passes:
|
||||
break
|
||||
except Exception as exc:
|
||||
print(f"scanner failed: {exc}", file=sys.stderr)
|
||||
return 5
|
||||
finally:
|
||||
if probe is not None:
|
||||
try:
|
||||
probe.stop()
|
||||
probe.wait()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import math
|
||||
import re
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
try:
|
||||
import numpy as np
|
||||
except Exception as exc:
|
||||
print(f"numpy import failed: {exc}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
from gnuradio import blocks, gr
|
||||
import osmosdr
|
||||
except Exception as exc:
|
||||
print(f"gnuradio/osmosdr import failed: {exc}", file=sys.stderr)
|
||||
print("Run with the SDR venv, e.g. .venv-sdr/bin/python read_energy_wide.py", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
EPS = 1e-20
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScanWindow:
|
||||
seq: int
|
||||
start_mhz: float
|
||||
end_mhz: float
|
||||
low_mhz: float
|
||||
high_mhz: float
|
||||
center_mhz: float
|
||||
status: str = "INIT"
|
||||
rms: Optional[float] = None
|
||||
power_lin: Optional[float] = None
|
||||
dbfs: Optional[float] = None
|
||||
samples: int = 0
|
||||
updated_at: float = 0.0
|
||||
error: str = ""
|
||||
pass_no: int = 0
|
||||
|
||||
|
||||
class WideProbeTop(gr.top_block):
|
||||
def __init__(
|
||||
self,
|
||||
index: int,
|
||||
center_freq_hz: float,
|
||||
sample_rate: float,
|
||||
vec_len: int,
|
||||
gain: float,
|
||||
if_gain: float,
|
||||
bb_gain: float,
|
||||
):
|
||||
super().__init__("hackrf_energy_wide_probe")
|
||||
self.probe = blocks.probe_signal_vc(vec_len)
|
||||
self.stream_to_vec = blocks.stream_to_vector(gr.sizeof_gr_complex * 1, vec_len)
|
||||
self.src = osmosdr.source(args=f"numchan=1 hackrf={index}")
|
||||
self.src.set_time_unknown_pps(osmosdr.time_spec_t())
|
||||
self.src.set_sample_rate(sample_rate)
|
||||
self.src.set_center_freq(center_freq_hz, 0)
|
||||
try:
|
||||
self.src.set_freq_corr(0, 0)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
self.src.set_gain_mode(False, 0)
|
||||
except Exception:
|
||||
pass
|
||||
for fn, val in (("set_gain", gain), ("set_if_gain", if_gain), ("set_bb_gain", bb_gain)):
|
||||
try:
|
||||
getattr(self.src, fn)(val, 0)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
self.src.set_bandwidth(0, 0)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
self.src.set_antenna("", 0)
|
||||
except Exception:
|
||||
pass
|
||||
self.connect((self.src, 0), (self.stream_to_vec, 0))
|
||||
self.connect((self.stream_to_vec, 0), (self.probe, 0))
|
||||
|
||||
def tune(self, freq_hz: float) -> None:
|
||||
self.src.set_center_freq(freq_hz, 0)
|
||||
|
||||
def read_metrics(self) -> Tuple[float, float, float, int]:
|
||||
arr = np.asarray(self.probe.level(), dtype=np.complex64)
|
||||
if arr.size == 0:
|
||||
raise RuntimeError("no samples")
|
||||
power_lin = float(np.mean(arr.real * arr.real + arr.imag * arr.imag))
|
||||
rms = math.sqrt(max(power_lin, 0.0))
|
||||
dbfs = 10.0 * math.log10(max(power_lin, EPS))
|
||||
return rms, power_lin, dbfs, int(arr.size)
|
||||
|
||||
def read_window(self, settle: float, avg_reads: int, pause_between_reads: float) -> Tuple[float, float, float, int]:
|
||||
if settle > 0:
|
||||
time.sleep(settle)
|
||||
|
||||
read_count = max(1, avg_reads)
|
||||
powers: List[float] = []
|
||||
sample_sizes: List[int] = []
|
||||
last_error: Optional[Exception] = None
|
||||
|
||||
for idx in range(read_count):
|
||||
deadline = time.time() + 1.0
|
||||
while True:
|
||||
try:
|
||||
_, power_lin, _, samples = self.read_metrics()
|
||||
powers.append(power_lin)
|
||||
sample_sizes.append(samples)
|
||||
break
|
||||
except Exception as exc:
|
||||
last_error = exc
|
||||
if time.time() >= deadline:
|
||||
raise RuntimeError(str(last_error) if last_error else "no samples")
|
||||
time.sleep(0.02)
|
||||
if idx + 1 < read_count and pause_between_reads > 0:
|
||||
time.sleep(pause_between_reads)
|
||||
|
||||
power_lin = float(sum(powers) / len(powers))
|
||||
rms = math.sqrt(max(power_lin, 0.0))
|
||||
dbfs = 10.0 * math.log10(max(power_lin, EPS))
|
||||
samples = int(sum(sample_sizes) / len(sample_sizes))
|
||||
return rms, power_lin, dbfs, samples
|
||||
|
||||
|
||||
def parse_hackrf_info() -> Dict[str, int]:
|
||||
try:
|
||||
proc = subprocess.run(["hackrf_info"], capture_output=True, text=True, timeout=15)
|
||||
except FileNotFoundError:
|
||||
raise RuntimeError("hackrf_info not found")
|
||||
except subprocess.TimeoutExpired:
|
||||
raise RuntimeError("hackrf_info timeout")
|
||||
text = (proc.stdout or "") + "\n" + (proc.stderr or "")
|
||||
out: Dict[str, int] = {}
|
||||
cur_idx: Optional[int] = None
|
||||
for line in text.splitlines():
|
||||
m = re.search(r"^Index:\s*(\d+)", line)
|
||||
if m:
|
||||
cur_idx = int(m.group(1))
|
||||
continue
|
||||
m = re.search(r"^Serial number:\s*([0-9a-fA-F]+)", line)
|
||||
if m and cur_idx is not None:
|
||||
out[m.group(1).lower()] = cur_idx
|
||||
if not out:
|
||||
raise RuntimeError("no devices parsed from hackrf_info")
|
||||
return out
|
||||
|
||||
|
||||
def fmt(value: Optional[float], spec: str) -> str:
|
||||
return "-" if value is None else format(value, spec)
|
||||
|
||||
|
||||
def build_windows(base_mhz: float, roof_mhz: float, step_mhz: float) -> List[ScanWindow]:
|
||||
if step_mhz <= 0:
|
||||
raise ValueError("step must be > 0")
|
||||
if base_mhz == roof_mhz:
|
||||
raise ValueError("base and roof must be different")
|
||||
|
||||
direction = -1.0 if roof_mhz < base_mhz else 1.0
|
||||
edge = base_mhz
|
||||
seq = 1
|
||||
windows: List[ScanWindow] = []
|
||||
|
||||
while True:
|
||||
next_edge = edge + direction * step_mhz
|
||||
if direction < 0 and next_edge < roof_mhz:
|
||||
next_edge = roof_mhz
|
||||
if direction > 0 and next_edge > roof_mhz:
|
||||
next_edge = roof_mhz
|
||||
|
||||
low_mhz = min(edge, next_edge)
|
||||
high_mhz = max(edge, next_edge)
|
||||
center_mhz = (low_mhz + high_mhz) / 2.0
|
||||
windows.append(
|
||||
ScanWindow(
|
||||
seq=seq,
|
||||
start_mhz=edge,
|
||||
end_mhz=next_edge,
|
||||
low_mhz=low_mhz,
|
||||
high_mhz=high_mhz,
|
||||
center_mhz=center_mhz,
|
||||
)
|
||||
)
|
||||
|
||||
if next_edge == roof_mhz:
|
||||
break
|
||||
edge = next_edge
|
||||
seq += 1
|
||||
|
||||
return windows
|
||||
|
||||
|
||||
def render(
|
||||
windows: List[ScanWindow],
|
||||
serial: str,
|
||||
index: int,
|
||||
sample_rate: float,
|
||||
base_mhz: float,
|
||||
roof_mhz: float,
|
||||
step_mhz: float,
|
||||
started_at: float,
|
||||
pass_no: int,
|
||||
current_seq: int,
|
||||
) -> None:
|
||||
now = time.time()
|
||||
capture_bw_mhz = sample_rate / 1e6
|
||||
current_row = next((row for row in windows if row.seq == current_seq), None)
|
||||
best_row = max(
|
||||
(row for row in windows if row.status == "OK" and row.dbfs is not None),
|
||||
key=lambda row: row.dbfs if row.dbfs is not None else float("-inf"),
|
||||
default=None,
|
||||
)
|
||||
print("\x1b[2J\x1b[H", end="")
|
||||
print("HackRF Wide Energy Monitor (relative power: RMS / linear / dBFS)")
|
||||
print(
|
||||
f"serial: {serial} | idx: {index} | sample-rate: {capture_bw_mhz:.3f} MHz | "
|
||||
f"scan: {base_mhz:.3f}->{roof_mhz:.3f} MHz step {step_mhz:.3f} MHz | "
|
||||
f"pass: {pass_no} | uptime: {int(now-started_at)}s | {time.strftime('%Y-%m-%d %H:%M:%S')}"
|
||||
)
|
||||
print()
|
||||
header = (
|
||||
f"{'cur':>3} {'seq':>3} {'window MHz':>23} {'center':>9} {'status':>8} "
|
||||
f"{'dBFS':>9} {'rms':>10} {'power':>12} {'N':>5} {'age':>5} error"
|
||||
)
|
||||
print(header)
|
||||
print("-" * len(header))
|
||||
for row in windows:
|
||||
age = "-" if row.updated_at <= 0 else f"{(now-row.updated_at):.1f}"
|
||||
err = row.error
|
||||
if len(err) > 50:
|
||||
err = err[:47] + "..."
|
||||
marker = ">>>" if row.seq == current_seq else ""
|
||||
print(
|
||||
f"{marker:>3} {row.seq:>3} "
|
||||
f"{f'{row.high_mhz:.3f}-{row.low_mhz:.3f}':>23} {row.center_mhz:>9.3f} {row.status:>8} "
|
||||
f"{fmt(row.dbfs, '.2f'):>9} {fmt(row.rms, '.6f'):>10} {fmt(row.power_lin, '.8f'):>12} "
|
||||
f"{row.samples:>5} {age:>5} {err}"
|
||||
)
|
||||
print()
|
||||
if best_row is not None:
|
||||
best_age = "-" if best_row.updated_at <= 0 else f"{(now-best_row.updated_at):.1f}"
|
||||
print(
|
||||
f"{'':>3} {'MAX':>3} "
|
||||
f"{f'{best_row.high_mhz:.3f}-{best_row.low_mhz:.3f}':>23} {best_row.center_mhz:>9.3f} {best_row.status:>8} "
|
||||
f"{fmt(best_row.dbfs, '.2f'):>9} {fmt(best_row.rms, '.6f'):>10} {fmt(best_row.power_lin, '.8f'):>12} "
|
||||
f"{best_row.samples:>5} {best_age:>5} pass={best_row.pass_no}"
|
||||
)
|
||||
elif current_row is not None:
|
||||
current_age = "-" if current_row.updated_at <= 0 else f"{(now-current_row.updated_at):.1f}"
|
||||
print(
|
||||
f"{'':>3} {'MAX':>3} "
|
||||
f"{f'{current_row.high_mhz:.3f}-{current_row.low_mhz:.3f}':>23} {current_row.center_mhz:>9.3f} {'INIT':>8} "
|
||||
f"{fmt(None, '.2f'):>9} {fmt(None, '.6f'):>10} {fmt(None, '.8f'):>12} "
|
||||
f"{0:>5} {current_age:>5} no successful windows yet"
|
||||
)
|
||||
print("Ctrl+C to stop. Window width equals step; sample-rate must be >= step to cover each window.")
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="Retune one HackRF across a wide frequency range and measure energy")
|
||||
parser.add_argument("--serial", required=True, help="HackRF serial number from hackrf_info")
|
||||
parser.add_argument("--sample-rate", type=float, required=True, help="Sample rate in Hz")
|
||||
parser.add_argument("--base", type=float, required=True, help="Scan start edge in MHz")
|
||||
parser.add_argument("--roof", type=float, required=True, help="Scan end edge in MHz")
|
||||
parser.add_argument("--step", type=float, required=True, help="Window width / retune step in MHz")
|
||||
parser.add_argument("--vec-len", type=int, default=4096, help="Probe vector length")
|
||||
parser.add_argument("--settle", type=float, default=0.12, help="Wait time after retune before reading (s)")
|
||||
parser.add_argument("--avg-reads", type=int, default=3, help="How many probe reads to average per window")
|
||||
parser.add_argument("--pause-between-reads", type=float, default=0.02, help="Pause between averaged reads (s)")
|
||||
parser.add_argument("--passes", type=int, default=0, help="Number of sweep passes, 0 means infinite")
|
||||
parser.add_argument("--gain", type=float, default=16.0, help="General gain")
|
||||
parser.add_argument("--if-gain", type=float, default=16.0, help="IF gain")
|
||||
parser.add_argument("--bb-gain", type=float, default=16.0, help="BB gain")
|
||||
return parser
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = build_parser().parse_args()
|
||||
serial = args.serial.lower()
|
||||
|
||||
try:
|
||||
windows = build_windows(args.base, args.roof, args.step)
|
||||
except ValueError as exc:
|
||||
print(f"invalid scan range: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
step_hz = args.step * 1e6
|
||||
if args.sample_rate < step_hz:
|
||||
print(
|
||||
f"sample-rate {args.sample_rate:.0f} Hz is smaller than step window {step_hz:.0f} Hz; "
|
||||
"this would leave gaps in the scan",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 2
|
||||
|
||||
try:
|
||||
serial_to_index = parse_hackrf_info()
|
||||
except Exception as exc:
|
||||
print(f"hackrf discovery failed: {exc}", file=sys.stderr)
|
||||
return 3
|
||||
|
||||
index = serial_to_index.get(serial)
|
||||
if index is None:
|
||||
print(f"serial {serial} not found in hackrf_info", file=sys.stderr)
|
||||
print("available serials:", file=sys.stderr)
|
||||
for item_serial, item_index in sorted(serial_to_index.items(), key=lambda item: item[1]):
|
||||
print(f" idx={item_index} serial={item_serial}", file=sys.stderr)
|
||||
return 4
|
||||
|
||||
stop_requested = False
|
||||
|
||||
def on_signal(signum, frame):
|
||||
nonlocal stop_requested
|
||||
stop_requested = True
|
||||
|
||||
signal.signal(signal.SIGINT, on_signal)
|
||||
signal.signal(signal.SIGTERM, on_signal)
|
||||
|
||||
probe: Optional[WideProbeTop] = None
|
||||
started_at = time.time()
|
||||
pass_no = 0
|
||||
current_seq = windows[0].seq
|
||||
|
||||
try:
|
||||
probe = WideProbeTop(
|
||||
index=index,
|
||||
center_freq_hz=windows[0].center_mhz * 1e6,
|
||||
sample_rate=args.sample_rate,
|
||||
vec_len=args.vec_len,
|
||||
gain=args.gain,
|
||||
if_gain=args.if_gain,
|
||||
bb_gain=args.bb_gain,
|
||||
)
|
||||
probe.start()
|
||||
time.sleep(max(args.settle, 0.12))
|
||||
|
||||
while not stop_requested:
|
||||
pass_no += 1
|
||||
for row in windows:
|
||||
if stop_requested:
|
||||
break
|
||||
current_seq = row.seq
|
||||
try:
|
||||
probe.tune(row.center_mhz * 1e6)
|
||||
rms, power_lin, dbfs, samples = probe.read_window(
|
||||
settle=args.settle,
|
||||
avg_reads=args.avg_reads,
|
||||
pause_between_reads=args.pause_between_reads,
|
||||
)
|
||||
row.status = "OK"
|
||||
row.rms = rms
|
||||
row.power_lin = power_lin
|
||||
row.dbfs = dbfs
|
||||
row.samples = samples
|
||||
row.error = ""
|
||||
row.updated_at = time.time()
|
||||
row.pass_no = pass_no
|
||||
except Exception as exc:
|
||||
row.status = "ERR"
|
||||
row.error = str(exc)
|
||||
row.updated_at = time.time()
|
||||
render(
|
||||
windows=windows,
|
||||
serial=serial,
|
||||
index=index,
|
||||
sample_rate=args.sample_rate,
|
||||
base_mhz=args.base,
|
||||
roof_mhz=args.roof,
|
||||
step_mhz=args.step,
|
||||
started_at=started_at,
|
||||
pass_no=pass_no,
|
||||
current_seq=current_seq,
|
||||
)
|
||||
if args.passes > 0 and pass_no >= args.passes:
|
||||
break
|
||||
except Exception as exc:
|
||||
print(f"scanner failed: {exc}", file=sys.stderr)
|
||||
return 5
|
||||
finally:
|
||||
if probe is not None:
|
||||
try:
|
||||
probe.stop()
|
||||
probe.wait()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@ -0,0 +1,99 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from gnuradio import blocks
|
||||
from gnuradio import gr
|
||||
from gnuradio import zeromq
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
import embedded_868 as my_freq
|
||||
|
||||
from common.runtime import load_root_env
|
||||
from common.shared_stream_addrs import SHARED_868_ADDR, SHARED_VECTOR_LEN
|
||||
|
||||
|
||||
load_root_env(__file__)
|
||||
|
||||
|
||||
class get_center_freq(gr.top_block):
|
||||
def __init__(self):
|
||||
gr.top_block.__init__(self, 'get_center_freq')
|
||||
|
||||
self.prob_freq = 0
|
||||
self.poll_rate = 10000
|
||||
self.vector_len = SHARED_VECTOR_LEN
|
||||
self.center_freq = 0
|
||||
self.shared_addr = SHARED_868_ADDR
|
||||
self._stop_polling = threading.Event()
|
||||
self._prob_freq_thread = None
|
||||
|
||||
self.probSigVec = blocks.probe_signal_vc(self.vector_len)
|
||||
self.shared_source_0 = zeromq.pull_source(
|
||||
gr.sizeof_gr_complex,
|
||||
self.vector_len,
|
||||
self.shared_addr,
|
||||
100,
|
||||
False,
|
||||
-1,
|
||||
False,
|
||||
)
|
||||
self.connect((self.shared_source_0, 0), (self.probSigVec, 0))
|
||||
|
||||
def start_polling(self):
|
||||
if self._prob_freq_thread is not None:
|
||||
return
|
||||
|
||||
def _prob_freq_probe():
|
||||
while not self._stop_polling.is_set():
|
||||
self.set_prob_freq(self.probSigVec.level())
|
||||
time.sleep(1.0 / self.poll_rate)
|
||||
|
||||
self._prob_freq_thread = threading.Thread(target=_prob_freq_probe, daemon=True)
|
||||
self._prob_freq_thread.start()
|
||||
|
||||
def get_prob_freq(self):
|
||||
return self.prob_freq
|
||||
|
||||
def set_prob_freq(self, prob_freq):
|
||||
self.prob_freq = prob_freq
|
||||
self.center_freq = my_freq.work(self.prob_freq)
|
||||
|
||||
def get_center_freq(self):
|
||||
return self.center_freq
|
||||
|
||||
def set_center_freq(self, center_freq):
|
||||
self.center_freq = center_freq
|
||||
|
||||
def close(self):
|
||||
self._stop_polling.set()
|
||||
self.stop()
|
||||
self.wait()
|
||||
|
||||
|
||||
def main(top_block_cls=get_center_freq, options=None):
|
||||
tb = top_block_cls()
|
||||
|
||||
def sig_handler(sig=None, frame=None):
|
||||
tb.close()
|
||||
sys.exit(0)
|
||||
|
||||
signal.signal(signal.SIGINT, sig_handler)
|
||||
signal.signal(signal.SIGTERM, sig_handler)
|
||||
|
||||
tb.start()
|
||||
tb.start_polling()
|
||||
try:
|
||||
print('shared_pull_addr:', SHARED_868_ADDR)
|
||||
print('debug_flag:', my_freq.debug_flag)
|
||||
print('save_data_flag:', my_freq.save_data_flag)
|
||||
print('send_to_module_flag:', my_freq.send_to_module_flag)
|
||||
except EOFError:
|
||||
pass
|
||||
tb.wait()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -0,0 +1,122 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from gnuradio import blocks
|
||||
from gnuradio import gr
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
import osmosdr
|
||||
|
||||
from common.runtime import load_root_env, resolve_hackrf_index
|
||||
from common.shared_stream_addrs import SHARED_VECTOR_LEN
|
||||
from shared_router_868_915 import SharedRouter868915
|
||||
|
||||
|
||||
load_root_env(__file__)
|
||||
|
||||
|
||||
def get_hack_id():
|
||||
return resolve_hackrf_index('hack_868', 'src/main_868_915_router.py')
|
||||
|
||||
|
||||
class get_center_freq(gr.top_block):
|
||||
def __init__(self):
|
||||
gr.top_block.__init__(self, 'get_center_freq')
|
||||
|
||||
self.prob_freq = 0
|
||||
self.poll_rate = 10000
|
||||
self.vector_len = SHARED_VECTOR_LEN
|
||||
self.router = SharedRouter868915()
|
||||
self.active_lane = self.router.get_active_name()
|
||||
self.center_freq = self.router.get_start_freq()
|
||||
self._stop_polling = threading.Event()
|
||||
self._prob_freq_thread = None
|
||||
|
||||
self.probSigVec = blocks.probe_signal_vc(self.vector_len)
|
||||
self.rtlsdr_source_0 = osmosdr.source(
|
||||
args='numchan=' + str(1) + ' ' + 'hackrf=' + get_hack_id()
|
||||
)
|
||||
self.rtlsdr_source_0.set_time_unknown_pps(osmosdr.time_spec_t())
|
||||
self.rtlsdr_source_0.set_freq_corr(0, 0)
|
||||
self.rtlsdr_source_0.set_antenna('', 0)
|
||||
self.rtlsdr_source_0.set_min_output_buffer(self.vector_len)
|
||||
self.apply_active_frontend()
|
||||
self.rtlsdr_source_0.set_center_freq(self.center_freq, 0)
|
||||
|
||||
self.blocks_stream_to_vector_1 = blocks.stream_to_vector(gr.sizeof_gr_complex * 1, self.vector_len)
|
||||
self.connect((self.rtlsdr_source_0, 0), (self.blocks_stream_to_vector_1, 0))
|
||||
self.connect((self.blocks_stream_to_vector_1, 0), (self.probSigVec, 0))
|
||||
|
||||
def start_polling(self):
|
||||
if self._prob_freq_thread is not None:
|
||||
return
|
||||
|
||||
def _prob_freq_probe():
|
||||
while not self._stop_polling.is_set():
|
||||
self.set_prob_freq(self.probSigVec.level())
|
||||
time.sleep(1.0 / self.poll_rate)
|
||||
|
||||
self._prob_freq_thread = threading.Thread(target=_prob_freq_probe, daemon=True)
|
||||
self._prob_freq_thread.start()
|
||||
|
||||
def apply_active_frontend(self):
|
||||
frontend = self.router.get_active_frontend()
|
||||
self.rtlsdr_source_0.set_sample_rate(frontend['sample_rate'])
|
||||
self.rtlsdr_source_0.set_gain(frontend['gain'], 0)
|
||||
self.rtlsdr_source_0.set_if_gain(frontend['if_gain'], 0)
|
||||
self.rtlsdr_source_0.set_bb_gain(frontend['bb_gain'], 0)
|
||||
self.rtlsdr_source_0.set_bandwidth(frontend['bandwidth'], 0)
|
||||
|
||||
def get_prob_freq(self):
|
||||
return self.prob_freq
|
||||
|
||||
def set_prob_freq(self, prob_freq):
|
||||
self.prob_freq = prob_freq
|
||||
next_center, lane_switched = self.router.route_vector(self.prob_freq)
|
||||
if lane_switched:
|
||||
self.active_lane = self.router.get_active_name()
|
||||
self.apply_active_frontend()
|
||||
if next_center != self.center_freq:
|
||||
self.set_center_freq(next_center)
|
||||
|
||||
def get_center_freq(self):
|
||||
return self.center_freq
|
||||
|
||||
def set_center_freq(self, center_freq):
|
||||
self.center_freq = center_freq
|
||||
self.rtlsdr_source_0.set_center_freq(self.center_freq, 0)
|
||||
|
||||
def close(self):
|
||||
self._stop_polling.set()
|
||||
try:
|
||||
self.router.close()
|
||||
finally:
|
||||
self.stop()
|
||||
self.wait()
|
||||
|
||||
|
||||
def main(top_block_cls=get_center_freq, options=None):
|
||||
tb = top_block_cls()
|
||||
|
||||
def sig_handler(sig=None, frame=None):
|
||||
tb.close()
|
||||
sys.exit(0)
|
||||
|
||||
signal.signal(signal.SIGINT, sig_handler)
|
||||
signal.signal(signal.SIGTERM, sig_handler)
|
||||
|
||||
tb.start()
|
||||
tb.start_polling()
|
||||
try:
|
||||
print('shared_router_active_lane:', tb.router.get_active_name())
|
||||
print('shared_router_start_freq:', tb.get_center_freq())
|
||||
except EOFError:
|
||||
pass
|
||||
tb.wait()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -0,0 +1,173 @@
|
||||
import os
|
||||
|
||||
import numpy as np
|
||||
import zmq
|
||||
|
||||
from common.runtime import load_root_env
|
||||
from common.shared_stream_addrs import SHARED_868_ADDR, SHARED_915_ADDR
|
||||
from core.multichannelswitcher import MultiChannel
|
||||
from core.sig_n_medi_collect import Signal
|
||||
|
||||
|
||||
load_root_env(__file__)
|
||||
|
||||
|
||||
class Scheduler868:
|
||||
def __init__(self):
|
||||
self.signal_length = int(os.getenv('signal_length_868'))
|
||||
self.multi_channel = MultiChannel(
|
||||
[*map(float, os.getenv('f_step_868').split())],
|
||||
[*map(float, os.getenv('f_bases_868').split())],
|
||||
[*map(float, os.getenv('f_roofs_868').split())],
|
||||
)
|
||||
self.base_freq = float(self.multi_channel.init_f())
|
||||
self.signal = Signal()
|
||||
|
||||
def get_current_freq(self):
|
||||
return float(self.multi_channel.get_cur_channel())
|
||||
|
||||
def process(self, lvl):
|
||||
current_before = self.get_current_freq()
|
||||
metric = self.signal.fill_signal(lvl, self.signal_length)
|
||||
if metric == 0:
|
||||
return current_before, False
|
||||
|
||||
next_freq = float(self.multi_channel.change_channel())
|
||||
self.signal.clear()
|
||||
lane_complete = next_freq == self.base_freq and current_before != self.base_freq
|
||||
return next_freq, lane_complete
|
||||
|
||||
|
||||
class Scheduler915:
|
||||
def __init__(self):
|
||||
self.porog = float(os.getenv('POROG_915'))
|
||||
self.point_amount = 100_000
|
||||
self.split_size = 400_000
|
||||
self.show_amount = int(0.8 * self.point_amount)
|
||||
self.f_base = 0.91e9
|
||||
self.f_step = 20e6
|
||||
self.f_roof = 0.98e9
|
||||
self.f = self.f_base
|
||||
self.channel = 1
|
||||
self.flag = 0
|
||||
self.signal_arr = np.array([], dtype=np.complex64)
|
||||
|
||||
def get_current_freq(self):
|
||||
return float(self.f)
|
||||
|
||||
def _median(self, sig):
|
||||
samples = np.asarray(np.abs(np.array(sig, dtype=np.complex64)), dtype=np.float32)
|
||||
sorted_samples = sorted(samples)
|
||||
median = abs(float(np.median(sorted_samples[self.show_amount:])))
|
||||
self.flag = 0 if self.porog > median else 1
|
||||
|
||||
def _advance(self):
|
||||
next_freq = self.f + self.f_step
|
||||
if next_freq >= self.f_roof:
|
||||
self.f = self.f_base
|
||||
self.channel = 1
|
||||
return float(self.f), True
|
||||
|
||||
self.f = next_freq
|
||||
self.channel += 1
|
||||
return float(self.f), False
|
||||
|
||||
def process(self, lvl):
|
||||
y = np.asarray(lvl, dtype=np.complex64).ravel()
|
||||
self.signal_arr = np.concatenate((self.signal_arr, y), axis=None)
|
||||
|
||||
if self.flag == 0 and len(self.signal_arr) >= self.point_amount:
|
||||
self._median(self.signal_arr[:self.point_amount])
|
||||
self.signal_arr = np.array([], dtype=np.complex64)
|
||||
if self.flag == 0:
|
||||
return self._advance()
|
||||
|
||||
if len(self.signal_arr) >= self.split_size:
|
||||
self.flag = 0
|
||||
self.signal_arr = np.array([], dtype=np.complex64)
|
||||
return self._advance()
|
||||
|
||||
return float(self.f), False
|
||||
|
||||
|
||||
class SharedRouter868915:
|
||||
def __init__(self):
|
||||
self.frontends = {
|
||||
'868': {
|
||||
'sample_rate': 20e6,
|
||||
'gain': 24,
|
||||
'if_gain': 24,
|
||||
'bb_gain': 100,
|
||||
'bandwidth': 0,
|
||||
},
|
||||
'915': {
|
||||
'sample_rate': 20e6,
|
||||
'gain': 16,
|
||||
'if_gain': 16,
|
||||
'bb_gain': 0,
|
||||
'bandwidth': 0,
|
||||
},
|
||||
}
|
||||
self.schedulers = {
|
||||
'868': Scheduler868(),
|
||||
'915': Scheduler915(),
|
||||
}
|
||||
self.context = zmq.Context.instance()
|
||||
self.sockets = {
|
||||
'868': self._make_socket(SHARED_868_ADDR),
|
||||
'915': self._make_socket(SHARED_915_ADDR),
|
||||
}
|
||||
self.active_name = '868'
|
||||
self.switch_counter = 0
|
||||
|
||||
def _make_socket(self, addr):
|
||||
sock = self.context.socket(zmq.PUSH)
|
||||
sock.setsockopt(zmq.SNDHWM, 64)
|
||||
sock.setsockopt(zmq.LINGER, 0)
|
||||
sock.setsockopt(zmq.IMMEDIATE, 1)
|
||||
sock.bind(addr)
|
||||
return sock
|
||||
|
||||
def close(self):
|
||||
for sock in self.sockets.values():
|
||||
try:
|
||||
sock.close(0)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def get_active_name(self):
|
||||
return self.active_name
|
||||
|
||||
def get_active_freq(self):
|
||||
return self.schedulers[self.active_name].get_current_freq()
|
||||
|
||||
def get_active_frontend(self):
|
||||
return dict(self.frontends[self.active_name])
|
||||
|
||||
def get_start_freq(self):
|
||||
return self.get_active_freq()
|
||||
|
||||
def route_vector(self, lvl):
|
||||
vector = np.asarray(lvl, dtype=np.complex64).ravel()
|
||||
active_name = self.active_name
|
||||
|
||||
try:
|
||||
self.sockets[active_name].send(vector.tobytes(), zmq.NOBLOCK)
|
||||
except zmq.Again:
|
||||
pass
|
||||
|
||||
next_freq, lane_complete = self.schedulers[active_name].process(vector)
|
||||
lane_switched = False
|
||||
|
||||
if lane_complete:
|
||||
previous = self.active_name
|
||||
self.active_name = '915' if self.active_name == '868' else '868'
|
||||
next_freq = self.get_active_freq()
|
||||
self.switch_counter += 1
|
||||
lane_switched = True
|
||||
print(
|
||||
f'[shared-router-868-915] switch#{self.switch_counter}: '
|
||||
f'{previous} -> {self.active_name}, tune={next_freq}'
|
||||
)
|
||||
|
||||
return float(next_freq), lane_switched
|
||||
Binary file not shown.
@ -0,0 +1,284 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "4fdb98fc-65bb-467e-be0c-168fee9b0fca",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"cuda:0\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"<contextlib.ExitStack at 0x272800ef510>"
|
||||
]
|
||||
},
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import pandas as pd\n",
|
||||
"import numpy as np\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"import time\n",
|
||||
"import io\n",
|
||||
"import cv2\n",
|
||||
"import copy\n",
|
||||
"import os\n",
|
||||
"from tqdm import tqdm\n",
|
||||
"import torch.nn as nn\n",
|
||||
"import torch\n",
|
||||
"import torchvision\n",
|
||||
"from torch.utils.data import Dataset\n",
|
||||
"from torch import default_generator, randperm\n",
|
||||
"from PIL import Image\n",
|
||||
"#from torch._utils import _accumulate\n",
|
||||
"import csv\n",
|
||||
"from torch.utils.data.dataset import Subset\n",
|
||||
"from scipy import ndimage\n",
|
||||
"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
|
||||
"print(device)\n",
|
||||
"batch_size = 16\n",
|
||||
"momentum=0.9\n",
|
||||
"lr = 1e-3\n",
|
||||
"import random\n",
|
||||
"sub_sample = 0.5\n",
|
||||
"import matplotlib\n",
|
||||
"import gc\n",
|
||||
"import torchsig.utils as u\n",
|
||||
"import torchsig.transforms.transforms as T\n",
|
||||
"matplotlib.use('Agg')\n",
|
||||
"import matplotlib as mpl\n",
|
||||
"mpl.rcParams['agg.path.chunksize'] = 256*256\n",
|
||||
"plt.ioff()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "4848b066-2e09-4c1c-b8fa-8e3fa84d907a",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"s = T.Spectrogram(nperseg=1024)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "9267fbe1",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def sig2pic_with_spec(path_to_data, filename, specT=None,figsize=(16,16), dpi=16, resize = None):\n",
|
||||
" try:\n",
|
||||
" if specT is None:\n",
|
||||
" specT = T.Spectrogram(nperseg=1024)\n",
|
||||
" with open(path_to_data + filename, 'rb') as file:\n",
|
||||
" tmp = np.frombuffer(file.read(), dtype=np.complex64)\n",
|
||||
" signal = tmp\n",
|
||||
" spectr = np.array(specT(signal)['data']['samples'][:,:figsize[0] * dpi])\n",
|
||||
" mag = np.abs(signal)\n",
|
||||
" real = signal.real\n",
|
||||
"\n",
|
||||
" fig2 = plt.figure(figsize = figsize)\n",
|
||||
" plt.axes(ylim=(-1, 1))\n",
|
||||
"\n",
|
||||
" plt.plot(real, color='black')\n",
|
||||
" plt.gca().set_axis_off()\n",
|
||||
" plt.subplots_adjust(top = 1, bottom = 0, right = 1, left = 0, hspace = 0, wspace = 0)\n",
|
||||
" plt.margins(0,0)\n",
|
||||
" buf2 = io.BytesIO()\n",
|
||||
" fig2.savefig(buf2, format=\"png\", dpi=dpi)\n",
|
||||
" buf2.seek(0)\n",
|
||||
" img_arr2 = np.frombuffer(buf2.getvalue(), dtype=np.uint8)\n",
|
||||
" buf2.close()\n",
|
||||
" img2 = cv2.imdecode(img_arr2, 1)\n",
|
||||
" img2 = cv2.cvtColor(img2, cv2.COLOR_BGR2GRAY)\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" plt.close(fig2)\n",
|
||||
"\n",
|
||||
" fig3 = plt.figure(figsize = figsize)\n",
|
||||
" plt.axes(ylim=(-1, 1))\n",
|
||||
"\n",
|
||||
" plt.plot(mag, color='black')\n",
|
||||
" plt.gca().set_axis_off()\n",
|
||||
" plt.subplots_adjust(top = 1, bottom = 0, right = 1, left = 0, hspace = 0, wspace = 0)\n",
|
||||
" plt.margins(0,0)\n",
|
||||
" buf3 = io.BytesIO()\n",
|
||||
" fig3.savefig(buf3, format=\"png\", dpi=dpi)\n",
|
||||
" buf3.seek(0)\n",
|
||||
" img_arr3 = np.frombuffer(buf3.getvalue(), dtype=np.uint8)\n",
|
||||
" buf3.close()\n",
|
||||
" img3 = cv2.imdecode(img_arr3, 1)\n",
|
||||
" img3 = cv2.cvtColor(img3, cv2.COLOR_BGR2GRAY)\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" plt.close(fig3)\n",
|
||||
"\n",
|
||||
" if resize != None:\n",
|
||||
" resized_real = cv2.resize(img2, resize)\n",
|
||||
" resized_mag = cv2.resize(img3, resize)\n",
|
||||
" resized_spectr = cv2.resize(spectr, resize)\n",
|
||||
" img = np.asarray([resized_real, resized_mag, resized_spectr], dtype=np.float32)\n",
|
||||
" return img\n",
|
||||
" img = np.asarray([img2, img3, spectr], dtype=np.float32)\n",
|
||||
" return img\n",
|
||||
" except Exception as e:\n",
|
||||
" print(str(e))\n",
|
||||
" return None"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "448da74a-e0ae-44d8-9877-8dd1f257a24f",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"path_to_binaries = '//192.168.11.63/data/DATASETS/Energomash/1200'\n",
|
||||
"path_to_pictures = '//192.168.11.63/data/DATASETS/Energomash/1200_jpg'"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "ac4945a8-29c4-4da4-945f-08658953e3e5",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from tqdm import tqdm"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "6f226f86-5d72-4573-8af6-750128b70263",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" 0%| | 0/963 [00:00<?, ?it/s]C:\\Users\\snytk\\miniconda3\\envs\\python311\\Lib\\site-packages\\scipy\\signal\\_spectral_py.py:1936: RuntimeWarning: overflow encountered in multiply\n",
|
||||
" result = np.conjugate(result) * result\n",
|
||||
"C:\\Users\\snytk\\miniconda3\\envs\\python311\\Lib\\site-packages\\scipy\\signal\\_spectral_py.py:1938: RuntimeWarning: invalid value encountered in multiply\n",
|
||||
" result *= scale\n",
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████| 963/963 [17:40<00:00, 1.10s/it]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Dir: drone finished!\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████| 963/963 [51:41<00:00, 3.22s/it]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Dir: noise finished!\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"size = (256,256)\n",
|
||||
"if not os.path.exists(path_to_pictures):\n",
|
||||
" os.mkdir(path_to_pictures)\n",
|
||||
"for subdir in os.listdir(path_to_binaries):\n",
|
||||
" filepath = path_to_binaries + '/' + subdir + '/'\n",
|
||||
" if not os.path.exists(path_to_pictures +'/' + subdir):\n",
|
||||
" os.mkdir(path_to_pictures + '/' + subdir)\n",
|
||||
" files = os.listdir(filepath)\n",
|
||||
" for file in tqdm(files):\n",
|
||||
" savepath = path_to_pictures +'/' + subdir + '/' + file + '.npy'\n",
|
||||
" savepath_real_png = path_to_pictures +'/' + subdir + '/' + file + '_real' + '.png' \n",
|
||||
" savepath_imag_png = path_to_pictures +'/' + subdir + '/' + file + '_imag' + '.png' \n",
|
||||
" savepath_spec_png = path_to_pictures +'/' + subdir + '/' + file + '_spec' + '.png'\n",
|
||||
" if not os.path.exists(savepath):\n",
|
||||
" img = sig2pic_with_spec(path_to_data=filepath, filename=file, specT=s, resize = size)\n",
|
||||
" gc.collect()\n",
|
||||
" try:\n",
|
||||
" \n",
|
||||
" plt.imshow(img[0])\n",
|
||||
" plt.savefig(savepath_real_png)\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" \n",
|
||||
" plt.imshow(img[1])\n",
|
||||
" plt.savefig(savepath_imag_png)\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
"\n",
|
||||
" plt.imshow(img[2])\n",
|
||||
" plt.savefig(savepath_spec_png)\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" \n",
|
||||
" np.save(savepath, img)\n",
|
||||
" \n",
|
||||
" except Exception:\n",
|
||||
" continue\n",
|
||||
" print('Dir: ', subdir , ' finished!')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "6f4bf849",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.6"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@ -0,0 +1,270 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "4fdb98fc-65bb-467e-be0c-168fee9b0fca",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"cuda:0\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"<contextlib.ExitStack at 0x25775eabcd0>"
|
||||
]
|
||||
},
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import pandas as pd\n",
|
||||
"import numpy as np\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"import time\n",
|
||||
"import io\n",
|
||||
"import cv2\n",
|
||||
"import copy\n",
|
||||
"import os\n",
|
||||
"from tqdm import tqdm\n",
|
||||
"import torch.nn as nn\n",
|
||||
"import torch\n",
|
||||
"import torchvision\n",
|
||||
"from torch.utils.data import Dataset\n",
|
||||
"from torch import default_generator, randperm\n",
|
||||
"from PIL import Image\n",
|
||||
"#from torch._utils import _accumulate\n",
|
||||
"import csv\n",
|
||||
"from torch.utils.data.dataset import Subset\n",
|
||||
"from scipy import ndimage\n",
|
||||
"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
|
||||
"print(device)\n",
|
||||
"batch_size = 16\n",
|
||||
"momentum=0.9\n",
|
||||
"lr = 1e-3\n",
|
||||
"import random\n",
|
||||
"sub_sample = 0.5\n",
|
||||
"import matplotlib\n",
|
||||
"import gc\n",
|
||||
"import torchsig.utils as u\n",
|
||||
"import torchsig.transforms.transforms as T\n",
|
||||
"matplotlib.use('Agg')\n",
|
||||
"import matplotlib as mpl\n",
|
||||
"mpl.rcParams['agg.path.chunksize'] = 256*256\n",
|
||||
"plt.ioff()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "4848b066-2e09-4c1c-b8fa-8e3fa84d907a",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"s = T.Spectrogram(nperseg=256)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "9267fbe1",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def sig2pic_with_spec(path_to_data, filename, specT=None,figsize=(16,16), dpi=16, resize = None):\n",
|
||||
" try:\n",
|
||||
" if specT is None:\n",
|
||||
" specT = T.Spectrogram(nperseg=256)\n",
|
||||
" with open(path_to_data + filename, 'rb') as file:\n",
|
||||
" tmp = np.frombuffer(file.read(), dtype=np.complex64)\n",
|
||||
" signal = tmp\n",
|
||||
" spectr = np.array(specT(signal)['data']['samples'][:,:figsize[0] * dpi])\n",
|
||||
" mag = np.abs(signal)\n",
|
||||
" real = signal.real\n",
|
||||
"\n",
|
||||
" fig2 = plt.figure(figsize = figsize)\n",
|
||||
" plt.axes(ylim=(-1, 1))\n",
|
||||
"\n",
|
||||
" plt.plot(real, color='black')\n",
|
||||
" plt.gca().set_axis_off()\n",
|
||||
" plt.subplots_adjust(top = 1, bottom = 0, right = 1, left = 0, hspace = 0, wspace = 0)\n",
|
||||
" plt.margins(0,0)\n",
|
||||
" buf2 = io.BytesIO()\n",
|
||||
" fig2.savefig(buf2, format=\"png\", dpi=dpi)\n",
|
||||
" buf2.seek(0)\n",
|
||||
" img_arr2 = np.frombuffer(buf2.getvalue(), dtype=np.uint8)\n",
|
||||
" buf2.close()\n",
|
||||
" img2 = cv2.imdecode(img_arr2, 1)\n",
|
||||
" img2 = cv2.cvtColor(img2, cv2.COLOR_BGR2GRAY)\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" plt.close(fig2)\n",
|
||||
"\n",
|
||||
" fig3 = plt.figure(figsize = figsize)\n",
|
||||
" plt.axes(ylim=(-1, 1))\n",
|
||||
"\n",
|
||||
" plt.plot(mag, color='black')\n",
|
||||
" plt.gca().set_axis_off()\n",
|
||||
" plt.subplots_adjust(top = 1, bottom = 0, right = 1, left = 0, hspace = 0, wspace = 0)\n",
|
||||
" plt.margins(0,0)\n",
|
||||
" buf3 = io.BytesIO()\n",
|
||||
" fig3.savefig(buf3, format=\"png\", dpi=dpi)\n",
|
||||
" buf3.seek(0)\n",
|
||||
" img_arr3 = np.frombuffer(buf3.getvalue(), dtype=np.uint8)\n",
|
||||
" buf3.close()\n",
|
||||
" img3 = cv2.imdecode(img_arr3, 1)\n",
|
||||
" img3 = cv2.cvtColor(img3, cv2.COLOR_BGR2GRAY)\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" plt.close(fig3)\n",
|
||||
"\n",
|
||||
" if resize != None:\n",
|
||||
" resized_real = cv2.resize(img2, resize)\n",
|
||||
" resized_mag = cv2.resize(img3, resize)\n",
|
||||
" resized_spectr = cv2.resize(spectr, resize)\n",
|
||||
" img = np.asarray([resized_real, resized_mag, resized_spectr], dtype=np.float32)\n",
|
||||
" return img\n",
|
||||
" img = np.asarray([img2, img3, spectr], dtype=np.float32)\n",
|
||||
" return img\n",
|
||||
" except Exception as e:\n",
|
||||
" print(str(e))\n",
|
||||
" return None"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "448da74a-e0ae-44d8-9877-8dd1f257a24f",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"path_to_binaries = '//192.168.11.63/data/DATASETS/Energomash/2400'\n",
|
||||
"path_to_pictures = '//192.168.11.63/data/DATASETS/Energomash/2400_jpg'"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "ac4945a8-29c4-4da4-945f-08658953e3e5",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from tqdm import tqdm"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "6f226f86-5d72-4573-8af6-750128b70263",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" 0%| | 0/965 [00:00<?, ?it/s]C:\\Users\\snytk\\miniconda3\\envs\\python311\\Lib\\site-packages\\scipy\\signal\\_spectral_py.py:1936: RuntimeWarning: overflow encountered in multiply\n",
|
||||
" result = np.conjugate(result) * result\n",
|
||||
"C:\\Users\\snytk\\miniconda3\\envs\\python311\\Lib\\site-packages\\scipy\\signal\\_spectral_py.py:1938: RuntimeWarning: invalid value encountered in multiply\n",
|
||||
" result *= scale\n",
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████| 965/965 [28:11<00:00, 1.75s/it]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Dir: noise finished!\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"size = (256,256)\n",
|
||||
"if not os.path.exists(path_to_pictures):\n",
|
||||
" os.mkdir(path_to_pictures)\n",
|
||||
"for subdir in os.listdir(path_to_binaries):\n",
|
||||
" filepath = path_to_binaries + '/' + subdir + '/'\n",
|
||||
" if not os.path.exists(path_to_pictures +'/' + subdir):\n",
|
||||
" os.mkdir(path_to_pictures + '/' + subdir)\n",
|
||||
" files = os.listdir(filepath)\n",
|
||||
" for file in tqdm(files):\n",
|
||||
" savepath = path_to_pictures +'/' + subdir + '/' + file + '.npy'\n",
|
||||
" savepath_real_png = path_to_pictures +'/' + subdir + '/' + file + '_real' + '.png' \n",
|
||||
" savepath_imag_png = path_to_pictures +'/' + subdir + '/' + file + '_imag' + '.png' \n",
|
||||
" savepath_spec_png = path_to_pictures +'/' + subdir + '/' + file + '_spec' + '.png'\n",
|
||||
" if not os.path.exists(savepath):\n",
|
||||
" img = sig2pic_with_spec(path_to_data=filepath, filename=file, specT=s, resize = size)\n",
|
||||
" gc.collect()\n",
|
||||
" try:\n",
|
||||
" \n",
|
||||
" plt.imshow(img[0])\n",
|
||||
" plt.savefig(savepath_real_png)\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" \n",
|
||||
" plt.imshow(img[1])\n",
|
||||
" plt.savefig(savepath_imag_png)\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
"\n",
|
||||
" plt.imshow(img[2])\n",
|
||||
" plt.savefig(savepath_spec_png)\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" \n",
|
||||
" np.save(savepath, img)\n",
|
||||
" \n",
|
||||
" except Exception:\n",
|
||||
" continue\n",
|
||||
" print('Dir: ', subdir , ' finished!')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "106b1add",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.6"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@ -0,0 +1,270 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "4fdb98fc-65bb-467e-be0c-168fee9b0fca",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"cuda:0\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"<contextlib.ExitStack at 0x25775eabcd0>"
|
||||
]
|
||||
},
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import pandas as pd\n",
|
||||
"import numpy as np\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"import time\n",
|
||||
"import io\n",
|
||||
"import cv2\n",
|
||||
"import copy\n",
|
||||
"import os\n",
|
||||
"from tqdm import tqdm\n",
|
||||
"import torch.nn as nn\n",
|
||||
"import torch\n",
|
||||
"import torchvision\n",
|
||||
"from torch.utils.data import Dataset\n",
|
||||
"from torch import default_generator, randperm\n",
|
||||
"from PIL import Image\n",
|
||||
"#from torch._utils import _accumulate\n",
|
||||
"import csv\n",
|
||||
"from torch.utils.data.dataset import Subset\n",
|
||||
"from scipy import ndimage\n",
|
||||
"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
|
||||
"print(device)\n",
|
||||
"batch_size = 16\n",
|
||||
"momentum=0.9\n",
|
||||
"lr = 1e-3\n",
|
||||
"import random\n",
|
||||
"sub_sample = 0.5\n",
|
||||
"import matplotlib\n",
|
||||
"import gc\n",
|
||||
"import torchsig.utils as u\n",
|
||||
"import torchsig.transforms.transforms as T\n",
|
||||
"matplotlib.use('Agg')\n",
|
||||
"import matplotlib as mpl\n",
|
||||
"mpl.rcParams['agg.path.chunksize'] = 256*256\n",
|
||||
"plt.ioff()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "4848b066-2e09-4c1c-b8fa-8e3fa84d907a",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"s = T.Spectrogram(nperseg=256)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "9267fbe1",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def sig2pic_with_spec(path_to_data, filename, specT=None,figsize=(16,16), dpi=16, resize = None):\n",
|
||||
" try:\n",
|
||||
" if specT is None:\n",
|
||||
" specT = T.Spectrogram(nperseg=256)\n",
|
||||
" with open(path_to_data + filename, 'rb') as file:\n",
|
||||
" tmp = np.frombuffer(file.read(), dtype=np.complex64)\n",
|
||||
" signal = tmp\n",
|
||||
" spectr = np.array(specT(signal)['data']['samples'][:,:figsize[0] * dpi])\n",
|
||||
" mag = np.abs(signal)\n",
|
||||
" real = signal.real\n",
|
||||
"\n",
|
||||
" fig2 = plt.figure(figsize = figsize)\n",
|
||||
" plt.axes(ylim=(-1, 1))\n",
|
||||
"\n",
|
||||
" plt.plot(real, color='black')\n",
|
||||
" plt.gca().set_axis_off()\n",
|
||||
" plt.subplots_adjust(top = 1, bottom = 0, right = 1, left = 0, hspace = 0, wspace = 0)\n",
|
||||
" plt.margins(0,0)\n",
|
||||
" buf2 = io.BytesIO()\n",
|
||||
" fig2.savefig(buf2, format=\"png\", dpi=dpi)\n",
|
||||
" buf2.seek(0)\n",
|
||||
" img_arr2 = np.frombuffer(buf2.getvalue(), dtype=np.uint8)\n",
|
||||
" buf2.close()\n",
|
||||
" img2 = cv2.imdecode(img_arr2, 1)\n",
|
||||
" img2 = cv2.cvtColor(img2, cv2.COLOR_BGR2GRAY)\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" plt.close(fig2)\n",
|
||||
"\n",
|
||||
" fig3 = plt.figure(figsize = figsize)\n",
|
||||
" plt.axes(ylim=(-1, 1))\n",
|
||||
"\n",
|
||||
" plt.plot(mag, color='black')\n",
|
||||
" plt.gca().set_axis_off()\n",
|
||||
" plt.subplots_adjust(top = 1, bottom = 0, right = 1, left = 0, hspace = 0, wspace = 0)\n",
|
||||
" plt.margins(0,0)\n",
|
||||
" buf3 = io.BytesIO()\n",
|
||||
" fig3.savefig(buf3, format=\"png\", dpi=dpi)\n",
|
||||
" buf3.seek(0)\n",
|
||||
" img_arr3 = np.frombuffer(buf3.getvalue(), dtype=np.uint8)\n",
|
||||
" buf3.close()\n",
|
||||
" img3 = cv2.imdecode(img_arr3, 1)\n",
|
||||
" img3 = cv2.cvtColor(img3, cv2.COLOR_BGR2GRAY)\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" plt.close(fig3)\n",
|
||||
"\n",
|
||||
" if resize != None:\n",
|
||||
" resized_real = cv2.resize(img2, resize)\n",
|
||||
" resized_mag = cv2.resize(img3, resize)\n",
|
||||
" resized_spectr = cv2.resize(spectr, resize)\n",
|
||||
" img = np.asarray([resized_real, resized_mag, resized_spectr], dtype=np.float32)\n",
|
||||
" return img\n",
|
||||
" img = np.asarray([img2, img3, spectr], dtype=np.float32)\n",
|
||||
" return img\n",
|
||||
" except Exception as e:\n",
|
||||
" print(str(e))\n",
|
||||
" return None"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "448da74a-e0ae-44d8-9877-8dd1f257a24f",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"path_to_binaries = '//192.168.11.63/data/DATASETS/Energomash/2400'\n",
|
||||
"path_to_pictures = '//192.168.11.63/data/DATASETS/Energomash/2400_jpg'"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "ac4945a8-29c4-4da4-945f-08658953e3e5",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from tqdm import tqdm"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "6f226f86-5d72-4573-8af6-750128b70263",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" 0%| | 0/965 [00:00<?, ?it/s]C:\\Users\\snytk\\miniconda3\\envs\\python311\\Lib\\site-packages\\scipy\\signal\\_spectral_py.py:1936: RuntimeWarning: overflow encountered in multiply\n",
|
||||
" result = np.conjugate(result) * result\n",
|
||||
"C:\\Users\\snytk\\miniconda3\\envs\\python311\\Lib\\site-packages\\scipy\\signal\\_spectral_py.py:1938: RuntimeWarning: invalid value encountered in multiply\n",
|
||||
" result *= scale\n",
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████| 965/965 [28:11<00:00, 1.75s/it]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Dir: noise finished!\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"size = (256,256)\n",
|
||||
"if not os.path.exists(path_to_pictures):\n",
|
||||
" os.mkdir(path_to_pictures)\n",
|
||||
"for subdir in os.listdir(path_to_binaries):\n",
|
||||
" filepath = path_to_binaries + '/' + subdir + '/'\n",
|
||||
" if not os.path.exists(path_to_pictures +'/' + subdir):\n",
|
||||
" os.mkdir(path_to_pictures + '/' + subdir)\n",
|
||||
" files = os.listdir(filepath)\n",
|
||||
" for file in tqdm(files):\n",
|
||||
" savepath = path_to_pictures +'/' + subdir + '/' + file + '.npy'\n",
|
||||
" savepath_real_png = path_to_pictures +'/' + subdir + '/' + file + '_real' + '.png' \n",
|
||||
" savepath_imag_png = path_to_pictures +'/' + subdir + '/' + file + '_imag' + '.png' \n",
|
||||
" savepath_spec_png = path_to_pictures +'/' + subdir + '/' + file + '_spec' + '.png'\n",
|
||||
" if not os.path.exists(savepath):\n",
|
||||
" img = sig2pic_with_spec(path_to_data=filepath, filename=file, specT=s, resize = size)\n",
|
||||
" gc.collect()\n",
|
||||
" try:\n",
|
||||
" \n",
|
||||
" plt.imshow(img[0])\n",
|
||||
" plt.savefig(savepath_real_png)\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" \n",
|
||||
" plt.imshow(img[1])\n",
|
||||
" plt.savefig(savepath_imag_png)\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
"\n",
|
||||
" plt.imshow(img[2])\n",
|
||||
" plt.savefig(savepath_spec_png)\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" \n",
|
||||
" np.save(savepath, img)\n",
|
||||
" \n",
|
||||
" except Exception:\n",
|
||||
" continue\n",
|
||||
" print('Dir: ', subdir , ' finished!')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "106b1add",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.6"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@ -0,0 +1,428 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "4fdb98fc-65bb-467e-be0c-168fee9b0fca",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"cuda:0\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"<contextlib.ExitStack at 0x73285e4f6300>"
|
||||
]
|
||||
},
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import pandas as pd\n",
|
||||
"import numpy as np\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"import time\n",
|
||||
"import io\n",
|
||||
"import cv2\n",
|
||||
"import copy\n",
|
||||
"import os\n",
|
||||
"from tqdm import tqdm\n",
|
||||
"import torch.nn as nn\n",
|
||||
"import torch\n",
|
||||
"import torchvision\n",
|
||||
"from torch.utils.data import Dataset\n",
|
||||
"from torch import default_generator, randperm\n",
|
||||
"from PIL import Image\n",
|
||||
"#from torch._utils import _accumulate\n",
|
||||
"import csv\n",
|
||||
"from torch.utils.data.dataset import Subset\n",
|
||||
"from scipy import ndimage\n",
|
||||
"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
|
||||
"print(device)\n",
|
||||
"batch_size = 16\n",
|
||||
"momentum=0.9\n",
|
||||
"lr = 1e-3\n",
|
||||
"import random\n",
|
||||
"sub_sample = 0.5\n",
|
||||
"import matplotlib\n",
|
||||
"import gc\n",
|
||||
"import torchsig.utils as u\n",
|
||||
"import torchsig.transforms.transforms as T\n",
|
||||
"matplotlib.use('Agg')\n",
|
||||
"import matplotlib as mpl\n",
|
||||
"mpl.rcParams['agg.path.chunksize'] = 256*256\n",
|
||||
"plt.ioff()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "4848b066-2e09-4c1c-b8fa-8e3fa84d907a",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"s = T.Spectrogram(nperseg=256)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "9267fbe1",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def sig2pic_with_spec(path_to_data, filename, specT=None, figsize=(16,16), dpi=16, resize = None):\n",
|
||||
" def standartize_signal(signal):\n",
|
||||
" mean = np.mean(signal)\n",
|
||||
" std = np.std(signal)\n",
|
||||
" standardized_signal = (signal - mean) / std\n",
|
||||
" return standardized_signal\n",
|
||||
" \n",
|
||||
" try:\n",
|
||||
" if specT is None:\n",
|
||||
" specT = T.Spectrogram(nperseg=256)\n",
|
||||
" with open(path_to_data + filename, 'rb') as file:\n",
|
||||
" tmp = np.frombuffer(file.read(), dtype=np.complex64)\n",
|
||||
" signal = tmp\n",
|
||||
" print(len(signal))\n",
|
||||
" spectr = np.array(specT(signal)['data']['samples'][:,:figsize[0] * dpi])\n",
|
||||
" mag = np.abs(signal)\n",
|
||||
" mag = standartize_signal(mag)\n",
|
||||
" real = signal.real\n",
|
||||
" real = standartize_signal(real)\n",
|
||||
"\n",
|
||||
" fig2 = plt.figure(figsize = figsize)\n",
|
||||
" plt.axes(ylim=(-1, 1))\n",
|
||||
"\n",
|
||||
" plt.plot(real, color='black')\n",
|
||||
" plt.gca().set_axis_off()\n",
|
||||
" plt.subplots_adjust(top = 1, bottom = 0, right = 1, left = 0, hspace = 0, wspace = 0)\n",
|
||||
" plt.margins(0,0)\n",
|
||||
" buf2 = io.BytesIO()\n",
|
||||
" fig2.savefig(buf2, format=\"png\", dpi=dpi)\n",
|
||||
" buf2.seek(0)\n",
|
||||
" img_arr2 = np.frombuffer(buf2.getvalue(), dtype=np.uint8)\n",
|
||||
" buf2.close()\n",
|
||||
" img2 = cv2.imdecode(img_arr2, 1)\n",
|
||||
" img2 = cv2.cvtColor(img2, cv2.COLOR_BGR2GRAY)\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" plt.close(fig2)\n",
|
||||
"\n",
|
||||
" fig3 = plt.figure(figsize = figsize)\n",
|
||||
" plt.axes(ylim=(-1, 1))\n",
|
||||
"\n",
|
||||
" plt.plot(mag, color='black')\n",
|
||||
" plt.gca().set_axis_off()\n",
|
||||
" plt.subplots_adjust(top = 1, bottom = 0, right = 1, left = 0, hspace = 0, wspace = 0)\n",
|
||||
" plt.margins(0,0)\n",
|
||||
" buf3 = io.BytesIO()\n",
|
||||
" fig3.savefig(buf3, format=\"png\", dpi=dpi)\n",
|
||||
" buf3.seek(0)\n",
|
||||
" img_arr3 = np.frombuffer(buf3.getvalue(), dtype=np.uint8)\n",
|
||||
" buf3.close()\n",
|
||||
" img3 = cv2.imdecode(img_arr3, 1)\n",
|
||||
" img3 = cv2.cvtColor(img3, cv2.COLOR_BGR2GRAY)\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" plt.close(fig3)\n",
|
||||
"\n",
|
||||
" if resize != None:\n",
|
||||
" resized_real = cv2.resize(img2, resize)\n",
|
||||
" resized_mag = cv2.resize(img3, resize)\n",
|
||||
" resized_spectr = cv2.resize(spectr, resize)\n",
|
||||
" img = np.asarray([resized_real, resized_mag, resized_spectr], dtype=np.float32)\n",
|
||||
" return img\n",
|
||||
" img = np.asarray([img2, img3, spectr], dtype=np.float32)\n",
|
||||
" return img\n",
|
||||
" except Exception as e:\n",
|
||||
" print(str(e))\n",
|
||||
" return None\n",
|
||||
"\n",
|
||||
"def plot_signal_and_magnitude(path_to_data, filename, filename_signal):\n",
|
||||
" def remove_outliers(signal, threshold):\n",
|
||||
" filtered_signal = np.where(np.abs(signal) <= threshold, signal, np.nan)\n",
|
||||
" return np.nan_to_num(filtered_signal)\n",
|
||||
" \n",
|
||||
" def standartize_signal(signal):\n",
|
||||
" mean = np.mean(signal)\n",
|
||||
" std = np.std(signal)\n",
|
||||
" standardized_signal = (signal - mean) / std\n",
|
||||
" return standardized_signal\n",
|
||||
" \n",
|
||||
" with open(path_to_data + filename, 'rb') as file:\n",
|
||||
" signal = np.frombuffer(file.read(), dtype=np.complex64)\n",
|
||||
" print(max(np.real(signal)))\n",
|
||||
" print(signal[:100])\n",
|
||||
" plt.figure(figsize=(12, 6))\n",
|
||||
" plt.subplot(2, 1, 1)\n",
|
||||
" plt.plot(remove_outliers(standartize_signal(np.real(signal)),1)[10000:], label='Real Part')\n",
|
||||
" plt.plot(remove_outliers(standartize_signal(np.imag(signal)),1)[10000:], label='Imaginary Part')\n",
|
||||
" plt.title('QAM Signal')\n",
|
||||
" plt.legend()\n",
|
||||
" plt.subplot(2, 1, 2)\n",
|
||||
" plt.plot(np.abs(signal), label='Magnitude')\n",
|
||||
" plt.title('Magnitude of QAM Signal')\n",
|
||||
" plt.legend()\n",
|
||||
" plt.tight_layout()\n",
|
||||
" plt.savefig(filename_signal)\n",
|
||||
" plt.close()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "448da74a-e0ae-44d8-9877-8dd1f257a24f",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"path_to_binaries = '/home/sibscience/Datasets/915_9K'\n",
|
||||
"path_to_pictures = '/home/sibscience/Datasets/915_9K_jpg'"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "ac4945a8-29c4-4da4-945f-08658953e3e5",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from tqdm import tqdm"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "6a5f4c51",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"ename": "FileNotFoundError",
|
||||
"evalue": "[Errno 2] No such file or directory: '/home/sibscience/Datasets/915_9K_jpg'",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[31m---------------------------------------------------------------------------\u001b[39m",
|
||||
"\u001b[31mFileNotFoundError\u001b[39m Traceback (most recent call last)",
|
||||
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[7]\u001b[39m\u001b[32m, line 3\u001b[39m\n\u001b[32m 1\u001b[39m size = (\u001b[32m256\u001b[39m,\u001b[32m256\u001b[39m)\n\u001b[32m 2\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m os.path.exists(path_to_pictures):\n\u001b[32m----> \u001b[39m\u001b[32m3\u001b[39m \u001b[43mos\u001b[49m\u001b[43m.\u001b[49m\u001b[43mmkdir\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath_to_pictures\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 4\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m subdir \u001b[38;5;129;01min\u001b[39;00m os.listdir(path_to_binaries):\n\u001b[32m 5\u001b[39m filepath = path_to_binaries + \u001b[33m'\u001b[39m\u001b[33m/\u001b[39m\u001b[33m'\u001b[39m + subdir + \u001b[33m'\u001b[39m\u001b[33m/\u001b[39m\u001b[33m'\u001b[39m\n",
|
||||
"\u001b[31mFileNotFoundError\u001b[39m: [Errno 2] No such file or directory: '/home/sibscience/Datasets/915_9K_jpg'"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"size = (256,256)\n",
|
||||
"if not os.path.exists(path_to_pictures):\n",
|
||||
" os.mkdir(path_to_pictures)\n",
|
||||
"for subdir in os.listdir(path_to_binaries):\n",
|
||||
" filepath = path_to_binaries + '/' + subdir + '/'\n",
|
||||
" if not os.path.exists(path_to_pictures +'/' + subdir):\n",
|
||||
" os.mkdir(path_to_pictures + '/' + subdir)\n",
|
||||
" files = os.listdir(filepath)\n",
|
||||
" for file in tqdm(files):\n",
|
||||
" savepath = path_to_pictures +'/' + subdir + '/' + file + '.npy'\n",
|
||||
" savepath_real_png = path_to_pictures +'/' + subdir + '/' + file + '.png' \n",
|
||||
" if not os.path.exists(savepath):\n",
|
||||
" img = plot_signal_and_magnitude(path_to_data=filepath, filename=file, filename_signal= savepath_real_png)\n",
|
||||
" gc.collect()\n",
|
||||
" print('Dir: ', subdir , ' finished!')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "6f226f86-5d72-4573-8af6-750128b70263",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\r\n",
|
||||
" 0%| | 0/565 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"800016\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\r\n",
|
||||
" 28%|█████████████████████▉ | 157/565 [00:01<00:02, 145.94it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"800016\n",
|
||||
"800016\n",
|
||||
"800016\n",
|
||||
"800016\n",
|
||||
"800016\n",
|
||||
"800016\n",
|
||||
"800016\n",
|
||||
"800016\n",
|
||||
"800016\n",
|
||||
"800016\n",
|
||||
"800016\n",
|
||||
"800016\n",
|
||||
"800016\n",
|
||||
"800016\n",
|
||||
"800016\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\r\n",
|
||||
" 30%|████████████████████████▎ | 172/565 [00:15<00:46, 8.38it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"800016\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" 31%|████████████████████████▍ | 173/565 [00:16<00:37, 10.59it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"800016\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ename": "KeyboardInterrupt",
|
||||
"evalue": "",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[1;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)",
|
||||
"Cell \u001b[1;32mIn[10], line 15\u001b[0m\n\u001b[0;32m 13\u001b[0m savepath_spec_png \u001b[38;5;241m=\u001b[39m path_to_pictures \u001b[38;5;241m+\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/\u001b[39m\u001b[38;5;124m'\u001b[39m \u001b[38;5;241m+\u001b[39m subdir \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/\u001b[39m\u001b[38;5;124m'\u001b[39m \u001b[38;5;241m+\u001b[39m file \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m_spec\u001b[39m\u001b[38;5;124m'\u001b[39m \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m.png\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[0;32m 14\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mexists(savepath):\n\u001b[1;32m---> 15\u001b[0m img \u001b[38;5;241m=\u001b[39m \u001b[43msig2pic_with_spec\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath_to_data\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfilepath\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfilename\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfile\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mspecT\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43ms\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mresize\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43msize\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 16\u001b[0m gc\u001b[38;5;241m.\u001b[39mcollect()\n\u001b[0;32m 17\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n",
|
||||
"Cell \u001b[1;32mIn[7], line 29\u001b[0m, in \u001b[0;36msig2pic_with_spec\u001b[1;34m(path_to_data, filename, specT, figsize, dpi, resize)\u001b[0m\n\u001b[0;32m 27\u001b[0m plt\u001b[38;5;241m.\u001b[39mmargins(\u001b[38;5;241m0\u001b[39m,\u001b[38;5;241m0\u001b[39m)\n\u001b[0;32m 28\u001b[0m buf2 \u001b[38;5;241m=\u001b[39m io\u001b[38;5;241m.\u001b[39mBytesIO()\n\u001b[1;32m---> 29\u001b[0m \u001b[43mfig2\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msavefig\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbuf2\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mformat\u001b[39;49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mpng\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdpi\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdpi\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 30\u001b[0m buf2\u001b[38;5;241m.\u001b[39mseek(\u001b[38;5;241m0\u001b[39m)\n\u001b[0;32m 31\u001b[0m img_arr2 \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mfrombuffer(buf2\u001b[38;5;241m.\u001b[39mgetvalue(), dtype\u001b[38;5;241m=\u001b[39mnp\u001b[38;5;241m.\u001b[39muint8)\n",
|
||||
"File \u001b[1;32m~\\miniconda3\\envs\\python311\\Lib\\site-packages\\matplotlib\\figure.py:3390\u001b[0m, in \u001b[0;36mFigure.savefig\u001b[1;34m(self, fname, transparent, **kwargs)\u001b[0m\n\u001b[0;32m 3388\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m ax \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39maxes:\n\u001b[0;32m 3389\u001b[0m _recursively_make_axes_transparent(stack, ax)\n\u001b[1;32m-> 3390\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcanvas\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mprint_figure\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfname\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
|
||||
"File \u001b[1;32m~\\miniconda3\\envs\\python311\\Lib\\site-packages\\matplotlib\\backend_bases.py:2193\u001b[0m, in \u001b[0;36mFigureCanvasBase.print_figure\u001b[1;34m(self, filename, dpi, facecolor, edgecolor, orientation, format, bbox_inches, pad_inches, bbox_extra_artists, backend, **kwargs)\u001b[0m\n\u001b[0;32m 2189\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m 2190\u001b[0m \u001b[38;5;66;03m# _get_renderer may change the figure dpi (as vector formats\u001b[39;00m\n\u001b[0;32m 2191\u001b[0m \u001b[38;5;66;03m# force the figure dpi to 72), so we need to set it again here.\u001b[39;00m\n\u001b[0;32m 2192\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m cbook\u001b[38;5;241m.\u001b[39m_setattr_cm(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfigure, dpi\u001b[38;5;241m=\u001b[39mdpi):\n\u001b[1;32m-> 2193\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[43mprint_method\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m 2194\u001b[0m \u001b[43m \u001b[49m\u001b[43mfilename\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 2195\u001b[0m \u001b[43m \u001b[49m\u001b[43mfacecolor\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfacecolor\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 2196\u001b[0m \u001b[43m \u001b[49m\u001b[43medgecolor\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43medgecolor\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 2197\u001b[0m \u001b[43m \u001b[49m\u001b[43morientation\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43morientation\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 2198\u001b[0m \u001b[43m \u001b[49m\u001b[43mbbox_inches_restore\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_bbox_inches_restore\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 2199\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 2200\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[0;32m 2201\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m bbox_inches \u001b[38;5;129;01mand\u001b[39;00m restore_bbox:\n",
|
||||
"File \u001b[1;32m~\\miniconda3\\envs\\python311\\Lib\\site-packages\\matplotlib\\backend_bases.py:2043\u001b[0m, in \u001b[0;36mFigureCanvasBase._switch_canvas_and_return_print_method.<locals>.<lambda>\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m 2039\u001b[0m optional_kws \u001b[38;5;241m=\u001b[39m { \u001b[38;5;66;03m# Passed by print_figure for other renderers.\u001b[39;00m\n\u001b[0;32m 2040\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdpi\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfacecolor\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124medgecolor\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124morientation\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m 2041\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mbbox_inches_restore\u001b[39m\u001b[38;5;124m\"\u001b[39m}\n\u001b[0;32m 2042\u001b[0m skip \u001b[38;5;241m=\u001b[39m optional_kws \u001b[38;5;241m-\u001b[39m {\u001b[38;5;241m*\u001b[39minspect\u001b[38;5;241m.\u001b[39msignature(meth)\u001b[38;5;241m.\u001b[39mparameters}\n\u001b[1;32m-> 2043\u001b[0m print_method \u001b[38;5;241m=\u001b[39m functools\u001b[38;5;241m.\u001b[39mwraps(meth)(\u001b[38;5;28;01mlambda\u001b[39;00m \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs: \u001b[43mmeth\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m 2044\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43m{\u001b[49m\u001b[43mk\u001b[49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mv\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mv\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mkwargs\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mitems\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mnot\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mskip\u001b[49m\u001b[43m}\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[0;32m 2045\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m: \u001b[38;5;66;03m# Let third-parties do as they see fit.\u001b[39;00m\n\u001b[0;32m 2046\u001b[0m print_method \u001b[38;5;241m=\u001b[39m meth\n",
|
||||
"File \u001b[1;32m~\\miniconda3\\envs\\python311\\Lib\\site-packages\\matplotlib\\backends\\backend_agg.py:497\u001b[0m, in \u001b[0;36mFigureCanvasAgg.print_png\u001b[1;34m(self, filename_or_obj, metadata, pil_kwargs)\u001b[0m\n\u001b[0;32m 450\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mprint_png\u001b[39m(\u001b[38;5;28mself\u001b[39m, filename_or_obj, \u001b[38;5;241m*\u001b[39m, metadata\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, pil_kwargs\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[0;32m 451\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m 452\u001b[0m \u001b[38;5;124;03m Write the figure to a PNG file.\u001b[39;00m\n\u001b[0;32m 453\u001b[0m \n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 495\u001b[0m \u001b[38;5;124;03m *metadata*, including the default 'Software' key.\u001b[39;00m\n\u001b[0;32m 496\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[1;32m--> 497\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_print_pil\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfilename_or_obj\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mpng\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpil_kwargs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmetadata\u001b[49m\u001b[43m)\u001b[49m\n",
|
||||
"File \u001b[1;32m~\\miniconda3\\envs\\python311\\Lib\\site-packages\\matplotlib\\backends\\backend_agg.py:445\u001b[0m, in \u001b[0;36mFigureCanvasAgg._print_pil\u001b[1;34m(self, filename_or_obj, fmt, pil_kwargs, metadata)\u001b[0m\n\u001b[0;32m 440\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_print_pil\u001b[39m(\u001b[38;5;28mself\u001b[39m, filename_or_obj, fmt, pil_kwargs, metadata\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[0;32m 441\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m 442\u001b[0m \u001b[38;5;124;03m Draw the canvas, then save it using `.image.imsave` (to which\u001b[39;00m\n\u001b[0;32m 443\u001b[0m \u001b[38;5;124;03m *pil_kwargs* and *metadata* are forwarded).\u001b[39;00m\n\u001b[0;32m 444\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[1;32m--> 445\u001b[0m \u001b[43mFigureCanvasAgg\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdraw\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[0;32m 446\u001b[0m mpl\u001b[38;5;241m.\u001b[39mimage\u001b[38;5;241m.\u001b[39mimsave(\n\u001b[0;32m 447\u001b[0m filename_or_obj, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbuffer_rgba(), \u001b[38;5;28mformat\u001b[39m\u001b[38;5;241m=\u001b[39mfmt, origin\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mupper\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m 448\u001b[0m dpi\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfigure\u001b[38;5;241m.\u001b[39mdpi, metadata\u001b[38;5;241m=\u001b[39mmetadata, pil_kwargs\u001b[38;5;241m=\u001b[39mpil_kwargs)\n",
|
||||
"File \u001b[1;32m~\\miniconda3\\envs\\python311\\Lib\\site-packages\\matplotlib\\backends\\backend_agg.py:388\u001b[0m, in \u001b[0;36mFigureCanvasAgg.draw\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 385\u001b[0m \u001b[38;5;66;03m# Acquire a lock on the shared font cache.\u001b[39;00m\n\u001b[0;32m 386\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtoolbar\u001b[38;5;241m.\u001b[39m_wait_cursor_for_draw_cm() \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtoolbar\n\u001b[0;32m 387\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m nullcontext()):\n\u001b[1;32m--> 388\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfigure\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdraw\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrenderer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 389\u001b[0m \u001b[38;5;66;03m# A GUI class may be need to update a window using this draw, so\u001b[39;00m\n\u001b[0;32m 390\u001b[0m \u001b[38;5;66;03m# don't forget to call the superclass.\u001b[39;00m\n\u001b[0;32m 391\u001b[0m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mdraw()\n",
|
||||
"File \u001b[1;32m~\\miniconda3\\envs\\python311\\Lib\\site-packages\\matplotlib\\artist.py:95\u001b[0m, in \u001b[0;36m_finalize_rasterization.<locals>.draw_wrapper\u001b[1;34m(artist, renderer, *args, **kwargs)\u001b[0m\n\u001b[0;32m 93\u001b[0m \u001b[38;5;129m@wraps\u001b[39m(draw)\n\u001b[0;32m 94\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdraw_wrapper\u001b[39m(artist, renderer, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m---> 95\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[43mdraw\u001b[49m\u001b[43m(\u001b[49m\u001b[43martist\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mrenderer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 96\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m renderer\u001b[38;5;241m.\u001b[39m_rasterizing:\n\u001b[0;32m 97\u001b[0m renderer\u001b[38;5;241m.\u001b[39mstop_rasterizing()\n",
|
||||
"File \u001b[1;32m~\\miniconda3\\envs\\python311\\Lib\\site-packages\\matplotlib\\artist.py:72\u001b[0m, in \u001b[0;36mallow_rasterization.<locals>.draw_wrapper\u001b[1;34m(artist, renderer)\u001b[0m\n\u001b[0;32m 69\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m artist\u001b[38;5;241m.\u001b[39mget_agg_filter() \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m 70\u001b[0m renderer\u001b[38;5;241m.\u001b[39mstart_filter()\n\u001b[1;32m---> 72\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mdraw\u001b[49m\u001b[43m(\u001b[49m\u001b[43martist\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mrenderer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 73\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[0;32m 74\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m artist\u001b[38;5;241m.\u001b[39mget_agg_filter() \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
|
||||
"File \u001b[1;32m~\\miniconda3\\envs\\python311\\Lib\\site-packages\\matplotlib\\figure.py:3154\u001b[0m, in \u001b[0;36mFigure.draw\u001b[1;34m(self, renderer)\u001b[0m\n\u001b[0;32m 3151\u001b[0m \u001b[38;5;66;03m# ValueError can occur when resizing a window.\u001b[39;00m\n\u001b[0;32m 3153\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpatch\u001b[38;5;241m.\u001b[39mdraw(renderer)\n\u001b[1;32m-> 3154\u001b[0m \u001b[43mmimage\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_draw_list_compositing_images\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m 3155\u001b[0m \u001b[43m \u001b[49m\u001b[43mrenderer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43martists\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msuppressComposite\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 3157\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m sfig \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msubfigs:\n\u001b[0;32m 3158\u001b[0m sfig\u001b[38;5;241m.\u001b[39mdraw(renderer)\n",
|
||||
"File \u001b[1;32m~\\miniconda3\\envs\\python311\\Lib\\site-packages\\matplotlib\\image.py:132\u001b[0m, in \u001b[0;36m_draw_list_compositing_images\u001b[1;34m(renderer, parent, artists, suppress_composite)\u001b[0m\n\u001b[0;32m 130\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m not_composite \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m has_images:\n\u001b[0;32m 131\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m a \u001b[38;5;129;01min\u001b[39;00m artists:\n\u001b[1;32m--> 132\u001b[0m \u001b[43ma\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdraw\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrenderer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 133\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m 134\u001b[0m \u001b[38;5;66;03m# Composite any adjacent images together\u001b[39;00m\n\u001b[0;32m 135\u001b[0m image_group \u001b[38;5;241m=\u001b[39m []\n",
|
||||
"File \u001b[1;32m~\\miniconda3\\envs\\python311\\Lib\\site-packages\\matplotlib\\artist.py:72\u001b[0m, in \u001b[0;36mallow_rasterization.<locals>.draw_wrapper\u001b[1;34m(artist, renderer)\u001b[0m\n\u001b[0;32m 69\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m artist\u001b[38;5;241m.\u001b[39mget_agg_filter() \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m 70\u001b[0m renderer\u001b[38;5;241m.\u001b[39mstart_filter()\n\u001b[1;32m---> 72\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mdraw\u001b[49m\u001b[43m(\u001b[49m\u001b[43martist\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mrenderer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 73\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[0;32m 74\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m artist\u001b[38;5;241m.\u001b[39mget_agg_filter() \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
|
||||
"File \u001b[1;32m~\\miniconda3\\envs\\python311\\Lib\\site-packages\\matplotlib\\axes\\_base.py:3070\u001b[0m, in \u001b[0;36m_AxesBase.draw\u001b[1;34m(self, renderer)\u001b[0m\n\u001b[0;32m 3067\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m artists_rasterized:\n\u001b[0;32m 3068\u001b[0m _draw_rasterized(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfigure, artists_rasterized, renderer)\n\u001b[1;32m-> 3070\u001b[0m \u001b[43mmimage\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_draw_list_compositing_images\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m 3071\u001b[0m \u001b[43m \u001b[49m\u001b[43mrenderer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43martists\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfigure\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msuppressComposite\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 3073\u001b[0m renderer\u001b[38;5;241m.\u001b[39mclose_group(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124maxes\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m 3074\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstale \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n",
|
||||
"File \u001b[1;32m~\\miniconda3\\envs\\python311\\Lib\\site-packages\\matplotlib\\image.py:132\u001b[0m, in \u001b[0;36m_draw_list_compositing_images\u001b[1;34m(renderer, parent, artists, suppress_composite)\u001b[0m\n\u001b[0;32m 130\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m not_composite \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m has_images:\n\u001b[0;32m 131\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m a \u001b[38;5;129;01min\u001b[39;00m artists:\n\u001b[1;32m--> 132\u001b[0m \u001b[43ma\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdraw\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrenderer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 133\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m 134\u001b[0m \u001b[38;5;66;03m# Composite any adjacent images together\u001b[39;00m\n\u001b[0;32m 135\u001b[0m image_group \u001b[38;5;241m=\u001b[39m []\n",
|
||||
"File \u001b[1;32m~\\miniconda3\\envs\\python311\\Lib\\site-packages\\matplotlib\\artist.py:72\u001b[0m, in \u001b[0;36mallow_rasterization.<locals>.draw_wrapper\u001b[1;34m(artist, renderer)\u001b[0m\n\u001b[0;32m 69\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m artist\u001b[38;5;241m.\u001b[39mget_agg_filter() \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m 70\u001b[0m renderer\u001b[38;5;241m.\u001b[39mstart_filter()\n\u001b[1;32m---> 72\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mdraw\u001b[49m\u001b[43m(\u001b[49m\u001b[43martist\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mrenderer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 73\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[0;32m 74\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m artist\u001b[38;5;241m.\u001b[39mget_agg_filter() \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
|
||||
"File \u001b[1;32m~\\miniconda3\\envs\\python311\\Lib\\site-packages\\matplotlib\\lines.py:801\u001b[0m, in \u001b[0;36mLine2D.draw\u001b[1;34m(self, renderer)\u001b[0m\n\u001b[0;32m 798\u001b[0m gc\u001b[38;5;241m.\u001b[39mset_foreground(lc_rgba, isRGBA\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[0;32m 800\u001b[0m gc\u001b[38;5;241m.\u001b[39mset_dashes(\u001b[38;5;241m*\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_dash_pattern)\n\u001b[1;32m--> 801\u001b[0m \u001b[43mrenderer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdraw_path\u001b[49m\u001b[43m(\u001b[49m\u001b[43mgc\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtpath\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43maffine\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrozen\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 802\u001b[0m gc\u001b[38;5;241m.\u001b[39mrestore()\n\u001b[0;32m 804\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_marker \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_markersize \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n",
|
||||
"File \u001b[1;32m~\\miniconda3\\envs\\python311\\Lib\\site-packages\\matplotlib\\backends\\backend_agg.py:117\u001b[0m, in \u001b[0;36mRendererAgg.draw_path\u001b[1;34m(self, gc, path, transform, rgbFace)\u001b[0m\n\u001b[0;32m 115\u001b[0m p\u001b[38;5;241m.\u001b[39msimplify_threshold \u001b[38;5;241m=\u001b[39m path\u001b[38;5;241m.\u001b[39msimplify_threshold\n\u001b[0;32m 116\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m--> 117\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_renderer\u001b[38;5;241m.\u001b[39mdraw_path(gc, p, transform, rgbFace)\n\u001b[0;32m 118\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mOverflowError\u001b[39;00m:\n\u001b[0;32m 119\u001b[0m msg \u001b[38;5;241m=\u001b[39m (\n\u001b[0;32m 120\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mExceeded cell block limit in Agg.\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 121\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPlease reduce the value of \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 127\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mpath\u001b[38;5;241m.\u001b[39msimplify_threshold\u001b[38;5;132;01m:\u001b[39;00m\u001b[38;5;124m.2f\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m on the input).\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 128\u001b[0m )\n",
|
||||
"\u001b[1;31mKeyboardInterrupt\u001b[0m: "
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"size = (256,256)\n",
|
||||
"if not os.path.exists(path_to_pictures):\n",
|
||||
" os.mkdir(path_to_pictures)\n",
|
||||
"for subdir in os.listdir(path_to_binaries):\n",
|
||||
" filepath = path_to_binaries + '/' + subdir + '/'\n",
|
||||
" if not os.path.exists(path_to_pictures +'/' + subdir):\n",
|
||||
" os.mkdir(path_to_pictures + '/' + subdir)\n",
|
||||
" files = os.listdir(filepath)\n",
|
||||
" for file in tqdm(files):\n",
|
||||
" savepath = path_to_pictures +'/' + subdir + '/' + file + '.npy'\n",
|
||||
" savepath_real_png = path_to_pictures +'/' + subdir + '/' + file + '_real' + '.png' \n",
|
||||
" savepath_imag_png = path_to_pictures +'/' + subdir + '/' + file + '_imag' + '.png' \n",
|
||||
" savepath_spec_png = path_to_pictures +'/' + subdir + '/' + file + '_spec' + '.png'\n",
|
||||
" if not os.path.exists(savepath):\n",
|
||||
" img = sig2pic_with_spec(path_to_data=filepath, filename=file, specT=s, resize = size)\n",
|
||||
" gc.collect()\n",
|
||||
" try:\n",
|
||||
" \n",
|
||||
" plt.imshow(img[0])\n",
|
||||
" plt.savefig(savepath_real_png)\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" \n",
|
||||
" plt.imshow(img[1])\n",
|
||||
" plt.savefig(savepath_imag_png)\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
"\n",
|
||||
" plt.imshow(img[2])\n",
|
||||
" plt.savefig(savepath_spec_png)\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" \n",
|
||||
" np.save(savepath, img)\n",
|
||||
" \n",
|
||||
" except Exception:\n",
|
||||
" continue\n",
|
||||
" print('Dir: ', subdir , ' finished!')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "58ff5fbd",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "9f9ad366",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": ".venv-train",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.12.3"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,222 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "4fdb98fc-65bb-467e-be0c-168fee9b0fca",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"cuda:0\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"<contextlib.ExitStack at 0x249d580ff50>"
|
||||
]
|
||||
},
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import pandas as pd\n",
|
||||
"import numpy as np\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"import time\n",
|
||||
"import io\n",
|
||||
"import cv2\n",
|
||||
"import copy\n",
|
||||
"import os\n",
|
||||
"from tqdm import tqdm\n",
|
||||
"import torch.nn as nn\n",
|
||||
"import torch\n",
|
||||
"import torchvision\n",
|
||||
"from torch.utils.data import Dataset\n",
|
||||
"from torch import default_generator, randperm\n",
|
||||
"from PIL import Image\n",
|
||||
"#from torch._utils import _accumulate\n",
|
||||
"import csv\n",
|
||||
"from torch.utils.data.dataset import Subset\n",
|
||||
"from scipy import ndimage\n",
|
||||
"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
|
||||
"print(device)\n",
|
||||
"batch_size = 16\n",
|
||||
"momentum=0.9\n",
|
||||
"lr = 1e-3\n",
|
||||
"import random\n",
|
||||
"sub_sample = 0.5\n",
|
||||
"import matplotlib\n",
|
||||
"import gc\n",
|
||||
"matplotlib.use('Agg')\n",
|
||||
"import matplotlib as mpl\n",
|
||||
"mpl.rcParams['agg.path.chunksize'] = 256*256\n",
|
||||
"plt.ioff()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "9267fbe1",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def sig2pic_with_spec(path_to_data, filename, figsize=(16,16), dpi=16, resize = None):\n",
|
||||
" try:\n",
|
||||
" array = np.load(path_to_data+filename)\n",
|
||||
"\n",
|
||||
" if resize != None:\n",
|
||||
" resized_real = cv2.resize(array[0], resize)\n",
|
||||
" resized_mag = cv2.resize(array[1], resize)\n",
|
||||
" resized_spectr = cv2.resize(array[2], resize)\n",
|
||||
" img = np.asarray(array, dtype=np.float32)\n",
|
||||
" return img\n",
|
||||
" \n",
|
||||
" img = np.asarray(array, dtype=np.float32)\n",
|
||||
" return img\n",
|
||||
" \n",
|
||||
" except Exception as e:\n",
|
||||
" print(str(e))\n",
|
||||
" return None"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"id": "448da74a-e0ae-44d8-9877-8dd1f257a24f",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"path_to_binaries = 'C:/Users/snytk/Lerning_NN_for_work/datasets/2.4_learning'\n",
|
||||
"path_to_pictures = 'C:/Users/snytk/Lerning_NN_for_work/datasets_jpg/2.4_jpg_learning'"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"id": "ac4945a8-29c4-4da4-945f-08658953e3e5",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from tqdm import tqdm"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"id": "6f226f86-5d72-4573-8af6-750128b70263",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████| 8751/8751 [1:01:54<00:00, 2.36it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Dir: drone finished!\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████████████████████████████████████████████████████████████████████| 11202/11202 [1:19:30<00:00, 2.35it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Dir: noise finished!\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"size = (256,256)\n",
|
||||
"if not os.path.exists(path_to_pictures):\n",
|
||||
" os.mkdir(path_to_pictures)\n",
|
||||
"for subdir in os.listdir(path_to_binaries):\n",
|
||||
" filepath = path_to_binaries + '/' + subdir + '/'\n",
|
||||
" if not os.path.exists(path_to_pictures +'/' + subdir):\n",
|
||||
" os.mkdir(path_to_pictures + '/' + subdir)\n",
|
||||
" files = os.listdir(filepath)\n",
|
||||
" for file in tqdm(files):\n",
|
||||
" savepath_real_png = path_to_pictures +'/' + subdir + '/' + file + '_real' + '.jpg' \n",
|
||||
" savepath_imag_png = path_to_pictures +'/' + subdir + '/' + file + '_imag' + '.jpg' \n",
|
||||
" savepath_spec_png = path_to_pictures +'/' + subdir + '/' + file + '_spec' + '.jpg'\n",
|
||||
" if not os.path.exists(savepath_real_png):\n",
|
||||
" img = sig2pic_with_spec(path_to_data=filepath, filename=file, resize = size)\n",
|
||||
" gc.collect()\n",
|
||||
" \n",
|
||||
" try:\n",
|
||||
" plt.imshow(img[0])\n",
|
||||
" plt.savefig(savepath_real_png)\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" \n",
|
||||
" plt.imshow(img[1])\n",
|
||||
" plt.savefig(savepath_imag_png)\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
"\n",
|
||||
" plt.imshow(img[2])\n",
|
||||
" plt.savefig(savepath_spec_png)\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" \n",
|
||||
" except Exception:\n",
|
||||
" continue\n",
|
||||
" print('Dir: ', subdir , ' finished!')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "cfbd309d",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.6"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@ -0,0 +1,360 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "4fdb98fc-65bb-467e-be0c-168fee9b0fca",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"cuda:0\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"<contextlib.ExitStack at 0x20af0408250>"
|
||||
]
|
||||
},
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import pandas as pd\n",
|
||||
"import numpy as np\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"import time\n",
|
||||
"import io\n",
|
||||
"import cv2\n",
|
||||
"import copy\n",
|
||||
"import os\n",
|
||||
"from tqdm import tqdm\n",
|
||||
"import torch.nn as nn\n",
|
||||
"import torch\n",
|
||||
"import torchvision\n",
|
||||
"from torch.utils.data import Dataset\n",
|
||||
"from torch import default_generator, randperm\n",
|
||||
"from PIL import Image\n",
|
||||
"#from torch._utils import _accumulate\n",
|
||||
"import csv\n",
|
||||
"from torch.utils.data.dataset import Subset\n",
|
||||
"from scipy import ndimage\n",
|
||||
"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
|
||||
"print(device)\n",
|
||||
"batch_size = 16\n",
|
||||
"momentum=0.9\n",
|
||||
"lr = 1e-3\n",
|
||||
"import random\n",
|
||||
"sub_sample = 0.5\n",
|
||||
"import matplotlib\n",
|
||||
"import gc\n",
|
||||
"matplotlib.use('Agg')\n",
|
||||
"import matplotlib as mpl\n",
|
||||
"mpl.rcParams['agg.path.chunksize'] = 256*256\n",
|
||||
"plt.ioff()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "9267fbe1",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def sig2pic_with_spec(path_to_data, filename, figsize=(16,16), dpi=16, resize = None):\n",
|
||||
" try:\n",
|
||||
" array = np.load(path_to_data+filename)\n",
|
||||
"\n",
|
||||
" if resize != None:\n",
|
||||
" resized_real = cv2.resize(array[0], resize)\n",
|
||||
" resized_mag = cv2.resize(array[1], resize)\n",
|
||||
" resized_spectr = cv2.resize(array[2], resize)\n",
|
||||
" img = np.asarray(array, dtype=np.float32)\n",
|
||||
" return img\n",
|
||||
" \n",
|
||||
" img = np.asarray(array, dtype=np.float32)\n",
|
||||
" return img\n",
|
||||
" \n",
|
||||
" except Exception as e:\n",
|
||||
" print(str(e))\n",
|
||||
" return None\n",
|
||||
" \n",
|
||||
" \n",
|
||||
" \n",
|
||||
"\n",
|
||||
"def pre_func_ensemble(data=None, src ='', ind_inference=0):\n",
|
||||
" try:\n",
|
||||
" import matplotlib.pyplot as plt\n",
|
||||
" matplotlib.use('Agg')\n",
|
||||
" plt.ioff()\n",
|
||||
"\n",
|
||||
" figsize = (16, 16)\n",
|
||||
" dpi = 16\n",
|
||||
"\n",
|
||||
" signal = np.vectorize(complex)(data[0], data[1])\n",
|
||||
" #np.save(src + '_inference_2400_' + str(ind_inference) + '.npy', signal)\n",
|
||||
" spec = transform.Spectrogram(nperseg=256)\n",
|
||||
" spectr = np.array(spec(signal)[:,:figsize[0] * dpi])\n",
|
||||
" fig1 = plt.figure(figsize = figsize)\n",
|
||||
" plt.axes(ylim=(-1, 1))\n",
|
||||
" sigr = signal.real\n",
|
||||
" sigi = signal.imag\n",
|
||||
" \n",
|
||||
" plt.plot(sigr, color='black')\n",
|
||||
" plt.gca().set_axis_off()\n",
|
||||
" plt.subplots_adjust(top = 1, bottom = 0, right = 1, left = 0, hspace = 0, wspace = 0)\n",
|
||||
" plt.margins(0,0)\n",
|
||||
" buf1 = io.BytesIO()\n",
|
||||
" fig1.savefig(buf1, format=\"png\", dpi=dpi)\n",
|
||||
" buf1.seek(0)\n",
|
||||
" img_arr1 = np.frombuffer(buf1.getvalue(), dtype=np.uint8)\n",
|
||||
" buf1.close()\n",
|
||||
" img1 = cv2.imdecode(img_arr1, 1)\n",
|
||||
" img1 = cv2.cvtColor(img1, cv2.COLOR_BGR2GRAY)\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" plt.close(fig1)\n",
|
||||
"\n",
|
||||
" fig2 = plt.figure(figsize = figsize)\n",
|
||||
" plt.axes(ylim=(-1, 1))\n",
|
||||
" sigr = signal.real\n",
|
||||
" sigi = signal.imag\n",
|
||||
" \n",
|
||||
" plt.plot(sigi, color='black')\n",
|
||||
" plt.gca().set_axis_off()\n",
|
||||
" plt.subplots_adjust(top = 1, bottom = 0, right = 1, left = 0, hspace = 0, wspace = 0)\n",
|
||||
" plt.margins(0,0)\n",
|
||||
" buf = io.BytesIO()\n",
|
||||
" fig2.savefig(buf, format=\"png\", dpi=dpi)\n",
|
||||
" buf.seek(0)\n",
|
||||
" img_arr = np.frombuffer(buf.getvalue(), dtype=np.uint8)\n",
|
||||
" buf.close()\n",
|
||||
" img = cv2.imdecode(img_arr, 1)\n",
|
||||
" img2 = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" plt.close(fig2)\n",
|
||||
"\n",
|
||||
" img = np.array([img1, img2, spectr])\n",
|
||||
" \n",
|
||||
" cv2.destroyAllWindows()\n",
|
||||
" del signal\n",
|
||||
" del spec\n",
|
||||
" del spectr\n",
|
||||
" del img1\n",
|
||||
" del img2\n",
|
||||
" del sigr\n",
|
||||
" del sigi\n",
|
||||
" del buf\n",
|
||||
" del buf1\n",
|
||||
" del img_arr\n",
|
||||
" del img_arr1\n",
|
||||
" cv2.destroyAllWindows()\n",
|
||||
" gc.collect()\n",
|
||||
"\n",
|
||||
" print('Подготовка данных завершена')\n",
|
||||
" print()\n",
|
||||
" return img\n",
|
||||
"\n",
|
||||
" except Exception as e:\n",
|
||||
" print(str(e))\n",
|
||||
" return None\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def build_func_ensemble(file_model='', file_config='', num_classes=None):\n",
|
||||
" try:\n",
|
||||
" import matplotlib.pyplot as plt\n",
|
||||
" matplotlib.use('Agg')\n",
|
||||
" plt.ioff()\n",
|
||||
" torch.cuda.empty_cache()\n",
|
||||
" model1 = models.resnet18(pretrained=False)\n",
|
||||
" model2 = models.resnet50(pretrained=False)\n",
|
||||
" model3 = models.resnet101(pretrained=False)\n",
|
||||
"\n",
|
||||
" num_classes = 2\n",
|
||||
"\n",
|
||||
" model1.fc = nn.Linear(model1.fc.in_features, num_classes)\n",
|
||||
" model2.fc = nn.Linear(model2.fc.in_features, num_classes)\n",
|
||||
" model3.fc = nn.Linear(model3.fc.in_features, num_classes)\n",
|
||||
"\n",
|
||||
" class Ensemble(nn.Module):\n",
|
||||
" def __init__(self, model1, model2, model3):\n",
|
||||
" super(Ensemble, self).__init__()\n",
|
||||
" self.model1 = model1\n",
|
||||
" self.model2 = model2\n",
|
||||
" self.model3 = model3\n",
|
||||
" self.fc = nn.Linear(3 * num_classes, num_classes)\n",
|
||||
"\n",
|
||||
" def forward(self, x):\n",
|
||||
" x1 = self.model1(x)\n",
|
||||
" x2 = self.model2(x)\n",
|
||||
" x3 = self.model3(x)\n",
|
||||
" x = torch.cat((x1, x2, x3), dim=1)\n",
|
||||
" x = self.fc(x)\n",
|
||||
" return x\n",
|
||||
"\n",
|
||||
" model = Ensemble(model1, model2, model3)\n",
|
||||
"\n",
|
||||
" device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
|
||||
" if device != 'cpu':\n",
|
||||
" model = model.to(device)\n",
|
||||
" model.load_state_dict(torch.load(file_model, map_location=device))\n",
|
||||
" model.eval()\n",
|
||||
"\n",
|
||||
" cv2.destroyAllWindows()\n",
|
||||
" del model1\n",
|
||||
" del model2\n",
|
||||
" del model3\n",
|
||||
" gc.collect()\n",
|
||||
"\n",
|
||||
" print('Инициализация модели завершена')\n",
|
||||
" print()\n",
|
||||
" return model\n",
|
||||
"\n",
|
||||
" except Exception as exc:\n",
|
||||
" print(str(exc))\n",
|
||||
" return None\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "448da74a-e0ae-44d8-9877-8dd1f257a24f",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"path_to_binaries = 'C:/Users/snytk/Lerning_NN_for_work/datasets/1.2_learning'\n",
|
||||
"path_to_pictures = 'C:/Users/snytk/Lerning_NN_for_work/datasets_jpg/1.2_jpg_learning'"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "ac4945a8-29c4-4da4-945f-08658953e3e5",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from tqdm import tqdm"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "6f226f86-5d72-4573-8af6-750128b70263",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████| 963/963 [06:36<00:00, 2.43it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Dir: drone finished!\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████| 1724/1724 [11:41<00:00, 2.46it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Dir: noise finished!\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"size = (256,256)\n",
|
||||
"if not os.path.exists(path_to_pictures):\n",
|
||||
" os.mkdir(path_to_pictures)\n",
|
||||
"for subdir in os.listdir(path_to_binaries):\n",
|
||||
" filepath = path_to_binaries + '/' + subdir + '/'\n",
|
||||
" if not os.path.exists(path_to_pictures +'/' + subdir):\n",
|
||||
" os.mkdir(path_to_pictures + '/' + subdir)\n",
|
||||
" files = os.listdir(filepath)\n",
|
||||
" for file in tqdm(files):\n",
|
||||
" savepath_real_png = path_to_pictures +'/' + subdir + '/' + file + '_real' + '.jpg' \n",
|
||||
" savepath_imag_png = path_to_pictures +'/' + subdir + '/' + file + '_imag' + '.jpg' \n",
|
||||
" savepath_spec_png = path_to_pictures +'/' + subdir + '/' + file + '_spec' + '.jpg'\n",
|
||||
" if not os.path.exists(savepath_real_png):\n",
|
||||
" img = sig2pic_with_spec(path_to_data=filepath, filename=file, resize = size)\n",
|
||||
" gc.collect()\n",
|
||||
" \n",
|
||||
" try:\n",
|
||||
" plt.imshow(img[0])\n",
|
||||
" plt.savefig(savepath_real_png)\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" \n",
|
||||
" plt.imshow(img[1])\n",
|
||||
" plt.savefig(savepath_imag_png)\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
"\n",
|
||||
" plt.imshow(img[2])\n",
|
||||
" plt.savefig(savepath_spec_png)\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" \n",
|
||||
" except Exception:\n",
|
||||
" continue\n",
|
||||
" print('Dir: ', subdir , ' finished!')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "871d7ab6",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.6"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@ -0,0 +1,194 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "4fdb98fc-65bb-467e-be0c-168fee9b0fca",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"cuda:0\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"<contextlib.ExitStack at 0x249d580ff50>"
|
||||
]
|
||||
},
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import pandas as pd\n",
|
||||
"import numpy as np\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"import time\n",
|
||||
"import io\n",
|
||||
"import cv2\n",
|
||||
"import copy\n",
|
||||
"import os\n",
|
||||
"from tqdm import tqdm\n",
|
||||
"import torch.nn as nn\n",
|
||||
"import torch\n",
|
||||
"import torchvision\n",
|
||||
"from torch.utils.data import Dataset\n",
|
||||
"from torch import default_generator, randperm\n",
|
||||
"from PIL import Image\n",
|
||||
"#from torch._utils import _accumulate\n",
|
||||
"import csv\n",
|
||||
"from torch.utils.data.dataset import Subset\n",
|
||||
"from scipy import ndimage\n",
|
||||
"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
|
||||
"print(device)\n",
|
||||
"batch_size = 16\n",
|
||||
"momentum=0.9\n",
|
||||
"lr = 1e-3\n",
|
||||
"import random\n",
|
||||
"sub_sample = 0.5\n",
|
||||
"import matplotlib\n",
|
||||
"import gc\n",
|
||||
"matplotlib.use('Agg')\n",
|
||||
"import matplotlib as mpl\n",
|
||||
"mpl.rcParams['agg.path.chunksize'] = 256*256\n",
|
||||
"plt.ioff()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "9267fbe1",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def sig2pic_with_spec(path_to_data, filename, figsize=(16,16), dpi=16, resize = None):\n",
|
||||
" try:\n",
|
||||
" array = np.load(path_to_data+filename)\n",
|
||||
"\n",
|
||||
" if resize != None:\n",
|
||||
" resized_real = cv2.resize(array[0], resize)\n",
|
||||
" resized_mag = cv2.resize(array[1], resize)\n",
|
||||
" resized_spectr = cv2.resize(array[2], resize)\n",
|
||||
" img = np.asarray(array, dtype=np.float32)\n",
|
||||
" return img\n",
|
||||
" \n",
|
||||
" img = np.asarray(array, dtype=np.float32)\n",
|
||||
" return img\n",
|
||||
" \n",
|
||||
" except Exception as e:\n",
|
||||
" print(str(e))\n",
|
||||
" return None"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"id": "448da74a-e0ae-44d8-9877-8dd1f257a24f",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"path_to_binaries = 'C:/Users/snytk/Lerning_NN_for_work/datasets/2.4_learning'\n",
|
||||
"path_to_pictures = 'C:/Users/snytk/Lerning_NN_for_work/datasets_jpg/2.4_jpg_learning'"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"id": "ac4945a8-29c4-4da4-945f-08658953e3e5",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from tqdm import tqdm"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "6f226f86-5d72-4573-8af6-750128b70263",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" 2%|█▍ | 158/8751 [01:06<1:01:14, 2.34it/s]"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"size = (256,256)\n",
|
||||
"if not os.path.exists(path_to_pictures):\n",
|
||||
" os.mkdir(path_to_pictures)\n",
|
||||
"for subdir in os.listdir(path_to_binaries):\n",
|
||||
" filepath = path_to_binaries + '/' + subdir + '/'\n",
|
||||
" if not os.path.exists(path_to_pictures +'/' + subdir):\n",
|
||||
" os.mkdir(path_to_pictures + '/' + subdir)\n",
|
||||
" files = os.listdir(filepath)\n",
|
||||
" for file in tqdm(files):\n",
|
||||
" savepath_real_png = path_to_pictures +'/' + subdir + '/' + file + '_real' + '.jpg' \n",
|
||||
" savepath_imag_png = path_to_pictures +'/' + subdir + '/' + file + '_imag' + '.jpg' \n",
|
||||
" savepath_spec_png = path_to_pictures +'/' + subdir + '/' + file + '_spec' + '.jpg'\n",
|
||||
" if not os.path.exists(savepath_real_png):\n",
|
||||
" img = sig2pic_with_spec(path_to_data=filepath, filename=file, resize = size)\n",
|
||||
" gc.collect()\n",
|
||||
" \n",
|
||||
" try:\n",
|
||||
" plt.imshow(img[0])\n",
|
||||
" plt.savefig(savepath_real_png)\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" \n",
|
||||
" plt.imshow(img[1])\n",
|
||||
" plt.savefig(savepath_imag_png)\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
"\n",
|
||||
" plt.imshow(img[2])\n",
|
||||
" plt.savefig(savepath_spec_png)\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" \n",
|
||||
" except Exception:\n",
|
||||
" continue\n",
|
||||
" print('Dir: ', subdir , ' finished!')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "871d7ab6",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.6"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@ -0,0 +1,194 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "4fdb98fc-65bb-467e-be0c-168fee9b0fca",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"cuda:0\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"<contextlib.ExitStack at 0x249d580ff50>"
|
||||
]
|
||||
},
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import pandas as pd\n",
|
||||
"import numpy as np\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"import time\n",
|
||||
"import io\n",
|
||||
"import cv2\n",
|
||||
"import copy\n",
|
||||
"import os\n",
|
||||
"from tqdm import tqdm\n",
|
||||
"import torch.nn as nn\n",
|
||||
"import torch\n",
|
||||
"import torchvision\n",
|
||||
"from torch.utils.data import Dataset\n",
|
||||
"from torch import default_generator, randperm\n",
|
||||
"from PIL import Image\n",
|
||||
"#from torch._utils import _accumulate\n",
|
||||
"import csv\n",
|
||||
"from torch.utils.data.dataset import Subset\n",
|
||||
"from scipy import ndimage\n",
|
||||
"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
|
||||
"print(device)\n",
|
||||
"batch_size = 16\n",
|
||||
"momentum=0.9\n",
|
||||
"lr = 1e-3\n",
|
||||
"import random\n",
|
||||
"sub_sample = 0.5\n",
|
||||
"import matplotlib\n",
|
||||
"import gc\n",
|
||||
"matplotlib.use('Agg')\n",
|
||||
"import matplotlib as mpl\n",
|
||||
"mpl.rcParams['agg.path.chunksize'] = 256*256\n",
|
||||
"plt.ioff()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "9267fbe1",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def sig2pic_with_spec(path_to_data, filename, figsize=(16,16), dpi=16, resize = None):\n",
|
||||
" try:\n",
|
||||
" array = np.load(path_to_data+filename)\n",
|
||||
"\n",
|
||||
" if resize != None:\n",
|
||||
" resized_real = cv2.resize(array[0], resize)\n",
|
||||
" resized_mag = cv2.resize(array[1], resize)\n",
|
||||
" resized_spectr = cv2.resize(array[2], resize)\n",
|
||||
" img = np.asarray(array, dtype=np.float32)\n",
|
||||
" return img\n",
|
||||
" \n",
|
||||
" img = np.asarray(array, dtype=np.float32)\n",
|
||||
" return img\n",
|
||||
" \n",
|
||||
" except Exception as e:\n",
|
||||
" print(str(e))\n",
|
||||
" return None"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"id": "448da74a-e0ae-44d8-9877-8dd1f257a24f",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"path_to_binaries = 'C:/Users/snytk/Lerning_NN_for_work/datasets/2.4_learning'\n",
|
||||
"path_to_pictures = 'C:/Users/snytk/Lerning_NN_for_work/datasets_jpg/2.4_jpg_learning'"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"id": "ac4945a8-29c4-4da4-945f-08658953e3e5",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from tqdm import tqdm"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "6f226f86-5d72-4573-8af6-750128b70263",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" 2%|█▍ | 158/8751 [01:06<1:01:14, 2.34it/s]"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"size = (256,256)\n",
|
||||
"if not os.path.exists(path_to_pictures):\n",
|
||||
" os.mkdir(path_to_pictures)\n",
|
||||
"for subdir in os.listdir(path_to_binaries):\n",
|
||||
" filepath = path_to_binaries + '/' + subdir + '/'\n",
|
||||
" if not os.path.exists(path_to_pictures +'/' + subdir):\n",
|
||||
" os.mkdir(path_to_pictures + '/' + subdir)\n",
|
||||
" files = os.listdir(filepath)\n",
|
||||
" for file in tqdm(files):\n",
|
||||
" savepath_real_png = path_to_pictures +'/' + subdir + '/' + file + '_real' + '.jpg' \n",
|
||||
" savepath_imag_png = path_to_pictures +'/' + subdir + '/' + file + '_imag' + '.jpg' \n",
|
||||
" savepath_spec_png = path_to_pictures +'/' + subdir + '/' + file + '_spec' + '.jpg'\n",
|
||||
" if not os.path.exists(savepath_real_png):\n",
|
||||
" img = sig2pic_with_spec(path_to_data=filepath, filename=file, resize = size)\n",
|
||||
" gc.collect()\n",
|
||||
" \n",
|
||||
" try:\n",
|
||||
" plt.imshow(img[0])\n",
|
||||
" plt.savefig(savepath_real_png)\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" \n",
|
||||
" plt.imshow(img[1])\n",
|
||||
" plt.savefig(savepath_imag_png)\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
"\n",
|
||||
" plt.imshow(img[2])\n",
|
||||
" plt.savefig(savepath_spec_png)\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" \n",
|
||||
" except Exception:\n",
|
||||
" continue\n",
|
||||
" print('Dir: ', subdir , ' finished!')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "871d7ab6",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.6"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@ -0,0 +1,236 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "4fdb98fc-65bb-467e-be0c-168fee9b0fca",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"cuda:0\n",
|
||||
"['Solarize_Light2', '_classic_test_patch', '_mpl-gallery', '_mpl-gallery-nogrid', 'bmh', 'classic', 'dark_background', 'fast', 'fivethirtyeight', 'ggplot', 'grayscale', 'seaborn-v0_8', 'seaborn-v0_8-bright', 'seaborn-v0_8-colorblind', 'seaborn-v0_8-dark', 'seaborn-v0_8-dark-palette', 'seaborn-v0_8-darkgrid', 'seaborn-v0_8-deep', 'seaborn-v0_8-muted', 'seaborn-v0_8-notebook', 'seaborn-v0_8-paper', 'seaborn-v0_8-pastel', 'seaborn-v0_8-poster', 'seaborn-v0_8-talk', 'seaborn-v0_8-ticks', 'seaborn-v0_8-white', 'seaborn-v0_8-whitegrid', 'tableau-colorblind10']\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"<contextlib.ExitStack at 0x27e4cbae550>"
|
||||
]
|
||||
},
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import pandas as pd\n",
|
||||
"import numpy as np\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"import time\n",
|
||||
"import io\n",
|
||||
"import cv2\n",
|
||||
"import copy\n",
|
||||
"import os\n",
|
||||
"from tqdm import tqdm\n",
|
||||
"import torch.nn as nn\n",
|
||||
"import torch\n",
|
||||
"import torchvision\n",
|
||||
"from torch.utils.data import Dataset\n",
|
||||
"from torch import default_generator, randperm\n",
|
||||
"from PIL import Image\n",
|
||||
"#from torch._utils import _accumulate\n",
|
||||
"import csv\n",
|
||||
"from torch.utils.data.dataset import Subset\n",
|
||||
"from scipy import ndimage\n",
|
||||
"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
|
||||
"print(device)\n",
|
||||
"batch_size = 16\n",
|
||||
"momentum=0.9\n",
|
||||
"lr = 1e-3\n",
|
||||
"import random\n",
|
||||
"sub_sample = 0.5\n",
|
||||
"import matplotlib\n",
|
||||
"import gc\n",
|
||||
"matplotlib.use('Agg')\n",
|
||||
"import matplotlib as mpl\n",
|
||||
"mpl.rcParams['agg.path.chunksize'] = 256*256\n",
|
||||
"#plt.style.use('mplstyle')\n",
|
||||
"plt.style.use('ggplot')\n",
|
||||
"plt.grid(None)\n",
|
||||
"plt.rcParams[\"axes.grid\"] = False\n",
|
||||
"print(plt.style.available)\n",
|
||||
"plt.ioff()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "9267fbe1",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def sig2pic_with_spec(path_to_data, filename, figsize=(16,16), dpi=16, resize = None):\n",
|
||||
" try:\n",
|
||||
" array = np.load(path_to_data+filename)\n",
|
||||
"\n",
|
||||
" if resize != None:\n",
|
||||
" resized_real = cv2.resize(array[0], resize)\n",
|
||||
" resized_mag = cv2.resize(array[1], resize)\n",
|
||||
" resized_spectr = cv2.resize(array[2], resize)\n",
|
||||
" img = np.asarray(array, dtype=np.float32)\n",
|
||||
" return img\n",
|
||||
" \n",
|
||||
" img = np.asarray(array, dtype=np.float32)\n",
|
||||
" return img\n",
|
||||
" \n",
|
||||
" except Exception as e:\n",
|
||||
" print(str(e))\n",
|
||||
" return None"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "448da74a-e0ae-44d8-9877-8dd1f257a24f",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"path_to_binaries = 'C:/Users/snytk/Lerning_NN_for_work/datasets/915_learning'\n",
|
||||
"path_to_pictures = 'C:/Users/snytk/Lerning_NN_for_work/datasets_jpg/915_jpg_learning'"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "ac4945a8-29c4-4da4-945f-08658953e3e5",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from tqdm import tqdm"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "6f226f86-5d72-4573-8af6-750128b70263",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████| 856/856 [10:50<00:00, 1.32it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Dir: drone finished!\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████| 627/627 [08:40<00:00, 1.20it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Dir: noise finished!\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"size = (256,256)\n",
|
||||
"if not os.path.exists(path_to_pictures):\n",
|
||||
" os.mkdir(path_to_pictures)\n",
|
||||
"for subdir in os.listdir(path_to_binaries):\n",
|
||||
" filepath = path_to_binaries + '/' + subdir + '/'\n",
|
||||
" if not os.path.exists(path_to_pictures +'/' + subdir):\n",
|
||||
" os.mkdir(path_to_pictures + '/' + subdir)\n",
|
||||
" files = os.listdir(filepath)\n",
|
||||
" for file in tqdm(files):\n",
|
||||
" savepath_real_png = path_to_pictures +'/' + subdir + '/' + file + '_real' + '.jpg' \n",
|
||||
" savepath_imag_png = path_to_pictures +'/' + subdir + '/' + file + '_imag' + '.jpg' \n",
|
||||
" savepath_spec_png = path_to_pictures +'/' + subdir + '/' + file + '_spec' + '.jpg'\n",
|
||||
" if not os.path.exists(savepath_real_png):\n",
|
||||
" img = sig2pic_with_spec(path_to_data=filepath, filename=file, resize = size)\n",
|
||||
" gc.collect()\n",
|
||||
" \n",
|
||||
" try:\n",
|
||||
" plt.imshow(img[0])\n",
|
||||
" plt.savefig(savepath_real_png)\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" \n",
|
||||
" plt.imshow(img[1])\n",
|
||||
" plt.savefig(savepath_imag_png)\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
"\n",
|
||||
" plt.imshow(img[2])\n",
|
||||
" plt.savefig(savepath_spec_png)\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" \n",
|
||||
" except Exception:\n",
|
||||
" continue\n",
|
||||
" print('Dir: ', subdir , ' finished!')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "871d7ab6",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "e080bb07",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.6"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@ -0,0 +1,7 @@
|
||||
Для ящика на выставки:
|
||||
aleksandr@192.168.3.85 19751975
|
||||
aleksandr@192.168.3.86 19751975
|
||||
|
||||
Для ящика на Липецк:
|
||||
aleksandr@192.168.3.85 19751975
|
||||
aleksandr@192.168.3.86 19751975
|
||||
@ -0,0 +1,55 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "a89c0273",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import torch\n",
|
||||
"import torchvision\n",
|
||||
"\n",
|
||||
"def convert_pth_to_pt(pth_path, pt_path, model_class):\n",
|
||||
" state_dict = torch.load(pth_path)\n",
|
||||
" model = model_class()\n",
|
||||
" model.load_state_dict(state_dict)\n",
|
||||
" torch.save(model, pt_path)\n",
|
||||
" print(f'Model saved to {pt_path}')\n",
|
||||
"\n",
|
||||
"class ModelClass(torch.nn.Module):\n",
|
||||
" def __init__(self):\n",
|
||||
" super(ModelClass, self).__init__()\n",
|
||||
"\n",
|
||||
" def forward(self, x):\n",
|
||||
" pass\n",
|
||||
"\n",
|
||||
"pth_path = 'model.pth'\n",
|
||||
"pt_path = 'model.pt'\n",
|
||||
"\n",
|
||||
"convert_pth_to_pt(pth_path, pt_path, ModelClass)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.8"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@ -0,0 +1,463 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "5a13ad6b-56c9-4381-b376-1765f6dd7553",
|
||||
"metadata": {
|
||||
"slideshow": {
|
||||
"slide_type": ""
|
||||
},
|
||||
"tags": []
|
||||
},
|
||||
"source": [
|
||||
"# Импортирование библиотек"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "7311cb4a-5bf3-4268-b431-43eea10e9ed6",
|
||||
"metadata": {
|
||||
"slideshow": {
|
||||
"slide_type": ""
|
||||
},
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"cuda\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ename": "error",
|
||||
"evalue": "OpenCV(4.10.0) D:\\a\\opencv-python\\opencv-python\\opencv\\modules\\highgui\\src\\window.cpp:1295: error: (-2:Unspecified error) The function is not implemented. Rebuild the library with Windows, GTK+ 2.x or Cocoa support. If you are on Ubuntu or Debian, install libgtk2.0-dev and pkg-config, then re-run cmake or configure script in function 'cvDestroyAllWindows'\n",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[1;31merror\u001b[0m Traceback (most recent call last)",
|
||||
"Cell \u001b[1;32mIn[1], line 37\u001b[0m\n\u001b[0;32m 35\u001b[0m \u001b[38;5;28mprint\u001b[39m(device)\n\u001b[0;32m 36\u001b[0m torch\u001b[38;5;241m.\u001b[39mcuda\u001b[38;5;241m.\u001b[39mempty_cache()\n\u001b[1;32m---> 37\u001b[0m \u001b[43mcv2\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdestroyAllWindows\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 38\u001b[0m gc\u001b[38;5;241m.\u001b[39mcollect()\n",
|
||||
"\u001b[1;31merror\u001b[0m: OpenCV(4.10.0) D:\\a\\opencv-python\\opencv-python\\opencv\\modules\\highgui\\src\\window.cpp:1295: error: (-2:Unspecified error) The function is not implemented. Rebuild the library with Windows, GTK+ 2.x or Cocoa support. If you are on Ubuntu or Debian, install libgtk2.0-dev and pkg-config, then re-run cmake or configure script in function 'cvDestroyAllWindows'\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from sklearn.model_selection import train_test_split\n",
|
||||
"from torch.utils.data import Dataset, DataLoader\n",
|
||||
"from torch import default_generator, randperm\n",
|
||||
"from torch.utils.data.dataset import Subset\n",
|
||||
"import torchvision.transforms as transforms\n",
|
||||
"from torchvision.io import read_image\n",
|
||||
"from importlib import import_module\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"from torchvision import models\n",
|
||||
"import torch, torchvision\n",
|
||||
"from pathlib import Path\n",
|
||||
"from PIL import Image\n",
|
||||
"import torch.nn as nn\n",
|
||||
"from tqdm import tqdm\n",
|
||||
"import pandas as pd\n",
|
||||
"import numpy as np\n",
|
||||
"import matplotlib\n",
|
||||
"import os, shutil\n",
|
||||
"import mlconfig\n",
|
||||
"import random\n",
|
||||
"import shutil\n",
|
||||
"import timeit\n",
|
||||
"import copy\n",
|
||||
"import time\n",
|
||||
"import cv2\n",
|
||||
"import csv\n",
|
||||
"import sys\n",
|
||||
"import io\n",
|
||||
"import gc\n",
|
||||
"\n",
|
||||
"plt.rcParams[\"savefig.bbox\"] = 'tight'\n",
|
||||
"torch.manual_seed(1)\n",
|
||||
"#matplotlib.use('Agg')\n",
|
||||
"device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
|
||||
"print(device)\n",
|
||||
"torch.cuda.empty_cache()\n",
|
||||
"cv2.destroyAllWindows()\n",
|
||||
"gc.collect()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "384de097-82c6-41f5-bda9-b2f54bc99593",
|
||||
"metadata": {
|
||||
"slideshow": {
|
||||
"slide_type": ""
|
||||
},
|
||||
"tags": []
|
||||
},
|
||||
"source": [
|
||||
"# Подготовка и обучение детектирование"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "46e4dc99-6994-4fee-a32e-f3983bd991bd",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def prepare_and_learning_detection(num_classes, num_samples, path_dataset, model_name, config_name, model):\n",
|
||||
" num_samples_per_class = num_samples // num_classes\n",
|
||||
"\n",
|
||||
" #----------Создаём папку для сохранения результатов обучения--------------\n",
|
||||
" \n",
|
||||
" ind = 1\n",
|
||||
" while True:\n",
|
||||
" if os.path.exists(\"models/\" + model_name + str(ind)):\n",
|
||||
" ind += 1\n",
|
||||
" else:\n",
|
||||
" os.mkdir(\"models/\" + model_name + str(ind))\n",
|
||||
" path_res = \"models/\" + model_name + str(ind) + '/'\n",
|
||||
" break\n",
|
||||
" \n",
|
||||
" #----------Создаём файл dataset.csv для обучения--------------\n",
|
||||
" \n",
|
||||
" pd_columns = ['file_name']\n",
|
||||
" df = pd.DataFrame(columns=pd_columns)\n",
|
||||
" \n",
|
||||
" subdirs = os.listdir(path_dataset)\n",
|
||||
" for subdir in subdirs:\n",
|
||||
" files = os.listdir(path_dataset + subdir + '/')\n",
|
||||
" num_samples_per_class = min(num_samples_per_class, len(files))\n",
|
||||
" for subdir in subdirs:\n",
|
||||
" files = os.listdir(path_dataset + subdir + '/')\n",
|
||||
" random.shuffle(files)\n",
|
||||
" files_to_process = files[:num_samples_per_class]\n",
|
||||
" for file in files_to_process:\n",
|
||||
" row = pd.DataFrame({pd_columns[0]: [str(path_dataset + subdir + '/' + file)]})\n",
|
||||
" df = pd.concat([df, row], ignore_index=True)\n",
|
||||
" \n",
|
||||
" df.to_csv(path_res + 'dataset.csv', index=False)\n",
|
||||
" \n",
|
||||
" #----------Импортируем параметры для обучения--------------\n",
|
||||
" \n",
|
||||
" def load_function(attr):\n",
|
||||
" module_, func = attr.rsplit('.', maxsplit=1)\n",
|
||||
" return getattr(import_module(module_), func)\n",
|
||||
" \n",
|
||||
" config = mlconfig.load('config_' + config_name + '.yaml')\n",
|
||||
" \n",
|
||||
" #----------Создаём класс датасета--------------\n",
|
||||
" \n",
|
||||
" class MyDataset(Dataset):\n",
|
||||
" def __init__(self, path_dataset, csv_file):\n",
|
||||
" data=[]\n",
|
||||
" with open(path_dataset + csv_file, newline='') as csvfile:\n",
|
||||
" reader = csv.reader(csvfile, delimiter=' ', quotechar='|')\n",
|
||||
" for row in list(reader)[1:]:\n",
|
||||
" row = str(row)\n",
|
||||
" data.append(row[2: len(row)-2])\n",
|
||||
" self.sig_filenames = data\n",
|
||||
" self.path_dataset = path_dataset\n",
|
||||
" \n",
|
||||
" def __len__(self):\n",
|
||||
" return len(self.sig_filenames)\n",
|
||||
" \n",
|
||||
" def __getitem__(self, idx):\n",
|
||||
" data_file = np.asarray(np.load(self.sig_filenames[idx], 'r+'), dtype=np.float32)\n",
|
||||
" if 'drone' in list(self.sig_filenames[idx].split('/')):\n",
|
||||
" label = torch.tensor(0)\n",
|
||||
" if 'noise' in list(self.sig_filenames[idx].split('/')):\n",
|
||||
" label = torch.tensor(1)\n",
|
||||
" return data_file, label\n",
|
||||
" \n",
|
||||
" #----------Создаём датасет--------------\n",
|
||||
" \n",
|
||||
" dataset = MyDataset(path_dataset=path_res, csv_file='dataset.csv')\n",
|
||||
" train_set, valid_set = torch.utils.data.random_split(dataset, [0.7, 0.3], generator=torch.Generator().manual_seed(42))\n",
|
||||
" batch_size = config.batch_size\n",
|
||||
" train_dataloader = torch.utils.data.DataLoader(train_set, batch_size=batch_size, shuffle=True, drop_last=True)\n",
|
||||
" valid_dataloader = torch.utils.data.DataLoader(valid_set, batch_size=batch_size, shuffle=True, drop_last=True)\n",
|
||||
" \n",
|
||||
" dataloaders = {}\n",
|
||||
" dataloaders['train'] = train_dataloader\n",
|
||||
" dataloaders['val'] = valid_dataloader\n",
|
||||
" dataset_sizes = {}\n",
|
||||
" dataset_sizes['train'] = len(train_set)\n",
|
||||
" dataset_sizes['val'] = len(valid_set)\n",
|
||||
"\n",
|
||||
" #----------Обучаем модель--------------\n",
|
||||
"\n",
|
||||
" val_loss = []\n",
|
||||
" val_acc = []\n",
|
||||
" train_loss = []\n",
|
||||
" train_acc = []\n",
|
||||
" epochs = config.epoch\n",
|
||||
" \n",
|
||||
" best_acc = 0.0\n",
|
||||
" best_model = copy.deepcopy(model.state_dict())\n",
|
||||
" limit = config.limit\n",
|
||||
" epoch_limit = epochs\n",
|
||||
" \n",
|
||||
" start = timeit.default_timer()\n",
|
||||
" for epoch in range(1, epochs+1):\n",
|
||||
" print(f\"Epoch : {epoch}\\n\")\n",
|
||||
" dataloader = None\n",
|
||||
" \n",
|
||||
" for phase in ['train', 'val']:\n",
|
||||
" running_loss = 0.0\n",
|
||||
" running_corrects = 0\n",
|
||||
" \n",
|
||||
" for (img, label) in tqdm(dataloaders[phase]):\n",
|
||||
" img, label = img.to(device), label.to(device)\n",
|
||||
" optimizer.zero_grad()\n",
|
||||
" \n",
|
||||
" with torch.set_grad_enabled(phase == 'train'):\n",
|
||||
" output = model(img)\n",
|
||||
" _, pred = torch.max(output.data, 1)\n",
|
||||
" loss = criterion(output, label)\n",
|
||||
" if phase=='train' :\n",
|
||||
" loss.backward()\n",
|
||||
" optimizer.step()\n",
|
||||
" \n",
|
||||
" running_loss += loss.item() * img.size(0)\n",
|
||||
" running_corrects += torch.sum(pred == label.data)\n",
|
||||
" \n",
|
||||
" epoch_loss = running_loss / dataset_sizes[phase]\n",
|
||||
" epoch_acc = running_corrects.double() / dataset_sizes[phase]\n",
|
||||
" \n",
|
||||
" print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))\n",
|
||||
" \n",
|
||||
" if phase=='train' :\n",
|
||||
" train_loss.append(epoch_loss)\n",
|
||||
" train_acc.append(epoch_acc)\n",
|
||||
" else :\n",
|
||||
" val_loss.append(epoch_loss)\n",
|
||||
" val_acc.append(epoch_acc)\n",
|
||||
" if val_acc[-1] > best_acc :\n",
|
||||
" ind_limit = 0\n",
|
||||
" best_acc = val_acc[-1]\n",
|
||||
" best_model = copy.deepcopy(model.state_dict())\n",
|
||||
" torch.save(best_model, path_res + model_name + '.pth')\n",
|
||||
" else:\n",
|
||||
" ind_limit += 1\n",
|
||||
" \n",
|
||||
" if ind_limit >= limit:\n",
|
||||
" break\n",
|
||||
" \n",
|
||||
" if ind_limit >= limit:\n",
|
||||
" epoch_limit = epoch\n",
|
||||
" break\n",
|
||||
" \n",
|
||||
" print()\n",
|
||||
" \n",
|
||||
" end = timeit.default_timer()\n",
|
||||
" print(f\"Total time elapsed = {end - start} seconds\")\n",
|
||||
" epoch_limit += 1\n",
|
||||
" \n",
|
||||
" #----------Вывод графиков и сохранение результатов обучения--------------\n",
|
||||
" \n",
|
||||
" train_acc = np.asarray(list(map(lambda x: x.item(), train_acc)))\n",
|
||||
" val_acc = np.asarray(list(map(lambda x: x.item(), val_acc)))\n",
|
||||
" \n",
|
||||
" np.save(path_res+'train_acc.npy', train_acc)\n",
|
||||
" np.save(path_res+'val_acc.npy', val_acc)\n",
|
||||
" np.save(path_res+'train_loss.npy', train_loss)\n",
|
||||
" np.save(path_res+'val_loss.npy', val_loss)\n",
|
||||
" \n",
|
||||
" plt.figure()\n",
|
||||
" plt.plot(range(1,epoch_limit), train_loss, color='blue')\n",
|
||||
" plt.plot(range(1,epoch_limit), val_loss, color='red')\n",
|
||||
" plt.xlabel('Epoch')\n",
|
||||
" plt.ylabel('Loss')\n",
|
||||
" plt.title('Loss Curve')\n",
|
||||
" plt.legend(['Train Loss', 'Validation Loss'])\n",
|
||||
" plt.show()\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" \n",
|
||||
" plt.figure()\n",
|
||||
" plt.plot(range(1,epoch_limit), train_acc, color='blue')\n",
|
||||
" plt.plot(range(1,epoch_limit), val_acc, color='red')\n",
|
||||
" plt.xlabel('Epoch')\n",
|
||||
" plt.ylabel('Accuracy')\n",
|
||||
" plt.title('Accuracy Curve')\n",
|
||||
" plt.legend(['Train Accuracy', 'Validation Accuracy'])\n",
|
||||
" plt.show()\n",
|
||||
" \n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" torch.cuda.empty_cache()\n",
|
||||
" cv2.destroyAllWindows()\n",
|
||||
" del model\n",
|
||||
" gc.collect()\n",
|
||||
"\n",
|
||||
" return path_res, model_name"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "93c136ee",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Ensemble"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "52e8d4c5",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"C:\\Users\\snytk\\miniconda3\\envs\\python311\\Lib\\site-packages\\torchvision\\models\\_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
|
||||
" warnings.warn(\n",
|
||||
"C:\\Users\\snytk\\miniconda3\\envs\\python311\\Lib\\site-packages\\torchvision\\models\\_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=None`.\n",
|
||||
" warnings.warn(msg)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Epoch : 1\n",
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████| 437/437 [23:49<00:00, 3.27s/it]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"train Loss: 0.0884 Acc: 0.9634\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████| 187/187 [09:15<00:00, 2.97s/it]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"val Loss: 0.0342 Acc: 0.9873\n",
|
||||
"\n",
|
||||
"Epoch : 2\n",
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" 84%|██████████████████████████████████████████████████████████████████▊ | 365/437 [19:28<04:18, 3.59s/it]"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"#----------Инициализируем модель и параметры обучения--------------\n",
|
||||
"\n",
|
||||
"torch.cuda.empty_cache()\n",
|
||||
"cv2.destroyAllWindows()\n",
|
||||
"gc.collect()\n",
|
||||
"\n",
|
||||
"num_classes = 3\n",
|
||||
"config_name = \"ensemble\"\n",
|
||||
" \n",
|
||||
"def load_function(attr):\n",
|
||||
" module_, func = attr.rsplit('.', maxsplit=1)\n",
|
||||
" return getattr(import_module(module_), func)\n",
|
||||
" \n",
|
||||
"config = mlconfig.load('config_' + config_name + '.yaml')\n",
|
||||
"\n",
|
||||
"model1 = models.resnet18(pretrained=False)\n",
|
||||
"model2 = models.resnet50(pretrained=False)\n",
|
||||
"model3 = models.resnet101(pretrained=False)\n",
|
||||
"\n",
|
||||
"num_classes = 2\n",
|
||||
"\n",
|
||||
"model1.fc = nn.Linear(model1.fc.in_features, num_classes)\n",
|
||||
"model2.fc = nn.Linear(model2.fc.in_features, num_classes)\n",
|
||||
"model3.fc = nn.Linear(model3.fc.in_features, num_classes)\n",
|
||||
"\n",
|
||||
"class Ensemble(nn.Module):\n",
|
||||
" def __init__(self, model1, model2, model3):\n",
|
||||
" super(Ensemble, self).__init__()\n",
|
||||
" self.model1 = model1\n",
|
||||
" self.model2 = model2\n",
|
||||
" self.model3 = model3\n",
|
||||
" self.fc = nn.Linear(3 * num_classes, num_classes)\n",
|
||||
"\n",
|
||||
" def forward(self, x):\n",
|
||||
" x1 = self.model1(x)\n",
|
||||
" x2 = self.model2(x)\n",
|
||||
" x3 = self.model3(x)\n",
|
||||
" x = torch.cat((x1, x2, x3), dim=1)\n",
|
||||
" x = self.fc(x)\n",
|
||||
" return x\n",
|
||||
"\n",
|
||||
"model = Ensemble(model1, model2, model3)\n",
|
||||
"\n",
|
||||
"optimizer = load_function(config.optimizer.name)(model.parameters(), lr=config.optimizer.lr)\n",
|
||||
"criterion = load_function(config.loss_function.name)()\n",
|
||||
"scheduler = load_function(config.scheduler.name)(optimizer, step_size=config.scheduler.step_size, gamma=config.scheduler.gamma)\n",
|
||||
"\n",
|
||||
"if device != 'cpu':\n",
|
||||
" model = model.to(device)\n",
|
||||
"\n",
|
||||
"#----------Создания датасета и обучение модели--------------\n",
|
||||
"\n",
|
||||
"path_res, model_name = prepare_and_learning_detection(num_classes = num_classes, num_samples = 10000, path_dataset = \"//192.168.1.64/data/DATASETS/2.4/2.4_learning/\", \n",
|
||||
" model_name = config_name+\"_2.4_\", config_name = config_name, model=model)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"torch.cuda.empty_cache()\n",
|
||||
"cv2.destroyAllWindows()\n",
|
||||
"del model\n",
|
||||
"gc.collect()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "57d18676",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"celltoolbar": "Отсутствует",
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.6"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@ -0,0 +1,523 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "e1db882b",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/home/sibscience-4/from_ssh/DroneDetector/.venv-train/lib/python3.12/site-packages/matplotlib/projections/__init__.py:63: UserWarning: Unable to import Axes3D. This may be due to multiple versions of Matplotlib being installed (e.g. as a system package and as a pip package). As a result, the 3D projection is not available.\n",
|
||||
" warnings.warn(\"Unable to import Axes3D. This may be due to multiple versions of \"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"cuda\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"220"
|
||||
]
|
||||
},
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from sklearn.model_selection import train_test_split\n",
|
||||
"from torch.utils.data import Dataset, DataLoader\n",
|
||||
"from torch import default_generator, randperm\n",
|
||||
"from torch.utils.data.dataset import Subset\n",
|
||||
"import torchvision.transforms as transforms\n",
|
||||
"from torchvision.io import read_image\n",
|
||||
"from importlib import import_module\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"from torchvision import models\n",
|
||||
"import torch, torchvision\n",
|
||||
"from pathlib import Path\n",
|
||||
"from PIL import Image\n",
|
||||
"import torch.nn as nn\n",
|
||||
"from tqdm import tqdm\n",
|
||||
"import pandas as pd\n",
|
||||
"import numpy as np\n",
|
||||
"import matplotlib\n",
|
||||
"import os, shutil\n",
|
||||
"import mlconfig\n",
|
||||
"import random\n",
|
||||
"import shutil\n",
|
||||
"import timeit\n",
|
||||
"import copy\n",
|
||||
"import time\n",
|
||||
"import cv2\n",
|
||||
"import csv\n",
|
||||
"import sys\n",
|
||||
"import io\n",
|
||||
"import gc\n",
|
||||
"\n",
|
||||
"plt.rcParams[\"savefig.bbox\"] = 'tight'\n",
|
||||
"torch.manual_seed(1)\n",
|
||||
"#matplotlib.use('Agg')\n",
|
||||
"device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
|
||||
"print(device)\n",
|
||||
"torch.cuda.empty_cache()\n",
|
||||
"cv2.destroyAllWindows()\n",
|
||||
"gc.collect()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "8e009995",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def prepare_and_learning_detection(num_classes, num_samples, path_dataset, model_name, config_name, model,selected_freq):\n",
|
||||
" num_samples_per_class = num_samples // num_classes\n",
|
||||
"\n",
|
||||
" #----------Создаём папку для сохранения результатов обучения--------------\n",
|
||||
" os.makedirs(\"models\", exist_ok=True)\n",
|
||||
" ind = 1\n",
|
||||
" while True:\n",
|
||||
" if os.path.exists(\"models/\" + model_name + str(ind)):\n",
|
||||
" ind += 1\n",
|
||||
" else:\n",
|
||||
" os.mkdir(\"models/\" + model_name + str(ind))\n",
|
||||
" path_res = \"models/\" + model_name + str(ind) + '/'\n",
|
||||
" break\n",
|
||||
" \n",
|
||||
" #----------Создаём файл dataset.csv для обучения--------------\n",
|
||||
" \n",
|
||||
" pd_columns = ['file_name']\n",
|
||||
" df = pd.DataFrame(columns=pd_columns)\n",
|
||||
" \n",
|
||||
" subdirs = os.listdir(path_dataset)\n",
|
||||
" \n",
|
||||
" for subdir in subdirs:\n",
|
||||
" freq_dir = os.path.join(path_dataset, subdir, str(selected_freq)+\"_jpg\")\n",
|
||||
" if not os.path.isdir(freq_dir):\n",
|
||||
" print(\"Error1\")\n",
|
||||
" continue\n",
|
||||
" \n",
|
||||
" files_k=[f for f in os.listdir(freq_dir)]\n",
|
||||
" print(len(files_k))\n",
|
||||
" \n",
|
||||
" files = [\n",
|
||||
" f for f in os.listdir(freq_dir)\n",
|
||||
" if os.path.isfile(os.path.join(freq_dir, f)) and f.endswith('imag.png')\n",
|
||||
" ]\n",
|
||||
" num_samples_per_class = min(num_samples_per_class, len(files))\n",
|
||||
" print(f\"num_samples per class {subdir} is {num_samples_per_class}\")\n",
|
||||
"\n",
|
||||
" for subdir in subdirs:\n",
|
||||
" freq_dir = os.path.join(path_dataset, subdir, str(selected_freq)+\"_jpg\")\n",
|
||||
" if not os.path.isdir(freq_dir):\n",
|
||||
" print(\"Error1\")\n",
|
||||
" continue\n",
|
||||
"\n",
|
||||
" files = [\n",
|
||||
" f for f in os.listdir(freq_dir)\n",
|
||||
" if os.path.isfile(os.path.join(freq_dir, f)) and f.endswith('imag.png')\n",
|
||||
" ]\n",
|
||||
" random.shuffle(files)\n",
|
||||
" files_to_process = files[:num_samples_per_class]\n",
|
||||
"\n",
|
||||
" for file in files_to_process:\n",
|
||||
" row = pd.DataFrame({\n",
|
||||
" pd_columns[0]: [str(os.path.join(freq_dir, file))]\n",
|
||||
" })\n",
|
||||
" df = pd.concat([df, row], ignore_index=True)\n",
|
||||
"\n",
|
||||
" dataset_csv_path = os.path.join(path_res, 'dataset.csv')\n",
|
||||
" df.to_csv(dataset_csv_path, index=False)\n",
|
||||
"\n",
|
||||
" if not os.path.exists(dataset_csv_path):\n",
|
||||
" raise RuntimeError(f'dataset.csv was not created: {dataset_csv_path}')\n",
|
||||
" #----------Импортируем параметры для обучения--------------\n",
|
||||
" \n",
|
||||
" def load_function(attr):\n",
|
||||
" module_, func = attr.rsplit('.', maxsplit=1)\n",
|
||||
" return getattr(import_module(module_), func)\n",
|
||||
" \n",
|
||||
" config = mlconfig.load('config_' + config_name + '.yaml')\n",
|
||||
" \n",
|
||||
" #----------Создаём класс датасета--------------\n",
|
||||
" \n",
|
||||
" class MyDataset(Dataset):\n",
|
||||
" def __init__(self, path_dataset, csv_file):\n",
|
||||
" data=[]\n",
|
||||
" with open(os.path.join(path_dataset, csv_file), newline='') as csvfile:\n",
|
||||
" reader = csv.reader(csvfile, delimiter=' ', quotechar='|')\n",
|
||||
" for row in list(reader)[1:]:\n",
|
||||
" row = str(row)\n",
|
||||
" data.append(row[2: len(row)-2])\n",
|
||||
" self.path_dataset = path_dataset\n",
|
||||
" self.target_shape = None\n",
|
||||
" self.target_hw = None\n",
|
||||
" self.sig_filenames = self._validate_files(data)\n",
|
||||
"\n",
|
||||
" def _pair_paths(self, filename):\n",
|
||||
" base = os.path.splitext(filename)[0]\n",
|
||||
" if base.endswith(\"real\"):\n",
|
||||
" return base + \".png\", base[:-4] + \"imag.png\"\n",
|
||||
" if base.endswith(\"imag\"):\n",
|
||||
" return base[:-4] + \"real.png\", base + \".png\"\n",
|
||||
" return None, None\n",
|
||||
"\n",
|
||||
" @staticmethod\n",
|
||||
" def _read_shape(path):\n",
|
||||
" img = cv2.imread(path)\n",
|
||||
" if img is None:\n",
|
||||
" return None\n",
|
||||
" return img.shape\n",
|
||||
"\n",
|
||||
" def _validate_files(self, filenames):\n",
|
||||
" from collections import Counter\n",
|
||||
"\n",
|
||||
" candidates = []\n",
|
||||
" dropped = []\n",
|
||||
" shape_counter = Counter()\n",
|
||||
"\n",
|
||||
" for filename in filenames:\n",
|
||||
" real_path, imag_path = self._pair_paths(filename)\n",
|
||||
" if real_path is None or imag_path is None:\n",
|
||||
" dropped.append((filename, \"bad file suffix\"))\n",
|
||||
" continue\n",
|
||||
"\n",
|
||||
" real_shape = self._read_shape(real_path)\n",
|
||||
" imag_shape = self._read_shape(imag_path)\n",
|
||||
" if real_shape is None or imag_shape is None:\n",
|
||||
" dropped.append((filename, f\"read failed real={real_shape} imag={imag_shape}\"))\n",
|
||||
" continue\n",
|
||||
" if real_shape != imag_shape:\n",
|
||||
" dropped.append((filename, f\"pair shape mismatch real={real_shape} imag={imag_shape}\"))\n",
|
||||
" continue\n",
|
||||
"\n",
|
||||
" shape_counter[real_shape] += 1\n",
|
||||
" candidates.append((filename, real_shape))\n",
|
||||
"\n",
|
||||
" if not candidates:\n",
|
||||
" raise RuntimeError(\"No valid image pairs left after shape validation\")\n",
|
||||
"\n",
|
||||
" preferred_shape = (1600, 1600, 3)\n",
|
||||
" if shape_counter.get(preferred_shape, 0) > 0:\n",
|
||||
" target_shape = preferred_shape\n",
|
||||
" else:\n",
|
||||
" target_shape = shape_counter.most_common(1)[0][0]\n",
|
||||
"\n",
|
||||
" self.target_shape = target_shape\n",
|
||||
" self.target_hw = target_shape[:2]\n",
|
||||
" valid = [filename for filename, _shape in candidates]\n",
|
||||
" resized_count = sum(1 for _filename, shape in candidates if shape != target_shape)\n",
|
||||
"\n",
|
||||
" print(f\"[dataset-shape-filter] shape_distribution={dict(shape_counter)}\")\n",
|
||||
" print(\n",
|
||||
" f\"[dataset-shape-filter] target_shape={target_shape} \"\n",
|
||||
" f\"kept={len(valid)} dropped={len(dropped)} will_resize={resized_count}\"\n",
|
||||
" )\n",
|
||||
" for filename, reason in dropped[:20]:\n",
|
||||
" print(f\"[dataset-shape-filter] drop {filename}: {reason}\")\n",
|
||||
" if len(dropped) > 20:\n",
|
||||
" print(f\"[dataset-shape-filter] ... {len(dropped) - 20} more dropped\")\n",
|
||||
"\n",
|
||||
" return valid\n",
|
||||
"\n",
|
||||
" def __len__(self):\n",
|
||||
" return len(self.sig_filenames)\n",
|
||||
"\n",
|
||||
" def _read_image_chw(self, path):\n",
|
||||
" img = cv2.imread(path)\n",
|
||||
" if img is None:\n",
|
||||
" raise RuntimeError(f\"failed to read image: {path}\")\n",
|
||||
" if img.shape[:2] != self.target_hw:\n",
|
||||
" img = cv2.resize(\n",
|
||||
" img,\n",
|
||||
" (self.target_hw[1], self.target_hw[0]),\n",
|
||||
" interpolation=cv2.INTER_AREA,\n",
|
||||
" )\n",
|
||||
" return np.asarray(cv2.split(img), dtype=np.float32)\n",
|
||||
"\n",
|
||||
" def __getitem__(self, idx):\n",
|
||||
" real_path, imag_path = self._pair_paths(self.sig_filenames[idx])\n",
|
||||
" image_real = self._read_image_chw(real_path)\n",
|
||||
" image_imag = self._read_image_chw(imag_path)\n",
|
||||
"\n",
|
||||
" path_parts = set(self.sig_filenames[idx].split('/'))\n",
|
||||
" if 'drone' in path_parts:\n",
|
||||
" label = torch.tensor(0)\n",
|
||||
" elif 'noise' in path_parts:\n",
|
||||
" label = torch.tensor(1)\n",
|
||||
" else:\n",
|
||||
" raise RuntimeError(f\"cannot infer label from path: {self.sig_filenames[idx]}\")\n",
|
||||
"\n",
|
||||
" return image_real, image_imag, label\n",
|
||||
"\n",
|
||||
" #----------Создаём датасет--------------\n",
|
||||
" \n",
|
||||
" dataset = MyDataset(path_dataset=path_res, csv_file='dataset.csv')\n",
|
||||
" train_set, valid_set = torch.utils.data.random_split(dataset, [0.7, 0.3], generator=torch.Generator().manual_seed(42))\n",
|
||||
" batch_size = config.batch_size\n",
|
||||
" train_dataloader = torch.utils.data.DataLoader(train_set, batch_size=batch_size, shuffle=True, drop_last=True)\n",
|
||||
" valid_dataloader = torch.utils.data.DataLoader(valid_set, batch_size=batch_size, shuffle=False, drop_last=False)\n",
|
||||
" \n",
|
||||
" dataloaders = {}\n",
|
||||
" dataloaders['train'] = train_dataloader\n",
|
||||
" dataloaders['val'] = valid_dataloader\n",
|
||||
" dataset_sizes = {}\n",
|
||||
" dataset_sizes['train'] = len(train_set)\n",
|
||||
" dataset_sizes['val'] = len(valid_set)\n",
|
||||
"\n",
|
||||
" #----------Обучаем модель--------------\n",
|
||||
"\n",
|
||||
" val_loss = []\n",
|
||||
" val_acc = []\n",
|
||||
" train_loss = []\n",
|
||||
" train_acc = []\n",
|
||||
" epochs = config.epoch\n",
|
||||
" min_delta = 1e-4\n",
|
||||
" \n",
|
||||
" best_val_loss = float('inf')\n",
|
||||
" best_model = copy.deepcopy(model.state_dict())\n",
|
||||
" limit = config.limit\n",
|
||||
" ind_limit = 0\n",
|
||||
" epoch_limit = epochs\n",
|
||||
" \n",
|
||||
" start = timeit.default_timer()\n",
|
||||
" for epoch in range(1, epochs+1):\n",
|
||||
" print(f\"Epoch : {epoch}\\n\")\n",
|
||||
" \n",
|
||||
" for phase in ['train', 'val']:\n",
|
||||
" if phase == 'train':\n",
|
||||
" model.train()\n",
|
||||
" else:\n",
|
||||
" model.eval()\n",
|
||||
"\n",
|
||||
" running_loss = 0.0\n",
|
||||
" running_corrects = 0\n",
|
||||
" \n",
|
||||
" for (img1, img2, label) in tqdm(dataloaders[phase]):\n",
|
||||
" img1, img2, label = img1.to(device), img2.to(device), label.to(device)\n",
|
||||
" optimizer.zero_grad()\n",
|
||||
" \n",
|
||||
" with torch.set_grad_enabled(phase == 'train'):\n",
|
||||
" output = model([img1, img2])\n",
|
||||
" _, pred = torch.max(output.data, 1)\n",
|
||||
" loss = criterion(output, label)\n",
|
||||
" if phase == 'train':\n",
|
||||
" loss.backward()\n",
|
||||
" optimizer.step()\n",
|
||||
" \n",
|
||||
" running_loss += loss.item() * img1.size(0)\n",
|
||||
" running_corrects += torch.sum(pred == label.data)\n",
|
||||
" \n",
|
||||
" epoch_loss = running_loss / dataset_sizes[phase]\n",
|
||||
" epoch_acc = running_corrects.double() / dataset_sizes[phase]\n",
|
||||
" \n",
|
||||
" print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))\n",
|
||||
" \n",
|
||||
" if phase == 'train':\n",
|
||||
" train_loss.append(epoch_loss)\n",
|
||||
" train_acc.append(epoch_acc)\n",
|
||||
" else:\n",
|
||||
" val_loss.append(epoch_loss)\n",
|
||||
" val_acc.append(epoch_acc)\n",
|
||||
" scheduler.step(epoch_loss)\n",
|
||||
"\n",
|
||||
" current_lr = optimizer.param_groups[0]['lr']\n",
|
||||
" print(f'val lr: {current_lr:.8f}')\n",
|
||||
"\n",
|
||||
" if epoch_loss < (best_val_loss - min_delta):\n",
|
||||
" ind_limit = 0\n",
|
||||
" best_val_loss = epoch_loss\n",
|
||||
" best_model = copy.deepcopy(model.state_dict())\n",
|
||||
" torch.save(best_model, path_res + model_name + '.pth')\n",
|
||||
" print(f'saved best model with val_loss={best_val_loss:.4f}')\n",
|
||||
" else:\n",
|
||||
" ind_limit += 1\n",
|
||||
" print(f'early stopping patience: {ind_limit}/{limit}')\n",
|
||||
" \n",
|
||||
" if ind_limit >= limit:\n",
|
||||
" break\n",
|
||||
" \n",
|
||||
" if ind_limit >= limit:\n",
|
||||
" epoch_limit = epoch\n",
|
||||
" break\n",
|
||||
" \n",
|
||||
" print()\n",
|
||||
" \n",
|
||||
" end = timeit.default_timer()\n",
|
||||
" print(f\"Total time elapsed = {end - start} seconds\")\n",
|
||||
" epoch_limit += 1\n",
|
||||
" \n",
|
||||
" #----------Вывод графиков и сохранение результатов обучения--------------\n",
|
||||
" \n",
|
||||
" train_acc = np.asarray(list(map(lambda x: x.item(), train_acc)))\n",
|
||||
" val_acc = np.asarray(list(map(lambda x: x.item(), val_acc)))\n",
|
||||
" \n",
|
||||
" np.save(path_res+'train_acc.npy', train_acc)\n",
|
||||
" np.save(path_res+'val_acc.npy', val_acc)\n",
|
||||
" np.save(path_res+'train_loss.npy', train_loss)\n",
|
||||
" np.save(path_res+'val_loss.npy', val_loss)\n",
|
||||
" \n",
|
||||
" plt.figure()\n",
|
||||
" plt.plot(range(1,epoch_limit), train_loss, color='blue')\n",
|
||||
" plt.plot(range(1,epoch_limit), val_loss, color='red')\n",
|
||||
" plt.xlabel('Epoch')\n",
|
||||
" plt.ylabel('Loss') \n",
|
||||
" \n",
|
||||
" plt.title('Loss Curve')\n",
|
||||
" plt.legend(['Train Loss', 'Validation Loss'])\n",
|
||||
" plt.show()\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" \n",
|
||||
" plt.figure()\n",
|
||||
" plt.plot(range(1,epoch_limit), train_acc, color='blue')\n",
|
||||
" plt.plot(range(1,epoch_limit), val_acc, color='red')\n",
|
||||
" plt.xlabel('Epoch')\n",
|
||||
" plt.ylabel('Accuracy')\n",
|
||||
" plt.title('Accuracy Curve')\n",
|
||||
" plt.legend(['Train Accuracy', 'Validation Accuracy'])\n",
|
||||
" plt.show()\n",
|
||||
" \n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" torch.cuda.empty_cache()\n",
|
||||
" cv2.destroyAllWindows()\n",
|
||||
" del model\n",
|
||||
" gc.collect()\n",
|
||||
"\n",
|
||||
" return path_res, model_name"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "bbbe7fea",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/home/sibscience-4/from_ssh/DroneDetector/.venv-train/lib/python3.12/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
|
||||
" warnings.warn(\n",
|
||||
"/home/sibscience-4/from_ssh/DroneDetector/.venv-train/lib/python3.12/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=None`.\n",
|
||||
" warnings.warn(msg)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Error1\n",
|
||||
"Error1\n",
|
||||
"Error1\n",
|
||||
"Error1\n",
|
||||
"Error1\n",
|
||||
"Error1\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ename": "RuntimeError",
|
||||
"evalue": "No valid image pairs left after shape validation",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[31m---------------------------------------------------------------------------\u001b[39m",
|
||||
"\u001b[31mRuntimeError\u001b[39m Traceback (most recent call last)",
|
||||
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[4]\u001b[39m\u001b[32m, line 45\u001b[39m\n\u001b[32m 41\u001b[39m model = model.to(device)\n\u001b[32m 42\u001b[39m \n\u001b[32m 43\u001b[39m \u001b[38;5;66;03m#----------Создания датасета и обучение модели--------------\u001b[39;00m\n\u001b[32m 44\u001b[39m \n\u001b[32m---> \u001b[39m\u001b[32m45\u001b[39m path_res, model_name = prepare_and_learning_detection(num_classes = num_classes, num_samples = 10000, path_dataset = \"/mnt/data/Dataset_overlay\", \n\u001b[32m 46\u001b[39m selected_freq=\u001b[32m1200\u001b[39m,model_name = config_name+\u001b[33m\"1200_\"\u001b[39m, config_name = config_name, model=model)\n\u001b[32m 47\u001b[39m \n\u001b[32m 48\u001b[39m \n",
|
||||
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[2]\u001b[39m\u001b[32m, line 183\u001b[39m, in \u001b[36mprepare_and_learning_detection\u001b[39m\u001b[34m(num_classes, num_samples, path_dataset, model_name, config_name, model, selected_freq)\u001b[39m\n\u001b[32m 179\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m image_real, image_imag, label\n\u001b[32m 180\u001b[39m \n\u001b[32m 181\u001b[39m \u001b[38;5;66;03m#----------Создаём датасет--------------\u001b[39;00m\n\u001b[32m 182\u001b[39m \n\u001b[32m--> \u001b[39m\u001b[32m183\u001b[39m dataset = MyDataset(path_dataset=path_res, csv_file=\u001b[33m'dataset.csv'\u001b[39m)\n\u001b[32m 184\u001b[39m train_set, valid_set = torch.utils.data.random_split(dataset, [\u001b[32m0.7\u001b[39m, \u001b[32m0.3\u001b[39m], generator=torch.Generator().manual_seed(\u001b[32m42\u001b[39m))\n\u001b[32m 185\u001b[39m batch_size = config.batch_size\n\u001b[32m 186\u001b[39m train_dataloader = torch.utils.data.DataLoader(train_set, batch_size=batch_size, shuffle=\u001b[38;5;28;01mTrue\u001b[39;00m, drop_last=\u001b[38;5;28;01mTrue\u001b[39;00m)\n",
|
||||
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[2]\u001b[39m\u001b[32m, line 83\u001b[39m, in \u001b[36mprepare_and_learning_detection.<locals>.MyDataset.__init__\u001b[39m\u001b[34m(self, path_dataset, csv_file)\u001b[39m\n\u001b[32m 79\u001b[39m data.append(row[\u001b[32m2\u001b[39m: len(row)-\u001b[32m2\u001b[39m])\n\u001b[32m 80\u001b[39m self.path_dataset = path_dataset\n\u001b[32m 81\u001b[39m self.target_shape = \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[32m 82\u001b[39m self.target_hw = \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[32m---> \u001b[39m\u001b[32m83\u001b[39m self.sig_filenames = self._validate_files(data)\n",
|
||||
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[2]\u001b[39m\u001b[32m, line 126\u001b[39m, in \u001b[36mprepare_and_learning_detection.<locals>.MyDataset._validate_files\u001b[39m\u001b[34m(self, filenames)\u001b[39m\n\u001b[32m 122\u001b[39m shape_counter[real_shape] += \u001b[32m1\u001b[39m\n\u001b[32m 123\u001b[39m candidates.append((filename, real_shape))\n\u001b[32m 124\u001b[39m \n\u001b[32m 125\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28;01mnot\u001b[39;00m candidates:\n\u001b[32m--> \u001b[39m\u001b[32m126\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m RuntimeError(\u001b[33m\"No valid image pairs left after shape validation\"\u001b[39m)\n\u001b[32m 127\u001b[39m \n\u001b[32m 128\u001b[39m preferred_shape = (\u001b[32m1600\u001b[39m, \u001b[32m1600\u001b[39m, \u001b[32m3\u001b[39m)\n\u001b[32m 129\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m shape_counter.get(preferred_shape, \u001b[32m0\u001b[39m) > \u001b[32m0\u001b[39m:\n",
|
||||
"\u001b[31mRuntimeError\u001b[39m: No valid image pairs left after shape validation"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"torch.cuda.empty_cache()\n",
|
||||
"cv2.destroyAllWindows()\n",
|
||||
"gc.collect()\n",
|
||||
"\n",
|
||||
"config_name = \"ensemble\"\n",
|
||||
" \n",
|
||||
"def load_function(attr):\n",
|
||||
" module_, func = attr.rsplit('.', maxsplit=1)\n",
|
||||
" return getattr(import_module(module_), func)\n",
|
||||
" \n",
|
||||
"config = mlconfig.load('config_' + config_name + '.yaml')\n",
|
||||
"\n",
|
||||
"model1 = models.resnet18(pretrained=False)\n",
|
||||
"model2 = models.resnet50(pretrained=False)\n",
|
||||
"\n",
|
||||
"num_classes = 2\n",
|
||||
"\n",
|
||||
"model1.fc = nn.Linear(model1.fc.in_features, num_classes)\n",
|
||||
"model2.fc = nn.Linear(model2.fc.in_features, num_classes)\n",
|
||||
"\n",
|
||||
"class Ensemble(nn.Module):\n",
|
||||
" def __init__(self, model1, model2):\n",
|
||||
" super(Ensemble, self).__init__()\n",
|
||||
" self.model1 = model1\n",
|
||||
" self.model2 = model2\n",
|
||||
" self.fc = nn.Linear(2 * num_classes, num_classes)\n",
|
||||
"\n",
|
||||
" def forward(self, x):\n",
|
||||
" x1 = self.model1(x[0])\n",
|
||||
" x2 = self.model2(x[1])\n",
|
||||
" x = torch.cat((x1, x2), dim=1)\n",
|
||||
" x = self.fc(x)\n",
|
||||
" return x\n",
|
||||
"model = Ensemble(model1, model2)\n",
|
||||
"\n",
|
||||
"optimizer = load_function(config.optimizer.name)(model.parameters(), lr=config.optimizer.lr)\n",
|
||||
"criterion = load_function(config.loss_function.name)()\n",
|
||||
"scheduler = load_function(config.scheduler.name)(optimizer, step_size=config.scheduler.step_size, gamma=config.scheduler.gamma)\n",
|
||||
"\n",
|
||||
"if device != 'cpu':\n",
|
||||
" model = model.to(device)\n",
|
||||
"\n",
|
||||
"#----------Создания датасета и обучение модели--------------\n",
|
||||
"\n",
|
||||
"path_res, model_name = prepare_and_learning_detection(num_classes = num_classes, num_samples = 10000, path_dataset = \"/mnt/data/Dataset_overlay\", \n",
|
||||
" selected_freq=2400,model_name = config_name+\"2400_\", config_name = config_name, model=model)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"torch.cuda.empty_cache()\n",
|
||||
"cv2.destroyAllWindows()\n",
|
||||
"del model\n",
|
||||
"gc.collect()"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "usr",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.12.3"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@ -0,0 +1,465 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "5a13ad6b-56c9-4381-b376-1765f6dd7553",
|
||||
"metadata": {
|
||||
"slideshow": {
|
||||
"slide_type": ""
|
||||
},
|
||||
"tags": []
|
||||
},
|
||||
"source": [
|
||||
"# Импортирование библиотек"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "7311cb4a-5bf3-4268-b431-43eea10e9ed6",
|
||||
"metadata": {
|
||||
"slideshow": {
|
||||
"slide_type": ""
|
||||
},
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"cuda\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"0"
|
||||
]
|
||||
},
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from sklearn.model_selection import train_test_split\n",
|
||||
"from torch.utils.data import Dataset, DataLoader\n",
|
||||
"from torch import default_generator, randperm\n",
|
||||
"from torch.utils.data.dataset import Subset\n",
|
||||
"import torchvision.transforms as transforms\n",
|
||||
"from torchvision.io import read_image\n",
|
||||
"from importlib import import_module\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"from torchvision import models\n",
|
||||
"import torch, torchvision\n",
|
||||
"from pathlib import Path\n",
|
||||
"from PIL import Image\n",
|
||||
"import torch.nn as nn\n",
|
||||
"from tqdm import tqdm\n",
|
||||
"import pandas as pd\n",
|
||||
"import numpy as np\n",
|
||||
"import matplotlib\n",
|
||||
"import os, shutil\n",
|
||||
"import mlconfig\n",
|
||||
"import random\n",
|
||||
"import shutil\n",
|
||||
"import timeit\n",
|
||||
"import copy\n",
|
||||
"import time\n",
|
||||
"import cv2\n",
|
||||
"import csv\n",
|
||||
"import sys\n",
|
||||
"import io\n",
|
||||
"import gc\n",
|
||||
"\n",
|
||||
"plt.rcParams[\"savefig.bbox\"] = 'tight'\n",
|
||||
"torch.manual_seed(1)\n",
|
||||
"#matplotlib.use('Agg')\n",
|
||||
"device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
|
||||
"print(device)\n",
|
||||
"torch.cuda.empty_cache()\n",
|
||||
"cv2.destroyAllWindows()\n",
|
||||
"gc.collect()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "384de097-82c6-41f5-bda9-b2f54bc99593",
|
||||
"metadata": {
|
||||
"slideshow": {
|
||||
"slide_type": ""
|
||||
},
|
||||
"tags": []
|
||||
},
|
||||
"source": [
|
||||
"# Подготовка и обучение детектирование"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "46e4dc99-6994-4fee-a32e-f3983bd991bd",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def prepare_and_learning_detection(num_classes, num_samples, path_dataset, model_name, config_name, model):\n",
|
||||
" num_samples_per_class = num_samples // num_classes\n",
|
||||
"\n",
|
||||
" #----------Создаём папку для сохранения результатов обучения--------------\n",
|
||||
" \n",
|
||||
" ind = 1\n",
|
||||
" while True:\n",
|
||||
" if os.path.exists(\"models/\" + model_name + str(ind)):\n",
|
||||
" ind += 1\n",
|
||||
" else:\n",
|
||||
" os.mkdir(\"models/\" + model_name + str(ind))\n",
|
||||
" path_res = \"models/\" + model_name + str(ind) + '/'\n",
|
||||
" break\n",
|
||||
" \n",
|
||||
" #----------Создаём файл dataset.csv для обучения--------------\n",
|
||||
" \n",
|
||||
" pd_columns = ['file_name']\n",
|
||||
" df = pd.DataFrame(columns=pd_columns)\n",
|
||||
" \n",
|
||||
" subdirs = os.listdir(path_dataset)\n",
|
||||
" for subdir in subdirs:\n",
|
||||
" files = os.listdir(path_dataset + subdir + '/')\n",
|
||||
" num_samples_per_class = min(num_samples_per_class, len(files))\n",
|
||||
" for subdir in subdirs:\n",
|
||||
" files = os.listdir(path_dataset + subdir + '/')\n",
|
||||
" random.shuffle(files)\n",
|
||||
" files_to_process = files[:num_samples_per_class]\n",
|
||||
" for file in files_to_process:\n",
|
||||
" row = pd.DataFrame({pd_columns[0]: [str(path_dataset + subdir + '/' + file)]})\n",
|
||||
" df = pd.concat([df, row], ignore_index=True)\n",
|
||||
" \n",
|
||||
" df.to_csv(path_res + 'dataset.csv', index=False)\n",
|
||||
" \n",
|
||||
" #----------Импортируем параметры для обучения--------------\n",
|
||||
" \n",
|
||||
" def load_function(attr):\n",
|
||||
" module_, func = attr.rsplit('.', maxsplit=1)\n",
|
||||
" return getattr(import_module(module_), func)\n",
|
||||
" \n",
|
||||
" config = mlconfig.load('config_' + config_name + '.yaml')\n",
|
||||
" \n",
|
||||
" #----------Создаём класс датасета--------------\n",
|
||||
" \n",
|
||||
" class MyDataset(Dataset):\n",
|
||||
" def __init__(self, path_dataset, csv_file):\n",
|
||||
" data=[]\n",
|
||||
" with open(path_dataset + csv_file, newline='') as csvfile:\n",
|
||||
" reader = csv.reader(csvfile, delimiter=' ', quotechar='|')\n",
|
||||
" for row in list(reader)[1:]:\n",
|
||||
" row = str(row)\n",
|
||||
" data.append(row[2: len(row)-2])\n",
|
||||
" self.sig_filenames = data\n",
|
||||
" self.path_dataset = path_dataset\n",
|
||||
" \n",
|
||||
" def __len__(self):\n",
|
||||
" return len(self.sig_filenames)\n",
|
||||
" \n",
|
||||
" def __getitem__(self, idx):\n",
|
||||
" image_real = np.asarray(cv2.split(cv2.imread(self.sig_filenames[idx][:-8]+'real.jpg')), dtype=np.float32)\n",
|
||||
" if 'drone' in list(self.sig_filenames[idx].split('/')):\n",
|
||||
" label = torch.tensor(0)\n",
|
||||
" if 'noise' in list(self.sig_filenames[idx].split('/')):\n",
|
||||
" label = torch.tensor(1)\n",
|
||||
" return image_real, label\n",
|
||||
" \n",
|
||||
" #----------Создаём датасет--------------\n",
|
||||
" \n",
|
||||
" dataset = MyDataset(path_dataset=path_res, csv_file='dataset.csv')\n",
|
||||
" train_set, valid_set = torch.utils.data.random_split(dataset, [0.7, 0.3], generator=torch.Generator().manual_seed(42))\n",
|
||||
" batch_size = config.batch_size\n",
|
||||
" train_dataloader = torch.utils.data.DataLoader(train_set, batch_size=batch_size, shuffle=True, drop_last=True)\n",
|
||||
" valid_dataloader = torch.utils.data.DataLoader(valid_set, batch_size=batch_size, shuffle=True, drop_last=True)\n",
|
||||
" \n",
|
||||
" dataloaders = {}\n",
|
||||
" dataloaders['train'] = train_dataloader\n",
|
||||
" dataloaders['val'] = valid_dataloader\n",
|
||||
" dataset_sizes = {}\n",
|
||||
" dataset_sizes['train'] = len(train_set)\n",
|
||||
" dataset_sizes['val'] = len(valid_set)\n",
|
||||
"\n",
|
||||
" #----------Обучаем модель--------------\n",
|
||||
"\n",
|
||||
" val_loss = []\n",
|
||||
" val_acc = []\n",
|
||||
" train_loss = []\n",
|
||||
" train_acc = []\n",
|
||||
" epochs = config.epoch\n",
|
||||
" \n",
|
||||
" best_acc = 0.0\n",
|
||||
" best_model = copy.deepcopy(model.state_dict())\n",
|
||||
" limit = config.limit\n",
|
||||
" epoch_limit = epochs\n",
|
||||
" \n",
|
||||
" start = timeit.default_timer()\n",
|
||||
" for epoch in range(1, epochs+1):\n",
|
||||
" print(f\"Epoch : {epoch}\\n\")\n",
|
||||
" dataloader = None\n",
|
||||
" \n",
|
||||
" for phase in ['train', 'val']:\n",
|
||||
" running_loss = 0.0\n",
|
||||
" running_corrects = 0\n",
|
||||
" \n",
|
||||
" for (img, label) in tqdm(dataloaders[phase]):\n",
|
||||
" img, label = img.to(device), label.to(device)\n",
|
||||
" optimizer.zero_grad()\n",
|
||||
" \n",
|
||||
" with torch.set_grad_enabled(phase == 'train'):\n",
|
||||
" output = model(img)\n",
|
||||
" _, pred = torch.max(output.data, 1)\n",
|
||||
" loss = criterion(output, label)\n",
|
||||
" if phase=='train' :\n",
|
||||
" loss.backward()\n",
|
||||
" optimizer.step()\n",
|
||||
" \n",
|
||||
" running_loss += loss.item() * img.size(0)\n",
|
||||
" running_corrects += torch.sum(pred == label.data)\n",
|
||||
" \n",
|
||||
" epoch_loss = running_loss / dataset_sizes[phase]\n",
|
||||
" epoch_acc = running_corrects.double() / dataset_sizes[phase]\n",
|
||||
" \n",
|
||||
" print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))\n",
|
||||
" \n",
|
||||
" if phase=='train' :\n",
|
||||
" train_loss.append(epoch_loss)\n",
|
||||
" train_acc.append(epoch_acc)\n",
|
||||
" else :\n",
|
||||
" val_loss.append(epoch_loss)\n",
|
||||
" val_acc.append(epoch_acc)\n",
|
||||
" if val_acc[-1] > best_acc :\n",
|
||||
" ind_limit = 0\n",
|
||||
" best_acc = val_acc[-1]\n",
|
||||
" best_model = copy.deepcopy(model.state_dict())\n",
|
||||
" torch.save(best_model, path_res + model_name + '.pth')\n",
|
||||
" else:\n",
|
||||
" ind_limit += 1\n",
|
||||
" \n",
|
||||
" if ind_limit >= limit:\n",
|
||||
" break\n",
|
||||
" \n",
|
||||
" if ind_limit >= limit:\n",
|
||||
" epoch_limit = epoch\n",
|
||||
" break\n",
|
||||
" \n",
|
||||
" print()\n",
|
||||
" \n",
|
||||
" end = timeit.default_timer()\n",
|
||||
" print(f\"Total time elapsed = {end - start} seconds\")\n",
|
||||
" epoch_limit += 1\n",
|
||||
" \n",
|
||||
" #----------Вывод графиков и сохранение результатов обучения--------------\n",
|
||||
" \n",
|
||||
" train_acc = np.asarray(list(map(lambda x: x.item(), train_acc)))\n",
|
||||
" val_acc = np.asarray(list(map(lambda x: x.item(), val_acc)))\n",
|
||||
" \n",
|
||||
" np.save(path_res+'train_acc.npy', train_acc)\n",
|
||||
" np.save(path_res+'val_acc.npy', val_acc)\n",
|
||||
" np.save(path_res+'train_loss.npy', train_loss)\n",
|
||||
" np.save(path_res+'val_loss.npy', val_loss)\n",
|
||||
" \n",
|
||||
" plt.figure()\n",
|
||||
" plt.plot(range(1,epoch_limit), train_loss, color='blue')\n",
|
||||
" plt.plot(range(1,epoch_limit), val_loss, color='red')\n",
|
||||
" plt.xlabel('Epoch')\n",
|
||||
" plt.ylabel('Loss')\n",
|
||||
" plt.title('Loss Curve')\n",
|
||||
" plt.legend(['Train Loss', 'Validation Loss'])\n",
|
||||
" plt.show()\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" \n",
|
||||
" plt.figure()\n",
|
||||
" plt.plot(range(1,epoch_limit), train_acc, color='blue')\n",
|
||||
" plt.plot(range(1,epoch_limit), val_acc, color='red')\n",
|
||||
" plt.xlabel('Epoch')\n",
|
||||
" plt.ylabel('Accuracy')\n",
|
||||
" plt.title('Accuracy Curve')\n",
|
||||
" plt.legend(['Train Accuracy', 'Validation Accuracy'])\n",
|
||||
" plt.show()\n",
|
||||
" \n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" torch.cuda.empty_cache()\n",
|
||||
" cv2.destroyAllWindows()\n",
|
||||
" del model\n",
|
||||
" gc.collect()\n",
|
||||
"\n",
|
||||
" return path_res, model_name"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "93c136ee",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Ensemble"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"id": "52e8d4c5",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"C:\\Users\\snytk\\miniconda3\\envs\\python311\\Lib\\site-packages\\torchvision\\models\\_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
|
||||
" warnings.warn(\n",
|
||||
"C:\\Users\\snytk\\miniconda3\\envs\\python311\\Lib\\site-packages\\torchvision\\models\\_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=ResNet18_Weights.IMAGENET1K_V1`. You can also use `weights=ResNet18_Weights.DEFAULT` to get the most up-to-date weights.\n",
|
||||
" warnings.warn(msg)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Epoch : 1\n",
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" 0%| | 0/337 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"tensor([], device='cuda:0', size=(4, 0))\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ename": "RuntimeError",
|
||||
"evalue": "Expected 3D (unbatched) or 4D (batched) input to conv2d, but got input of size: [4, 0]",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[1;31mRuntimeError\u001b[0m Traceback (most recent call last)",
|
||||
"Cell \u001b[1;32mIn[10], line 42\u001b[0m\n\u001b[0;32m 38\u001b[0m model \u001b[38;5;241m=\u001b[39m model\u001b[38;5;241m.\u001b[39mto(device)\n\u001b[0;32m 40\u001b[0m \u001b[38;5;66;03m#----------Создания датасета и обучение модели--------------\u001b[39;00m\n\u001b[1;32m---> 42\u001b[0m path_res, model_name \u001b[38;5;241m=\u001b[39m \u001b[43mprepare_and_learning_detection\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnum_classes\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mnum_classes\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnum_samples\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m20000\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpath_dataset\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m//192.168.11.63/data/DATASETS/Energomash/2400_learning/\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[0;32m 43\u001b[0m \u001b[43m \u001b[49m\u001b[43mmodel_name\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mconfig_name\u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m_2.4_jpg_\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconfig_name\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mconfig_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmodel\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 46\u001b[0m torch\u001b[38;5;241m.\u001b[39mcuda\u001b[38;5;241m.\u001b[39mempty_cache()\n\u001b[0;32m 47\u001b[0m cv2\u001b[38;5;241m.\u001b[39mdestroyAllWindows()\n",
|
||||
"Cell \u001b[1;32mIn[2], line 108\u001b[0m, in \u001b[0;36mprepare_and_learning_detection\u001b[1;34m(num_classes, num_samples, path_dataset, model_name, config_name, model)\u001b[0m\n\u001b[0;32m 105\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mzero_grad()\n\u001b[0;32m 107\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mset_grad_enabled(phase \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtrain\u001b[39m\u001b[38;5;124m'\u001b[39m):\n\u001b[1;32m--> 108\u001b[0m output \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[43m(\u001b[49m\u001b[43mimg\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 109\u001b[0m _, pred \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mmax(output\u001b[38;5;241m.\u001b[39mdata, \u001b[38;5;241m1\u001b[39m)\n\u001b[0;32m 110\u001b[0m loss \u001b[38;5;241m=\u001b[39m criterion(output, label)\n",
|
||||
"File \u001b[1;32m~\\miniconda3\\envs\\python311\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1553\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m 1551\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[0;32m 1552\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m-> 1553\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
|
||||
"File \u001b[1;32m~\\miniconda3\\envs\\python311\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1562\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m 1557\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m 1558\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m 1559\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m 1560\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m 1561\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1562\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 1564\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m 1565\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
|
||||
"Cell \u001b[1;32mIn[10], line 28\u001b[0m, in \u001b[0;36mModel.forward\u001b[1;34m(self, x)\u001b[0m\n\u001b[0;32m 26\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, x):\n\u001b[0;32m 27\u001b[0m \u001b[38;5;28mprint\u001b[39m(x)\n\u001b[1;32m---> 28\u001b[0m x \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodel\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 29\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m x\n",
|
||||
"File \u001b[1;32m~\\miniconda3\\envs\\python311\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1553\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m 1551\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[0;32m 1552\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m-> 1553\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
|
||||
"File \u001b[1;32m~\\miniconda3\\envs\\python311\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1562\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m 1557\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m 1558\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m 1559\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m 1560\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m 1561\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1562\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 1564\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m 1565\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
|
||||
"File \u001b[1;32m~\\miniconda3\\envs\\python311\\Lib\\site-packages\\torchvision\\models\\resnet.py:285\u001b[0m, in \u001b[0;36mResNet.forward\u001b[1;34m(self, x)\u001b[0m\n\u001b[0;32m 284\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, x: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[1;32m--> 285\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_forward_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m)\u001b[49m\n",
|
||||
"File \u001b[1;32m~\\miniconda3\\envs\\python311\\Lib\\site-packages\\torchvision\\models\\resnet.py:268\u001b[0m, in \u001b[0;36mResNet._forward_impl\u001b[1;34m(self, x)\u001b[0m\n\u001b[0;32m 266\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_forward_impl\u001b[39m(\u001b[38;5;28mself\u001b[39m, x: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[0;32m 267\u001b[0m \u001b[38;5;66;03m# See note [TorchScript super()]\u001b[39;00m\n\u001b[1;32m--> 268\u001b[0m x \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconv1\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 269\u001b[0m x \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbn1(x)\n\u001b[0;32m 270\u001b[0m x \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrelu(x)\n",
|
||||
"File \u001b[1;32m~\\miniconda3\\envs\\python311\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1553\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m 1551\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[0;32m 1552\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m-> 1553\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
|
||||
"File \u001b[1;32m~\\miniconda3\\envs\\python311\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1562\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m 1557\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m 1558\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m 1559\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m 1560\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m 1561\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1562\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 1564\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m 1565\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
|
||||
"File \u001b[1;32m~\\miniconda3\\envs\\python311\\Lib\\site-packages\\torch\\nn\\modules\\conv.py:458\u001b[0m, in \u001b[0;36mConv2d.forward\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m 457\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[1;32m--> 458\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_conv_forward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbias\u001b[49m\u001b[43m)\u001b[49m\n",
|
||||
"File \u001b[1;32m~\\miniconda3\\envs\\python311\\Lib\\site-packages\\torch\\nn\\modules\\conv.py:454\u001b[0m, in \u001b[0;36mConv2d._conv_forward\u001b[1;34m(self, input, weight, bias)\u001b[0m\n\u001b[0;32m 450\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpadding_mode \u001b[38;5;241m!=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mzeros\u001b[39m\u001b[38;5;124m'\u001b[39m:\n\u001b[0;32m 451\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m F\u001b[38;5;241m.\u001b[39mconv2d(F\u001b[38;5;241m.\u001b[39mpad(\u001b[38;5;28minput\u001b[39m, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_reversed_padding_repeated_twice, mode\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpadding_mode),\n\u001b[0;32m 452\u001b[0m weight, bias, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstride,\n\u001b[0;32m 453\u001b[0m _pair(\u001b[38;5;241m0\u001b[39m), \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdilation, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgroups)\n\u001b[1;32m--> 454\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mF\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconv2d\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbias\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstride\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 455\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpadding\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdilation\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgroups\u001b[49m\u001b[43m)\u001b[49m\n",
|
||||
"\u001b[1;31mRuntimeError\u001b[0m: Expected 3D (unbatched) or 4D (batched) input to conv2d, but got input of size: [4, 0]"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"#----------Инициализируем модель и параметры обучения--------------\n",
|
||||
"\n",
|
||||
"torch.cuda.empty_cache()\n",
|
||||
"cv2.destroyAllWindows()\n",
|
||||
"gc.collect()\n",
|
||||
"\n",
|
||||
"config_name = \"ensemble\"\n",
|
||||
" \n",
|
||||
"def load_function(attr):\n",
|
||||
" module_, func = attr.rsplit('.', maxsplit=1)\n",
|
||||
" return getattr(import_module(module_), func)\n",
|
||||
" \n",
|
||||
"config = mlconfig.load('config_' + config_name + '.yaml')\n",
|
||||
"\n",
|
||||
"model = models.resnet18(pretrained=True)\n",
|
||||
"\n",
|
||||
"num_classes = 2\n",
|
||||
"\n",
|
||||
"model.fc = nn.Linear(model.fc.in_features, num_classes)\n",
|
||||
"\n",
|
||||
"class Model(nn.Module):\n",
|
||||
" def __init__(self, model):\n",
|
||||
" super(Model, self).__init__()\n",
|
||||
" self.model = model\n",
|
||||
"\n",
|
||||
" def forward(self, x):\n",
|
||||
" print(x)\n",
|
||||
" x = self.model(x)\n",
|
||||
" return x\n",
|
||||
"\n",
|
||||
"model = Model(model)\n",
|
||||
"\n",
|
||||
"optimizer = load_function(config.optimizer.name)(model.parameters(), lr=config.optimizer.lr)\n",
|
||||
"criterion = load_function(config.loss_function.name)()\n",
|
||||
"scheduler = load_function(config.scheduler.name)(optimizer, step_size=config.scheduler.step_size, gamma=config.scheduler.gamma)\n",
|
||||
"\n",
|
||||
"if device != 'cpu':\n",
|
||||
" model = model.to(device)\n",
|
||||
"\n",
|
||||
"#----------Создания датасета и обучение модели--------------\n",
|
||||
"\n",
|
||||
"path_res, model_name = prepare_and_learning_detection(num_classes = num_classes, num_samples = 20000, path_dataset = \"//192.168.11.63/data/DATASETS/Energomash/2400_learning/\", \n",
|
||||
" model_name = config_name+\"_2.4_jpg_\", config_name = config_name, model=model)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"torch.cuda.empty_cache()\n",
|
||||
"cv2.destroyAllWindows()\n",
|
||||
"del model\n",
|
||||
"gc.collect()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "57d18676",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "c10afb29",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"celltoolbar": "Отсутствует",
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.6"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@ -0,0 +1,465 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "5a13ad6b-56c9-4381-b376-1765f6dd7553",
|
||||
"metadata": {
|
||||
"slideshow": {
|
||||
"slide_type": ""
|
||||
},
|
||||
"tags": []
|
||||
},
|
||||
"source": [
|
||||
"# Импортирование библиотек"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "7311cb4a-5bf3-4268-b431-43eea10e9ed6",
|
||||
"metadata": {
|
||||
"slideshow": {
|
||||
"slide_type": ""
|
||||
},
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"cuda\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"0"
|
||||
]
|
||||
},
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from sklearn.model_selection import train_test_split\n",
|
||||
"from torch.utils.data import Dataset, DataLoader\n",
|
||||
"from torch import default_generator, randperm\n",
|
||||
"from torch.utils.data.dataset import Subset\n",
|
||||
"import torchvision.transforms as transforms\n",
|
||||
"from torchvision.io import read_image\n",
|
||||
"from importlib import import_module\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"from torchvision import models\n",
|
||||
"import torch, torchvision\n",
|
||||
"from pathlib import Path\n",
|
||||
"from PIL import Image\n",
|
||||
"import torch.nn as nn\n",
|
||||
"from tqdm import tqdm\n",
|
||||
"import pandas as pd\n",
|
||||
"import numpy as np\n",
|
||||
"import matplotlib\n",
|
||||
"import os, shutil\n",
|
||||
"import mlconfig\n",
|
||||
"import random\n",
|
||||
"import shutil\n",
|
||||
"import timeit\n",
|
||||
"import copy\n",
|
||||
"import time\n",
|
||||
"import cv2\n",
|
||||
"import csv\n",
|
||||
"import sys\n",
|
||||
"import io\n",
|
||||
"import gc\n",
|
||||
"\n",
|
||||
"plt.rcParams[\"savefig.bbox\"] = 'tight'\n",
|
||||
"torch.manual_seed(1)\n",
|
||||
"#matplotlib.use('Agg')\n",
|
||||
"device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
|
||||
"print(device)\n",
|
||||
"torch.cuda.empty_cache()\n",
|
||||
"cv2.destroyAllWindows()\n",
|
||||
"gc.collect()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "384de097-82c6-41f5-bda9-b2f54bc99593",
|
||||
"metadata": {
|
||||
"slideshow": {
|
||||
"slide_type": ""
|
||||
},
|
||||
"tags": []
|
||||
},
|
||||
"source": [
|
||||
"# Подготовка и обучение детектирование"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "46e4dc99-6994-4fee-a32e-f3983bd991bd",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def prepare_and_learning_detection(num_classes, num_samples, path_dataset, model_name, config_name, model):\n",
|
||||
" num_samples_per_class = num_samples // num_classes\n",
|
||||
"\n",
|
||||
" #----------Создаём папку для сохранения результатов обучения--------------\n",
|
||||
" \n",
|
||||
" ind = 1\n",
|
||||
" while True:\n",
|
||||
" if os.path.exists(\"models/\" + model_name + str(ind)):\n",
|
||||
" ind += 1\n",
|
||||
" else:\n",
|
||||
" os.mkdir(\"models/\" + model_name + str(ind))\n",
|
||||
" path_res = \"models/\" + model_name + str(ind) + '/'\n",
|
||||
" break\n",
|
||||
" \n",
|
||||
" #----------Создаём файл dataset.csv для обучения--------------\n",
|
||||
" \n",
|
||||
" pd_columns = ['file_name']\n",
|
||||
" df = pd.DataFrame(columns=pd_columns)\n",
|
||||
" \n",
|
||||
" subdirs = os.listdir(path_dataset)\n",
|
||||
" for subdir in subdirs:\n",
|
||||
" files = os.listdir(path_dataset + subdir + '/')\n",
|
||||
" num_samples_per_class = min(num_samples_per_class, len(files))\n",
|
||||
" for subdir in subdirs:\n",
|
||||
" files = os.listdir(path_dataset + subdir + '/')\n",
|
||||
" random.shuffle(files)\n",
|
||||
" files_to_process = files[:num_samples_per_class]\n",
|
||||
" for file in files_to_process:\n",
|
||||
" row = pd.DataFrame({pd_columns[0]: [str(path_dataset + subdir + '/' + file)]})\n",
|
||||
" df = pd.concat([df, row], ignore_index=True)\n",
|
||||
" \n",
|
||||
" df.to_csv(path_res + 'dataset.csv', index=False)\n",
|
||||
" \n",
|
||||
" #----------Импортируем параметры для обучения--------------\n",
|
||||
" \n",
|
||||
" def load_function(attr):\n",
|
||||
" module_, func = attr.rsplit('.', maxsplit=1)\n",
|
||||
" return getattr(import_module(module_), func)\n",
|
||||
" \n",
|
||||
" config = mlconfig.load('config_' + config_name + '.yaml')\n",
|
||||
" \n",
|
||||
" #----------Создаём класс датасета--------------\n",
|
||||
" \n",
|
||||
" class MyDataset(Dataset):\n",
|
||||
" def __init__(self, path_dataset, csv_file):\n",
|
||||
" data=[]\n",
|
||||
" with open(path_dataset + csv_file, newline='') as csvfile:\n",
|
||||
" reader = csv.reader(csvfile, delimiter=' ', quotechar='|')\n",
|
||||
" for row in list(reader)[1:]:\n",
|
||||
" row = str(row)\n",
|
||||
" data.append(row[2: len(row)-2])\n",
|
||||
" self.sig_filenames = data\n",
|
||||
" self.path_dataset = path_dataset\n",
|
||||
" \n",
|
||||
" def __len__(self):\n",
|
||||
" return len(self.sig_filenames)\n",
|
||||
" \n",
|
||||
" def __getitem__(self, idx):\n",
|
||||
" image_real = np.asarray(cv2.split(cv2.imread(self.sig_filenames[idx][:-8]+'real.jpg')), dtype=np.float32)\n",
|
||||
" if 'drone' in list(self.sig_filenames[idx].split('/')):\n",
|
||||
" label = torch.tensor(0)\n",
|
||||
" if 'noise' in list(self.sig_filenames[idx].split('/')):\n",
|
||||
" label = torch.tensor(1)\n",
|
||||
" return image_real, label\n",
|
||||
" \n",
|
||||
" #----------Создаём датасет--------------\n",
|
||||
" \n",
|
||||
" dataset = MyDataset(path_dataset=path_res, csv_file='dataset.csv')\n",
|
||||
" train_set, valid_set = torch.utils.data.random_split(dataset, [0.7, 0.3], generator=torch.Generator().manual_seed(42))\n",
|
||||
" batch_size = config.batch_size\n",
|
||||
" train_dataloader = torch.utils.data.DataLoader(train_set, batch_size=batch_size, shuffle=True, drop_last=True)\n",
|
||||
" valid_dataloader = torch.utils.data.DataLoader(valid_set, batch_size=batch_size, shuffle=True, drop_last=True)\n",
|
||||
" \n",
|
||||
" dataloaders = {}\n",
|
||||
" dataloaders['train'] = train_dataloader\n",
|
||||
" dataloaders['val'] = valid_dataloader\n",
|
||||
" dataset_sizes = {}\n",
|
||||
" dataset_sizes['train'] = len(train_set)\n",
|
||||
" dataset_sizes['val'] = len(valid_set)\n",
|
||||
"\n",
|
||||
" #----------Обучаем модель--------------\n",
|
||||
"\n",
|
||||
" val_loss = []\n",
|
||||
" val_acc = []\n",
|
||||
" train_loss = []\n",
|
||||
" train_acc = []\n",
|
||||
" epochs = config.epoch\n",
|
||||
" \n",
|
||||
" best_acc = 0.0\n",
|
||||
" best_model = copy.deepcopy(model.state_dict())\n",
|
||||
" limit = config.limit\n",
|
||||
" epoch_limit = epochs\n",
|
||||
" \n",
|
||||
" start = timeit.default_timer()\n",
|
||||
" for epoch in range(1, epochs+1):\n",
|
||||
" print(f\"Epoch : {epoch}\\n\")\n",
|
||||
" dataloader = None\n",
|
||||
" \n",
|
||||
" for phase in ['train', 'val']:\n",
|
||||
" running_loss = 0.0\n",
|
||||
" running_corrects = 0\n",
|
||||
" \n",
|
||||
" for (img, label) in tqdm(dataloaders[phase]):\n",
|
||||
" img, label = img.to(device), label.to(device)\n",
|
||||
" optimizer.zero_grad()\n",
|
||||
" \n",
|
||||
" with torch.set_grad_enabled(phase == 'train'):\n",
|
||||
" output = model(img)\n",
|
||||
" _, pred = torch.max(output.data, 1)\n",
|
||||
" loss = criterion(output, label)\n",
|
||||
" if phase=='train' :\n",
|
||||
" loss.backward()\n",
|
||||
" optimizer.step()\n",
|
||||
" \n",
|
||||
" running_loss += loss.item() * img.size(0)\n",
|
||||
" running_corrects += torch.sum(pred == label.data)\n",
|
||||
" \n",
|
||||
" epoch_loss = running_loss / dataset_sizes[phase]\n",
|
||||
" epoch_acc = running_corrects.double() / dataset_sizes[phase]\n",
|
||||
" \n",
|
||||
" print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))\n",
|
||||
" \n",
|
||||
" if phase=='train' :\n",
|
||||
" train_loss.append(epoch_loss)\n",
|
||||
" train_acc.append(epoch_acc)\n",
|
||||
" else :\n",
|
||||
" val_loss.append(epoch_loss)\n",
|
||||
" val_acc.append(epoch_acc)\n",
|
||||
" if val_acc[-1] > best_acc :\n",
|
||||
" ind_limit = 0\n",
|
||||
" best_acc = val_acc[-1]\n",
|
||||
" best_model = copy.deepcopy(model.state_dict())\n",
|
||||
" torch.save(best_model, path_res + model_name + '.pth')\n",
|
||||
" else:\n",
|
||||
" ind_limit += 1\n",
|
||||
" \n",
|
||||
" if ind_limit >= limit:\n",
|
||||
" break\n",
|
||||
" \n",
|
||||
" if ind_limit >= limit:\n",
|
||||
" epoch_limit = epoch\n",
|
||||
" break\n",
|
||||
" \n",
|
||||
" print()\n",
|
||||
" \n",
|
||||
" end = timeit.default_timer()\n",
|
||||
" print(f\"Total time elapsed = {end - start} seconds\")\n",
|
||||
" epoch_limit += 1\n",
|
||||
" \n",
|
||||
" #----------Вывод графиков и сохранение результатов обучения--------------\n",
|
||||
" \n",
|
||||
" train_acc = np.asarray(list(map(lambda x: x.item(), train_acc)))\n",
|
||||
" val_acc = np.asarray(list(map(lambda x: x.item(), val_acc)))\n",
|
||||
" \n",
|
||||
" np.save(path_res+'train_acc.npy', train_acc)\n",
|
||||
" np.save(path_res+'val_acc.npy', val_acc)\n",
|
||||
" np.save(path_res+'train_loss.npy', train_loss)\n",
|
||||
" np.save(path_res+'val_loss.npy', val_loss)\n",
|
||||
" \n",
|
||||
" plt.figure()\n",
|
||||
" plt.plot(range(1,epoch_limit), train_loss, color='blue')\n",
|
||||
" plt.plot(range(1,epoch_limit), val_loss, color='red')\n",
|
||||
" plt.xlabel('Epoch')\n",
|
||||
" plt.ylabel('Loss')\n",
|
||||
" plt.title('Loss Curve')\n",
|
||||
" plt.legend(['Train Loss', 'Validation Loss'])\n",
|
||||
" plt.show()\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" \n",
|
||||
" plt.figure()\n",
|
||||
" plt.plot(range(1,epoch_limit), train_acc, color='blue')\n",
|
||||
" plt.plot(range(1,epoch_limit), val_acc, color='red')\n",
|
||||
" plt.xlabel('Epoch')\n",
|
||||
" plt.ylabel('Accuracy')\n",
|
||||
" plt.title('Accuracy Curve')\n",
|
||||
" plt.legend(['Train Accuracy', 'Validation Accuracy'])\n",
|
||||
" plt.show()\n",
|
||||
" \n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" torch.cuda.empty_cache()\n",
|
||||
" cv2.destroyAllWindows()\n",
|
||||
" del model\n",
|
||||
" gc.collect()\n",
|
||||
"\n",
|
||||
" return path_res, model_name"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "93c136ee",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Ensemble"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"id": "52e8d4c5",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"C:\\Users\\snytk\\miniconda3\\envs\\python311\\Lib\\site-packages\\torchvision\\models\\_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
|
||||
" warnings.warn(\n",
|
||||
"C:\\Users\\snytk\\miniconda3\\envs\\python311\\Lib\\site-packages\\torchvision\\models\\_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=ResNet18_Weights.IMAGENET1K_V1`. You can also use `weights=ResNet18_Weights.DEFAULT` to get the most up-to-date weights.\n",
|
||||
" warnings.warn(msg)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Epoch : 1\n",
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" 0%| | 0/337 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"tensor([], device='cuda:0', size=(4, 0))\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ename": "RuntimeError",
|
||||
"evalue": "Expected 3D (unbatched) or 4D (batched) input to conv2d, but got input of size: [4, 0]",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[1;31mRuntimeError\u001b[0m Traceback (most recent call last)",
|
||||
"Cell \u001b[1;32mIn[10], line 42\u001b[0m\n\u001b[0;32m 38\u001b[0m model \u001b[38;5;241m=\u001b[39m model\u001b[38;5;241m.\u001b[39mto(device)\n\u001b[0;32m 40\u001b[0m \u001b[38;5;66;03m#----------Создания датасета и обучение модели--------------\u001b[39;00m\n\u001b[1;32m---> 42\u001b[0m path_res, model_name \u001b[38;5;241m=\u001b[39m \u001b[43mprepare_and_learning_detection\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnum_classes\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mnum_classes\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnum_samples\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m20000\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpath_dataset\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m//192.168.11.63/data/DATASETS/Energomash/2400_learning/\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[0;32m 43\u001b[0m \u001b[43m \u001b[49m\u001b[43mmodel_name\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mconfig_name\u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m_2.4_jpg_\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconfig_name\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mconfig_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmodel\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 46\u001b[0m torch\u001b[38;5;241m.\u001b[39mcuda\u001b[38;5;241m.\u001b[39mempty_cache()\n\u001b[0;32m 47\u001b[0m cv2\u001b[38;5;241m.\u001b[39mdestroyAllWindows()\n",
|
||||
"Cell \u001b[1;32mIn[2], line 108\u001b[0m, in \u001b[0;36mprepare_and_learning_detection\u001b[1;34m(num_classes, num_samples, path_dataset, model_name, config_name, model)\u001b[0m\n\u001b[0;32m 105\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mzero_grad()\n\u001b[0;32m 107\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mset_grad_enabled(phase \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtrain\u001b[39m\u001b[38;5;124m'\u001b[39m):\n\u001b[1;32m--> 108\u001b[0m output \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[43m(\u001b[49m\u001b[43mimg\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 109\u001b[0m _, pred \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mmax(output\u001b[38;5;241m.\u001b[39mdata, \u001b[38;5;241m1\u001b[39m)\n\u001b[0;32m 110\u001b[0m loss \u001b[38;5;241m=\u001b[39m criterion(output, label)\n",
|
||||
"File \u001b[1;32m~\\miniconda3\\envs\\python311\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1553\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m 1551\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[0;32m 1552\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m-> 1553\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
|
||||
"File \u001b[1;32m~\\miniconda3\\envs\\python311\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1562\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m 1557\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m 1558\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m 1559\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m 1560\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m 1561\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1562\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 1564\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m 1565\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
|
||||
"Cell \u001b[1;32mIn[10], line 28\u001b[0m, in \u001b[0;36mModel.forward\u001b[1;34m(self, x)\u001b[0m\n\u001b[0;32m 26\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, x):\n\u001b[0;32m 27\u001b[0m \u001b[38;5;28mprint\u001b[39m(x)\n\u001b[1;32m---> 28\u001b[0m x \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodel\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 29\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m x\n",
|
||||
"File \u001b[1;32m~\\miniconda3\\envs\\python311\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1553\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m 1551\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[0;32m 1552\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m-> 1553\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
|
||||
"File \u001b[1;32m~\\miniconda3\\envs\\python311\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1562\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m 1557\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m 1558\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m 1559\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m 1560\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m 1561\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1562\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 1564\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m 1565\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
|
||||
"File \u001b[1;32m~\\miniconda3\\envs\\python311\\Lib\\site-packages\\torchvision\\models\\resnet.py:285\u001b[0m, in \u001b[0;36mResNet.forward\u001b[1;34m(self, x)\u001b[0m\n\u001b[0;32m 284\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, x: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[1;32m--> 285\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_forward_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m)\u001b[49m\n",
|
||||
"File \u001b[1;32m~\\miniconda3\\envs\\python311\\Lib\\site-packages\\torchvision\\models\\resnet.py:268\u001b[0m, in \u001b[0;36mResNet._forward_impl\u001b[1;34m(self, x)\u001b[0m\n\u001b[0;32m 266\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_forward_impl\u001b[39m(\u001b[38;5;28mself\u001b[39m, x: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[0;32m 267\u001b[0m \u001b[38;5;66;03m# See note [TorchScript super()]\u001b[39;00m\n\u001b[1;32m--> 268\u001b[0m x \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconv1\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 269\u001b[0m x \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbn1(x)\n\u001b[0;32m 270\u001b[0m x \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrelu(x)\n",
|
||||
"File \u001b[1;32m~\\miniconda3\\envs\\python311\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1553\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m 1551\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[0;32m 1552\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m-> 1553\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
|
||||
"File \u001b[1;32m~\\miniconda3\\envs\\python311\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1562\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m 1557\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m 1558\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m 1559\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m 1560\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m 1561\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1562\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 1564\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m 1565\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
|
||||
"File \u001b[1;32m~\\miniconda3\\envs\\python311\\Lib\\site-packages\\torch\\nn\\modules\\conv.py:458\u001b[0m, in \u001b[0;36mConv2d.forward\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m 457\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[1;32m--> 458\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_conv_forward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbias\u001b[49m\u001b[43m)\u001b[49m\n",
|
||||
"File \u001b[1;32m~\\miniconda3\\envs\\python311\\Lib\\site-packages\\torch\\nn\\modules\\conv.py:454\u001b[0m, in \u001b[0;36mConv2d._conv_forward\u001b[1;34m(self, input, weight, bias)\u001b[0m\n\u001b[0;32m 450\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpadding_mode \u001b[38;5;241m!=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mzeros\u001b[39m\u001b[38;5;124m'\u001b[39m:\n\u001b[0;32m 451\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m F\u001b[38;5;241m.\u001b[39mconv2d(F\u001b[38;5;241m.\u001b[39mpad(\u001b[38;5;28minput\u001b[39m, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_reversed_padding_repeated_twice, mode\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpadding_mode),\n\u001b[0;32m 452\u001b[0m weight, bias, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstride,\n\u001b[0;32m 453\u001b[0m _pair(\u001b[38;5;241m0\u001b[39m), \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdilation, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgroups)\n\u001b[1;32m--> 454\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mF\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconv2d\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbias\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstride\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m 455\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpadding\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdilation\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgroups\u001b[49m\u001b[43m)\u001b[49m\n",
|
||||
"\u001b[1;31mRuntimeError\u001b[0m: Expected 3D (unbatched) or 4D (batched) input to conv2d, but got input of size: [4, 0]"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"#----------Инициализируем модель и параметры обучения--------------\n",
|
||||
"\n",
|
||||
"torch.cuda.empty_cache()\n",
|
||||
"cv2.destroyAllWindows()\n",
|
||||
"gc.collect()\n",
|
||||
"\n",
|
||||
"config_name = \"ensemble\"\n",
|
||||
" \n",
|
||||
"def load_function(attr):\n",
|
||||
" module_, func = attr.rsplit('.', maxsplit=1)\n",
|
||||
" return getattr(import_module(module_), func)\n",
|
||||
" \n",
|
||||
"config = mlconfig.load('config_' + config_name + '.yaml')\n",
|
||||
"\n",
|
||||
"model = models.resnet18(pretrained=True)\n",
|
||||
"\n",
|
||||
"num_classes = 2\n",
|
||||
"\n",
|
||||
"model.fc = nn.Linear(model.fc.in_features, num_classes)\n",
|
||||
"\n",
|
||||
"class Model(nn.Module):\n",
|
||||
" def __init__(self, model):\n",
|
||||
" super(Model, self).__init__()\n",
|
||||
" self.model = model\n",
|
||||
"\n",
|
||||
" def forward(self, x):\n",
|
||||
" print(x)\n",
|
||||
" x = self.model(x)\n",
|
||||
" return x\n",
|
||||
"\n",
|
||||
"model = Model(model)\n",
|
||||
"\n",
|
||||
"optimizer = load_function(config.optimizer.name)(model.parameters(), lr=config.optimizer.lr)\n",
|
||||
"criterion = load_function(config.loss_function.name)()\n",
|
||||
"scheduler = load_function(config.scheduler.name)(optimizer, step_size=config.scheduler.step_size, gamma=config.scheduler.gamma)\n",
|
||||
"\n",
|
||||
"if device != 'cpu':\n",
|
||||
" model = model.to(device)\n",
|
||||
"\n",
|
||||
"#----------Создания датасета и обучение модели--------------\n",
|
||||
"\n",
|
||||
"path_res, model_name = prepare_and_learning_detection(num_classes = num_classes, num_samples = 20000, path_dataset = \"//192.168.11.63/data/DATASETS/Energomash/2400_learning/\", \n",
|
||||
" model_name = config_name+\"_2.4_jpg_\", config_name = config_name, model=model)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"torch.cuda.empty_cache()\n",
|
||||
"cv2.destroyAllWindows()\n",
|
||||
"del model\n",
|
||||
"gc.collect()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "57d18676",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "c10afb29",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"celltoolbar": "Отсутствует",
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.6"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@ -0,0 +1,503 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "5a13ad6b-56c9-4381-b376-1765f6dd7553",
|
||||
"metadata": {
|
||||
"slideshow": {
|
||||
"slide_type": ""
|
||||
},
|
||||
"tags": []
|
||||
},
|
||||
"source": [
|
||||
"# Импортирование библиотек"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "7311cb4a-5bf3-4268-b431-43eea10e9ed6",
|
||||
"metadata": {
|
||||
"slideshow": {
|
||||
"slide_type": ""
|
||||
},
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"cuda\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"12"
|
||||
]
|
||||
},
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from sklearn.model_selection import train_test_split\n",
|
||||
"from torch.utils.data import Dataset, DataLoader\n",
|
||||
"from torch import default_generator, randperm\n",
|
||||
"from torch.utils.data.dataset import Subset\n",
|
||||
"import torchvision.transforms as transforms\n",
|
||||
"from torchvision.io import read_image\n",
|
||||
"from importlib import import_module\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"from torchvision import models\n",
|
||||
"import torch, torchvision\n",
|
||||
"from pathlib import Path\n",
|
||||
"from PIL import Image\n",
|
||||
"import torch.nn as nn\n",
|
||||
"from tqdm import tqdm\n",
|
||||
"import pandas as pd\n",
|
||||
"import numpy as np\n",
|
||||
"import matplotlib\n",
|
||||
"import os, shutil\n",
|
||||
"import mlconfig\n",
|
||||
"import random\n",
|
||||
"import shutil\n",
|
||||
"import timeit\n",
|
||||
"import copy\n",
|
||||
"import time\n",
|
||||
"import cv2\n",
|
||||
"import csv\n",
|
||||
"import sys\n",
|
||||
"import io\n",
|
||||
"import gc\n",
|
||||
"\n",
|
||||
"plt.rcParams[\"savefig.bbox\"] = 'tight'\n",
|
||||
"torch.manual_seed(1)\n",
|
||||
"#matplotlib.use('Agg')\n",
|
||||
"device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
|
||||
"print(device)\n",
|
||||
"torch.cuda.empty_cache()\n",
|
||||
"cv2.destroyAllWindows()\n",
|
||||
"gc.collect()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "384de097-82c6-41f5-bda9-b2f54bc99593",
|
||||
"metadata": {
|
||||
"slideshow": {
|
||||
"slide_type": ""
|
||||
},
|
||||
"tags": []
|
||||
},
|
||||
"source": [
|
||||
"# Подготовка и обучение детектирование"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "46e4dc99-6994-4fee-a32e-f3983bd991bd",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def prepare_and_learning_detection(num_classes, num_samples, path_dataset, model_name, config_name, model):\n",
|
||||
" num_samples_per_class = num_samples // num_classes\n",
|
||||
"\n",
|
||||
" #----------Создаём папку для сохранения результатов обучения--------------\n",
|
||||
" \n",
|
||||
" ind = 1\n",
|
||||
" while True:\n",
|
||||
" if os.path.exists(\"models/\" + model_name + str(ind)):\n",
|
||||
" ind += 1\n",
|
||||
" else:\n",
|
||||
" os.mkdir(\"models/\" + model_name + str(ind))\n",
|
||||
" path_res = \"models/\" + model_name + str(ind) + '/'\n",
|
||||
" break\n",
|
||||
" \n",
|
||||
" #----------Создаём файл dataset.csv для обучения--------------\n",
|
||||
" \n",
|
||||
" pd_columns = ['file_name']\n",
|
||||
" df = pd.DataFrame(columns=pd_columns)\n",
|
||||
" \n",
|
||||
" subdirs = os.listdir(path_dataset)\n",
|
||||
" for subdir in subdirs:\n",
|
||||
" files = os.listdir(path_dataset + subdir + '/')\n",
|
||||
" num_samples_per_class = min(num_samples_per_class, len(files))\n",
|
||||
" for subdir in subdirs:\n",
|
||||
" files = os.listdir(path_dataset + subdir + '/')\n",
|
||||
" random.shuffle(files)\n",
|
||||
" files_to_process = files[:num_samples_per_class]\n",
|
||||
" for file in files_to_process:\n",
|
||||
" row = pd.DataFrame({pd_columns[0]: [str(path_dataset + subdir + '/' + file)]})\n",
|
||||
" df = pd.concat([df, row], ignore_index=True)\n",
|
||||
" \n",
|
||||
" df.to_csv(path_res + 'dataset.csv', index=False)\n",
|
||||
" \n",
|
||||
" #----------Импортируем параметры для обучения--------------\n",
|
||||
" \n",
|
||||
" def load_function(attr):\n",
|
||||
" module_, func = attr.rsplit('.', maxsplit=1)\n",
|
||||
" return getattr(import_module(module_), func)\n",
|
||||
" \n",
|
||||
" config = mlconfig.load('config_' + config_name + '.yaml')\n",
|
||||
" \n",
|
||||
" #----------Создаём класс датасета--------------\n",
|
||||
" \n",
|
||||
" class MyDataset(Dataset):\n",
|
||||
" def __init__(self, path_dataset, csv_file):\n",
|
||||
" data=[]\n",
|
||||
" with open(path_dataset + csv_file, newline='') as csvfile:\n",
|
||||
" reader = csv.reader(csvfile, delimiter=' ', quotechar='|')\n",
|
||||
" for row in list(reader)[1:]:\n",
|
||||
" row = str(row)\n",
|
||||
" data.append(row[2: len(row)-2])\n",
|
||||
" self.sig_filenames = data\n",
|
||||
" self.path_dataset = path_dataset\n",
|
||||
" \n",
|
||||
" def __len__(self):\n",
|
||||
" return len(self.sig_filenames)\n",
|
||||
" \n",
|
||||
" def __getitem__(self, idx):\n",
|
||||
" image_real = np.asarray(cv2.split(cv2.imread(self.sig_filenames[idx][:-8]+'real.jpg')), dtype=np.float32)\n",
|
||||
" image_imag = np.asarray(cv2.split(cv2.imread(self.sig_filenames[idx][:-8]+'imag.jpg')), dtype=np.float32)\n",
|
||||
" image_spec = np.asarray(cv2.split(cv2.imread(self.sig_filenames[idx][:-8]+'spec.jpg')), dtype=np.float32)\n",
|
||||
" if 'drone' in list(self.sig_filenames[idx].split('/')):\n",
|
||||
" label = torch.tensor(0)\n",
|
||||
" if 'noise' in list(self.sig_filenames[idx].split('/')):\n",
|
||||
" label = torch.tensor(1)\n",
|
||||
" return image_real, image_imag, image_spec, label\n",
|
||||
" \n",
|
||||
" #----------Создаём датасет--------------\n",
|
||||
" \n",
|
||||
" dataset = MyDataset(path_dataset=path_res, csv_file='dataset.csv')\n",
|
||||
" train_set, valid_set = torch.utils.data.random_split(dataset, [0.7, 0.3], generator=torch.Generator().manual_seed(42))\n",
|
||||
" batch_size = config.batch_size\n",
|
||||
" train_dataloader = torch.utils.data.DataLoader(train_set, batch_size=batch_size, shuffle=True, drop_last=True)\n",
|
||||
" valid_dataloader = torch.utils.data.DataLoader(valid_set, batch_size=batch_size, shuffle=True, drop_last=True)\n",
|
||||
" \n",
|
||||
" dataloaders = {}\n",
|
||||
" dataloaders['train'] = train_dataloader\n",
|
||||
" dataloaders['val'] = valid_dataloader\n",
|
||||
" dataset_sizes = {}\n",
|
||||
" dataset_sizes['train'] = len(train_set)\n",
|
||||
" dataset_sizes['val'] = len(valid_set)\n",
|
||||
"\n",
|
||||
" #----------Обучаем модель--------------\n",
|
||||
"\n",
|
||||
" val_loss = []\n",
|
||||
" val_acc = []\n",
|
||||
" train_loss = []\n",
|
||||
" train_acc = []\n",
|
||||
" epochs = config.epoch\n",
|
||||
" \n",
|
||||
" best_acc = 0.0\n",
|
||||
" best_model = copy.deepcopy(model.state_dict())\n",
|
||||
" limit = config.limit\n",
|
||||
" epoch_limit = epochs\n",
|
||||
" \n",
|
||||
" start = timeit.default_timer()\n",
|
||||
" for epoch in range(1, epochs+1):\n",
|
||||
" print(f\"Epoch : {epoch}\\n\")\n",
|
||||
" dataloader = None\n",
|
||||
" \n",
|
||||
" for phase in ['train', 'val']:\n",
|
||||
" running_loss = 0.0\n",
|
||||
" running_corrects = 0\n",
|
||||
" \n",
|
||||
" for (img1, img2, img3, label) in tqdm(dataloaders[phase]):\n",
|
||||
" img1, img2, img3, label = img1.to(device), img2.to(device), img3.to(device), label.to(device)\n",
|
||||
" optimizer.zero_grad()\n",
|
||||
" \n",
|
||||
" with torch.set_grad_enabled(phase == 'train'):\n",
|
||||
" output = model([img1, img2, img3])\n",
|
||||
" _, pred = torch.max(output.data, 1)\n",
|
||||
" loss = criterion(output, label)\n",
|
||||
" if phase=='train' :\n",
|
||||
" loss.backward()\n",
|
||||
" optimizer.step()\n",
|
||||
" \n",
|
||||
" running_loss += loss.item() * 3 * img1.size(0)\n",
|
||||
" running_corrects += torch.sum(pred == label.data)\n",
|
||||
" \n",
|
||||
" epoch_loss = running_loss / dataset_sizes[phase]\n",
|
||||
" epoch_acc = running_corrects.double() / dataset_sizes[phase]\n",
|
||||
" \n",
|
||||
" print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))\n",
|
||||
" \n",
|
||||
" if phase=='train' :\n",
|
||||
" train_loss.append(epoch_loss)\n",
|
||||
" train_acc.append(epoch_acc)\n",
|
||||
" else :\n",
|
||||
" val_loss.append(epoch_loss)\n",
|
||||
" val_acc.append(epoch_acc)\n",
|
||||
" if val_acc[-1] > best_acc :\n",
|
||||
" ind_limit = 0\n",
|
||||
" best_acc = val_acc[-1]\n",
|
||||
" best_model = copy.deepcopy(model.state_dict())\n",
|
||||
" torch.save(best_model, path_res + model_name + '.pth')\n",
|
||||
" else:\n",
|
||||
" ind_limit += 1\n",
|
||||
" \n",
|
||||
" if ind_limit >= limit:\n",
|
||||
" break\n",
|
||||
" \n",
|
||||
" if ind_limit >= limit:\n",
|
||||
" epoch_limit = epoch\n",
|
||||
" break\n",
|
||||
" \n",
|
||||
" print()\n",
|
||||
" \n",
|
||||
" end = timeit.default_timer()\n",
|
||||
" print(f\"Total time elapsed = {end - start} seconds\")\n",
|
||||
" epoch_limit += 1\n",
|
||||
" \n",
|
||||
" #----------Вывод графиков и сохранение результатов обучения--------------\n",
|
||||
" \n",
|
||||
" train_acc = np.asarray(list(map(lambda x: x.item(), train_acc)))\n",
|
||||
" val_acc = np.asarray(list(map(lambda x: x.item(), val_acc)))\n",
|
||||
" \n",
|
||||
" np.save(path_res+'train_acc.npy', train_acc)\n",
|
||||
" np.save(path_res+'val_acc.npy', val_acc)\n",
|
||||
" np.save(path_res+'train_loss.npy', train_loss)\n",
|
||||
" np.save(path_res+'val_loss.npy', val_loss)\n",
|
||||
" \n",
|
||||
" plt.figure()\n",
|
||||
" plt.plot(range(1,epoch_limit), train_loss, color='blue')\n",
|
||||
" plt.plot(range(1,epoch_limit), val_loss, color='red')\n",
|
||||
" plt.xlabel('Epoch')\n",
|
||||
" plt.ylabel('Loss')\n",
|
||||
" plt.title('Loss Curve')\n",
|
||||
" plt.legend(['Train Loss', 'Validation Loss'])\n",
|
||||
" plt.show()\n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" \n",
|
||||
" plt.figure()\n",
|
||||
" plt.plot(range(1,epoch_limit), train_acc, color='blue')\n",
|
||||
" plt.plot(range(1,epoch_limit), val_acc, color='red')\n",
|
||||
" plt.xlabel('Epoch')\n",
|
||||
" plt.ylabel('Accuracy')\n",
|
||||
" plt.title('Accuracy Curve')\n",
|
||||
" plt.legend(['Train Accuracy', 'Validation Accuracy'])\n",
|
||||
" plt.show()\n",
|
||||
" \n",
|
||||
" plt.clf()\n",
|
||||
" plt.cla()\n",
|
||||
" plt.close()\n",
|
||||
" torch.cuda.empty_cache()\n",
|
||||
" cv2.destroyAllWindows()\n",
|
||||
" del model\n",
|
||||
" gc.collect()\n",
|
||||
"\n",
|
||||
" return path_res, model_name"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "93c136ee",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Ensemble"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "52e8d4c5",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"C:\\Users\\snytk\\miniconda3\\envs\\python311\\Lib\\site-packages\\torchvision\\models\\_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
|
||||
" warnings.warn(\n",
|
||||
"C:\\Users\\snytk\\miniconda3\\envs\\python311\\Lib\\site-packages\\torchvision\\models\\_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=None`.\n",
|
||||
" warnings.warn(msg)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Epoch : 1\n",
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████| 658/658 [1:00:26<00:00, 5.51s/it]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"train Loss: 0.6663 Acc: 0.9241\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████| 282/282 [02:45<00:00, 1.71it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"val Loss: 0.4023 Acc: 0.9557\n",
|
||||
"\n",
|
||||
"Epoch : 2\n",
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████| 658/658 [43:11<00:00, 3.94s/it]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"train Loss: 0.4096 Acc: 0.9514\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|████████████████████████████████████████████████████████████████████████████████| 282/282 [00:47<00:00, 5.98it/s]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"val Loss: 0.3390 Acc: 0.9574\n",
|
||||
"\n",
|
||||
"Epoch : 3\n",
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" 8%|██████▍ | 52/658 [04:26<51:43, 5.12s/it]"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"#----------Инициализируем модель и параметры обучения--------------\n",
|
||||
"\n",
|
||||
"torch.cuda.empty_cache()\n",
|
||||
"cv2.destroyAllWindows()\n",
|
||||
"gc.collect()\n",
|
||||
"\n",
|
||||
"num_classes = 3\n",
|
||||
"config_name = \"ensemble\"\n",
|
||||
" \n",
|
||||
"def load_function(attr):\n",
|
||||
" module_, func = attr.rsplit('.', maxsplit=1)\n",
|
||||
" return getattr(import_module(module_), func)\n",
|
||||
" \n",
|
||||
"config = mlconfig.load('config_' + config_name + '.yaml')\n",
|
||||
"\n",
|
||||
"model1 = models.resnet18(pretrained=False)\n",
|
||||
"model2 = models.resnet50(pretrained=False)\n",
|
||||
"model3 = models.resnet101(pretrained=False)\n",
|
||||
"\n",
|
||||
"num_classes = 2\n",
|
||||
"\n",
|
||||
"model1.fc = nn.Linear(model1.fc.in_features, num_classes)\n",
|
||||
"model2.fc = nn.Linear(model2.fc.in_features, num_classes)\n",
|
||||
"model3.fc = nn.Linear(model3.fc.in_features, num_classes)\n",
|
||||
"\n",
|
||||
"class Ensemble(nn.Module):\n",
|
||||
" def __init__(self, model1, model2, model3):\n",
|
||||
" super(Ensemble, self).__init__()\n",
|
||||
" self.model1 = model1\n",
|
||||
" self.model2 = model2\n",
|
||||
" self.model3 = model3\n",
|
||||
" self.fc = nn.Linear(3 * num_classes, num_classes)\n",
|
||||
"\n",
|
||||
" def forward(self, x):\n",
|
||||
" x1 = self.model1(x[0])\n",
|
||||
" x2 = self.model2(x[1])\n",
|
||||
" x3 = self.model3(x[2])\n",
|
||||
" x = torch.cat((x1, x2, x3), dim=1)\n",
|
||||
" x = self.fc(x)\n",
|
||||
" return x\n",
|
||||
"\n",
|
||||
"model = Ensemble(model1, model2, model3)\n",
|
||||
"\n",
|
||||
"optimizer = load_function(config.optimizer.name)(model.parameters(), lr=config.optimizer.lr)\n",
|
||||
"criterion = load_function(config.loss_function.name)()\n",
|
||||
"scheduler = load_function(config.scheduler.name)(optimizer, step_size=config.scheduler.step_size, gamma=config.scheduler.gamma)\n",
|
||||
"\n",
|
||||
"if device != 'cpu':\n",
|
||||
" model = model.to(device)\n",
|
||||
"\n",
|
||||
"#----------Создания датасета и обучение модели--------------\n",
|
||||
"\n",
|
||||
"path_res, model_name = prepare_and_learning_detection(num_classes = num_classes, num_samples = 5000, path_dataset = \"C:/Users/snytk/Lerning_NN_for_work/datasets_jpg/915_jpg_learning/\", \n",
|
||||
" model_name = config_name+\"_915_jpg_\", config_name = config_name, model=model)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"torch.cuda.empty_cache()\n",
|
||||
"cv2.destroyAllWindows()\n",
|
||||
"del model\n",
|
||||
"gc.collect()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "57d18676",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "eab69324",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"celltoolbar": "Отсутствует",
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.6"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@ -0,0 +1,217 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import csv
|
||||
import os
|
||||
import random
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
import cv2
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
from tqdm import tqdm
|
||||
|
||||
|
||||
DEFAULT_FREQS = (1200, 2400)
|
||||
PNG_SUFFIXES = ("_real.png", "_imag.png", "_spec.png")
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Build a two-class image dataset with drone signatures overlaid on noise images. "
|
||||
"The output is ready for Training_models2pic_val_loss.ipynb."
|
||||
)
|
||||
)
|
||||
parser.add_argument("--drone-root", default="/mnt/data/Dataset/drone")
|
||||
parser.add_argument("--noise-img-root", default="/mnt/data/Dataset_img/noise")
|
||||
parser.add_argument("--output-root", default="/mnt/data/Dataset_overlay")
|
||||
parser.add_argument("--freqs", default=",".join(str(v) for v in DEFAULT_FREQS))
|
||||
parser.add_argument("--alpha", type=float, default=1.0, help="Overlay strength: 1.0 keeps the darkest drone/noise pixels.")
|
||||
parser.add_argument("--limit-per-freq", type=int, default=0, help="0 means use all available noise images per frequency.")
|
||||
parser.add_argument("--seed", type=int, default=42)
|
||||
parser.add_argument("--copy-noise", action="store_true", help="Copy noise files instead of hardlinking them.")
|
||||
parser.add_argument("--overwrite", action="store_true")
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
parser.add_argument("--no-progress", action="store_true", help="Disable tqdm progress bars.")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def parse_freqs(value):
|
||||
return [int(item.strip()) for item in value.split(",") if item.strip()]
|
||||
|
||||
|
||||
def collect_drone_npys(root, freq):
|
||||
root = Path(root)
|
||||
candidates = []
|
||||
candidates.extend((root / str(freq)).rglob("*.npy"))
|
||||
candidates.extend((root / f"{freq}_jpg").glob("*.npy"))
|
||||
return sorted({p for p in candidates if p.is_file()})
|
||||
|
||||
|
||||
def collect_noise_npys(root, freq):
|
||||
root = Path(root)
|
||||
candidates = []
|
||||
candidates.extend((root / f"{freq}_jpg").glob("*.npy"))
|
||||
candidates.extend((root / str(freq)).rglob("*.npy"))
|
||||
return sorted({p for p in candidates if p.is_file()})
|
||||
|
||||
|
||||
def load_image_tensor(path):
|
||||
arr = np.load(path)
|
||||
if arr.ndim == 2:
|
||||
arr = np.stack([arr, arr, arr], axis=0)
|
||||
if arr.ndim == 3 and arr.shape[-1] in (1, 3) and arr.shape[0] not in (1, 3):
|
||||
arr = np.moveaxis(arr, -1, 0)
|
||||
if arr.ndim != 3:
|
||||
raise ValueError(f"expected 3D image tensor, got shape={arr.shape} path={path}")
|
||||
if arr.shape[0] == 1:
|
||||
arr = np.repeat(arr, 3, axis=0)
|
||||
if arr.shape[0] < 3:
|
||||
raise ValueError(f"expected at least 3 channels, got shape={arr.shape} path={path}")
|
||||
return arr[:3].astype(np.float32, copy=False)
|
||||
|
||||
|
||||
def resize_like(arr, shape):
|
||||
if arr.shape == shape:
|
||||
return arr
|
||||
channels, height, width = shape
|
||||
resized = []
|
||||
for channel in arr[:channels]:
|
||||
resized.append(cv2.resize(channel, (width, height), interpolation=cv2.INTER_LINEAR))
|
||||
return np.asarray(resized, dtype=np.float32)
|
||||
|
||||
|
||||
def overlay_tensors(noise, drone, alpha):
|
||||
drone = resize_like(drone, noise.shape)
|
||||
bright_overlay = np.minimum(noise, drone)
|
||||
mixed = (1.0 - alpha) * noise + alpha * bright_overlay
|
||||
return np.clip(mixed, 0, 255).astype(np.float32)
|
||||
|
||||
|
||||
def to_uint8(channel):
|
||||
arr = np.asarray(channel, dtype=np.float32)
|
||||
finite = arr[np.isfinite(arr)]
|
||||
if finite.size == 0:
|
||||
return np.zeros(arr.shape, dtype=np.uint8)
|
||||
min_v = float(finite.min())
|
||||
max_v = float(finite.max())
|
||||
if min_v >= 0.0 and max_v <= 255.0:
|
||||
return np.clip(arr, 0, 255).astype(np.uint8)
|
||||
if max_v == min_v:
|
||||
return np.zeros(arr.shape, dtype=np.uint8)
|
||||
norm = (arr - min_v) / (max_v - min_v)
|
||||
return np.clip(norm * 255.0, 0, 255).astype(np.uint8)
|
||||
|
||||
|
||||
def save_notebook_style_png(path, channel):
|
||||
fig = plt.figure(figsize=(16, 16))
|
||||
plt.imshow(channel)
|
||||
plt.savefig(path)
|
||||
plt.clf()
|
||||
plt.cla()
|
||||
plt.close()
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def save_sample(base_path, tensor):
|
||||
np.save(str(base_path) + ".npy", tensor.astype(np.float32))
|
||||
for idx, suffix in enumerate(PNG_SUFFIXES):
|
||||
save_notebook_style_png(str(base_path) + suffix, tensor[idx])
|
||||
|
||||
|
||||
def link_or_copy(src, dst, copy_file):
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
if dst.exists():
|
||||
return
|
||||
if copy_file:
|
||||
shutil.copy2(src, dst)
|
||||
return
|
||||
try:
|
||||
os.link(src, dst)
|
||||
except OSError:
|
||||
shutil.copy2(src, dst)
|
||||
|
||||
|
||||
def copy_noise_family(noise_npy, out_dir, copy_file):
|
||||
base_name = noise_npy.name[:-4] if noise_npy.name.endswith(".npy") else noise_npy.name
|
||||
link_or_copy(noise_npy, out_dir / noise_npy.name, copy_file)
|
||||
for suffix in PNG_SUFFIXES:
|
||||
sidecar = noise_npy.with_name(base_name + suffix)
|
||||
if sidecar.exists():
|
||||
link_or_copy(sidecar, out_dir / sidecar.name, copy_file)
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
random.seed(args.seed)
|
||||
freqs = parse_freqs(args.freqs)
|
||||
output_root = Path(args.output_root)
|
||||
manifest_rows = []
|
||||
|
||||
if args.overwrite and output_root.exists() and not args.dry_run:
|
||||
shutil.rmtree(output_root)
|
||||
|
||||
for freq in freqs:
|
||||
drone_files = collect_drone_npys(args.drone_root, freq)
|
||||
noise_files = collect_noise_npys(args.noise_img_root, freq)
|
||||
if not drone_files:
|
||||
raise RuntimeError(f"no drone npy files found for freq={freq} under {args.drone_root}")
|
||||
if not noise_files:
|
||||
raise RuntimeError(f"no noise image npy files found for freq={freq} under {args.noise_img_root}")
|
||||
|
||||
count = len(noise_files) if args.limit_per_freq <= 0 else min(args.limit_per_freq, len(noise_files))
|
||||
selected_noise = noise_files[:]
|
||||
random.shuffle(selected_noise)
|
||||
selected_noise = selected_noise[:count]
|
||||
|
||||
out_drone_dir = output_root / "drone" / f"{freq}_jpg"
|
||||
out_noise_dir = output_root / "noise" / f"{freq}_jpg"
|
||||
|
||||
print(f"freq={freq}: drone_source={len(drone_files)} noise_source={len(noise_files)} output_per_class={count}")
|
||||
if args.dry_run:
|
||||
continue
|
||||
|
||||
out_drone_dir.mkdir(parents=True, exist_ok=True)
|
||||
out_noise_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
iterator = tqdm(
|
||||
enumerate(selected_noise),
|
||||
total=count,
|
||||
desc=f"overlay freq={freq}",
|
||||
unit="sample",
|
||||
disable=args.no_progress,
|
||||
)
|
||||
for idx, noise_path in iterator:
|
||||
drone_path = drone_files[idx % len(drone_files)]
|
||||
noise_tensor = load_image_tensor(noise_path)
|
||||
drone_tensor = load_image_tensor(drone_path)
|
||||
mixed = overlay_tensors(noise_tensor, drone_tensor, args.alpha)
|
||||
|
||||
out_base = out_drone_dir / f"overlay_{freq}_{idx:06d}"
|
||||
save_sample(out_base, mixed)
|
||||
copy_noise_family(noise_path, out_noise_dir, args.copy_noise)
|
||||
|
||||
manifest_rows.append({
|
||||
"freq": freq,
|
||||
"output": str(out_base) + ".npy",
|
||||
"noise_source": str(noise_path),
|
||||
"drone_source": str(drone_path),
|
||||
"alpha": args.alpha,
|
||||
})
|
||||
|
||||
if not args.dry_run:
|
||||
manifest_path = output_root / "overlay_manifest.csv"
|
||||
with manifest_path.open("w", newline="", encoding="utf-8") as fh:
|
||||
writer = csv.DictWriter(fh, fieldnames=["freq", "output", "noise_source", "drone_source", "alpha"])
|
||||
writer.writeheader()
|
||||
writer.writerows(manifest_rows)
|
||||
print(f"wrote {len(manifest_rows)} overlay samples")
|
||||
print(f"manifest: {manifest_path}")
|
||||
print(f"dataset: {output_root}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -0,0 +1,58 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "537ea5d0-1d6c-423e-9417-171b70a76c66",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"import pandas as pd\n",
|
||||
"\n",
|
||||
"path = '//192.168.11.63/data/DATASETS/Energomash/915_learning'\n",
|
||||
"pd_columns = ['file_name']\n",
|
||||
"df = pd.DataFrame(columns=pd_columns)\n",
|
||||
"\n",
|
||||
"p = 0\n",
|
||||
"for i in os.walk(path):\n",
|
||||
" p+=1\n",
|
||||
" if p != 1:\n",
|
||||
" for j in i[2]:\n",
|
||||
" row = pd.DataFrame({pd_columns[0]: [str(str(i[0]) + '\\\\' + str(j)).replace('\\\\', '/')]})\n",
|
||||
" df = pd.concat([df, row], ignore_index=True)\n",
|
||||
"\n",
|
||||
"df.to_csv(path + '\\dataset.csv', index=False)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "8f6e1ff8",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.6"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
Binary file not shown.
@ -0,0 +1,91 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "689613d2",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"80.58750000000002\n",
|
||||
"(array([8.97705408, 4.95 , 0. ]), array([-8.97705408, 4.95 , 0. ]))\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import numpy \n",
|
||||
"from math import sqrt as square\n",
|
||||
"from numpy import sqrt, dot, cross \n",
|
||||
"from numpy.linalg import norm \n",
|
||||
"\n",
|
||||
"#rssi = [rssi, max_rssi, min_rssi, gamma]\n",
|
||||
"\n",
|
||||
"def dist(rssi):\n",
|
||||
" rssi = list(map(float, rssi))\n",
|
||||
" return square(abs(rssi[0]-rssi[1]))*rssi[3]/square(abs(rssi[0]-rssi[2]))\n",
|
||||
"\n",
|
||||
"def sol(x1,x2,x3,rssi1,rssi2,rssi3):\n",
|
||||
" r1 = dist(rssi1)\n",
|
||||
" r2 = dist(rssi2)\n",
|
||||
" r3 = dist(rssi3)\n",
|
||||
" x1=numpy.array(x1)\n",
|
||||
" x2=numpy.array(x2)\n",
|
||||
" x3=numpy.array(x3)\n",
|
||||
" temp1 = x2-x1 \n",
|
||||
" e_x = temp1/norm(temp1) \n",
|
||||
" temp2 = x3-x1 \n",
|
||||
" i = dot(e_x,temp2) \n",
|
||||
" temp3 = temp2 - i*e_x \n",
|
||||
" e_y = temp3/norm(temp3) \n",
|
||||
" e_z = cross(e_x,e_y) \n",
|
||||
" d = norm(x2-x1) \n",
|
||||
" j = dot(e_y,temp2) \n",
|
||||
" x = (r1*r1 - r2*r2 + d*d) / (2*d) \n",
|
||||
" y = (r1*r1 - r3*r3 -2*i*x + i*i + j*j) / (2*j) \n",
|
||||
" temp4 = r1*r1 - x*x - y*y \n",
|
||||
" print(temp4)\n",
|
||||
" if temp4<0: \n",
|
||||
" return \"Нет пересечения!\"\n",
|
||||
" z = sqrt(temp4) \n",
|
||||
" p_12_a = x1 + x*e_x + y*e_y + z*e_z \n",
|
||||
" p_12_b = x1 + x*e_x + y*e_y - z*e_z \n",
|
||||
" return p_12_a,p_12_b\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"print(sol([0,0,1],[0,0,-1],[0,10,0],[50,100,0,10.3],[50,100,0,10.3],[50,100,0,10.3]))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "0a68f35a",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.6"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@ -0,0 +1,57 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "f18afe4b",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import numpy \n",
|
||||
"from math import sqrt as square\n",
|
||||
"from numpy import sqrt, dot, cross \n",
|
||||
"from numpy.linalg import norm \n",
|
||||
"\n",
|
||||
"#rssi = [rssi, max_rssi, min_rssi, gamma]\n",
|
||||
"\n",
|
||||
"def dist(rssi):\n",
|
||||
" rssi = list(map(float, rssi))\n",
|
||||
" return square(abs(rssi[0]-rssi[1]))*rssi[3]/square(abs(rssi[0]-rssi[2]))\n",
|
||||
"\n",
|
||||
"def sol(x1,x2,x3,rssi1,rssi2,rssi3):\n",
|
||||
" r1 = dist(rssi1)\n",
|
||||
" r2 = dist(rssi2)\n",
|
||||
" r3 = dist(rssi3)\n",
|
||||
" x1=numpy.array(x1)\n",
|
||||
" x2=numpy.array(x2)\n",
|
||||
" x3=numpy.array(x3)\n",
|
||||
" \n",
|
||||
" return sector.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"print(sol([0,0,1],[0,0,-1],[0,10,0],[50,100,0,10.3],[50,100,0,10.3],[50,100,0,10.3]))"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.6"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
Loading…
Reference in New Issue