diff --git a/logs/analysis.ipynb b/logs/analysis.ipynb new file mode 100644 index 0000000..a28f089 --- /dev/null +++ b/logs/analysis.ipynb @@ -0,0 +1,673 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "215a6c3a", + "metadata": {}, + "source": [ + "# NN inference analysis\n", + "\n", + "Анализ CSV с результатами инференса: доля класса `drone`, частоты срабатываний, уверенность модели и интервалы между `drone`-классификациями." + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "4e8cff32", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CSV path: /home/sibscience-4/from_ssh/DroneDetector/logs/nn_results_live_6gb.csv\n", + "Rows: 27258\n", + "Time range: 2026-05-04 17:35:21.019627763+07:00 -> 2026-05-05 12:17:19.369858371+07:00\n", + "Missing freq rows: 0\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
docker_timestampevent_time_isoevent_time_epochfreqmodel_idmodel_typepredictionprobabilitytslocal_time
02026-05-04T10:35:21.019627763Z2026-05-04T17:35:21+07:001.777891e+0924002ensemble_2400_v44drone0.992026-05-04 10:35:21.019627763+00:002026-05-04 17:35:21.019627763+07:00
12026-05-04T10:35:21.019631281Z2026-05-04T17:35:21+07:001.777891e+0912001ensemble_1200_v44noise1.002026-05-04 10:35:21.019631281+00:002026-05-04 17:35:21.019631281+07:00
22026-05-04T10:35:27.048188525Z2026-05-04T17:35:27+07:001.777891e+0924002ensemble_2400_v44drone0.992026-05-04 10:35:27.048188525+00:002026-05-04 17:35:27.048188525+07:00
32026-05-04T10:35:29.238925690Z2026-05-04T17:35:29+07:001.777891e+0912001ensemble_1200_v44noise1.002026-05-04 10:35:29.238925690+00:002026-05-04 17:35:29.238925690+07:00
42026-05-04T10:35:32.842234116Z2026-05-04T17:35:32+07:001.777891e+0924002ensemble_2400_v44drone0.922026-05-04 10:35:32.842234116+00:002026-05-04 17:35:32.842234116+07:00
\n", + "
" + ], + "text/plain": [ + " docker_timestamp event_time_iso \\\n", + "0 2026-05-04T10:35:21.019627763Z 2026-05-04T17:35:21+07:00 \n", + "1 2026-05-04T10:35:21.019631281Z 2026-05-04T17:35:21+07:00 \n", + "2 2026-05-04T10:35:27.048188525Z 2026-05-04T17:35:27+07:00 \n", + "3 2026-05-04T10:35:29.238925690Z 2026-05-04T17:35:29+07:00 \n", + "4 2026-05-04T10:35:32.842234116Z 2026-05-04T17:35:32+07:00 \n", + "\n", + " event_time_epoch freq model_id model_type prediction \\\n", + "0 1.777891e+09 2400 2 ensemble_2400_v44 drone \n", + "1 1.777891e+09 1200 1 ensemble_1200_v44 noise \n", + "2 1.777891e+09 2400 2 ensemble_2400_v44 drone \n", + "3 1.777891e+09 1200 1 ensemble_1200_v44 noise \n", + "4 1.777891e+09 2400 2 ensemble_2400_v44 drone \n", + "\n", + " probability ts \\\n", + "0 0.99 2026-05-04 10:35:21.019627763+00:00 \n", + "1 1.00 2026-05-04 10:35:21.019631281+00:00 \n", + "2 0.99 2026-05-04 10:35:27.048188525+00:00 \n", + "3 1.00 2026-05-04 10:35:29.238925690+00:00 \n", + "4 0.92 2026-05-04 10:35:32.842234116+00:00 \n", + "\n", + " local_time \n", + "0 2026-05-04 17:35:21.019627763+07:00 \n", + "1 2026-05-04 17:35:21.019631281+07:00 \n", + "2 2026-05-04 17:35:27.048188525+07:00 \n", + "3 2026-05-04 17:35:29.238925690+07:00 \n", + "4 2026-05-04 17:35:32.842234116+07:00 " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from pathlib import Path\n", + "import pandas as pd\n", + "\n", + "csv_path = Path('/home/sibscience-4/from_ssh/DroneDetector/logs/nn_results_live_6gb.csv')\n", + "df = pd.read_csv(csv_path)\n", + "\n", + "df['ts'] = pd.to_datetime(df['docker_timestamp'], utc=True)\n", + "df['local_time'] = df['ts'].dt.tz_convert('Asia/Novosibirsk')\n", + "df['freq'] = pd.to_numeric(df['freq'], errors='coerce').astype('Int64')\n", + "df['probability'] = pd.to_numeric(df['probability'], errors='coerce')\n", + "df = df.sort_values('ts').reset_index(drop=True)\n", + "\n", + "print(f'CSV path: {csv_path}')\n", + "print(f'Rows: {len(df)}')\n", + "print(f'Time range: {df[\"local_time\"].min()} -> {df[\"local_time\"].max()}')\n", + "print(f'Missing freq rows: {df[\"freq\"].isna().sum()}')\n", + "display(df.head())" + ] + }, + { + "cell_type": "markdown", + "id": "0fcc6dd6", + "metadata": {}, + "source": [ + "## Общая сводка по частотам и классам" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "7bf0fc3f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
freqpredictioncountavg_probabilitymin_probabilitymax_probabilityfreq_totalclass_rate
01200drone30.8400000.780.91136320.000220
11200noise136290.9974360.911.00136320.999780
22400drone119210.8680130.501.00136260.874872
32400noise17050.6491850.501.00136260.125128
\n", + "
" + ], + "text/plain": [ + " freq prediction count avg_probability min_probability max_probability \\\n", + "0 1200 drone 3 0.840000 0.78 0.91 \n", + "1 1200 noise 13629 0.997436 0.91 1.00 \n", + "2 2400 drone 11921 0.868013 0.50 1.00 \n", + "3 2400 noise 1705 0.649185 0.50 1.00 \n", + "\n", + " freq_total class_rate \n", + "0 13632 0.000220 \n", + "1 13632 0.999780 \n", + "2 13626 0.874872 \n", + "3 13626 0.125128 " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "class_summary = (\n", + " df.groupby(['freq', 'prediction'], dropna=False)\n", + " .agg(\n", + " count=('prediction', 'size'),\n", + " avg_probability=('probability', 'mean'),\n", + " min_probability=('probability', 'min'),\n", + " max_probability=('probability', 'max'),\n", + " )\n", + " .reset_index()\n", + ")\n", + "\n", + "freq_total = df.groupby('freq', dropna=False).size().rename('freq_total').reset_index()\n", + "class_summary = class_summary.merge(freq_total, on='freq', how='left')\n", + "class_summary['class_rate'] = class_summary['count'] / class_summary['freq_total']\n", + "class_summary = class_summary.sort_values(['freq', 'prediction']).reset_index(drop=True)\n", + "\n", + "display(class_summary)" + ] + }, + { + "cell_type": "markdown", + "id": "039fcfa6", + "metadata": {}, + "source": [ + "## Статистика по классу drone" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "c1ed2bc4", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
freqtotal_inferencesdrone_countavg_drone_probabilitymedian_drone_probabilitymin_drone_probabilitymax_drone_probabilityfirst_drone_timelast_drone_timedrone_ratedataset_duration_mindrone_per_min
012001363230.8400000.830.780.912026-05-05 03:18:55.374394280+07:002026-05-05 08:39:23.676669045+07:000.0002201121.9725040.002674
1240013626119210.8680130.930.501.002026-05-04 17:35:21.019627763+07:002026-05-05 12:17:19.369858371+07:000.8748721121.97250410.625038
\n", + "
" + ], + "text/plain": [ + " freq total_inferences drone_count avg_drone_probability \\\n", + "0 1200 13632 3 0.840000 \n", + "1 2400 13626 11921 0.868013 \n", + "\n", + " median_drone_probability min_drone_probability max_drone_probability \\\n", + "0 0.83 0.78 0.91 \n", + "1 0.93 0.50 1.00 \n", + "\n", + " first_drone_time last_drone_time \\\n", + "0 2026-05-05 03:18:55.374394280+07:00 2026-05-05 08:39:23.676669045+07:00 \n", + "1 2026-05-04 17:35:21.019627763+07:00 2026-05-05 12:17:19.369858371+07:00 \n", + "\n", + " drone_rate dataset_duration_min drone_per_min \n", + "0 0.000220 1121.972504 0.002674 \n", + "1 0.874872 1121.972504 10.625038 " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "drone = df[df['prediction'].eq('drone')].copy()\n", + "\n", + "total_by_freq = df.groupby('freq', dropna=False).size().rename('total_inferences')\n", + "drone_by_freq = drone.groupby('freq', dropna=False).agg(\n", + " drone_count=('prediction', 'size'),\n", + " avg_drone_probability=('probability', 'mean'),\n", + " median_drone_probability=('probability', 'median'),\n", + " min_drone_probability=('probability', 'min'),\n", + " max_drone_probability=('probability', 'max'),\n", + " first_drone_time=('local_time', 'min'),\n", + " last_drone_time=('local_time', 'max'),\n", + ")\n", + "\n", + "drone_stats = total_by_freq.to_frame().join(drone_by_freq, how='left').fillna({'drone_count': 0})\n", + "drone_stats['drone_count'] = drone_stats['drone_count'].astype(int)\n", + "drone_stats['drone_rate'] = drone_stats['drone_count'] / drone_stats['total_inferences']\n", + "\n", + "if len(df) > 1:\n", + " duration_min = (df['ts'].max() - df['ts'].min()).total_seconds() / 60\n", + "else:\n", + " duration_min = 0\n", + "\n", + "drone_stats['dataset_duration_min'] = duration_min\n", + "drone_stats['drone_per_min'] = drone_stats['drone_count'] / duration_min if duration_min > 0 else 0\n", + "drone_stats = drone_stats.reset_index().sort_values('freq').reset_index(drop=True)\n", + "\n", + "display(drone_stats)" + ] + }, + { + "cell_type": "markdown", + "id": "56ae5e2b", + "metadata": {}, + "source": [ + "## Интервалы между drone-классификациями" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "0c43eb07", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
freqinterval_countavg_interval_secmedian_interval_secmin_interval_secmax_interval_secp90_interval_secp95_interval_sec
0120029614.1511379614.151137619.19611218609.10616316810.11515817709.610661
12400119205.6475134.9741570.0000001210.1079509.20372810.079097
\n", + "
" + ], + "text/plain": [ + " freq interval_count avg_interval_sec median_interval_sec \\\n", + "0 1200 2 9614.151137 9614.151137 \n", + "1 2400 11920 5.647513 4.974157 \n", + "\n", + " min_interval_sec max_interval_sec p90_interval_sec p95_interval_sec \n", + "0 619.196112 18609.106163 16810.115158 17709.610661 \n", + "1 0.000000 1210.107950 9.203728 10.079097 " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "if drone.empty:\n", + " print('No drone predictions found')\n", + " drone_interval_stats = pd.DataFrame()\n", + "else:\n", + " drone = drone.sort_values(['freq', 'ts']).copy()\n", + " drone['dt_drone_freq_sec'] = drone.groupby('freq')['ts'].diff().dt.total_seconds()\n", + " drone_interval_stats = (\n", + " drone.groupby('freq', dropna=False)['dt_drone_freq_sec']\n", + " .agg(\n", + " interval_count='count',\n", + " avg_interval_sec='mean',\n", + " median_interval_sec='median',\n", + " min_interval_sec='min',\n", + " max_interval_sec='max',\n", + " p90_interval_sec=lambda s: s.quantile(0.90),\n", + " p95_interval_sec=lambda s: s.quantile(0.95),\n", + " )\n", + " .reset_index()\n", + " )\n", + " display(drone_interval_stats)" + ] + }, + { + "cell_type": "markdown", + "id": "2ecf96f0", + "metadata": {}, + "source": [ + "## Drone-события" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ff14a339", + "metadata": {}, + "outputs": [], + "source": [ + "drone_events = drone[[\n", + " 'local_time',\n", + " 'docker_timestamp',\n", + " 'freq',\n", + " 'model_id',\n", + " 'model_type',\n", + " 'prediction',\n", + " 'probability',\n", + "]].sort_values('local_time').reset_index(drop=True)\n", + "\n", + "display(drone_events.head(50))\n", + "display(drone_events.tail(50))" + ] + }, + { + "cell_type": "markdown", + "id": "376b84d0", + "metadata": {}, + "source": [ + "## Частота drone-классификаций по минутам" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5f82f5c1", + "metadata": {}, + "outputs": [], + "source": [ + "if drone.empty:\n", + " print('No drone predictions found')\n", + "else:\n", + " drone_per_minute = (\n", + " drone.set_index('local_time')\n", + " .groupby('freq')\n", + " .resample('1min')\n", + " .size()\n", + " .rename('drone_count')\n", + " .reset_index()\n", + " )\n", + " display(drone_per_minute.tail(100))\n", + "\n", + " pivot = drone_per_minute.pivot_table(\n", + " index='local_time',\n", + " columns='freq',\n", + " values='drone_count',\n", + " fill_value=0,\n", + " )\n", + " display(pivot.tail(50))" + ] + }, + { + "cell_type": "markdown", + "id": "ad570d05", + "metadata": {}, + "source": [ + "## Быстрые графики" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "895550a1", + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "if not drone.empty:\n", + " ax = drone_stats.set_index('freq')['drone_rate'].plot(kind='bar', figsize=(8, 4), title='Drone rate by frequency')\n", + " ax.set_ylabel('drone_count / total_inferences')\n", + " plt.show()\n", + "\n", + " ax = drone.boxplot(column='probability', by='freq', figsize=(8, 4))\n", + " ax.set_title('Drone probability by frequency')\n", + " ax.set_xlabel('freq')\n", + " ax.set_ylabel('probability')\n", + " plt.suptitle('')\n", + " plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}