{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# --- CSS STYLE ---\nfrom IPython.core.display import HTML\ndef css_styling():\n styles = open(\"../input/2020-cost-of-living/alerts.css\", \"r\").read()\n return HTML(\"\")\ncss_styling()","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2021-07-30T19:03:26.463204Z","iopub.execute_input":"2021-07-30T19:03:26.463585Z","iopub.status.idle":"2021-07-30T19:03:26.493020Z","shell.execute_reply.started":"2021-07-30T19:03:26.463490Z","shell.execute_reply":"2021-07-30T19:03:26.492261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"![](\"https://i.imgur.com/k8NA44c.png\")
\n\nš Searching the Sky - Explore & Understand š
\n\n# 1. Introduction\n\nUiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii!\n\nI was hoping to have some fun with a notebook. Haven't done some proper artistic EDA in a while, and the theme of this competition is absolute perfection.\n\n\nš Competition Goal: detect GW (Gravitational Wave) signals from the mergers of binary black holes from simulated GW time-series data, created from a network of Earth-based detectors.\n
\n\nš Let's get started!\n\n### Libraries ā¬","metadata":{}},{"cell_type":"code","source":"!pip install -q nnAudio -qq\n\n# Libraries\nimport os\nimport re\nimport gc\nimport wandb\nimport time\nfrom tqdm import tqdm\nimport glob\nimport pickle\nimport warnings\nimport pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib as mpl\nimport matplotlib.patches as patches\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport matplotlib.gridspec as gridspec\nfrom matplotlib.offsetbox import AnnotationBbox, OffsetImage\nfrom pylab import text\nimport torch\nfrom nnAudio.Spectrogram import CQT1992v2\n\n# Librosa\nimport librosa\nfrom librosa.feature import melspectrogram\nimport librosa.display\n\n# Environment check\nwarnings.filterwarnings(\"ignore\")\nos.environ[\"WANDB_SILENT\"] = \"true\"\nCONFIG = {'competition': 'g2net', '_wandb_kernel': 'aot'}\n\n# Secrets š¤«\nfrom kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\nsecret_value_0 = user_secrets.get_secret(\"wandb\")\n\n# Custom colors\nclass color:\n S = '\\033[1m' + '\\033[93m'\n E = '\\033[0m'\n \nmy_colors = [\"#E7C84B\", \"#4EE4EA\", \"#4EA9EA\", \"#242179\", \"#AB51E9\", \"#E051E9\"]\nprint(color.S+\"Notebook Color Scheme:\"+color.E)\nsns.palplot(sns.color_palette(my_colors))\n\n# Set Style\nsns.set_style(\"white\")\nmpl.rcParams['xtick.labelsize'] = 16\nmpl.rcParams['ytick.labelsize'] = 16\nmpl.rcParams['axes.spines.left'] = False\nmpl.rcParams['axes.spines.right'] = False\nmpl.rcParams['axes.spines.top'] = False\nplt.rcParams.update({'font.size': 17})","metadata":{"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2021-07-30T19:03:26.494472Z","iopub.execute_input":"2021-07-30T19:03:26.494844Z","iopub.status.idle":"2021-07-30T19:03:38.076363Z","shell.execute_reply.started":"2021-07-30T19:03:26.494793Z","shell.execute_reply":"2021-07-30T19:03:38.075574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> š **Note**: If this line throws an error, try using wandb.login() instead. It will ask for the API key to login, which you can get from your [W&B profile](https://wandb.ai/site) (click on Profile -> Settings -> scroll to API keys).\n\n***You can find my W&B Dashboard here -> https://wandb.ai/andrada/g2net?workspace=user-andrada***","metadata":{}},{"cell_type":"code","source":"! wandb login $secret_value_0","metadata":{"execution":{"iopub.status.busy":"2021-07-30T19:03:38.078436Z","iopub.execute_input":"2021-07-30T19:03:38.078804Z","iopub.status.idle":"2021-07-30T19:03:40.003622Z","shell.execute_reply.started":"2021-07-30T19:03:38.078766Z","shell.execute_reply":"2021-07-30T19:03:40.002558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Custom Functions ā¬","metadata":{}},{"cell_type":"code","source":"def show_values_on_bars(axs, h_v=\"v\", space=0.4):\n '''Plots the value at the end of the a seaborn barplot.\n axs: the ax of the plot\n h_v: weather or not the barplot is vertical/ horizontal'''\n \n def _show_on_single_plot(ax):\n if h_v == \"v\":\n for p in ax.patches:\n _x = p.get_x() + p.get_width() / 2\n _y = p.get_y() + p.get_height()\n value = int(p.get_height())\n ax.text(_x, _y, format(value, ','), ha=\"center\") \n elif h_v == \"h\":\n for p in ax.patches:\n _x = p.get_x() + p.get_width() + float(space)\n _y = p.get_y() + p.get_height()\n value = int(p.get_width())\n ax.text(_x, _y, format(value, ','), ha=\"left\")\n\n if isinstance(axs, np.ndarray):\n for idx, ax in np.ndenumerate(axs):\n _show_on_single_plot(ax)\n else:\n _show_on_single_plot(axs)\n \n \ndef offset_png(x, y, path, ax, zoom, offset):\n '''For adding other .png images to the graph.\n source: https://stackoverflow.com/questions/61971090/how-can-i-add-images-to-bars-in-axes-matplotlib'''\n \n img = plt.imread(f\"../input/g2net-gravitational-wave-dataset/pngs/{path}.png\")\n im = OffsetImage(img, zoom=zoom)\n im.image.axes = ax\n x_offset = offset\n ab = AnnotationBbox(im, (x, y), xybox=(x_offset, 0), frameon=False,\n xycoords='data', boxcoords=\"offset points\", pad=0)\n ax.add_artist(ab)\n \n \ndef save_dataset_artifact(run_name, artifact_name, path):\n '''Saves dataset to W&B Artifactory.\n run_name: name of the experiment\n artifact_name: under what name should the dataset be stored\n path: path to the dataset'''\n \n run = wandb.init(project='g2net', \n name=run_name, \n config=CONFIG, anonymous=\"allow\")\n artifact = wandb.Artifact(name=artifact_name, \n type='dataset')\n artifact.add_file(path)\n\n wandb.log_artifact(artifact)\n wandb.finish()\n print(\"Artifact has been saved successfully.\")\n \n \ndef create_wandb_plot(x_data=None, y_data=None, x_name=None, y_name=None, title=None, log=None, plot=\"line\"):\n '''Create and save lineplot/barplot in W&B Environment.\n x_data & y_data: Pandas Series containing x & y data\n x_name & y_name: strings containing axis names\n title: title of the graph\n log: string containing name of log'''\n \n data = [[label, val] for (label, val) in zip(x_data, y_data)]\n table = wandb.Table(data=data, columns = [x_name, y_name])\n \n if plot == \"line\":\n wandb.log({log : wandb.plot.line(table, x_name, y_name, title=title)})\n elif plot == \"bar\":\n wandb.log({log : wandb.plot.bar(table, x_name, y_name, title=title)})\n elif plot == \"scatter\":\n wandb.log({log : wandb.plot.scatter(table, x_name, y_name, title=title)})\n \n \ndef create_wandb_hist(x_data=None, x_name=None, title=None, log=None):\n '''Create and save histogram in W&B Environment.\n x_data: Pandas Series containing x values\n x_name: strings containing axis name\n title: title of the graph\n log: string containing name of log'''\n \n data = [[x] for x in x_data]\n table = wandb.Table(data=data, columns=[x_name])\n wandb.log({log : wandb.plot.histogram(table, x_name, title=title)})","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2021-07-30T19:03:40.005682Z","iopub.execute_input":"2021-07-30T19:03:40.006025Z","iopub.status.idle":"2021-07-30T19:03:40.024312Z","shell.execute_reply.started":"2021-07-30T19:03:40.005986Z","shell.execute_reply":"2021-07-30T19:03:40.023254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. š« The Data\n\nThe `training_labels.csv` file contains the file id and the `target`, meaning a flag that is:\n* 0: if there is no signal\n* 1: is there is any signal","metadata":{}},{"cell_type":"code","source":"# Read in the training data\ntrain = pd.read_csv(\"../input/g2net-gravitational-wave-detection/training_labels.csv\")\n\n# Print some useful information\nprint(color.S+\"Train Data has:\"+color.E, \"{:,}\".format(train.shape[0]), \"observations.\", \"\\n\" +\n color.S+\"Number of Missing Values:\"+color.E, train.isna().sum()[0], \"\\n\" +\n \"\\n\" +\n color.S+\"Head of Training Data:\"+color.E)\ntrain.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-07-30T19:03:40.025700Z","iopub.execute_input":"2021-07-30T19:03:40.026396Z","iopub.status.idle":"2021-07-30T19:03:40.456885Z","shell.execute_reply.started":"2021-07-30T19:03:40.026356Z","shell.execute_reply":"2021-07-30T19:03:40.455922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save data to W&B Dashboard\nsave_dataset_artifact(run_name='save-training_labels',\n artifact_name='training_labels', \n path=\"../input/g2net-gravitational-wave-detection/training_labels.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-07-30T19:03:40.458513Z","iopub.execute_input":"2021-07-30T19:03:40.458908Z","iopub.status.idle":"2021-07-30T19:03:51.468040Z","shell.execute_reply.started":"2021-07-30T19:03:40.458870Z","shell.execute_reply":"2021-07-30T19:03:51.466329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.1 The Target - is there a black hole?\n\n> š **Note**: The targets are splitted almost 50% - 50%. This is because the **data itself is simulated**, so there's the benefit that you can purposely simulate a black hole as many times as you want. However, as the description tells us, signals of black holes are **very rare**.","metadata":{}},{"cell_type":"code","source":"run = wandb.init(project='g2net', name='explore', config=CONFIG, anonymous=\"allow\")","metadata":{"execution":{"iopub.status.busy":"2021-07-30T19:03:51.469745Z","iopub.execute_input":"2021-07-30T19:03:51.470132Z","iopub.status.idle":"2021-07-30T19:03:55.242247Z","shell.execute_reply.started":"2021-07-30T19:03:51.470089Z","shell.execute_reply":"2021-07-30T19:03:55.241098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20, 12))\nax = sns.countplot(data=train, y=\"target\", palette=my_colors)\n\nshow_values_on_bars(ax, h_v=\"h\", space=0.4)\nax.set_xlabel(\"Frequency\", size = 22)\nax.set_ylabel(\"Target\", size = 22)\nax.set_title(\"- Frequency of Target variable -\", size = 26, weight='bold')\nplt.yticks(ticks=[0, 1], labels=[\"signal not present\", \"signal present\"])\nplt.xticks([])\nsns.despine(left=True, bottom=True)\n\noffset_png(x=243000, y=1, path=\"black_hole\", ax=ax, zoom=0.3, offset=0)","metadata":{"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2021-07-30T19:03:55.245458Z","iopub.execute_input":"2021-07-30T19:03:55.245840Z","iopub.status.idle":"2021-07-30T19:03:56.347031Z","shell.execute_reply.started":"2021-07-30T19:03:55.245782Z","shell.execute_reply":"2021-07-30T19:03:56.346071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create W&B Plot\ncreate_wandb_plot(x_data=[\"signal not present\", \"signal present\"], \n y_data=train[\"target\"].value_counts().values, \n x_name=\"Target\", y_name=\"Frequency\", \n title=\"- Frequency of Target variable -\", \n log=\"target_plot\", plot=\"bar\")","metadata":{"execution":{"iopub.status.busy":"2021-07-30T19:03:56.348922Z","iopub.execute_input":"2021-07-30T19:03:56.349349Z","iopub.status.idle":"2021-07-30T19:03:59.291696Z","shell.execute_reply.started":"2021-07-30T19:03:56.349306Z","shell.execute_reply":"2021-07-30T19:03:59.290634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Add info about total number of observations\nwandb.log({\"total_obs\" : np.int(train.shape[0])})","metadata":{"execution":{"iopub.status.busy":"2021-07-30T19:03:59.296736Z","iopub.execute_input":"2021-07-30T19:03:59.299215Z","iopub.status.idle":"2021-07-30T19:03:59.750181Z","shell.execute_reply.started":"2021-07-30T19:03:59.299167Z","shell.execute_reply":"2021-07-30T19:03:59.749259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.2 The .npy files\n\n> š **Note**: The **simulated GW** (Gravitational Waves) are coming from 3 different Observatories:\n* LIGO Hanford: below in purple\n* LIGO Livingston: below in yellow\n* VIRGO: below in green\n\n![](\"https://i.imgur.com/IJZyBGJ.jpg\")
","metadata":{}},{"cell_type":"code","source":"# Get the full paths to the files and create a df\npaths = glob.glob(\"../input/g2net-gravitational-wave-detection/train/*/*/*/*\")\nids = [path.split(\"/\")[-1].split(\".\")[0] for path in paths]\npaths_df = pd.DataFrame({\"path\":paths, \"id\": ids})\n\n# Append the full path as a new column\ntrain_df = pd.merge(left=train, right=paths_df, on=\"id\")\n\nprint(color.S+\"train_df:\"+color.E)\ntrain_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-07-30T19:03:59.753194Z","iopub.execute_input":"2021-07-30T19:03:59.753454Z","iopub.status.idle":"2021-07-30T19:05:42.420925Z","shell.execute_reply.started":"2021-07-30T19:03:59.753429Z","shell.execute_reply":"2021-07-30T19:05:42.419471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"OK!\n\nEach file has a shape of **`(3, 4096)`** - meaning 3 different GW coming from the 3 sites around the globe, of a length of 4096. The length of 4096 spans for 2 seconds and it is sampled at 2,048 Hz.\n\n> š **Note**: Keep in mind this data is becoming pretty big. We have **560,000 observation x 3 sites x 4,096 time series length** => **6,881,280,000** (that's 6 billion datapoints ... with a B)\n\nHence, I will rename these 3 as `Site1`, `Site2` and `Site3`, like the one and only [Heads or Tails](https://www.kaggle.com/headsortails) did in his notebook [right here](https://www.kaggle.com/headsortails/when-stars-collide-g2net-eda).","metadata":{}},{"cell_type":"code","source":"def get_npy_df(path):\n '''Returns a df of the 3 site information for a particular file.\n path: a string containing the full path to the file'''\n \n df = pd.DataFrame({\"Site1\" : np.load(path)[0],\n \"Site2\" : np.load(path)[1],\n \"Site3\" : np.load(path)[2]})\n \n return df","metadata":{"execution":{"iopub.status.busy":"2021-07-30T19:05:42.422879Z","iopub.execute_input":"2021-07-30T19:05:42.423215Z","iopub.status.idle":"2021-07-30T19:05:43.091452Z","shell.execute_reply.started":"2021-07-30T19:05:42.423177Z","shell.execute_reply":"2021-07-30T19:05:43.090607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. š©āš Explore ...\n\n## 3.1 The Gravitational Waves\n\nLet's take a look at the Gravitational Waves and see what insights we can find about them, before starting creating an actual model.\n\n### š GW when there is **NO** signal present:\n\nThe 3 sites have fairly similar distribution, with the third one having fewer outliers than the rest.","metadata":{}},{"cell_type":"code","source":"# Get a sample data with TARGET == 0\nno_target = list(train_df.loc[train_df[\"target\"] == 0, \"path\"])[23]\nno_target = get_npy_df(path = no_target)\n\n# Plot\nfig = plt.figure(figsize=(20, 12))\nouter = gridspec.GridSpec(1, 3, wspace=0.2, hspace=0.2)\nfig.suptitle('- GW Fluctuation: Target = 0 -', size = 26, weight='bold')\nsites = [\"Site1\", \"Site2\", \"Site3\"]\ncolors = [my_colors[2], my_colors[3], my_colors[4]]\npeaks = [8e19, 6.2e19, 2.3e20]\npngs = [\"blue\", \"yellow\", \"pink\"]\nsize = [0.08, 0.07, 0.05]\n\nfor i, site, col, p, png, s in zip(range(3), sites, colors, peaks, pngs, size):\n inner = gridspec.GridSpecFromSubplotSpec(2, 1,\n subplot_spec=outer[i], \n wspace=0.1, hspace=0.1,\n height_ratios= (.15, .85))\n ax1 = plt.Subplot(fig, inner[0])\n ax2 = plt.Subplot(fig, inner[1])\n mean = no_target[site].mean()\n \n sns.boxplot(no_target[site], ax=ax1, color=col)\n sns.kdeplot(data=no_target, x=site, ax=ax2, color=col, shade=True, \n lw=2, alpha=0.5)\n ax2.axvline(x=mean, color=col, lw=3, ls=\"--\")\n ax2.text(x=mean, y=p, s=f'{mean}', size=13, color=col, weight='bold')\n \n fig.add_subplot(ax1)\n fig.add_subplot(ax2)\n ax1.set(xlabel='')\n axs = [ax1, ax2]\n for ax in axs:\n ax.set_xticks([])\n ax.set_ylabel(\"\")\n sns.despine(bottom=True, left=True)\n offset_png(x=mean, y=p/5, path=png, ax=ax2, zoom=s, offset=0)","metadata":{"_kg_hide-input":true,"_kg_hide-output":false,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2021-07-30T19:05:43.094495Z","iopub.execute_input":"2021-07-30T19:05:43.094754Z","iopub.status.idle":"2021-07-30T19:05:44.959882Z","shell.execute_reply.started":"2021-07-30T19:05:43.094729Z","shell.execute_reply":"2021-07-30T19:05:44.958352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### š GW when there **IS** a signal present:\n\nThe distributions look similar, however there is some more fluctuation at the peak of density and and the extremes, especially for Site 2.","metadata":{}},{"cell_type":"code","source":"# Get a sample data with TARGET == 1\nno_target = list(train_df.loc[train_df[\"target\"] == 1, \"path\"])[23]\nno_target = get_npy_df(path = no_target)\n\n# Plot\nfig = plt.figure(figsize=(20, 12))\nouter = gridspec.GridSpec(1, 3, wspace=0.2, hspace=0.2)\nfig.suptitle('- GW Fluctuation: Target = 1 -', size = 26, weight='bold')\nsites = [\"Site1\", \"Site2\", \"Site3\"]\ncolors = [my_colors[0], my_colors[1], my_colors[2]]\npeaks = [6.5e19, 9e19, 2.3e20]\npngs = [\"newborn\", \"earth\", \"blue2\"]\nsize = [0.02, 0.045, 0.045]\n\nfor i, site, col, p, png, s in zip(range(3), sites, colors, peaks, pngs, size):\n inner = gridspec.GridSpecFromSubplotSpec(2, 1,\n subplot_spec=outer[i], \n wspace=0.1, hspace=0.1,\n height_ratios= (.15, .85))\n ax1 = plt.Subplot(fig, inner[0])\n ax2 = plt.Subplot(fig, inner[1])\n mean = no_target[site].mean()\n \n sns.boxplot(no_target[site], ax=ax1, color=col)\n sns.kdeplot(data=no_target, x=site, ax=ax2, color=col, shade=True, \n lw=2, alpha=0.5)\n ax2.axvline(x=mean, color=col, lw=3, ls=\"--\")\n ax2.text(x=mean, y=p, s=f'{mean}', size=13, color=col, weight='bold')\n \n fig.add_subplot(ax1)\n fig.add_subplot(ax2)\n ax1.set(xlabel='')\n axs = [ax1, ax2]\n for ax in axs:\n ax.set_xticks([])\n ax.set_ylabel(\"\")\n sns.despine(bottom=True, left=True)\n offset_png(x=mean, y=p/5, path=png, ax=ax2, zoom=s, offset=0)","metadata":{"_kg_hide-input":true,"_kg_hide-output":false,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2021-07-30T19:05:44.961321Z","iopub.execute_input":"2021-07-30T19:05:44.961645Z","iopub.status.idle":"2021-07-30T19:05:48.486649Z","shell.execute_reply.started":"2021-07-30T19:05:44.961610Z","shell.execute_reply":"2021-07-30T19:05:48.485842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3.2 Signals in time\n\nOk, now we can look at how these waves look in time, comparing the 3 sites we already know and are familiar with: LIGO, Hanford, LIGO Livingston and VIRGO.\n\n> š **Note**: There are **indeed** some differences, that can be seen a bit more clear than by looking only at histograms. The signals with no target have bigger fluctuations, while the other ones have smaller more consistent ones. However, these diferences are **very tiny**.","metadata":{}},{"cell_type":"code","source":"# Target Sample\nwith_target = list(train_df.loc[train_df[\"target\"] == 1, \"path\"])[23]\nwith_target = get_npy_df(path = with_target)\n# No Target Sample\nno_target = list(train_df.loc[train_df[\"target\"] == 0, \"path\"])[23]\nno_target = get_npy_df(path = no_target)\n# size = [6.5e19, 9e19, 2.3e20]\n\n# Plot\nfig, ((ax1, ax2), (ax3, ax4), (ax5, ax6)) = plt.subplots(3, 2, figsize=(20, 12), sharey=True, sharex=True)\nfig.suptitle('- GW Signals in Time -', size = 26, weight='bold')\n\nsns.lineplot(y=with_target[\"Site1\"], x=range(len(with_target)), ax=ax1,\n lw=3, color=my_colors[1])\nsns.lineplot(y=no_target[\"Site1\"], x=range(len(no_target)), ax=ax2,\n lw=3, color=my_colors[2])\nsns.lineplot(y=with_target[\"Site2\"], x=range(len(with_target)), ax=ax3,\n lw=3, color=my_colors[5])\nsns.lineplot(y=no_target[\"Site2\"], x=range(len(no_target)), ax=ax4,\n lw=3, color=my_colors[4])\nsns.lineplot(y=with_target[\"Site3\"], x=range(len(with_target)), ax=ax5,\n lw=3, color=my_colors[0])\nsns.lineplot(y=no_target[\"Site3\"], x=range(len(no_target)), ax=ax6,\n lw=3, color=my_colors[3])\n\nax1.title.set_text('With Target')\nax2.title.set_text('No Target');\n\n# Images\noffset_png(x=2700, y=2e-20, path=\"astronaut\", ax=ax2, zoom=0.15, offset=0)\noffset_png(x=1700, y=1.7e-20, path=\"satellite\", ax=ax3, zoom=0.2, offset=0)\noffset_png(x=500, y=-1.3e-20, path=\"spaceshut\", ax=ax6, zoom=0.065, offset=0)","metadata":{"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2021-07-30T19:05:48.490086Z","iopub.execute_input":"2021-07-30T19:05:48.490943Z","iopub.status.idle":"2021-07-30T19:05:51.187711Z","shell.execute_reply.started":"2021-07-30T19:05:48.490904Z","shell.execute_reply":"2021-07-30T19:05:51.186903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3.3 The MEL Spectrogram\n\nš **What is a Spectrogram?** - A spectrogram is a **visual representation** of the spectrum of frequencies of a signal as it varies with *time*.\n\nš **What is a Mel Spectrogram?** - A mel spectrogram is a spectrogram where the **frequencies are converted to the mel scale**.\n\nš **Why should we use it?** - In sound processing, the mel-frequency cepstrum (MFC) is a representation of the short-term power spectrum of a sound, based on a linear cosine transform of a log power spectrum on a nonlinear mel scale of frequency. This frequency warping *can allow for better representation of sound*.","metadata":{}},{"cell_type":"code","source":"def make_spectrogram(path, prints=False):\n '''Creates a MEL spectrogram.'''\n \n # Get the waves from the 3 sites\n waves = np.load(path).astype(np.float32)\n if prints:\n print(color.S+\"Waves Shape:\"+color.E, waves.shape)\n \n # Loop and make spectrogram\n spectrograms = []\n \n for i in range(3):\n # Compute a mel-scaled spectrogram.\n spec = melspectrogram(waves[i] / max(waves[i]), sr=4096, \n n_mels=128, fmin=20, fmax=2048)\n # Convert a power spectrogram (amplitude squared) to decibel (dB) units\n spec = librosa.power_to_db(spec).transpose((1, 0))\n spectrograms.append(spec)\n \n return spectrograms","metadata":{"execution":{"iopub.status.busy":"2021-07-30T19:05:51.191887Z","iopub.execute_input":"2021-07-30T19:05:51.192146Z","iopub.status.idle":"2021-07-30T19:05:51.691462Z","shell.execute_reply.started":"2021-07-30T19:05:51.192120Z","shell.execute_reply":"2021-07-30T19:05:51.690661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Sample\n\nFirst let's look at how the function above works on a simple sample from the data.","metadata":{}},{"cell_type":"code","source":"path = train_df[\"path\"][0]\n\n# Get the spectrogram\nspectrogram = make_spectrogram(path, prints=True)\n \n# Plot it\nimg = np.vstack(spectrogram)\n\nplt.figure(figsize=(22, 10))\nplt.title('Sample Mel Spectrogram', size = 20, weight='bold')\nplt.imshow(img, cmap=\"cool\")\nplt.axis(\"off\");","metadata":{"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2021-07-30T19:05:51.692766Z","iopub.execute_input":"2021-07-30T19:05:51.693152Z","iopub.status.idle":"2021-07-30T19:05:52.407689Z","shell.execute_reply.started":"2021-07-30T19:05:51.693113Z","shell.execute_reply":"2021-07-30T19:05:52.406901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### š GW Signals Spectrogram - With Target vs No Target\n\nGood! Looks nice! Now we can do a proper comparison between a few samples that contain the Target Signal vs samples that don't.\n\n> š **Note**: You can notice that it is very **hard to observe any kind of difference** between the images, as the fluctuation is so unperceptable by the naked eye.","metadata":{}},{"cell_type":"code","source":"# Samples per category\nn=3\n\n# Sample 6 paths with target and no target available\npaths_no_target = train_df[train_df[\"target\"] == 0][\"path\"].sample(n, random_state=23).values\npaths_with_target = train_df[train_df[\"target\"] == 1][\"path\"].sample(n, random_state=23).values\n\nall_paths = np.append(paths_no_target, paths_with_target)\n\n# Plot\nfig, axes = plt.subplots(nrows=2, ncols=n, figsize=(21,5))\nwandb_logs = []\n\n# Enumerate & plot\nfor i, path in enumerate(all_paths):\n if i < n: title = \"No Target\" \n else: title=\"With Target\"\n \n spec = make_spectrogram(path, prints=False)\n img = np.vstack(spec)\n \n x = i // n\n y = i % n\n \n axes[x, y].imshow(img, cmap=\"cool\")\n axes[x, y].set_title(title)\n axes[x, y].axis('off');\n \n # Save to W&B\n wandb_logs.append(wandb.Image(img, caption=f\"{title}_{i}\"))\n \n \nplt.subplots_adjust(left=None, bottom=None, right=None, top=None, wspace=0.07, hspace=0.0)\nwandb.log({\"spectrograms\": wandb_logs})","metadata":{"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2021-07-30T19:05:52.409039Z","iopub.execute_input":"2021-07-30T19:05:52.409423Z","iopub.status.idle":"2021-07-30T19:05:54.351582Z","shell.execute_reply.started":"2021-07-30T19:05:52.409382Z","shell.execute_reply":"2021-07-30T19:05:54.350842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3.4 nnAudio()\n\n**`nnAudio`**: is an audio processing toolbox using `PyTorch` CNN as its backend. By doing so, spectrograms can be generated from audio on-the-fly during neural network training and the Fourier kernels (e.g. or CQT kernels) can be trained ([more info on this pachage here](https://github.com/KinWaiCheuk/nnAudio)).\n\n\nš Special thanks to
Y.Nakama and
his notebook here, from where I took my inspiration to use this library.\n
![](\"https://i.imgur.com/MAerCzs.png\")
\n\nš PyTorch EffNet Model + Feature Metadata š
\n\n### Competition Metric\n\n> š **AUC - ROC curve**: is a performance measurement for the *classification problems* at various threshold settings. **ROC is a probability curve** and **AUC represents the degree or measure of separability**. *Higher the AUC, the better the model is at predicting the classes*.\n\nBelow is a sample example: the goal is tu have the \"Area Under the Curve\" (AUC) be as big as possible - meaning that the line should aim to be as closer to the X and Y axis as possible.","metadata":{}},{"cell_type":"code","source":"# Libraries\nfrom sklearn.datasets import make_classification\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_curve, roc_auc_score \n\n# Generate Sample Dataset\nX, y = make_classification(n_samples=1000, n_classes=2, random_state=1)\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=2)\n\n# \"no skill\" prediction\nns_probs = [0 for _ in range(len(y_test))]\n# Fit Ensemble\nmodel = RandomForestClassifier()\nmodel.fit(X_train, y_train)\n\n# Predict probabilities\nrf_probs = model.predict_proba(X_test)\nrf_probs = rf_probs[:, 1]\n\n# Comparison\nns_auc = roc_auc_score(y_test, ns_probs)\nrf_auc = roc_auc_score(y_test, rf_probs)\n\nprint(color.S+'No Skill: ROC AUC=%.3f' % (ns_auc)+color.E)\nprint(color.S+'Random Forest: ROC AUC=%.3f' % (rf_auc)+color.E)\n\n# Plot\nfig, ax = plt.subplots(figsize=(21, 10))\nplt.title('- Example of ROC AUC -', size = 26, weight='bold')\nns_fpr, ns_tpr, _ = roc_curve(y_test, ns_probs)\nrf_fpr, rf_tpr, _ = roc_curve(y_test, rf_probs)\n\nax.plot(ns_fpr, ns_tpr, ls=\"dotted\", label='NoSkill', lw=6, color=my_colors[0])\nax.plot(rf_fpr, rf_tpr, ls=\"dashdot\", label='RandomForest', lw=6, color=my_colors[4])\n\nplt.xlabel('False Positive Rate')\nplt.ylabel('True Positive Rate')\nplt.legend();\n\noffset_png(x=0.88, y=0.3, path=\"astronaut2\", ax=ax, zoom=0.35, offset=0)","metadata":{"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2021-07-30T19:07:26.290775Z","iopub.execute_input":"2021-07-30T19:07:26.291129Z","iopub.status.idle":"2021-07-30T19:07:26.976789Z","shell.execute_reply.started":"2021-07-30T19:07:26.291092Z","shell.execute_reply":"2021-07-30T19:07:26.975806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### ā¬ More Libraries & Functions","metadata":{}},{"cell_type":"code","source":"!pip install efficientnet_pytorch -qq\n\nimport random\nimport torch\nfrom torch import nn\nfrom torch.utils.data import Dataset, DataLoader\nimport torch.nn.functional as F\nfrom torch.optim import Adam\nfrom torch.autograd import Variable\nfrom torch.cuda.amp import GradScaler, autocast\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom efficientnet_pytorch import EfficientNet\n\nfrom sklearn.model_selection import StratifiedKFold, train_test_split\nfrom sklearn import model_selection as sk_model_selection\n\n# Set \ndef set_seed(seed = 23):\n '''Sets the seed of the entire notebook so results are the same every time we run.\n This is for REPRODUCIBILITY.'''\n np.random.seed(seed)\n random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n torch.backends.cudnn.deterministic = True\n os.environ['PYTHONHASHSEED'] = str(seed)\n \nset_seed()\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint('Device available now:', device)\n\n\n# ~~~~~~~~~~~~~~~~~~~\n# ~~~~~FUNCTIONS~~~~~\n# ~~~~~~~~~~~~~~~~~~~\ndef plot_loss_graph(train_losses, valid_losses, epoch, fold):\n '''Lineplot of the training/validation losses.'''\n \n fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(10, 2.5))\n fig.suptitle(f\"Fold {fold} | Epoch {epoch}\", fontsize=12, y=1.05)\n axes = [ax1, ax2]\n data = [train_losses, valid_losses]\n sns.lineplot(y=train_losses, x=range(len(train_losses)),\n lw=2.3, ls=\":\", color=my_colors[3], ax=ax1)\n sns.lineplot(y=valid_losses, x=range(len(valid_losses)),\n lw=2.3, ls=\"-\", color=my_colors[5], ax=ax2)\n for ax, t, d in zip(axes, [\"Train\", \"Valid\"], data):\n ax.set_title(f\"{t} Evolution\", size=12, weight='bold')\n ax.set_xlabel(\"Iteration\", weight='bold', size=9)\n ax.set_ylabel(\"Loss\", weight='bold', size=9)\n ax.tick_params(labelsize=9)\n plt.show()\n \n \ndef get_auc_score(valid_preds, valid_targets, gpu=True):\n '''Compute ROC AUC score.'''\n if gpu:\n predictions = torch.cat(valid_preds).cpu().detach().numpy().tolist()\n else:\n predictions = torch.cat(valid_preds).detach().numpy().tolist()\n actuals = [int(x) for x in valid_targets]\n\n roc_auc = roc_auc_score(actuals, predictions)\n return roc_auc","metadata":{"_kg_hide-input":true,"_kg_hide-output":false,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2021-07-30T19:07:26.978413Z","iopub.execute_input":"2021-07-30T19:07:26.978813Z","iopub.status.idle":"2021-07-30T19:07:34.616721Z","shell.execute_reply.started":"2021-07-30T19:07:26.978767Z","shell.execute_reply":"2021-07-30T19:07:34.615674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"../input/g2net-gravitational-wave-dataset/training_labels_features.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-07-30T19:07:34.618311Z","iopub.execute_input":"2021-07-30T19:07:34.619092Z","iopub.status.idle":"2021-07-30T19:07:37.059218Z","shell.execute_reply.started":"2021-07-30T19:07:34.619042Z","shell.execute_reply":"2021-07-30T19:07:37.058150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5. šØāš PyTorch Dataset\n\nFirst we must create the `PyTorch Dataset`, which will be a class that will take the paths and targets, compute the numpy arrays' spectrograms and return the result.\n\nThis class is also helpful within the `Dataloader` tool, so we can iterate through multiple files at once.\n\n> š **Bonus**: I added the features from the 3 sites too on a later iteration - now we can use the additional information for better serults. ;)","metadata":{}},{"cell_type":"code","source":"class G2Dataset(Dataset):\n \n def __init__(self, path, features, target=None, test=False, prints=False):\n '''Initiate the arguments & import the numpy file/s.'''\n self.path = path\n self.features = features\n self.target = target\n self.test = test\n self.prints = prints\n \n def __len__(self):\n return len(self.path)\n \n def __transform__(self, np_file):\n '''Transforms the np_file into spectrogram.'''\n spectrogram = []\n TRANSFORM = CQT1992v2(sr=2048, fmin=20, \n fmax=1024, hop_length=32, \n verbose=False)\n \n # Create an image with 3 channels - for the 3 sites\n for i in range(3):\n waves = np_file[i] / np.max(np_file[i])\n waves = torch.from_numpy(waves).float()\n channel = TRANSFORM(waves).squeeze().numpy()\n spectrogram.append(channel)\n \n spectrogram = torch.tensor(spectrogram).float()\n \n if self.prints:\n plt.figure(figsize=(5, 5))\n plot = spectrogram.detach().cpu().numpy()\n plot = np.transpose(plot, (1, 2, 0))\n plt.imshow(plot)\n plt.axis(\"off\")\n plt.show();\n\n return spectrogram\n \n def __getitem__(self, i):\n \n # Load the numpy file\n np_file = np.load(self.path[i])\n # Create the spectrograms\n spectrograms = self.__transform__(np_file)\n # Select the features\n metadata = np.array(self.features.iloc[i].values, dtype=np.float32)\n \n # Return the images & target if available\n if self.test==False:\n y = torch.tensor(self.target[i], dtype=torch.float)\n return {\"spectrogram\": spectrograms,\n \"metadata\": metadata,\n \"targets\": y}\n else:\n return {\"spectrogram\": spectrograms,\n \"metadata\": metadata}","metadata":{"execution":{"iopub.status.busy":"2021-07-30T19:07:37.060741Z","iopub.execute_input":"2021-07-30T19:07:37.061141Z","iopub.status.idle":"2021-07-30T19:07:37.074044Z","shell.execute_reply.started":"2021-07-30T19:07:37.061100Z","shell.execute_reply":"2021-07-30T19:07:37.072949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### ~ Test the Dataset function ~\n\nGood! Now that we've created our `Dataset` class, we can test it by using a simple sample of 4 observations:\n* 4 distinct paths pointing to the numpy arrays (split in 2 batches of size 2)\n* 4 distinct targets, which are the labels of these paths\n\n> š **Note**: I am also lotting here the 3 channel spectrogram that is created from the 3 sites: hence, we're making 1 image with 3 channels, instead of 3 images with only 1 channel. This is how they look!\n\n![](\"https://i.imgur.com/IHHFq75.png\")
","metadata":{}},{"cell_type":"code","source":"# Sample\npath = train[\"path\"][:4].values\nfeatures = train.iloc[:4, 3:]\ntarget = train[\"target\"][:4].values\n\n# Initiate the Dataset\ndataset = G2Dataset(path=path, target=target, features=features,\n test=False, prints=True)\n\n# Initiate the Dataloader\ndataloader = DataLoader(dataset, batch_size=2, shuffle=False)\n\n# Output of the Dataloader\nfor k, data in enumerate(dataloader):\n spectrograms, features, targets = data.values()\n print(color.S + f\"Batch: {k}\" + color.E, \"\\n\" +\n color.S + \"Spectrograms:\" + color.E, spectrograms.shape, \"\\n\" +\n color.S + f\"Features:\" + color.E, features.shape, \"\\n\" +\n color.S + \"Target:\" + color.E, targets, \"\\n\" +\n \"=\"*50)","metadata":{"execution":{"iopub.status.busy":"2021-07-30T19:07:37.075383Z","iopub.execute_input":"2021-07-30T19:07:37.076093Z","iopub.status.idle":"2021-07-30T19:07:37.797362Z","shell.execute_reply.started":"2021-07-30T19:07:37.075938Z","shell.execute_reply":"2021-07-30T19:07:37.796543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 6. š PyTorch EfficientNet\n\nNow we need to create a `Module` class which will help us take the output from the `Dataset` class and train it to predict out target variable.","metadata":{}},{"cell_type":"code","source":"class G2EffNet(nn.Module):\n \n def __init__(self, no_features, no_neurons=250):\n super().__init__()\n \n # NN for the spectrogram - out layer = 2560\n self.spectrogram = EfficientNet.from_pretrained('efficientnet-b7')\n \n # NN for the features\n self.metadata = nn.Sequential(nn.Linear(no_features, no_neurons),\n nn.BatchNorm1d(no_neurons),\n nn.ReLU(),\n nn.Dropout(p=0.2),\n \n nn.Linear(no_neurons, no_neurons),\n nn.BatchNorm1d(no_neurons),\n nn.ReLU(),\n nn.Dropout(p=0.2))\n \n # Final NN for classification\n # Combination of spectrogram + features\n self.classification = nn.Sequential(nn.Linear(2560 + no_neurons, 1))\n \n def forward(self, spectrogram, features, prints=False):\n \n if prints: print(color.S+'Spectrogram In:'+color.E, spectrogram.shape, '\\n'+\n color.S+'Features In:'+color.E, features.shape, '\\n' +\n '='*40)\n \n # Spectrogram\n spectrogram = self.spectrogram.extract_features(spectrogram)\n if prints: print(color.S+'Spectrogram Out:'+color.E, spectrogram.shape)\n \n spectrogram = F.avg_pool2d(spectrogram, spectrogram.size()[2:]).reshape(-1, 2560)\n if prints: print(color.S+'Spectrogram Reshaped:'+color.E, spectrogram.shape)\n \n # Features\n features = self.metadata(features)\n if prints: print(color.S+'Features Out:'+color.E, features.shape)\n \n # Combine Layers\n concatenated = torch.cat((spectrogram, features), dim=1)\n out = self.classification(concatenated)\n if prints: print(color.S+'Concat shape:'+color.E, concatenated.shape, \"\\n\" + \n color.S+'Out shape:'+color.E, out.shape)\n \n return torch.sigmoid(out)","metadata":{"execution":{"iopub.status.busy":"2021-07-30T19:07:37.798753Z","iopub.execute_input":"2021-07-30T19:07:37.799163Z","iopub.status.idle":"2021-07-30T19:07:37.811228Z","shell.execute_reply.started":"2021-07-30T19:07:37.799117Z","shell.execute_reply":"2021-07-30T19:07:37.810357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### ~ How it works? ~\n\nGoooood! :) Let's see how it works! Below is a schema to help you better grasp how the model works:\n\n![](\"https://i.imgur.com/Kir64Dy.png\")
","metadata":{}},{"cell_type":"code","source":"# Create an example model - Effnet\nmodel_example = G2EffNet(no_features=15, no_neurons=250)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-07-30T19:07:37.812676Z","iopub.execute_input":"2021-07-30T19:07:37.813304Z","iopub.status.idle":"2021-07-30T19:07:46.909609Z","shell.execute_reply.started":"2021-07-30T19:07:37.813265Z","shell.execute_reply":"2021-07-30T19:07:46.908553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We'll use previous datasets & dataloader\n# example for 1 batch\nfor k, data in enumerate(dataloader):\n spectrograms, features, targets = data.values()\n break\n \n# Outputs\nout = model_example(spectrograms, features, prints=True)\n\n# Criterion\ncriterion_example = nn.BCEWithLogitsLoss()\n# Unsqueeze(1) from shape=[3] => shape=[3, 1]\nloss = criterion_example(out, targets.unsqueeze(1)) \nprint(color.S+'LOSS:'+color.E, loss.item())","metadata":{"execution":{"iopub.status.busy":"2021-07-30T19:07:46.911010Z","iopub.execute_input":"2021-07-30T19:07:46.911344Z","iopub.status.idle":"2021-07-30T19:07:48.402331Z","shell.execute_reply.started":"2021-07-30T19:07:46.911315Z","shell.execute_reply":"2021-07-30T19:07:48.401289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 7. š Training ...\n\n## 7.1 Training Function\n\nUsually this part can get quite long and weird; this is why I usually choose to visualize it with a schema, so I can better know at a later date what I did here.\n\nš **As a summary**:\n\n* First we initiate a new **W&B experiment**, where we store all the hyperparameters we'll be using - this way we know how to reproduce everything afterwards.\n* Then we split the data into folds\n* For each fold:\n * We initiate a `G2Dataset()`, the model, loss criterion, optimizer etc.\n * We start the training loop (epochs):\n * train on the training data (`model.train()`), we compute the loss and then optimize\n * evaluate how the model did (`model.eval()`)\n * Compute a `roc_auc` score and, if better than the last one, we save the model\n* Repeat\n\n![](\"https://i.imgur.com/6Pme9rZ.png\")
\n\n### Full Training Function below ā¬","metadata":{}},{"cell_type":"code","source":"def train_effnet(name, epochs, splits, batch_size, no_neurons, lr, weight_decay, sample):\n\n # === W&B Experiment ===\n s = time.time()\n params = dict(model=name, epochs=epochs, split=splits, \n batch=batch_size, neurons=no_neurons, \n lr=lr, weight_decay=weight_decay, sample=sample)\n CONFIG.update(params)\n run = wandb.init(project='g2net', name=f\"effnet_{name}\", config=CONFIG, anonymous=\"allow\")\n\n\n # === CV Split ===\n df = train.sample(sample, random_state=23)\n cv = StratifiedKFold(n_splits=splits)\n cv_splits = cv.split(X=df, y=df['target'].values)\n\n\n\n for fold, (train_i, valid_i) in enumerate(cv_splits):\n\n print(\"~\"*25)\n print(\"~\"*8, color.S+f\"FOLD {fold}\"+color.E, \"~\"*8)\n print(\"~\"*25)\n\n train_df = df.iloc[train_i, :]\n # To go quicker through validation\n valid_df = df.iloc[valid_i, :].sample(int(sample*(splits/10)*0.6),\n random_state=23)\n\n # Datasets & Dataloader\n train_dataset = G2Dataset(path=train_df[\"path\"].values, target=train_df[\"target\"].values,\n features=train_df.iloc[:, 3:], test=False)\n valid_dataset = G2Dataset(path=valid_df[\"path\"].values, target=valid_df[\"target\"].values,\n features=valid_df.iloc[:, 3:], test=False)\n\n train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n valid_loader = DataLoader(valid_dataset, batch_size=batch_size, shuffle=False)\n\n # Model/ Optimizer/ Criterion/ Scheduler\n model = G2EffNet(no_features=15, no_neurons=no_neurons).to(device)\n optimizer = Adam(model.parameters(), lr=lr, \n weight_decay=weight_decay, amsgrad=False)\n criterion = nn.BCEWithLogitsLoss()\n # scheduler = ReduceLROnPlateau(optimizer=optimizer, mode='max', verbose=True,\n # patience=VAR.patience, factor=VAR.factor)\n scaler = GradScaler()\n\n # ~~~~~~~~~~~~\n # ~~~ LOOP ~~~\n # ~~~~~~~~~~~~\n BEST_SCORE = 0.0\n\n for epoch in range(epochs):\n print(\"=\"*8, color.S+f\"Epoch {epoch}\"+color.E, \"=\"*8)\n\n # === TRAIN ===\n model.train()\n train_losses = []\n for k, data in enumerate(train_loader):\n spectrograms, features, targets = data.values()\n spectrograms, features, targets = spectrograms.to(device), features.to(device), targets.to(device)\n\n with autocast():\n out = model(spectrograms, features)\n loss = criterion(out, targets.unsqueeze(1))\n train_losses.append(loss.cpu().detach().numpy().tolist())\n\n scaler.scale(loss).backward()\n scaler.step(optimizer)\n scaler.update()\n\n mean_train_loss = np.mean(train_losses)\n print(color.S+\"Mean Train Loss:\"+color.E, mean_train_loss)\n wandb.log({\"mean_train_loss\": np.float(mean_train_loss)}, step=epoch)\n\n\n # === EVAL ===\n model.eval()\n valid_losses, valid_preds, valid_targets = [], [], []\n with torch.no_grad():\n for k, data in enumerate(valid_loader):\n spectrograms, features, targets = data.values()\n valid_targets.extend(targets.detach().numpy().tolist())\n spectrograms, features, targets = spectrograms.to(device), features.to(device), targets.to(device)\n\n out = model(spectrograms, features)\n\n valid_preds.extend(out)\n loss = criterion(out, targets.unsqueeze(1))\n valid_losses.append(loss.cpu().detach().numpy().tolist())\n\n mean_valid_loss = np.mean(valid_losses)\n print(color.S+\"Mean Valid Loss:\"+color.E, mean_valid_loss)\n wandb.log({\"mean_valid_loss\": np.float(mean_valid_loss)}, step=epoch)\n plot_loss_graph(train_losses, valid_losses, epoch, fold)\n\n\n # === UPDATES ===\n roc_auc = get_auc_score(valid_preds, valid_targets, gpu=torch.cuda.is_available())\n print(color.S+\"ROC AUC:\"+color.E, roc_auc)\n wandb.log({\"roc_auc\": np.float(roc_auc)}, step=epoch)\n\n if roc_auc > BEST_SCORE: \n print(\"! Saving model in fold {} | epoch {} ...\".format(fold, epoch), \"\\n\")\n torch.save(model.state_dict(), f\"Baseline_fold_{fold}_auc_{round(roc_auc, 5)}.pt\")\n\n BEST_SCORE = roc_auc\n\n\n del model, optimizer, criterion, spectrograms, features, targets\n torch.cuda.empty_cache()\n gc.collect()\n\n wandb.finish()\n print(color.S+f\"Time to run: {round((time.time() - s)/60, 2)} minutes\"+color.E)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-30T19:07:48.403981Z","iopub.execute_input":"2021-07-30T19:07:48.404376Z","iopub.status.idle":"2021-07-30T19:07:48.431157Z","shell.execute_reply.started":"2021-07-30T19:07:48.404333Z","shell.execute_reply":"2021-07-30T19:07:48.429203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 7.2 Experiments","metadata":{}},{"cell_type":"code","source":"# class VAR:\n# name = \"60k_samples\"\n# splits = 3\n# epochs = 2\n# batch_size = 64\n# no_neurons = 250\n# lr = 0.0001\n# weight_decay = 0.000001\n# patience = 1\n# factor = 0.01\n# sample=60000\n \n \n# train_effnet(name=VAR.name, epochs=VAR.epochs, splits=VAR.splits, \n# batch_size=VAR.batch_size, no_neurons=VAR.no_neurons, lr=VAR.lr, \n# weight_decay=VAR.weight_decay, sample=VAR.sample)","metadata":{"execution":{"iopub.status.busy":"2021-07-30T19:07:48.433874Z","iopub.execute_input":"2021-07-30T19:07:48.434246Z","iopub.status.idle":"2021-07-30T19:07:49.141938Z","shell.execute_reply.started":"2021-07-30T19:07:48.434210Z","shell.execute_reply":"2021-07-30T19:07:49.140908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# === TEST CELL - runs faster ===\nclass VAR:\n name = \"test\"\n splits = 3\n epochs = 2\n batch_size = 64\n no_neurons = 250\n lr = 0.0001\n weight_decay = 0.000001\n patience = 1\n factor = 0.01\n sample=2000\n \n \ntrain_effnet(name=VAR.name, epochs=VAR.epochs, splits=VAR.splits, \n batch_size=VAR.batch_size, no_neurons=VAR.no_neurons, lr=VAR.lr, \n weight_decay=VAR.weight_decay, sample=VAR.sample)","metadata":{"execution":{"iopub.status.busy":"2021-07-30T19:07:49.143419Z","iopub.execute_input":"2021-07-30T19:07:49.143856Z","iopub.status.idle":"2021-07-30T19:24:14.922006Z","shell.execute_reply.started":"2021-07-30T19:07:49.143801Z","shell.execute_reply":"2021-07-30T19:24:14.920899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### W&B Dashboard\n\nYou can check the evolution of the experiments here -> https://wandb.ai/andrada/g2net?workspace=user-andrada\n\n> š Below is a *sneak peak* of the dashboard:\n","metadata":{}},{"cell_type":"markdown","source":"# 8. šŖ Submission\n\nWe're at the end of the line folks!\n\nI've put here a simple submission code for this notebook.\n\nš **Steps to submission**:\n* Retrieve the pretrained model/s\n* Create a new `Dataset` & `Dataloader` - careful here; you don't have the target anymore\n* Predict using the trained models\n* Blend the predictions if you want into a final output\n* Submit\n* Have a snack, you're done š","metadata":{}},{"cell_type":"code","source":"# Sample submission containing extracted features\ntest = pd.read_csv(\"../input/g2net-gravitational-wave-dataset/sample_submission_features.csv\")\ntest = test.head(20) ### SMALLER TO RUN FASTER - ERASE LINE TO GET FULL SUBMISSION","metadata":{"execution":{"iopub.status.busy":"2021-07-30T19:24:14.923551Z","iopub.execute_input":"2021-07-30T19:24:14.923920Z","iopub.status.idle":"2021-07-30T19:24:16.870966Z","shell.execute_reply.started":"2021-07-30T19:24:14.923880Z","shell.execute_reply":"2021-07-30T19:24:16.870089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Retrieve all pretrained models\nnames = [\"Baseline_fold_0_auc_0.79091\", \"Baseline_fold_1_auc_0.78462\",\n \"Baseline_fold_2_auc_0.7886\"]\nmodels = []\n\nfor i in range(len(names)):\n model = G2EffNet(no_features=15, no_neurons=250).to(device)\n model.load_state_dict(torch.load(f\"../input/g2net-gravitational-wave-dataset/{names[i]}.pt\",\n map_location=torch.device(device)))\n model.eval()\n models.append(model)","metadata":{"execution":{"iopub.status.busy":"2021-07-30T19:24:16.872223Z","iopub.execute_input":"2021-07-30T19:24:16.872650Z","iopub.status.idle":"2021-07-30T19:24:28.817481Z","shell.execute_reply.started":"2021-07-30T19:24:16.872518Z","shell.execute_reply":"2021-07-30T19:24:28.816645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Test Dataset & Dataloader\ndataset = G2Dataset(path=test[\"path\"].values, target=None,\n features=test.iloc[:, 3:], test=True)\ndataloader = DataLoader(dataset, batch_size=10, shuffle=False)\n\n# === Loop ===\nall_preds = []\n\n# Disable gradients\nwith torch.no_grad():\n for k, data in enumerate(dataloader):\n \n spectrograms, features = data.values()\n spectrograms, features = spectrograms.to(device), features.to(device)\n \n # Predict with each of the 3 models\n out0 = models[0](spectrograms, features).cpu().numpy().squeeze()\n out1 = models[1](spectrograms, features).cpu().numpy().squeeze()\n out2 = models[2](spectrograms, features).cpu().numpy().squeeze()\n \n # Blend the predictions\n all_preds.extend((out0 + out1 + out2)/3)","metadata":{"execution":{"iopub.status.busy":"2021-07-30T19:24:28.818750Z","iopub.execute_input":"2021-07-30T19:24:28.819106Z","iopub.status.idle":"2021-07-30T19:24:30.480220Z","shell.execute_reply.started":"2021-07-30T19:24:28.819071Z","shell.execute_reply":"2021-07-30T19:24:30.479274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Submission\n\n> š **Note**: For the purpose of this notebook running faster, I'll make the prediction on only the first 20 observations within the test data. Delete `test = test.head(20)` line to get the full prediction; which is save within my G2net dataset as well. :)","metadata":{}},{"cell_type":"code","source":"ss = pd.read_csv(\"../input/g2net-gravitational-wave-detection/sample_submission.csv\").head(20)\nss[\"target\"] = all_preds\n\nss.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-30T19:41:43.456121Z","iopub.execute_input":"2021-07-30T19:41:43.456447Z","iopub.status.idle":"2021-07-30T19:41:43.635576Z","shell.execute_reply.started":"2021-07-30T19:41:43.456416Z","shell.execute_reply":"2021-07-30T19:41:43.634552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"actual_submission = pd.read_csv(\"../input/g2net-gravitational-wave-dataset/60k_submission.csv\")\nactual_submission.to_csv(\"60k_submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-30T20:14:14.240716Z","iopub.execute_input":"2021-07-30T20:14:14.242217Z","iopub.status.idle":"2021-07-30T20:14:14.392762Z","shell.execute_reply.started":"2021-07-30T20:14:14.241951Z","shell.execute_reply":"2021-07-30T20:14:14.391203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"![](\"https://i.imgur.com/LzL1Srh.png\")
","metadata":{}},{"cell_type":"code","source":"# TODO: make KFold Validation - DONE\n# TODO: Change graph representation - DONE\n# TODO: create train() function - DONE\n# TODO: create schema for train() function - DONE\n# TODO: preprocess submission data - DONE\n# TODO: train on more data & submit with model - DONE\n# TODO: dataset add spectrogram augmentation","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-07-30T19:42:19.418071Z","iopub.execute_input":"2021-07-30T19:42:19.418381Z","iopub.status.idle":"2021-07-30T19:42:19.422247Z","shell.execute_reply.started":"2021-07-30T19:42:19.418352Z","shell.execute_reply":"2021-07-30T19:42:19.420904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"![](\"https://i.imgur.com/cUQXtS7.png\")
\n\n# My Specs\n\n* š„ **Z8 G4** Workstation\n* š¾ 2 CPUs & 96GB Memory\n* š® **NVIDIA** Quadro RTX 8000\n* š» **Zbook** Studio G7 on the go","metadata":{}}]}