{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Training Notebook for the G2Net competition. This implements a Bi-directional GRU using Keras, using preprocessed spectrogram.\n\nThis uses Yasufumi Nakama's spectrogram preprocessing notebooks and datasets:\n* Train: [Notebook](https://www.kaggle.com/yasufuminakama/g2net-spectrogram-generation-train), [Dataset](https://www.kaggle.com/yasufuminakama/g2net-n-mels-128-train-images)\n* Test: [Notebook](https://www.kaggle.com/yasufuminakama/g2net-spectrogram-generation-test), [Dataset](https://www.kaggle.com/yasufuminakama/g2net-n-mels-128-test-images)","metadata":{}},{"cell_type":"code","source":"import os\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nimport tensorflow as tf\nfrom tensorflow.keras import layers\n# !pip install -q nnAudio -qq\n# import torch\n# from nnAudio.Spectrogram import CQT1992v2\n# Q_TRANSFORM = CQT1992v2(sr=2048, fmin=20, fmax=1024, hop_length=32)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-06T09:31:53.877570Z","iopub.execute_input":"2021-08-06T09:31:53.877948Z","iopub.status.idle":"2021-08-06T09:31:53.882567Z","shell.execute_reply.started":"2021-08-06T09:31:53.877909Z","shell.execute_reply":"2021-08-06T09:31:53.881548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomDataset(tf.keras.utils.Sequence):\n    def __init__(self, df, directory, batch_size=32, random_state=42, shuffle=True, target=True, ext='.npy'):\n        np.random.seed(random_state)\n        \n        self.directory = directory\n        self.df = df\n        self.shuffle = shuffle\n        self.target = target\n        self.batch_size = batch_size\n        self.ext = ext\n#         self.q_transform = CQT1992v2(\n#             sr=2048, fmin=20, fmax=1024, hop_length=32\n#         )\n        \n        self.on_epoch_end()\n    \n    def __len__(self):\n        return np.ceil(self.df.shape[0] / self.batch_size).astype(int)\n    \n    \n    \n    def __get_qtransform(self, x):\n#         image = []\n#         for i in range(3):\n        waves = x / np.max(x)\n        waves = torch.from_numpy(waves).float()\n        image = self.q_transform(waves).squeeze().numpy()\n#         image.append(channel)\n#         image = np.stack(image)\n        \n        return image\n    def __getitem__(self, idx):\n        start_idx = idx * self.batch_size\n        batch = self.df[start_idx: start_idx + self.batch_size]\n        \n        signals = []\n\n        for fname in batch.id:\n            path = os.path.join(self.directory, fname + self.ext)\n            data = np.load(path)\n#             data = self.__get_qtransform(data)\n            signals.append(data)\n        \n        signals = np.stack(signals).astype('float32')\n        signals = np.expand_dims(signals, axis=3)\n        \n        if self.target:\n            return signals, batch.target.values\n        else:\n            return signals\n    \n    def on_epoch_end(self):\n        if self.shuffle:\n            self.df = self.df.sample(frac=1).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2021-08-06T09:31:53.902561Z","iopub.execute_input":"2021-08-06T09:31:53.902892Z","iopub.status.idle":"2021-08-06T09:31:53.916123Z","shell.execute_reply.started":"2021-08-06T09:31:53.902861Z","shell.execute_reply":"2021-08-06T09:31:53.914824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_model():\n#     inputs = layers.Input(shape=(27, 128, 1))\n    inputs = layers.Input(shape=(27, 128, 1))\n\n    conv0 = layers.Conv2D(filters = 32, kernel_size = (3,5), strides = (1,1), activation = 'relu')\n    norm0 = layers.BatchNormalization()\n    conv1 = layers.Conv2D(filters = 32, kernel_size = (3,5), strides = (1,1), activation = 'relu')\n    norm1 = layers.BatchNormalization()\n    pool0 = layers.MaxPooling2D(pool_size=(2, 2), strides=(1, 3), padding='valid')\n    \n    conv2 = layers.Conv2D(filters = 64, kernel_size = (3,1), strides = (1,1), activation = 'relu')\n    norm2 = layers.BatchNormalization()\n    conv3 = layers.Conv2D(filters = 64, kernel_size = (3,1), strides = (1,1), activation = 'relu')\n    norm3 = layers.BatchNormalization()\n    pool1 = layers.MaxPooling2D(pool_size=(2, 2), strides=(1, 1), padding='valid')\n    \n    conv4 = layers.Conv2D(filters = 128, kernel_size = (1,5), strides = (1,1), activation = 'relu')\n    norm4 = layers.BatchNormalization()\n    conv5 = layers.Conv2D(filters = 128, kernel_size = (1,5), strides = (1,1), activation = 'relu')\n    norm5= layers.BatchNormalization()\n    pool2 = layers.MaxPooling2D(pool_size=(2, 2), strides=(3, 3), padding='valid')\n    \n    conv6 = layers.Conv2D(filters = 256, kernel_size = (3,3), strides = (1,1), activation = 'relu')\n    norm6 = layers.BatchNormalization()\n    conv7 = layers.Conv2D(filters = 256, kernel_size = (3,3), strides = (1,1), activation = 'relu')\n    norm7 = layers.BatchNormalization()\n    pool3 = layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid')\n    \n#     gru0 = layers.Bidirectional(layers.GRU(256, return_sequences=True))\n#     drop0 = layers.SpatialDropout1D(0.25)\n#     gru1 = layers.Bidirectional(layers.GRU(256, return_sequences=True))\n#     drop1 = layers.SpatialDropout1D(0.25)\n    \n    avgpool = layers.GlobalAveragePooling1D(name='avg_pool')\n    maxpool = layers.GlobalMaxPooling1D(name='max_pool')\n    \n        \n    x = conv0(inputs)\n    x = norm0(x)\n    x = conv1(x)\n    x = norm1(x)\n    x = pool0(x)\n    \n    x = conv2(x)\n    x = norm2(x)\n    x = conv3(x)\n    x = norm3(x)\n    x = pool1(x)\n    \n    x = conv4(x)\n    x = norm4(x)\n    x = conv5(x)\n    x = norm5(x)\n    x = pool2(x)\n    \n    x = conv6(x)\n    x = norm6(x)\n    x = conv7(x)\n    x = norm7(x)\n    x = pool3(x)\n    \n    x = tf.squeeze(x, [1])\n    \n#     x = gru0(x)\n#     x = drop0(x)\n#     x = gru1(x)\n#     x = drop1(x)\n\n    x = tf.keras.layers.Concatenate()([avgpool(x), maxpool(x)])\n    \n    x = layers.Dense(512, activation=\"relu\")(x)\n    x = layers.Dense(256, activation=\"relu\")(x)\n    x = layers.Dense(128, activation=\"relu\")(x)\n    x = layers.Dense(64, activation=\"relu\")(x)\n    x = layers.Dense(1, activation=\"sigmoid\", name=\"sigmoid\")(x)\n\n    model = tf.keras.Model(inputs=inputs, outputs=x)\n    \n    return model\n\nmodel = build_model()\nmodel.compile(\"adam\", loss=\"binary_crossentropy\", metrics=[tf.keras.metrics.AUC()])\nmodel.summary()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-08-06T09:31:53.950279Z","iopub.execute_input":"2021-08-06T09:31:53.950655Z","iopub.status.idle":"2021-08-06T09:31:54.205292Z","shell.execute_reply.started":"2021-08-06T09:31:53.950622Z","shell.execute_reply":"2021-08-06T09:31:54.204313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/g2net-gravitational-wave-detection/training_labels.csv')\n# df_split = np.array_split(train, 7)\n# train = df_split[0]\nsub = pd.read_csv('../input/g2net-gravitational-wave-detection/sample_submission.csv')\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-06T09:31:54.208756Z","iopub.execute_input":"2021-08-06T09:31:54.209042Z","iopub.status.idle":"2021-08-06T09:31:54.660885Z","shell.execute_reply.started":"2021-08-06T09:31:54.209014Z","shell.execute_reply":"2021-08-06T09:31:54.660022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df = train.sample(frac=1).reset_index(drop=True)\n\nsplit = int(sample_df.shape[0] * 0.8)\n\ntrain_df = sample_df[:split]\nvalid_df = sample_df[split:]\n\n# train_df = train_df[:10000]\n# valid_df = valid_df[:2000]","metadata":{"execution":{"iopub.status.busy":"2021-08-06T09:31:54.662730Z","iopub.execute_input":"2021-08-06T09:31:54.663018Z","iopub.status.idle":"2021-08-06T09:31:54.759688Z","shell.execute_reply.started":"2021-08-06T09:31:54.662984Z","shell.execute_reply":"2021-08-06T09:31:54.758688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dset = CustomDataset(\n    train_df, '../input/g2net-n-mels-128-train-images', batch_size=64)\n\nvalid_dset = CustomDataset(\n    valid_df, '../input/g2net-n-mels-128-train-images', batch_size=64, shuffle=False)\n\ntest_dset = CustomDataset(\n    sub, \"../input/g2net-n-mels-128-test-images\", batch_size=64, target=False, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-06T09:31:54.761539Z","iopub.execute_input":"2021-08-06T09:31:54.761920Z","iopub.status.idle":"2021-08-06T09:31:54.859894Z","shell.execute_reply.started":"2021-08-06T09:31:54.761878Z","shell.execute_reply":"2021-08-06T09:31:54.858750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import matplotlib.pyplot as plt\n\n# for elem in train_dset:\n#     plt.imshow(elem[0][0])\n#     plt.show()\n# #   print(elem[0].shape)","metadata":{"execution":{"iopub.status.busy":"2021-08-06T09:31:54.861435Z","iopub.execute_input":"2021-08-06T09:31:54.862028Z","iopub.status.idle":"2021-08-06T09:31:54.865907Z","shell.execute_reply.started":"2021-08-06T09:31:54.861982Z","shell.execute_reply":"2021-08-06T09:31:54.864900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ckpt = tf.keras.callbacks.ModelCheckpoint(\n    \"model_weights.h5\", save_best_only=True, save_weights_only=True,\n)\n\ntrain_history = model.fit(\n    train_dset, \n    use_multiprocessing=True, \n    workers=4, \n    epochs=20,\n    validation_data=valid_dset,\n    callbacks=[ckpt],\n)","metadata":{"execution":{"iopub.status.busy":"2021-08-06T09:31:54.867503Z","iopub.execute_input":"2021-08-06T09:31:54.868190Z","iopub.status.idle":"2021-08-06T09:32:08.190408Z","shell.execute_reply.started":"2021-08-06T09:31:54.868147Z","shell.execute_reply":"2021-08-06T09:32:08.185216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.load_weights('model_weights.h5')","metadata":{"execution":{"iopub.status.busy":"2021-08-06T09:32:08.193115Z","iopub.status.idle":"2021-08-06T09:32:08.220408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = model.predict(\n    test_dset, use_multiprocessing=True, workers=4, verbose=1\n)","metadata":{"execution":{"iopub.status.busy":"2021-08-06T09:32:08.230868Z","iopub.status.idle":"2021-08-06T09:32:08.232827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub['target'] = y_pred\nsub.to_csv('submission.csv', index=False)\nsub.to_csv('/kaggle/working/submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-06T09:32:08.236847Z","iopub.status.idle":"2021-08-06T09:32:08.239449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install kaggle --upgrade\n# !kaggle competitions submit -c g2net-gravitational-wave-detection -f /kaggle/working/submission.csv -m \"Message\"","metadata":{"execution":{"iopub.status.busy":"2021-08-06T09:32:08.241018Z","iopub.status.idle":"2021-08-06T09:32:08.246576Z"},"trusted":true},"execution_count":null,"outputs":[]}]}