{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install nnAudio\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport tensorflow as tf\nfrom sklearn.model_selection import train_test_split\n\n####Spectrogram creators####\nfrom scipy import signal as sig\nfrom nnAudio.Spectrogram import CQT1992v2\nfrom nnAudio.Spectrogram import CQT2010v2\n############################\n \nimport matplotlib.pyplot as plt\nimport torch\nfrom glob import glob\nroot = '../input/g2net-gravitational-wave-detection/'\nfiles_paths = glob(root + '/train/*/*/*/*')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-28T16:02:38.035352Z","iopub.execute_input":"2021-11-28T16:02:38.036115Z","iopub.status.idle":"2021-11-28T16:02:48.944733Z","shell.execute_reply.started":"2021-11-28T16:02:38.036077Z","shell.execute_reply":"2021-11-28T16:02:48.943923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Zawartość pojedyńczego pliku z danymi","metadata":{}},{"cell_type":"code","source":"train_labels = pd.read_csv(root+'/training_labels.csv')\ndisplay(train_labels)\npos_labels=0\nneg_labels=0\nfor index, row in train_labels.iterrows():\n    if(row['target'] == 1):\n        pos_labels+=1\n    else:\n        neg_labels+=1\nplt.bar([0, 1], [neg_labels, pos_labels])\nplt.xticks([0, 1], ['NIE', 'TAK'])\nplt.xlabel(\"Czy była została fala grawitacyjna?\")\nplt.ylabel(\"Ilość rekordów\")\nplt.title('Opis Train_labels')","metadata":{"execution":{"iopub.status.busy":"2021-11-28T16:02:48.948562Z","iopub.execute_input":"2021-11-28T16:02:48.949005Z","iopub.status.idle":"2021-11-28T16:03:14.061007Z","shell.execute_reply.started":"2021-11-28T16:02:48.948968Z","shell.execute_reply":"2021-11-28T16:03:14.060188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"root = '../input/g2net-gravitational-wave-detection'\nfilename = root + '/train/0/0/0/00000e74ad.npy'\nfiledata = np.load(filename)\nprint(filedata.shape)\nfig, (ax1, ax2, ax3) = plt.subplots(3, sharex=True)\nax1.plot(np.arange(4096), filedata[0])\nax2.plot(np.arange(4096), filedata[1])\nax3.plot(np.arange(4096), filedata[2])","metadata":{"execution":{"iopub.status.busy":"2021-11-28T16:03:14.062484Z","iopub.execute_input":"2021-11-28T16:03:14.063091Z","iopub.status.idle":"2021-11-28T16:03:14.433809Z","shell.execute_reply.started":"2021-11-28T16:03:14.063052Z","shell.execute_reply":"2021-11-28T16:03:14.433132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Wyciągnie rekordów dla nauczania się sieci i ich podstawowa analiza","metadata":{}},{"cell_type":"code","source":"ids_from_files_paths = [path.split('/')[-1].split('.')[0] for path in files_paths]\nids_paths = pd.DataFrame({\n    'id':ids_from_files_paths,\n    'path':files_paths\n})\ndisplay(ids_paths)","metadata":{"execution":{"iopub.status.busy":"2021-11-28T16:03:14.435519Z","iopub.execute_input":"2021-11-28T16:03:14.436088Z","iopub.status.idle":"2021-11-28T16:03:14.89457Z","shell.execute_reply.started":"2021-11-28T16:03:14.43605Z","shell.execute_reply":"2021-11-28T16:03:14.893827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_desc = pd.merge(left = train_labels, right = ids_paths, on = 'id')\ndisplay(train_desc)","metadata":{"execution":{"iopub.status.busy":"2021-11-28T16:03:14.895738Z","iopub.execute_input":"2021-11-28T16:03:14.897453Z","iopub.status.idle":"2021-11-28T16:03:15.447341Z","shell.execute_reply.started":"2021-11-28T16:03:14.89741Z","shell.execute_reply":"2021-11-28T16:03:15.446647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Tworzenie spectrogramu","metadata":{}},{"cell_type":"markdown","source":"### Funkcji przejścia w dziedzinę częstotliwości","metadata":{}},{"cell_type":"code","source":"def create_record_spectrogram_1992(data_from_record):\n    data_combined = np.hstack(data_from_record)\n    data_combined = data_combined/np.max(data_combined)\n    spectrogram_creator = CQT1992v2(\n        sr=2048, fmin=20, fmax=1024, hop_length=64, verbose = False\n    )\n    return spectrogram_creator(torch.from_numpy(data_combined).float())\nplt.imshow(create_record_spectrogram_1992(data_from_record)[0])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-28T16:03:15.448492Z","iopub.execute_input":"2021-11-28T16:03:15.449208Z","iopub.status.idle":"2021-11-28T16:03:15.687635Z","shell.execute_reply.started":"2021-11-28T16:03:15.449169Z","shell.execute_reply":"2021-11-28T16:03:15.68599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_record_spectrogram_2010(data_from_record):\n    data_combined = np.hstack(data_from_record)\n    data_combined = data_combined/np.max(data_combined)\n    spectrogram_creator = CQT2010v2(\n        sr=2048, n_bins=50\n    )\n    return spectrogram_creator(torch.from_numpy(data_combined).float())\nplt.imshow(create_record_spectrogram_2010(data_from_record)[0])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-28T16:04:29.689484Z","iopub.execute_input":"2021-11-28T16:04:29.69001Z","iopub.status.idle":"2021-11-28T16:04:29.718537Z","shell.execute_reply.started":"2021-11-28T16:04:29.689973Z","shell.execute_reply":"2021-11-28T16:04:29.717618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_record_spectrogram(data_from_record):\n    data_combined = np.hstack(data_from_record)\n    data_combined = data_combined/np.max(data_combined)\n    f, t, Sxx = sig.spectrogram(\n        data_combined, fs = 2048, mode='psd')\n    return f, t, Sxx\nf, t, Sxx = create_record_spectrogram(data_from_record)\nprint(f.shape)\nprint(t.shape)\nprint(Sxx.shape)\nplt.pcolormesh(t, f, Sxx, shading='gouraud')\nplt.ylabel('Frequency [Hz]')\nplt.xlabel('Time [sec]')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-28T16:04:30.642407Z","iopub.execute_input":"2021-11-28T16:04:30.64294Z","iopub.status.idle":"2021-11-28T16:04:30.66946Z","shell.execute_reply.started":"2021-11-28T16:04:30.642905Z","shell.execute_reply":"2021-11-28T16:04:30.668563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(5):\n    data_from_record = np.load(train_desc.loc[i, 'path'])\n    print(create_record_spectrogram_1992(data_from_record)[0].shape)","metadata":{"execution":{"iopub.status.busy":"2021-11-28T16:04:32.140603Z","iopub.execute_input":"2021-11-28T16:04:32.140866Z","iopub.status.idle":"2021-11-28T16:04:32.420709Z","shell.execute_reply.started":"2021-11-28T16:04:32.140834Z","shell.execute_reply":"2021-11-28T16:04:32.419965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Size of data loaded from record`s paths: '+ str(data_from_record.shape))\nprint('Size of single one interferometer`s record: '+str(data_from_record[0].shape))\ndata_from_record_combined = np.hstack(data_from_record)\nprint('Combined interferometers record: '+str(data_from_record_combined.shape))","metadata":{"execution":{"iopub.status.busy":"2021-11-28T16:04:32.752244Z","iopub.execute_input":"2021-11-28T16:04:32.752909Z","iopub.status.idle":"2021-11-28T16:04:32.760288Z","shell.execute_reply.started":"2021-11-28T16:04:32.752875Z","shell.execute_reply":"2021-11-28T16:04:32.759356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### DataGenerator for tensorflow","metadata":{}},{"cell_type":"code","source":"class DataGenerator(tf.keras.utils.Sequence):\n    \n    def __init__(self, path, list_IDs, data, batch_size):\n            self.path = path\n            self.list_IDs = list_IDs\n            self.data = data\n            self.batch_size = batch_size\n            self.indexes = np.arange(len(self.list_IDs))\n    \n    def __len__(self):\n        len_ = int(len(self.list_IDs)/self.batch_size)\n        if len_ * self.batch_size < len(self.list_IDs):\n            len_ += 1\n        return len_\n    \n    def __data_generation(self, list_IDs_temp):\n        X = np.zeros((self.batch_size, 69, 193))\n        y = np.zeros((self.batch_size, 1))\n        for i, ID in enumerate(list_IDs_temp):\n            id_ = self.data.loc[ID, 'id']\n            file = id_+\".npy\"\n            path_in = '/'.join([self.path, id_[0], id_[1], id_[2]]) + '/'\n            data_array = np.load(path_in+file)\n            data_array = create_record_spectrogram_1992(data_array)\n            X[i, ] = data_array\n            y[i, ] = self.data.loc[ID, 'target']\n        return X, y\n    \n    def __getitem__(self, index):\n        indexes = self.indexes[index * self.batch_size : (index+1) * self.batch_size]\n        list_IDs_temp = [self.list_IDs[k] for k in indexes]\n        X, y = self.__data_generation(list_IDs_temp)\n        return X, y","metadata":{"execution":{"iopub.status.busy":"2021-11-28T16:04:39.983283Z","iopub.execute_input":"2021-11-28T16:04:39.984006Z","iopub.status.idle":"2021-11-28T16:04:40.829433Z","shell.execute_reply.started":"2021-11-28T16:04:39.98397Z","shell.execute_reply":"2021-11-28T16:04:40.828684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = pd.read_csv(root+'/sample_submission.csv')\ntrain_ids = train_labels['id'].values\ny = train_labels['target'].values","metadata":{"execution":{"iopub.status.busy":"2021-11-28T16:04:41.804675Z","iopub.execute_input":"2021-11-28T16:04:41.805392Z","iopub.status.idle":"2021-11-28T16:04:41.962795Z","shell.execute_reply.started":"2021-11-28T16:04:41.805357Z","shell.execute_reply":"2021-11-28T16:04:41.962068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_indices, validation_indices = train_test_split(list(train_labels.index), test_size=0.25, random_state=2021)\n# print(len(train_indices))\nprint(train_indices[3])\nprint(len(validation_indices))\ntest_indices = list(sample_submission.index)","metadata":{"execution":{"iopub.status.busy":"2021-11-28T16:04:42.326936Z","iopub.execute_input":"2021-11-28T16:04:42.327432Z","iopub.status.idle":"2021-11-28T16:04:42.59882Z","shell.execute_reply.started":"2021-11-28T16:04:42.327394Z","shell.execute_reply":"2021-11-28T16:04:42.596035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_gen = DataGenerator(root+'/train/', train_indices, train_labels, 32)\nvalid_gen = DataGenerator(root+'/train/', validation_indices, train_labels, 32)\ntest_gen = DataGenerator( root+'/test/', test_indices, sample_submission, 32)","metadata":{"execution":{"iopub.status.busy":"2021-11-28T16:11:25.531924Z","iopub.execute_input":"2021-11-28T16:11:25.532193Z","iopub.status.idle":"2021-11-28T16:11:25.539338Z","shell.execute_reply.started":"2021-11-28T16:11:25.532164Z","shell.execute_reply":"2021-11-28T16:11:25.538474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"root\n","metadata":{"execution":{"iopub.status.busy":"2021-11-28T16:04:45.149806Z","iopub.execute_input":"2021-11-28T16:04:45.15055Z","iopub.status.idle":"2021-11-28T16:04:45.156346Z","shell.execute_reply.started":"2021-11-28T16:04:45.150509Z","shell.execute_reply":"2021-11-28T16:04:45.155477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = tf.keras.Sequential()\nmodel.add(tf.keras.layers.Conv1D(\n    32,\n    kernel_size = 3, \n    activation = 'relu',\n    input_shape=(69, 193),\n)\n)\nmodel.add(tf.keras.layers.MaxPooling1D())\nmodel.add(tf.keras.layers.Conv1D(\n    64,\n    kernel_size = 3, \n    activation = 'relu',\n)\n)\nmodel.add(tf.keras.layers.MaxPooling1D())\n# model.add(tf.keras.layers.BatchNormalization())\nmodel.add(tf.keras.layers.Flatten())\nmodel.add(tf.keras.layers.Dense(\n    64,\n    activation=\"relu\"\n))\nmodel.add(tf.keras.layers.Dense(\n    32,\n    activation=\"relu\"\n)) \nmodel.add(tf.keras.layers.Dense(\n    1,\n    activation=\"sigmoid\"\n))\nmodel.compile(\n    optimizer = tf.keras.optimizers.Adam(learning_rate=2e-4),\n    loss = 'binary_crossentropy',\n    metrics=['accuracy']\n)\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-11-28T17:26:21.968992Z","iopub.execute_input":"2021-11-28T17:26:21.969534Z","iopub.status.idle":"2021-11-28T17:26:22.046605Z","shell.execute_reply.started":"2021-11-28T17:26:21.969491Z","shell.execute_reply":"2021-11-28T17:26:22.045807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit_generator(generator=train_gen, validation_data=valid_gen, epochs = 1, workers=-1)\n# 1 Epoch took around 2 and half hours\n\npredicted_test_seq_keras = model.predict_generator(test_gen, verbose=1)\n\nsample_submission['target'] = predicted_test_seq_keras[:len(sample_submission)]\n\nsample_submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-11-28T17:26:31.017137Z","iopub.execute_input":"2021-11-28T17:26:31.017709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}