{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom matplotlib import pyplot as plt\nimport tensorflow as tf\nimport keras\nimport keras.layers as L\nfrom keras.models import Model,Sequential\nimport math\nfrom keras.utils import Sequence\nfrom keras.preprocessing import image\nfrom random import shuffle\nfrom sklearn.model_selection import train_test_split\nimport plotly.express as px\nimport seaborn as sns\nfrom sklearn.utils import class_weight\nfrom librosa import cqt\nfrom skimage.transform import resize\nfrom random import shuffle\nimport glob\nimport pathlib\nimport cv2","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-29T08:19:02.355643Z","iopub.execute_input":"2021-08-29T08:19:02.355986Z","iopub.status.idle":"2021-08-29T08:19:11.291235Z","shell.execute_reply.started":"2021-08-29T08:19:02.355912Z","shell.execute_reply":"2021-08-29T08:19:11.290313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels =pd.read_csv('../input/g2net-gravitational-wave-detection/training_labels.csv')\nsample_submission = pd.read_csv('../input/g2net-gravitational-wave-detection/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2021-08-29T08:19:11.292787Z","iopub.execute_input":"2021-08-29T08:19:11.293096Z","iopub.status.idle":"2021-08-29T08:19:11.809513Z","shell.execute_reply.started":"2021-08-29T08:19:11.293063Z","shell.execute_reply":"2021-08-29T08:19:11.808603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"colors=(\"black\", \"red\", \"green\")\nsignal_names=(\"LIGO Hanford\", \"LIGO Livingston\", \"Virgo\")\nx = np.load('../input/g2net-gravitational-wave-detection/train/0/0/0/00000e74ad.npy').astype(np.float32)\nfig = plt.figure(figsize=(16, 10))\n\n    \nplt.subplot(4, 1, 4)\nplt.plot(x)\nplt.legend(signal_names, fontsize=12, loc=\"lower right\")","metadata":{"execution":{"iopub.status.busy":"2021-08-29T08:19:11.811291Z","iopub.execute_input":"2021-08-29T08:19:11.811632Z","iopub.status.idle":"2021-08-29T08:19:18.063917Z","shell.execute_reply.started":"2021-08-29T08:19:11.811597Z","shell.execute_reply":"2021-08-29T08:19:18.063101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def id_to_path(idx,train=True):\n    path = '../input/g2net-gravitational-wave-detection/'\n    if train:\n        folder = 'train/'\n    else:\n        folder = 'test/'\n    path+=folder+idx[0]+'/'+idx[1]+'/'+idx[2]+'/'+idx+'.npy'\n    return path","metadata":{"execution":{"iopub.status.busy":"2021-08-29T08:19:18.065352Z","iopub.execute_input":"2021-08-29T08:19:18.065666Z","iopub.status.idle":"2021-08-29T08:19:18.071607Z","shell.execute_reply.started":"2021-08-29T08:19:18.065634Z","shell.execute_reply":"2021-08-29T08:19:18.070787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def id_to_path2(idx,train=True):\n    path = '../input/g2net-constant-q-transformed/'\n    if train:\n        folder = 'train_cqt/train_cqt/'\n    else:\n        folder = 'test_cqt/test_cqt/'\n    path+=folder+idx+'.png'\n    return path","metadata":{"execution":{"iopub.status.busy":"2021-08-29T08:19:18.073155Z","iopub.execute_input":"2021-08-29T08:19:18.073762Z","iopub.status.idle":"2021-08-29T08:19:18.081196Z","shell.execute_reply.started":"2021-08-29T08:19:18.073726Z","shell.execute_reply":"2021-08-29T08:19:18.08014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def getimg(x):\n    cq=np.hstack(x)\n    cq=cq/np.max(cq)\n    cq = cqt(cq,sr=2048, fmin=20, n_bins=64,pad_mode='wrap')\n    return cq","metadata":{"execution":{"iopub.status.busy":"2021-08-29T08:19:18.082711Z","iopub.execute_input":"2021-08-29T08:19:18.083123Z","iopub.status.idle":"2021-08-29T08:19:18.089914Z","shell.execute_reply.started":"2021-08-29T08:19:18.083087Z","shell.execute_reply":"2021-08-29T08:19:18.088973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = getimg(x)\nimg.shape","metadata":{"execution":{"iopub.status.busy":"2021-08-29T08:19:18.09121Z","iopub.execute_input":"2021-08-29T08:19:18.091888Z","iopub.status.idle":"2021-08-29T08:19:18.714229Z","shell.execute_reply.started":"2021-08-29T08:19:18.091833Z","shell.execute_reply":"2021-08-29T08:19:18.713155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#img = resize(abs(img), (65,32))\nimg = img.reshape(50,32)\nimg2= np.stack([img,img,img],axis = -1)\nimg2.shape","metadata":{"execution":{"iopub.status.busy":"2021-08-29T08:19:18.717318Z","iopub.execute_input":"2021-08-29T08:19:18.717723Z","iopub.status.idle":"2021-08-29T08:19:18.726541Z","shell.execute_reply.started":"2021-08-29T08:19:18.717674Z","shell.execute_reply":"2021-08-29T08:19:18.72565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(abs(img2),aspect='auto', origin='lower')\nplt.xlabel('Time Steps')\nplt.ylabel('Freq bins')","metadata":{"execution":{"iopub.status.busy":"2021-08-29T08:19:18.728889Z","iopub.execute_input":"2021-08-29T08:19:18.729288Z","iopub.status.idle":"2021-08-29T08:19:18.871285Z","shell.execute_reply.started":"2021-08-29T08:19:18.72923Z","shell.execute_reply":"2021-08-29T08:19:18.870508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(abs(img2),aspect='auto', origin='lower')\nplt.xlabel('Time Steps')\nplt.ylabel('Freq bins')","metadata":{"execution":{"iopub.status.busy":"2021-08-29T08:19:18.872595Z","iopub.execute_input":"2021-08-29T08:19:18.872944Z","iopub.status.idle":"2021-08-29T08:19:19.010258Z","shell.execute_reply.started":"2021-08-29T08:19:18.872907Z","shell.execute_reply":"2021-08-29T08:19:19.009246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = cv2.imread('../input/g2net-constant-q-transformed/train_cqt/train_cqt/00000e74ad.png')\nplt.imshow(img)","metadata":{"execution":{"iopub.status.busy":"2021-08-29T08:19:19.01175Z","iopub.execute_input":"2021-08-29T08:19:19.012089Z","iopub.status.idle":"2021-08-29T08:19:19.202763Z","shell.execute_reply.started":"2021-08-29T08:19:19.012054Z","shell.execute_reply":"2021-08-29T08:19:19.201823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = cv2.imread(id_to_path2('00000e74ad'))\nplt.imshow(img)","metadata":{"execution":{"iopub.status.busy":"2021-08-29T08:19:19.204111Z","iopub.execute_input":"2021-08-29T08:19:19.204461Z","iopub.status.idle":"2021-08-29T08:19:19.368351Z","shell.execute_reply.started":"2021-08-29T08:19:19.204428Z","shell.execute_reply":"2021-08-29T08:19:19.367426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img.shape","metadata":{"execution":{"iopub.status.busy":"2021-08-29T08:19:19.369828Z","iopub.execute_input":"2021-08-29T08:19:19.370309Z","iopub.status.idle":"2021-08-29T08:19:19.376444Z","shell.execute_reply.started":"2021-08-29T08:19:19.370269Z","shell.execute_reply":"2021-08-29T08:19:19.37557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Dataset(Sequence):\n    def __init__(self,idx,directory,batch_size=16,shuffle=True):\n        self.idx = idx\n        self.batch_size = batch_size\n        self.shuffle = shuffle\n        self.directory = directory\n        self.is_train = True\n        self.y = []\n        if self.directory == 'test':\n            self.is_train = False\n        \n    def __len__(self):\n        return math.ceil(len(self.idx)/self.batch_size)\n    def getimg(self,x):\n        cq=cv2.imread(x)\n        return cq\n    def getlabel(self,x):\n        x=x.split('/')[4]\n        if x == 'ones':\n            return 1\n        else:\n            return 0\n                  \n    def __getitem__(self,ids):\n        batch_ids = self.idx[ids * self.batch_size:(ids + 1) * self.batch_size]    \n        list_x1 = np.array([self.getimg(x) for x in batch_ids])\n        if self.directory != 'test':\n            batch_y = np.array([self.getlabel(x) for x in batch_ids])\n            return list_x1, batch_y\n        else:\n            return list_x1","metadata":{"execution":{"iopub.status.busy":"2021-08-29T08:27:52.521372Z","iopub.execute_input":"2021-08-29T08:27:52.52174Z","iopub.status.idle":"2021-08-29T08:27:52.533197Z","shell.execute_reply.started":"2021-08-29T08:27:52.521681Z","shell.execute_reply":"2021-08-29T08:27:52.53212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ones = [file for file in sorted(glob.glob(\"../input/g2net-as-image/train/ones/*\"))]\ntrain_zero = [file for file in sorted(glob.glob(\"../input/g2net-as-image/train/zero/*\"))]\ntrain = train_ones+train_zero\nshuffle(train)","metadata":{"execution":{"iopub.status.busy":"2021-08-29T08:19:19.391195Z","iopub.execute_input":"2021-08-29T08:19:19.391578Z","iopub.status.idle":"2021-08-29T08:19:27.294976Z","shell.execute_reply.started":"2021-08-29T08:19:19.391543Z","shell.execute_reply":"2021-08-29T08:19:27.294105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_ones = [file for file in sorted(glob.glob(\"../input/g2net-as-image/valid/ones/*\"))]\nvalid_zero = [file for file in sorted(glob.glob(\"../input/g2net-as-image/valid/zero/*\"))]\nvalid = valid_ones + valid_zero\nshuffle(valid)","metadata":{"execution":{"iopub.status.busy":"2021-08-29T08:19:27.296222Z","iopub.execute_input":"2021-08-29T08:19:27.296726Z","iopub.status.idle":"2021-08-29T08:19:29.497847Z","shell.execute_reply.started":"2021-08-29T08:19:27.296676Z","shell.execute_reply":"2021-08-29T08:19:29.496933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = Dataset(train,directory = 'train')\nvalid_dataset = Dataset(valid,directory = 'valid', batch_size=64)","metadata":{"execution":{"iopub.status.busy":"2021-08-29T08:27:56.53143Z","iopub.execute_input":"2021-08-29T08:27:56.531773Z","iopub.status.idle":"2021-08-29T08:27:56.538726Z","shell.execute_reply.started":"2021-08-29T08:27:56.531739Z","shell.execute_reply":"2021-08-29T08:27:56.537553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install google-cloud-automl","metadata":{"execution":{"iopub.status.busy":"2021-08-29T08:19:29.505533Z","iopub.execute_input":"2021-08-29T08:19:29.505911Z","iopub.status.idle":"2021-08-29T08:19:36.636841Z","shell.execute_reply.started":"2021-08-29T08:19:29.505874Z","shell.execute_reply":"2021-08-29T08:19:36.635734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -U efficientnet","metadata":{"execution":{"iopub.status.busy":"2021-08-29T08:19:36.638435Z","iopub.execute_input":"2021-08-29T08:19:36.638875Z","iopub.status.idle":"2021-08-29T08:19:43.889638Z","shell.execute_reply.started":"2021-08-29T08:19:36.638837Z","shell.execute_reply":"2021-08-29T08:19:43.888548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import efficientnet.keras as efn","metadata":{"execution":{"iopub.status.busy":"2021-08-29T08:19:43.894381Z","iopub.execute_input":"2021-08-29T08:19:43.894644Z","iopub.status.idle":"2021-08-29T08:19:43.950128Z","shell.execute_reply.started":"2021-08-29T08:19:43.894615Z","shell.execute_reply":"2021-08-29T08:19:43.949272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ef=efn.EfficientNetB0(input_shape=(67, 193, 3),weights='imagenet',include_top=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-29T08:19:43.951341Z","iopub.execute_input":"2021-08-29T08:19:43.951827Z","iopub.status.idle":"2021-08-29T08:19:57.253633Z","shell.execute_reply.started":"2021-08-29T08:19:43.951789Z","shell.execute_reply":"2021-08-29T08:19:57.252787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#for layer in ef.layers[:]:\n#    layer.trainable = False\n","metadata":{"execution":{"iopub.status.busy":"2021-08-29T08:19:57.255741Z","iopub.execute_input":"2021-08-29T08:19:57.25612Z","iopub.status.idle":"2021-08-29T08:19:57.260093Z","shell.execute_reply.started":"2021-08-29T08:19:57.256081Z","shell.execute_reply":"2021-08-29T08:19:57.25914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install tensorflow-hub","metadata":{"execution":{"iopub.status.busy":"2021-08-29T08:19:57.2658Z","iopub.execute_input":"2021-08-29T08:19:57.266263Z","iopub.status.idle":"2021-08-29T08:20:03.397902Z","shell.execute_reply.started":"2021-08-29T08:19:57.266223Z","shell.execute_reply":"2021-08-29T08:20:03.396906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow_hub as hub","metadata":{"execution":{"iopub.status.busy":"2021-08-29T08:20:03.410342Z","iopub.execute_input":"2021-08-29T08:20:03.410745Z","iopub.status.idle":"2021-08-29T08:20:03.577798Z","shell.execute_reply.started":"2021-08-29T08:20:03.410688Z","shell.execute_reply":"2021-08-29T08:20:03.576972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hub_url = 'gs://cloud-tpu-checkpoints/efficientnet/v2/hub/efficientnetv2-l/feature-vector'","metadata":{"execution":{"iopub.status.busy":"2021-08-29T08:20:03.581654Z","iopub.execute_input":"2021-08-29T08:20:03.581953Z","iopub.status.idle":"2021-08-29T08:20:03.585221Z","shell.execute_reply.started":"2021-08-29T08:20:03.581918Z","shell.execute_reply":"2021-08-29T08:20:03.584308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inp=L.Input(shape=(67, 193, 3))\nl = ef(inp)\n#l = hub.KerasLayer(hub_url, trainable=True)(inp)\n#l = L.Conv2D(64,(3,3), activation = 'relu')(inp)\nl = L.Flatten()(l)\nl = L.Dense(1, activation='sigmoid')(l)\nmodel = Model(inputs= [inp],outputs=l)\nmodel.summary()\nmodel.compile(optimizer='adam',loss='binary_crossentropy', metrics=[keras.metrics.AUC()])","metadata":{"execution":{"iopub.status.busy":"2021-08-29T08:28:06.517537Z","iopub.execute_input":"2021-08-29T08:28:06.517889Z","iopub.status.idle":"2021-08-29T08:28:08.266643Z","shell.execute_reply.started":"2021-08-29T08:28:06.517858Z","shell.execute_reply":"2021-08-29T08:28:08.265642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(train_dataset,epochs=3,validation_data=valid_dataset)","metadata":{"execution":{"iopub.status.busy":"2021-08-29T08:28:13.407806Z","iopub.execute_input":"2021-08-29T08:28:13.40816Z","iopub.status.idle":"2021-08-29T10:33:32.724074Z","shell.execute_reply.started":"2021-08-29T08:28:13.408129Z","shell.execute_reply":"2021-08-29T10:33:32.721683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_idx = [file for file in sorted(glob.glob(\"../input/g2net-as-image/test/*\"))]","metadata":{"execution":{"iopub.status.busy":"2021-08-29T10:33:58.805462Z","iopub.execute_input":"2021-08-29T10:33:58.805787Z","iopub.status.idle":"2021-08-29T10:34:07.66652Z","shell.execute_reply.started":"2021-08-29T10:33:58.805754Z","shell.execute_reply":"2021-08-29T10:34:07.665676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset = Dataset(test_idx,directory = 'test', batch_size=64)","metadata":{"execution":{"iopub.status.busy":"2021-08-29T10:34:07.668043Z","iopub.execute_input":"2021-08-29T10:34:07.668373Z","iopub.status.idle":"2021-08-29T10:34:07.676245Z","shell.execute_reply.started":"2021-08-29T10:34:07.668335Z","shell.execute_reply":"2021-08-29T10:34:07.675422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = model.predict(test_dataset)","metadata":{"execution":{"iopub.status.busy":"2021-08-29T10:34:07.678156Z","iopub.execute_input":"2021-08-29T10:34:07.678564Z","iopub.status.idle":"2021-08-29T11:10:54.058854Z","shell.execute_reply.started":"2021-08-29T10:34:07.678527Z","shell.execute_reply":"2021-08-29T11:10:54.057913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = preds.reshape(-1)","metadata":{"execution":{"iopub.status.busy":"2021-08-29T11:11:21.811274Z","iopub.execute_input":"2021-08-29T11:11:21.811612Z","iopub.status.idle":"2021-08-29T11:11:21.815897Z","shell.execute_reply.started":"2021-08-29T11:11:21.811578Z","shell.execute_reply":"2021-08-29T11:11:21.815046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_samples = []","metadata":{"execution":{"iopub.status.busy":"2021-08-29T11:11:22.892517Z","iopub.execute_input":"2021-08-29T11:11:22.892865Z","iopub.status.idle":"2021-08-29T11:11:22.896828Z","shell.execute_reply.started":"2021-08-29T11:11:22.892833Z","shell.execute_reply":"2021-08-29T11:11:22.895724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def getid(path):\n    x = path.split('/')[4]\n    x = x.split('.')[0]\n    return x","metadata":{"execution":{"iopub.status.busy":"2021-08-29T11:13:00.530187Z","iopub.execute_input":"2021-08-29T11:13:00.530505Z","iopub.status.idle":"2021-08-29T11:13:00.536551Z","shell.execute_reply.started":"2021-08-29T11:13:00.530474Z","shell.execute_reply":"2021-08-29T11:13:00.535721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in test_idx:\n    test_samples.append(getid(i))","metadata":{"execution":{"iopub.status.busy":"2021-08-29T11:13:32.612494Z","iopub.execute_input":"2021-08-29T11:13:32.612846Z","iopub.status.idle":"2021-08-29T11:13:32.814782Z","shell.execute_reply.started":"2021-08-29T11:13:32.612814Z","shell.execute_reply":"2021-08-29T11:13:32.813896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(test_samples)","metadata":{"execution":{"iopub.status.busy":"2021-08-29T11:13:35.413418Z","iopub.execute_input":"2021-08-29T11:13:35.413759Z","iopub.status.idle":"2021-08-29T11:13:35.419562Z","shell.execute_reply.started":"2021-08-29T11:13:35.413724Z","shell.execute_reply":"2021-08-29T11:13:35.418488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({'id':test_samples,'target':preds})\nsubmission.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-29T11:13:38.374907Z","iopub.execute_input":"2021-08-29T11:13:38.375241Z","iopub.status.idle":"2021-08-29T11:13:39.206154Z","shell.execute_reply.started":"2021-08-29T11:13:38.37521Z","shell.execute_reply":"2021-08-29T11:13:39.205275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}