{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q timm\n!pip install -qq efficientnet_pytorch","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-09-28T08:33:03.115574Z","iopub.execute_input":"2021-09-28T08:33:03.116210Z","iopub.status.idle":"2021-09-28T08:33:18.799833Z","shell.execute_reply.started":"2021-09-28T08:33:03.116111Z","shell.execute_reply":"2021-09-28T08:33:18.798761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Special thanks to Grandmaster Abhishek thakur, for super useful pytorch trainer","metadata":{}},{"cell_type":"code","source":"tez_path = '../input/tez-modified-tqdm/'","metadata":{"execution":{"iopub.status.busy":"2021-09-28T08:33:18.802178Z","iopub.execute_input":"2021-09-28T08:33:18.802447Z","iopub.status.idle":"2021-09-28T08:33:18.807524Z","shell.execute_reply.started":"2021-09-28T08:33:18.802412Z","shell.execute_reply":"2021-09-28T08:33:18.806616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append(tez_path)","metadata":{"execution":{"iopub.status.busy":"2021-09-28T08:33:18.814935Z","iopub.execute_input":"2021-09-28T08:33:18.815247Z","iopub.status.idle":"2021-09-28T08:33:18.824139Z","shell.execute_reply.started":"2021-09-28T08:33:18.815210Z","shell.execute_reply":"2021-09-28T08:33:18.823191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.image as immg\nimport matplotlib.ticker as ticker\nimport seaborn as sns\nfrom PIL import Image\nimport matplotlib.image as immg\nimport os,gc,random\nimport albumentations\nimport tez\nimport timm\nimport torch\nimport torch.nn as nn\nfrom torch.nn import functional as F\nfrom efficientnet_pytorch import EfficientNet","metadata":{"execution":{"iopub.status.busy":"2021-09-28T08:33:18.825847Z","iopub.execute_input":"2021-09-28T08:33:18.826528Z","iopub.status.idle":"2021-09-28T08:33:23.746537Z","shell.execute_reply.started":"2021-09-28T08:33:18.826481Z","shell.execute_reply":"2021-09-28T08:33:23.745675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.mkdir('/kaggle/working/train/')","metadata":{"execution":{"iopub.status.busy":"2021-09-28T08:33:23.748059Z","iopub.execute_input":"2021-09-28T08:33:23.749054Z","iopub.status.idle":"2021-09-28T08:33:23.754496Z","shell.execute_reply.started":"2021-09-28T08:33:23.748991Z","shell.execute_reply":"2021-09-28T08:33:23.753343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!unzip -qq '../input/band-pass-traintest-g2net-audio-wave-data/TrainG2NET.zip' -d '/kaggle/working/train/'","metadata":{"execution":{"iopub.status.busy":"2021-09-28T08:33:23.755889Z","iopub.execute_input":"2021-09-28T08:33:23.756736Z","iopub.status.idle":"2021-09-28T08:35:49.314799Z","shell.execute_reply.started":"2021-09-28T08:33:23.756692Z","shell.execute_reply":"2021-09-28T08:35:49.313401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.mkdir('/kaggle/working/test/')","metadata":{"execution":{"iopub.status.busy":"2021-09-28T08:35:49.318666Z","iopub.execute_input":"2021-09-28T08:35:49.319041Z","iopub.status.idle":"2021-09-28T08:35:49.325010Z","shell.execute_reply.started":"2021-09-28T08:35:49.318980Z","shell.execute_reply":"2021-09-28T08:35:49.323967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!unzip -qq '../input/band-pass-traintest-g2net-audio-wave-data/TestG2NET.zip' -d '/kaggle/working/test/'","metadata":{"execution":{"iopub.status.busy":"2021-09-28T08:35:49.327890Z","iopub.execute_input":"2021-09-28T08:35:49.328154Z","iopub.status.idle":"2021-09-28T08:36:48.401754Z","shell.execute_reply.started":"2021-09-28T08:35:49.328125Z","shell.execute_reply":"2021-09-28T08:36:48.400819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2021-09-28T08:36:48.403213Z","iopub.execute_input":"2021-09-28T08:36:48.403580Z","iopub.status.idle":"2021-09-28T08:36:48.408570Z","shell.execute_reply.started":"2021-09-28T08:36:48.403527Z","shell.execute_reply":"2021-09-28T08:36:48.407494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading data","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/g2net-gravitational-wave-detection/training_labels.csv')\ntest = pd.read_csv('../input/g2net-gravitational-wave-detection/sample_submission.csv')\n\ndef get_train_file_path(image_id):\n    return \"../input/g2net-gravitational-wave-detection/train/{}/{}/{}/{}.npy\".format(\n        image_id[0], image_id[1], image_id[2], image_id)\n\ndef get_test_file_path(image_id):\n    return \"../input/g2net-gravitational-wave-detection/test/{}/{}/{}/{}.npy\".format(\n        image_id[0], image_id[1], image_id[2], image_id)","metadata":{"execution":{"iopub.status.busy":"2021-09-28T08:36:48.409955Z","iopub.execute_input":"2021-09-28T08:36:48.410603Z","iopub.status.idle":"2021-09-28T08:36:49.111448Z","shell.execute_reply.started":"2021-09-28T08:36:48.410557Z","shell.execute_reply":"2021-09-28T08:36:49.110615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['file_path'] = train['id'].apply(lambda x:\"/kaggle/working/train/\"+str(x)+\".png\")","metadata":{"execution":{"iopub.status.busy":"2021-09-28T08:36:49.112554Z","iopub.execute_input":"2021-09-28T08:36:49.112779Z","iopub.status.idle":"2021-09-28T08:36:49.452841Z","shell.execute_reply.started":"2021-09-28T08:36:49.112753Z","shell.execute_reply":"2021-09-28T08:36:49.452113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Stratified Kfold","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split,StratifiedKFold","metadata":{"execution":{"iopub.status.busy":"2021-09-28T08:41:10.301286Z","iopub.execute_input":"2021-09-28T08:41:10.301561Z","iopub.status.idle":"2021-09-28T08:41:10.305526Z","shell.execute_reply.started":"2021-09-28T08:41:10.301527Z","shell.execute_reply":"2021-09-28T08:41:10.304703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train.copy()\ntrain_df['kfold'] = -1\ny = train_df['target'].values\nkf = StratifiedKFold(n_splits=200,random_state = 2021,shuffle = True)\nfor fold ,(trn_,val_ )in enumerate(kf.split(X=train_df,y=y)):\n    train_df.loc[val_,'kfold'] = fold","metadata":{"execution":{"iopub.status.busy":"2021-09-28T08:41:40.375726Z","iopub.execute_input":"2021-09-28T08:41:40.376579Z","iopub.status.idle":"2021-09-28T08:41:41.515301Z","shell.execute_reply.started":"2021-09-28T08:41:40.376537Z","shell.execute_reply":"2021-09-28T08:41:41.514500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df.to_csv('cnn1d_cqt_train_kfold.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2021-09-28T08:41:41.516914Z","iopub.execute_input":"2021-09-28T08:41:41.517163Z","iopub.status.idle":"2021-09-28T08:41:41.519819Z","shell.execute_reply.started":"2021-09-28T08:41:41.517137Z","shell.execute_reply":"2021-09-28T08:41:41.519318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df = pd.read_csv('../input/effnetb0-3ch3-g2net/cnn1d_cqt_train_kfold.csv')","metadata":{"execution":{"iopub.status.busy":"2021-09-28T08:41:41.520798Z","iopub.execute_input":"2021-09-28T08:41:41.521122Z","iopub.status.idle":"2021-09-28T08:41:41.534464Z","shell.execute_reply.started":"2021-09-28T08:41:41.521095Z","shell.execute_reply":"2021-09-28T08:41:41.533345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[train_df.kfold==1].shape,train_df[train_df.kfold!=1].shape","metadata":{"execution":{"iopub.status.busy":"2021-09-28T08:42:00.517307Z","iopub.execute_input":"2021-09-28T08:42:00.518164Z","iopub.status.idle":"2021-09-28T08:42:00.557664Z","shell.execute_reply.started":"2021-09-28T08:42:00.518114Z","shell.execute_reply":"2021-09-28T08:42:00.556954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-28T08:41:42.730078Z","iopub.execute_input":"2021-09-28T08:41:42.730355Z","iopub.status.idle":"2021-09-28T08:41:42.740536Z","shell.execute_reply.started":"2021-09-28T08:41:42.730329Z","shell.execute_reply":"2021-09-28T08:41:42.739909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FOLD = 1","metadata":{"execution":{"iopub.status.busy":"2021-09-28T08:42:11.841325Z","iopub.execute_input":"2021-09-28T08:42:11.841607Z","iopub.status.idle":"2021-09-28T08:42:11.845861Z","shell.execute_reply.started":"2021-09-28T08:42:11.841580Z","shell.execute_reply":"2021-09-28T08:42:11.845026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = train_df[train_df['kfold']!=FOLD].reset_index(drop=True)\ndf_valid = train_df[train_df['kfold']==FOLD].reset_index(drop=True)\ntrain_targets = df_train['target'].values\nvalid_targets = df_valid['target'].values","metadata":{"execution":{"iopub.status.busy":"2021-09-28T08:42:12.794856Z","iopub.execute_input":"2021-09-28T08:42:12.795801Z","iopub.status.idle":"2021-09-28T08:42:12.863862Z","shell.execute_reply.started":"2021-09-28T08:42:12.795749Z","shell.execute_reply":"2021-09-28T08:42:12.863218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.shape[0],df_valid.shape[0]","metadata":{"execution":{"iopub.status.busy":"2021-09-28T08:42:13.209975Z","iopub.execute_input":"2021-09-28T08:42:13.210768Z","iopub.status.idle":"2021-09-28T08:42:13.217324Z","shell.execute_reply.started":"2021-09-28T08:42:13.210731Z","shell.execute_reply":"2021-09-28T08:42:13.216500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset","metadata":{}},{"cell_type":"code","source":"class G2NetDataset:\n    def __init__(self,paths,targets):\n        \n        self.paths = paths\n        self.targets = targets\n        \n    def __len__(self):\n        return len(self.paths)\n    \n    def __getitem__(self,item):\n        \n        targs = self.targets[item]\n        image = Image.open(self.paths[item])\n        image = np.array(image)/255.0\n        image123 = np.array([image[:,:129],image[:,129:129*2],image[:,129*2:]])\n        image_tensor = torch.tensor(image123,dtype=torch.float)\n        \n        return {\"image\": image_tensor,\n                \"targets\": torch.tensor(targs,dtype=torch.float)}","metadata":{"execution":{"iopub.status.busy":"2021-09-25T14:03:07.045797Z","iopub.execute_input":"2021-09-25T14:03:07.049366Z","iopub.status.idle":"2021-09-25T14:03:07.083491Z","shell.execute_reply.started":"2021-09-25T14:03:07.049328Z","shell.execute_reply":"2021-09-25T14:03:07.081257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = G2NetDataset(df_train.file_path,train_targets)\nvalid_dataset = G2NetDataset(df_valid.file_path,valid_targets)","metadata":{"execution":{"iopub.status.busy":"2021-09-25T14:03:07.08506Z","iopub.execute_input":"2021-09-25T14:03:07.085505Z","iopub.status.idle":"2021-09-25T14:03:07.154273Z","shell.execute_reply.started":"2021-09-25T14:03:07.08546Z","shell.execute_reply":"2021-09-25T14:03:07.152738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img  = train_dataset[0]['image']\ntar = train_dataset[0]['targets']","metadata":{"execution":{"iopub.status.busy":"2021-09-25T14:03:07.160332Z","iopub.execute_input":"2021-09-25T14:03:07.162377Z","iopub.status.idle":"2021-09-25T14:03:07.210215Z","shell.execute_reply.started":"2021-09-25T14:03:07.162329Z","shell.execute_reply":"2021-09-25T14:03:07.2093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sklearn.metrics as sklm\nimport torch.optim as optim","metadata":{"execution":{"iopub.status.busy":"2021-09-25T14:03:07.21192Z","iopub.execute_input":"2021-09-25T14:03:07.21222Z","iopub.status.idle":"2021-09-25T14:03:07.220904Z","shell.execute_reply.started":"2021-09-25T14:03:07.212177Z","shell.execute_reply":"2021-09-25T14:03:07.219884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model G2Net ResNest50d","metadata":{}},{"cell_type":"code","source":"class G2NetModel(tez.Model):\n    def __init__(self):\n        super().__init__()\n        \n        self.net = timm.create_model('resnest50d',pretrained=True)\n        #self.net = EfficientNet.from_pretrained(\"efficientnet-b4\")\n        self.dropout = nn.Dropout(0.1)\n        self.out = nn.Linear(1000,1)\n        \n        self.step_scheduler_after = \"epoch\"\n        self.step_scheduler_metric = \"valid_rocauc\"\n        \n\n    def monitor_metrics(self, outputs, targets):\n        outputs = torch.sigmoid(outputs).cpu().detach().numpy() \n        targets = targets.cpu().detach().numpy()\n        try:\n            res = sklm.roc_auc_score(targets, outputs)\n        except:\n            res = 0.5\n        return {\"rocauc\": res}\n\n    def fetch_optimizer(self):\n        opt = torch.optim.Adam(self.parameters(), lr=1e-3)\n        return opt\n\n    def fetch_scheduler(self):\n        rlr = torch.optim.lr_scheduler.ReduceLROnPlateau(\n            self.optimizer,\n            verbose=True,\n            factor=0.7,\n            mode=\"max\",\n            patience=2,\n            threshold=0.01,)\n        return rlr\n\n    def forward(self, image, targets=None):\n        \n        #batch_size, _, _, _ = image.shape\n        x = self.net(image)\n        #x = F.adaptive_avg_pool2d(x, 1).reshape(batch_size, -1)\n        outputs = self.out(self.dropout(x))\n        \n        if targets is not None:\n            loss = nn.BCEWithLogitsLoss()(outputs, targets.view(-1, 1))\n            metrics = self.monitor_metrics(outputs, targets)\n            return outputs, loss, metrics\n        return outputs, None, {}","metadata":{"execution":{"iopub.status.busy":"2021-09-25T14:03:07.22254Z","iopub.execute_input":"2021-09-25T14:03:07.223098Z","iopub.status.idle":"2021-09-25T14:03:07.238644Z","shell.execute_reply.started":"2021-09-25T14:03:07.223054Z","shell.execute_reply":"2021-09-25T14:03:07.237687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = G2NetModel()","metadata":{"execution":{"iopub.status.busy":"2021-09-25T14:03:07.240678Z","iopub.execute_input":"2021-09-25T14:03:07.241043Z","iopub.status.idle":"2021-09-25T14:03:10.92775Z","shell.execute_reply.started":"2021-09-25T14:03:07.241Z","shell.execute_reply":"2021-09-25T14:03:10.926793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tez.callbacks import EarlyStopping\nes = EarlyStopping(\n    monitor=\"valid_rocauc\", model_path=\"model.bin\", patience=5, mode=\"max\"\n)","metadata":{"execution":{"iopub.status.busy":"2021-09-25T14:03:10.929376Z","iopub.execute_input":"2021-09-25T14:03:10.929665Z","iopub.status.idle":"2021-09-25T14:03:10.934298Z","shell.execute_reply.started":"2021-09-25T14:03:10.929629Z","shell.execute_reply":"2021-09-25T14:03:10.933356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Starting Model Training:')","metadata":{"execution":{"iopub.status.busy":"2021-09-25T14:03:10.936304Z","iopub.execute_input":"2021-09-25T14:03:10.937192Z","iopub.status.idle":"2021-09-25T14:03:10.950563Z","shell.execute_reply.started":"2021-09-25T14:03:10.937123Z","shell.execute_reply":"2021-09-25T14:03:10.949248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Start Training","metadata":{}},{"cell_type":"code","source":"model.fit(\n        train_dataset,\n        valid_dataset=valid_dataset,\n        train_bs = 128,\n        valid_bs = 128,\n        device = \"cuda\",\n        epochs = 8,\n        callbacks = [es],\n        fp16 = True)","metadata":{"execution":{"iopub.status.busy":"2021-09-25T14:03:10.953004Z","iopub.execute_input":"2021-09-25T14:03:10.9534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test Dataset & Load Best Model","metadata":{}},{"cell_type":"code","source":"class G2NetModel(tez.Model):\n    def __init__(self):\n        super().__init__()\n        self.net = timm.create_model('resnest50d',pretrained=False)\n        self.dropout = nn.Dropout(0.1)\n        self.out = nn.Linear(1000,1)\n    def forward(self, image):\n        x = self.net(image)\n        outputs = self.out(self.dropout(x))\n        return outputs, None, {}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = G2NetModel()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.load(os.path.join('/kaggle/working', \"model.bin\"))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv('../input/g2net-gravitational-wave-detection/sample_submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ts_filepath = sub['id'].apply(lambda x:'/kaggle/working/test/'+str(x)+'.png').values","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TestDataset:\n    def __init__(self,paths):\n        \n        self.paths = paths\n        self.targets = None\n        \n    def __len__(self):\n        return len(self.paths)\n    \n    def __getitem__(self,item):\n        \n        image = Image.open(self.paths[item])\n        image = np.array(image)/255.0\n        image123 = np.array([image[:,:129],image[:,129:129*2],image[:,129*2:]])\n        image_tensor = torch.tensor(image123,dtype=torch.float)\n        \n        return {\"image\": image_tensor}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset = TestDataset(ts_filepath)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prediction on Test Set","metadata":{}},{"cell_type":"code","source":"preds = model.predict(\n    test_dataset, batch_size=64, n_jobs=2, device=\"cuda\"\n)\ntemp_preds = None\nfor p in preds:\n    if temp_preds is None:\n        temp_preds = p\n    else:\n        temp_preds = np.vstack((temp_preds, p))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub['target'] = torch.from_numpy(temp_preds).sigmoid().flatten().numpy()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv('submission_3ch.csv',index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -r '/kaggle/working/test/'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -r '/kaggle/working/train/'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}