{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q nnAudio","metadata":{"execution":{"iopub.status.busy":"2021-09-01T07:39:45.245427Z","iopub.execute_input":"2021-09-01T07:39:45.245753Z","iopub.status.idle":"2021-09-01T07:39:52.077972Z","shell.execute_reply.started":"2021-09-01T07:39:45.245723Z","shell.execute_reply":"2021-09-01T07:39:52.077009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# installing the torch-xla nightly version\n!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n!python pytorch-xla-env-setup.py --apt-packages libomp5 libopenblas-dev","metadata":{"execution":{"iopub.status.busy":"2021-09-01T07:40:06.318530Z","iopub.execute_input":"2021-09-01T07:40:06.318907Z","iopub.status.idle":"2021-09-01T07:41:14.788995Z","shell.execute_reply.started":"2021-09-01T07:40:06.318873Z","shell.execute_reply":"2021-09-01T07:41:14.787923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install pretrainedmodels","metadata":{"execution":{"iopub.status.busy":"2021-09-01T07:41:27.057890Z","iopub.execute_input":"2021-09-01T07:41:27.058283Z","iopub.status.idle":"2021-09-01T07:41:36.425360Z","shell.execute_reply.started":"2021-09-01T07:41:27.058254Z","shell.execute_reply":"2021-09-01T07:41:36.424186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd ../input/cpythonlibrary/cpython-master\nfrom Lib import copy\n%cd /kaggle/working","metadata":{"execution":{"iopub.status.busy":"2021-09-01T07:42:05.017890Z","iopub.execute_input":"2021-09-01T07:42:05.018264Z","iopub.status.idle":"2021-09-01T07:42:05.135035Z","shell.execute_reply.started":"2021-09-01T07:42:05.018232Z","shell.execute_reply":"2021-09-01T07:42:05.134273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install torch-summary","metadata":{"execution":{"iopub.status.busy":"2021-09-01T07:42:06.190398Z","iopub.execute_input":"2021-09-01T07:42:06.190896Z","iopub.status.idle":"2021-09-01T07:42:13.407327Z","shell.execute_reply.started":"2021-09-01T07:42:06.190864Z","shell.execute_reply":"2021-09-01T07:42:13.406223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch_xla\nimport torch_xla.distributed.parallel_loader as pl\nimport torch_xla.core.xla_model as xm\nimport torch_xla.distributed.xla_multiprocessing as xmp\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom scipy import signal\nimport torch\nfrom torch.utils.data import Dataset\nfrom nnAudio.Spectrogram import CQT # CQT is an alias of CQT1992v2\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n%matplotlib inline\n\nfrom torchsummary import summary\nfrom torchvision import transforms\nfrom torch.utils.data import Dataset\nfrom torch.utils.data import DataLoader\nfrom sklearn import model_selection\nfrom PIL import Image\nimport albumentations\nfrom torch.utils.data import DataLoader\nimport torch.nn.functional as F\nimport gc\nimport torch.nn as nn\nfrom sklearn.metrics import roc_auc_score\nfrom torch.utils.data.sampler import SequentialSampler\n\nimport pretrainedmodels\nTRAIN_BATCH_SIZE = 1\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2021-09-01T07:42:13.409068Z","iopub.execute_input":"2021-09-01T07:42:13.409416Z","iopub.status.idle":"2021-09-01T07:42:17.478482Z","shell.execute_reply.started":"2021-09-01T07:42:13.409382Z","shell.execute_reply":"2021-09-01T07:42:17.477714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cqt = CQT(sr=2048,        # sample rate\n            fmin=20,        # min freq\n            fmax=1024,      # max freq\n            hop_length=64,  # hop length\n            verbose=False)\n\nsample = np.load(\"../input/g2net-gravitational-wave-detection/test/0/1/0/01002036c9.npy\")\nsample = np.concatenate(sample, axis=0)\nsample = sample / np.max(sample)\nsample = torch.tensor(sample, dtype=torch.float)\n\nconst_q_transform = cqt(sample).squeeze()\nconst_q_transform = const_q_transform.repeat(3, 1, 1)\n\nplt.figure(figsize=(17, 3))\nplt.title(\"01002036c9.npy\")\nplt.pcolormesh(const_q_transform[0])\n\nconst_q_transform.size()","metadata":{"execution":{"iopub.status.busy":"2021-09-01T07:43:07.492280Z","iopub.execute_input":"2021-09-01T07:43:07.492826Z","iopub.status.idle":"2021-09-01T07:43:07.710618Z","shell.execute_reply.started":"2021-09-01T07:43:07.492784Z","shell.execute_reply":"2021-09-01T07:43:07.709962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"../input/g2net-gravitational-wave-detection/training_labels.csv\")\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-01T07:43:07.936483Z","iopub.execute_input":"2021-09-01T07:43:07.936978Z","iopub.status.idle":"2021-09-01T07:43:08.433769Z","shell.execute_reply.started":"2021-09-01T07:43:07.936947Z","shell.execute_reply":"2021-09-01T07:43:08.432869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"kfold\"] = -1    \ndf = df.sample(frac=1).reset_index(drop=True)\ny = df.target.values\nkf = model_selection.StratifiedKFold(n_splits=5)\n\nfor f, (t_, v_) in enumerate(kf.split(X=df, y=y)):\n    df.loc[v_, 'kfold'] = f\ndf.head(10)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T07:43:08.601476Z","iopub.execute_input":"2021-09-01T07:43:08.601862Z","iopub.status.idle":"2021-09-01T07:43:08.893723Z","shell.execute_reply.started":"2021-09-01T07:43:08.601798Z","shell.execute_reply":"2021-09-01T07:43:08.892840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class waveformClassification(Dataset):\n    def __init__(self, ids,tabular_data):\n        self.ids = ids\n        self.tabular_data = tabular_data\n        \n        \n    def __len__(self):\n        return len(self.ids)\n    \n    def __getitem__(self, index):\n        file_path = \"../input/g2net-gravitational-wave-detection/train/\"+ \\\n                    self.tabular_data[\"id\"][index][0]+\"/\"+ \\\n                    self.tabular_data[\"id\"][index][1] + \"/\" + \\\n                    self.tabular_data[\"id\"][index][2] + \"/\" + \\\n                    self.tabular_data[\"id\"][index] + \".npy\"\n        \n        sample = np.load(file_path)\n        sample = np.concatenate(sample, axis=0)\n        sample = sample / np.max(sample)\n        sample = torch.tensor(sample, dtype=torch.float)\n\n        const_q_transform = cqt(sample).squeeze()\n        const_q_transform = const_q_transform.repeat(3, 1, 1)\n        \n        return {\n            'file_name' :self.tabular_data[\"id\"][index] + \".npy\",\n            'tabular_data' : const_q_transform,\n            'label' : torch.tensor(self.tabular_data[\"target\"][index], dtype = torch.float)\n        }","metadata":{"execution":{"iopub.status.busy":"2021-09-01T07:50:10.843250Z","iopub.execute_input":"2021-09-01T07:50:10.843790Z","iopub.status.idle":"2021-09-01T07:50:10.853344Z","shell.execute_reply.started":"2021-09-01T07:50:10.843758Z","shell.execute_reply":"2021-09-01T07:50:10.852371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = waveformClassification(ids = [i for i in range(len(df))], \n                                  tabular_data = df)\n\nval_data = waveformClassification(ids = [i for i in range(len(df))], \n                                tabular_data = df)\n\n#dry run \nidx = 472226\nplt.figure(figsize=(17, 3))\nplt.title(val_data[idx][\"file_name\"])\nplt.pcolormesh(val_data[idx][\"tabular_data\"][0])\n\nprint(val_data[idx][\"label\"])","metadata":{"execution":{"iopub.status.busy":"2021-09-01T08:23:38.275856Z","iopub.execute_input":"2021-09-01T08:23:38.276274Z","iopub.status.idle":"2021-09-01T08:23:38.599551Z","shell.execute_reply.started":"2021-09-01T08:23:38.276241Z","shell.execute_reply":"2021-09-01T08:23:38.598150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ResNet18(nn.Module):\n    def __init__(self):\n        super(ResNet18, self).__init__()\n        self.model = pretrainedmodels.__dict__['resnet18'](pretrained=None)\n        self.dropout = nn.Dropout(0.1)\n        self.final_layer = nn.Linear(512 , 1)\n        \n    def forward(self, inputs):\n        batch_size, _, _, _ = inputs.shape\n        \n        x = self.model.features(inputs)\n        \n        x = F.adaptive_avg_pool2d(x, 1).reshape(batch_size, -1)\n        outputs = self.final_layer(self.dropout(x))\n\n        return outputs\n    \nmodel = ResNet18()\nmodel = model.to(xm.xla_device())","metadata":{"execution":{"iopub.status.busy":"2021-09-01T08:36:20.527115Z","iopub.execute_input":"2021-09-01T08:36:20.527566Z","iopub.status.idle":"2021-09-01T08:36:21.161403Z","shell.execute_reply.started":"2021-09-01T08:36:20.527524Z","shell.execute_reply":"2021-09-01T08:36:21.160090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"summary(model, (3, 69, 193))","metadata":{"execution":{"iopub.status.busy":"2021-09-01T08:36:22.436337Z","iopub.execute_input":"2021-09-01T08:36:22.436991Z","iopub.status.idle":"2021-09-01T08:36:23.282528Z","shell.execute_reply.started":"2021-09-01T08:36:22.436935Z","shell.execute_reply":"2021-09-01T08:36:23.281474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCHS = 5\n\noptimizer = torch.optim.Adam(model.parameters(), lr = 1e-3 * 0.95 * xm.xrt_world_size())\n\nloss_fn = torch.nn.BCEWithLogitsLoss()\n\nscheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size= 2, gamma=0.1)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T08:36:24.289954Z","iopub.execute_input":"2021-09-01T08:36:24.290320Z","iopub.status.idle":"2021-09-01T08:36:24.297466Z","shell.execute_reply.started":"2021-09-01T08:36:24.290291Z","shell.execute_reply":"2021-09-01T08:36:24.296286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# defining the training loop\ndef train_loop_fn(data_loader, model, optimizer, device, scheduler=None):\n    running_loss = 0.0\n    all_targets = 0\n    all_predictions = 0\n    \n    model.train()\n    \n    for batch_index,dataset in enumerate(data_loader):\n        input_data = dataset[\"tabular_data\"]\n        targets = dataset[\"label\"]\n        \n        input_data = input_data.to(device, dtype=torch.float32)\n        targets = targets.to(device, dtype=torch.float)\n        targets = targets.unsqueeze(1)\n        \n        optimizer.zero_grad()\n\n        outputs = model(input_data)\n        \n        y_true = targets.detach().cpu().numpy()\n        y_pred = outputs.detach().cpu().numpy()\n\n        \n        loss = loss_fn(outputs, targets)\n\n        loss.backward()\n        xm.optimizer_step(optimizer)\n        scheduler.step(loss)\n\n        running_loss += loss.item()\n        \n        if batch_index > 0:\n            all_targets = np.concatenate((all_targets, y_true), axis=0)\n            all_predictions = np.concatenate((all_predictions, y_pred), axis=0)\n        else:\n            all_targets = y_true\n            all_predictions = y_pred\n            \n    train_loss = running_loss / float(len(train_data))\n    train_roc_score = roc_auc_score(all_targets, all_predictions)\n    \n    return train_loss, train_roc_score","metadata":{"execution":{"iopub.status.busy":"2021-09-01T08:36:24.915782Z","iopub.execute_input":"2021-09-01T08:36:24.916149Z","iopub.status.idle":"2021-09-01T08:36:24.927198Z","shell.execute_reply.started":"2021-09-01T08:36:24.916119Z","shell.execute_reply":"2021-09-01T08:36:24.926041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def eval_loop_fn(data_loader, model, device):\n    running_loss = 0.0\n    all_targets = 0\n    all_predictions = 0\n    \n    model.eval()\n    \n    for batch_index,dataset in enumerate(data_loader):\n        input_data = dataset['tabular_data']\n        targets = dataset['label']\n        \n        input_data = input_data.to(device, dtype=torch.float32)\n        targets = targets.to(device, dtype=torch.float)\n        targets = targets.unsqueeze(1)\n\n        outputs = model(input_data)\n        \n        y_true = targets.detach().cpu().numpy()\n        y_pred = outputs.detach().cpu().numpy()\n        \n        loss = loss_fn(outputs, targets)\n\n        running_loss += loss.item()\n                    \n        if batch_index > 0:\n            all_targets = np.concatenate((all_targets, y_true), axis=0)\n            all_predictions = np.concatenate((all_predictions, y_pred), axis=0)\n        else:\n            all_targets = y_true\n            all_predictions = y_pred\n            \n    val_loss = running_loss / float(len(val_data))\n    val_roc_score = roc_auc_score(all_targets, all_predictions)\n    \n    return val_loss, val_roc_score","metadata":{"execution":{"iopub.status.busy":"2021-09-01T08:36:25.517527Z","iopub.execute_input":"2021-09-01T08:36:25.517911Z","iopub.status.idle":"2021-09-01T08:36:25.527783Z","shell.execute_reply.started":"2021-09-01T08:36:25.517875Z","shell.execute_reply":"2021-09-01T08:36:25.526486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def _run():\n    no_of_folds = 1\n    for i in range(no_of_folds):\n        a_string = \"*\" * 20\n\n        print(a_string, \" FOLD NUMBER \", i, a_string)\n        \n        i= 0 \n        df_train = df[df.kfold != i].reset_index(drop=True)\n        df_valid = df[df.kfold == i].reset_index(drop=True)\n\n        train_data = waveformClassification(ids = [k for k in range(len(df_train))], \n                                  tabular_data = df_train)\n        val_data = waveformClassification(ids = [m for m in range(len(df_valid))], \n                                        tabular_data = df_valid)\n\n        train_sampler = torch.utils.data.distributed.DistributedSampler(\n                  train_data,\n                  num_replicas=xm.xrt_world_size(),\n                  rank=xm.get_ordinal(),\n                  shuffle=True)\n\n        valid_sampler = torch.utils.data.distributed.DistributedSampler(\n                  val_data,\n                  num_replicas=xm.xrt_world_size(),\n                  rank=xm.get_ordinal(),\n                  shuffle=False)\n\n        training_dataloader = DataLoader(train_data,\n                                num_workers= 4,\n                                batch_size= TRAIN_BATCH_SIZE,\n                                sampler=train_sampler,\n                                drop_last=True\n                               )\n\n        val_dataloader = DataLoader(val_data,\n                                num_workers= 4,\n                                batch_size= TRAIN_BATCH_SIZE,\n                                sampler=valid_sampler,\n                                drop_last=False\n                               )\n        all_accuracies = []\n        \n        for epoch in range(EPOCHS):\n            xm.master_print(f\"Epoch --> {epoch+1} / {EPOCHS}\")\n            xm.master_print(f\"-------------------------------\")\n            \n            train_para_loader = pl.ParallelLoader(training_dataloader, [xm.xla_device()])\n            train_loss, train_roc = train_loop_fn(train_para_loader.per_device_loader(xm.xla_device()), model, optimizer, xm.xla_device(), scheduler)\n            xm.master_print(f'training Loss: {train_loss} & training ROC Score: {train_roc}.')\n            \n            val_para_loader = pl.ParallelLoader(val_dataloader, [xm.xla_device()])\n            valid_loss, val_roc = eval_loop_fn(val_para_loader.per_device_loader(xm.xla_device()), model, xm.xla_device())\n            xm.master_print(f'validation Loss: {valid_loss} & validation ROC Score: {val_roc} \\n')\n            \n            all_accuracies.append(val_roc)\n        xm.master_print('\\n')\n        \n        if i < 1:\n            best_accuracy = max(all_accuracies)\n            best_model = copy.deepcopy(model)\n        else:\n            if best_accuracy > max(all_accuracies):\n                continue\n            else:\n                best_accuracy = max(all_accuracies)\n                best_model = copy.deepcopy(model)\n        \n    torch.save(best_model.state_dict(),'./first_basic_model.bin')\n    xm.master_print()\n    xm.master_print(\"The highest ROC core that we got across all the folds is {:.2f}\".format(best_accuracy))\n    \n    return best_model","metadata":{"execution":{"iopub.status.busy":"2021-09-01T08:36:44.855604Z","iopub.execute_input":"2021-09-01T08:36:44.856246Z","iopub.status.idle":"2021-09-01T08:36:44.872694Z","shell.execute_reply.started":"2021-09-01T08:36:44.856165Z","shell.execute_reply":"2021-09-01T08:36:44.871986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# initializing the training of model\ndef _mp_fn(rank, flags):\n    torch.set_default_tensor_type('torch.FloatTensor')\n    a = _run()\n    \n# applying multiprocessing so that images get paralley trained in different cores of kaggle-tpu\nFLAGS={}\nxmp.spawn(_mp_fn, args=(FLAGS,), nprocs=1, start_method='fork')","metadata":{"execution":{"iopub.status.busy":"2021-09-01T08:36:45.141665Z","iopub.execute_input":"2021-09-01T08:36:45.142210Z","iopub.status.idle":"2021-09-01T08:36:45.743625Z","shell.execute_reply.started":"2021-09-01T08:36:45.142163Z","shell.execute_reply":"2021-09-01T08:36:45.740600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}