{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# About\n\nThis notebook is the inference part of [@hidehisaarai1213](https://www.kaggle.com/hidehisaarai1213)'s [G2Net: Read from TFRecord & Train with PyTorch](https://www.kaggle.com/hidehisaarai1213/g2net-read-from-tfrecord-train-with-pytorch) notebook. It is also based on [@yasufuminakama](https://www.kaggle.com/yasufuminakama)'s [G2Net / efficientnet_b7 / baseline [training]](https://www.kaggle.com/yasufuminakama/g2net-efficientnet-b7-baseline-training) notebook. \n\n\n**This code is based on again [@yasufuminakama](https://www.kaggle.com/yasufuminakama)'s original [G2Net / efficientnet_b7 / baseline [inference]\n](https://www.kaggle.com/yasufuminakama/g2net-efficientnet-b7-baseline-inference) notebook.**\n\nPlease show your support to original authors as well!!!","metadata":{}},{"cell_type":"code","source":"!pip install -q nnAudio\n!pip install -q timm","metadata":{"execution":{"iopub.status.busy":"2021-08-23T19:38:32.290677Z","iopub.execute_input":"2021-08-23T19:38:32.291122Z","iopub.status.idle":"2021-08-23T19:38:47.442156Z","shell.execute_reply.started":"2021-08-23T19:38:32.291032Z","shell.execute_reply":"2021-08-23T19:38:47.441162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Packages","metadata":{}},{"cell_type":"code","source":"import os\nimport time\nimport math\nimport glob\nimport random\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nimport scipy as sp\nimport tensorflow as tf  # for reading TFRecord Dataset\nimport tensorflow_datasets as tfds  # for making tf.data.Dataset to return numpy arrays\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport timm\nfrom kaggle_datasets import KaggleDatasets\nfrom nnAudio.Spectrogram import CQT1992v2\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import KFold\nfrom tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2021-08-23T19:38:47.444041Z","iopub.execute_input":"2021-08-23T19:38:47.444429Z","iopub.status.idle":"2021-08-23T19:38:55.165212Z","shell.execute_reply.started":"2021-08-23T19:38:47.444391Z","shell.execute_reply":"2021-08-23T19:38:55.164255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2021-08-23T19:38:55.167364Z","iopub.execute_input":"2021-08-23T19:38:55.167756Z","iopub.status.idle":"2021-08-23T19:38:55.218153Z","shell.execute_reply.started":"2021-08-23T19:38:55.167717Z","shell.execute_reply":"2021-08-23T19:38:55.216917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CFG","metadata":{}},{"cell_type":"code","source":"class CFG:\n    debug = False\n    print_freq = 50\n    num_workers = 4\n    model_name = \"tf_efficientnet_b0_ns\"\n    qtransform_params = {\"sr\": 2048, \"fmin\": 20, \"fmax\": 1024, \"hop_length\": 24, \"bins_per_octave\": 12}\n    scheduler = \"CosineAnnealingLR\"\n    epochs = 3\n    T_max = 3\n    lr = 1e-4\n    min_lr = 1e-7\n    batch_size = 64\n    weight_decay = 1e-3\n    gradient_accumulation_steps = 1\n    max_grad_norm = 1000\n    seed = 42\n    target_size = 1\n    target_col = \"target\"\n    n_fold = 5\n    trn_fold = [0, 1, 2, 3, 4]\n    train = True\n\nif CFG.debug:\n    CFG.epochs = 1","metadata":{"execution":{"iopub.status.busy":"2021-08-23T19:38:55.220137Z","iopub.execute_input":"2021-08-23T19:38:55.220594Z","iopub.status.idle":"2021-08-23T19:38:55.229588Z","shell.execute_reply.started":"2021-08-23T19:38:55.220536Z","shell.execute_reply":"2021-08-23T19:38:55.228526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_torch(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nseed_torch(seed=CFG.seed)","metadata":{"execution":{"iopub.status.busy":"2021-08-23T19:38:55.231014Z","iopub.execute_input":"2021-08-23T19:38:55.231597Z","iopub.status.idle":"2021-08-23T19:38:55.242363Z","shell.execute_reply.started":"2021-08-23T19:38:55.231523Z","shell.execute_reply":"2021-08-23T19:38:55.241523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"# ====================================================\n# Dataset\n# ====================================================\nclass TestDataset(torch.utils.data.Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df['file_path'].values\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def apply_transform(self, waves):\n        waves = waves / np.max(waves, axis=1)[:, None]\n        waves = torch.from_numpy(waves).float()\n        return waves\n\n    def __getitem__(self, idx):\n        file_path = self.file_names[idx]\n        waves = np.load(file_path)\n        image = self.apply_transform(waves)\n        image = image.squeeze().numpy()\n        if self.transform:\n            image = self.transform(image=image)['image']\n        return image # , label","metadata":{"execution":{"iopub.status.busy":"2021-08-23T19:38:55.243900Z","iopub.execute_input":"2021-08-23T19:38:55.244451Z","iopub.status.idle":"2021-08-23T19:38:55.254084Z","shell.execute_reply.started":"2021-08-23T19:38:55.244410Z","shell.execute_reply":"2021-08-23T19:38:55.253131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{"execution":{"iopub.status.busy":"2021-08-23T20:14:08.788843Z","iopub.execute_input":"2021-08-23T20:14:08.789264Z","iopub.status.idle":"2021-08-23T20:14:08.793306Z","shell.execute_reply.started":"2021-08-23T20:14:08.789230Z","shell.execute_reply":"2021-08-23T20:14:08.792196Z"}}},{"cell_type":"code","source":"class CustomModel(nn.Module):\n    def __init__(self, cfg, pretrained=False):\n        super().__init__()\n        self.cfg = cfg\n        self.wave_transform = CQT1992v2(**CFG.qtransform_params)\n        self.model = timm.create_model(self.cfg.model_name, pretrained=pretrained, in_chans=3)\n        self.n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(self.n_features, self.cfg.target_size)\n\n    def forward(self, x):\n        waves = []\n        for i in range(3):\n            waves.append(self.wave_transform(x[:, i]))\n        x = torch.stack(waves, dim=1)\n        output = self.model(x)\n        return output\n","metadata":{"execution":{"iopub.status.busy":"2021-08-23T19:38:55.255790Z","iopub.execute_input":"2021-08-23T19:38:55.256256Z","iopub.status.idle":"2021-08-23T19:38:55.268374Z","shell.execute_reply.started":"2021-08-23T19:38:55.256172Z","shell.execute_reply":"2021-08-23T19:38:55.267408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_test_file_path(image_id):\n    return \"../input/g2net-gravitational-wave-detection/test/{}/{}/{}/{}.npy\".format(\n        image_id[0], image_id[1], image_id[2], image_id)","metadata":{"execution":{"iopub.status.busy":"2021-08-23T19:38:55.284727Z","iopub.execute_input":"2021-08-23T19:38:55.285126Z","iopub.status.idle":"2021-08-23T19:38:55.297311Z","shell.execute_reply.started":"2021-08-23T19:38:55.285087Z","shell.execute_reply":"2021-08-23T19:38:55.296384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub = pd.read_csv(\"../input/g2net-gravitational-wave-detection/sample_submission.csv\")\ndf_sub['file_path'] = df_sub['id'].apply(get_test_file_path)\n\ntest_dataset = TestDataset(df_sub)\ntest_loader = torch.utils.data.DataLoader(\n    test_dataset, \n    batch_size=512,\n    num_workers=CFG.num_workers, \n    shuffle=False,\n    pin_memory=True, \n    drop_last=False\n)\n\nbest_loss_models = [torch.load(state, map_location=device)[\"model\"] \n                    for state in sorted(glob.glob(\"../input/g2net-read-from-tfrecord-train-with-pytorch/*best_loss.pth\"))]\nbest_score_models = [torch.load(state, map_location=device)[\"model\"]\n                     for state in sorted(glob.glob(\"../input/g2net-read-from-tfrecord-train-with-pytorch/*best_score.pth\"))]\n\nbest_loss_model = CustomModel(CFG)\nbest_score_model = CustomModel(CFG)","metadata":{"execution":{"iopub.status.busy":"2021-08-23T19:38:55.298229Z","iopub.execute_input":"2021-08-23T19:38:55.298494Z","iopub.status.idle":"2021-08-23T19:38:55.679106Z","shell.execute_reply.started":"2021-08-23T19:38:55.298470Z","shell.execute_reply":"2021-08-23T19:38:55.678159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"def inference(model, states, data_loader, device):\n    \n    model.to(device)\n    tk0 = tqdm(enumerate(data_loader), total=len(data_loader))\n    probs = []\n    \n    for idx, images in tk0:\n        \n        images = images.to(device)\n        avg_preds = []\n        for state in states:\n            model.load_state_dict(state)\n            model.eval()\n            with torch.no_grad():\n                preds = model(images)\n            \n            avg_preds.append(preds.sigmoid().cpu().numpy())\n        \n        avg_preds = np.mean(avg_preds, axis=0)\n        probs.append(avg_preds)\n\n    probs = np.concatenate(probs)\n    return probs\n","metadata":{"execution":{"iopub.status.busy":"2021-08-23T19:39:09.941264Z","iopub.execute_input":"2021-08-23T19:39:09.941698Z","iopub.status.idle":"2021-08-23T19:39:09.949585Z","shell.execute_reply.started":"2021-08-23T19:39:09.941658Z","shell.execute_reply":"2021-08-23T19:39:09.948456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = inference(best_score_model, best_score_models, test_loader, device)\n# preds_2 = inference(best_loss_model, best_loss_models, test_loader, device)","metadata":{"execution":{"iopub.status.busy":"2021-08-23T19:39:30.596084Z","iopub.execute_input":"2021-08-23T19:39:30.596415Z","iopub.status.idle":"2021-08-23T19:47:54.057072Z","shell.execute_reply.started":"2021-08-23T19:39:30.596385Z","shell.execute_reply":"2021-08-23T19:47:54.054082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub[\"target\"] = preds\ndf_sub.drop([\"file_path\"], axis=1, inplace=True)\ndf_sub.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-23T20:12:23.845314Z","iopub.execute_input":"2021-08-23T20:12:23.845685Z","iopub.status.idle":"2021-08-23T20:12:24.515470Z","shell.execute_reply.started":"2021-08-23T20:12:23.845647Z","shell.execute_reply":"2021-08-23T20:12:24.514532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub","metadata":{"execution":{"iopub.status.busy":"2021-08-23T20:11:15.115570Z","iopub.execute_input":"2021-08-23T20:11:15.116032Z","iopub.status.idle":"2021-08-23T20:11:15.145878Z","shell.execute_reply.started":"2021-08-23T20:11:15.115987Z","shell.execute_reply":"2021-08-23T20:11:15.144310Z"},"trusted":true},"execution_count":null,"outputs":[]}]}