{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction\n\nThis is a minimal version of [G2 Net - EDA and Modelling](https://www.kaggle.com/ihelon/g2net-eda-and-modeling)\n\nI added some comments that may be useful for beginner to pytorch.","metadata":{}},{"cell_type":"markdown","source":"# 1.Import Library","metadata":{}},{"cell_type":"code","source":"import os\nimport json\nimport random\nimport collections\n\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport torch\n\n#Library for signal processing\n!pip install -q nnAudio -qq\nfrom nnAudio.Spectrogram import CQT1992v2\n\nimport time\n\nfrom torch import nn\nfrom torch.utils import data as torch_data\nfrom sklearn import model_selection as sk_model_selection\nfrom torch.nn import functional as torch_functional\nfrom torch.autograd import Variable\n!pip install efficientnet_pytorch -qq\nimport efficientnet_pytorch","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed_all(seed)\n        torch.backends.cudnn.deterministic = True\n\n\nset_seed(42)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2.Import Data","metadata":{}},{"cell_type":"code","source":"def convert_image_id_2_path(image_id: str, is_train: bool = True) -> str:\n    folder = \"train\" if is_train else \"test\"\n    return \"../input/g2net-gravitational-wave-detection/{}/{}/{}/{}/{}.npy\".format(\n        folder, image_id[0], image_id[1], image_id[2], image_id \n    )","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(\"../input/g2net-gravitational-wave-detection/training_labels.csv\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"global_transform = CQT1992v2(sr=2048, fmin=20, fmax=1024, \n                                     hop_length=16,bins_per_octave=16,pad_mode='constant')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3.Data Retriever and Data Loader\n\nThis allows us to easily get a batch of data each time we need in concatenated form.\nA batch is a set of data items. For example, each wave item has size (x,y,z). With the use of data retriever and data loader, we can easily get a batch of data which has size (b,x,y,z) with b is the batch size.","metadata":{}},{"cell_type":"code","source":"class DataRetriever(torch_data.Dataset):\n    def __init__(self, paths, targets):\n        self.paths = paths\n        self.targets = targets\n        \n        \n        self.q_transform = global_transform\n    def __len__(self):\n        return len(self.paths)\n    \n    def __get_qtransform(self, x):\n        image = []\n        for i in range(3):\n            waves = x[i] / np.max(x[i])\n            waves = torch.from_numpy(waves).float()\n            channel = self.q_transform(waves).squeeze().numpy()\n            image.append(channel)\n        out = torch.tensor(image).float()\n       \n        return out\n    \n    def __getitem__(self, index):\n        file_path = convert_image_id_2_path(self.paths[index])\n        x = np.load(file_path)\n        image = self.__get_qtransform(x)\n        \n        y = torch.tensor(self.targets[index], dtype=torch.float)\n            \n        return {\"X\": image, \"y\": y}","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Split train data frame into train set and validation set.\n\ndf_train, df_valid = sk_model_selection.train_test_split(\n    train_df, \n    test_size=0.2, \n    random_state=42, \n    stratify=train_df[\"target\"],\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Construct training data retriever and validation data retriever\n\ntrain_data_retriever = DataRetriever(\n    df_train[\"id\"].values, \n    df_train[\"target\"].values, \n)\n\nvalid_data_retriever = DataRetriever(\n    df_valid[\"id\"].values, \n    df_valid[\"target\"].values,\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#This is the data loader that allow us to load batch of data\n\ntrain_loader = torch_data.DataLoader(\n    train_data_retriever,\n    batch_size=32,\n    shuffle=True,\n    num_workers=8,\n)\n\nvalid_loader = torch_data.DataLoader(\n    valid_data_retriever, \n    batch_size=32,\n    shuffle=False,\n    num_workers=8,\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4.Model","metadata":{}},{"cell_type":"code","source":"#This is simply pass data that Q-transformed with the size (x,y,3) to an EfficientNet, \n#the fully connected layer near the end of EfficientNet replaced by another fully connected \n#layer that serve our purpose.\n\nclass Model(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.net = efficientnet_pytorch.EfficientNet.from_pretrained(\"efficientnet-b7\")\n        n_features = self.net._fc.in_features\n        self.net._fc = nn.Linear(in_features=n_features, out_features=1, bias=True)\n    \n    def forward(self, x):\n        out = self.net(x)\n        return out","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5.Accuracy Meter and Loss Meter\n\nThis calculate the accuracy of the model and loss after each step.\n","metadata":{}},{"cell_type":"code","source":"class LossMeter:\n    def __init__(self):\n        self.avg = 0\n        self.n = 0\n\n    def update(self, val):\n        self.n += 1\n        # incremental update\n        self.avg = val / self.n + (self.n - 1) / self.n * self.avg\n\n        \nclass AccMeter:\n    def __init__(self):\n        self.avg = 0\n        self.n = 0\n        \n    def update(self, y_true, y_pred):\n        y_true = y_true.cpu().numpy().astype(int)\n        y_pred = y_pred.cpu().numpy() >= 0\n        last_n = self.n\n        self.n += len(y_true)\n        true_count = np.sum(y_true == y_pred)\n        # incremental update\n        self.avg = true_count / self.n + last_n / self.n * self.avg","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 6.Train the model","metadata":{}},{"cell_type":"code","source":"class Trainer:\n    def __init__(\n        self, \n        model, \n        device, \n        optimizer, \n        criterion, \n        loss_meter, \n        score_meter\n    ):\n        self.model = model\n        self.device = device\n        self.optimizer = optimizer\n        self.criterion = criterion\n        self.loss_meter = loss_meter\n        self.score_meter = score_meter\n        \n        self.best_valid_score = -np.inf\n        self.n_patience = 0\n        \n        self.messages = {\n            \"epoch\": \"[Epoch {}: {}] loss: {:.5f}, score: {:.5f}, time: {} s\",\n            \"checkpoint\": \"The score improved from {:.5f} to {:.5f}. Save model to '{}'\",\n            \"patience\": \"\\nValid score didn't improve last {} epochs.\"\n        }\n    \n    def fit(self, epochs, train_loader, valid_loader, save_path, patience):        \n        for n_epoch in range(1, epochs + 1):\n            self.info_message(\"EPOCH: {}\", n_epoch)\n            #Train the model.\n            train_loss, train_score, train_time = self.train_epoch(train_loader)\n            #Test the performance of trained model on validation set.\n            valid_loss, valid_score, valid_time = self.valid_epoch(valid_loader)\n            \n            self.info_message(\n                self.messages[\"epoch\"], \"Train\", n_epoch, train_loss, train_score, train_time\n            )\n            \n            self.info_message(\n                self.messages[\"epoch\"], \"Valid\", n_epoch, valid_loss, valid_score, valid_time\n            )\n\n            \n            self.info_message(\n                self.messages[\"checkpoint\"], self.best_valid_score, valid_score, save_path\n            )\n            self.best_valid_score = valid_score\n            self.save_model(n_epoch, save_path)\n            self.n_patience = 0\n\n    # MAIN FUNCTION: Traing the model       \n    def train_epoch(self, train_loader):\n        self.model.train() #MUST DO: set the model in Training mode because the dropout layer\n        # behaves different in training and when using model to calculated result\n        t = time.time()\n        train_loss = self.loss_meter()\n        train_score = self.score_meter()\n        \n        for step, batch in enumerate(train_loader, 1):\n            #Load input and label\n            \n            X = batch[\"X\"].to(self.device)\n            targets = batch[\"y\"].to(self.device)\n            \n            #Set to zero_grad to reset gradient of current parameters inside model to zero.\n            #View this link for more info: https://stackoverflow.com/a/48009142\n            self.optimizer.zero_grad()\n            outputs = self.model(X).squeeze(1)\n            \n            loss = self.criterion(outputs, targets) #Calculate loss function, current criterion is in this link\n            #https://pytorch.org/docs/stable/generated/torch.nn.BCEWithLogitsLoss.html#torch.nn.BCEWithLogitsLoss\n            \n            loss.backward() #This step will calculate the gradient of model paramters and \n            #also have modification to reduce the loss\n\n            train_loss.update(loss.detach().item())\n            train_score.update(targets, outputs.detach())\n\n            self.optimizer.step() #In some optimizer like Adam, learning rate and other optimize params\n            #may not be constants. Therefore, they change after each step. \n            #This simple process these changes.\n            \n            _loss, _score = train_loss.avg, train_score.avg\n            message = 'Train Step {}/{}, train_loss: {:.5f}, train_score: {:.5f}'\n            if(step%1000==0):\n                self.info_message(message, step, len(train_loader), _loss, _score, end=\"\\r\")\n        \n        return train_loss.avg, train_score.avg, int(time.time() - t)\n    \n    def valid_epoch(self, valid_loader):\n        self.model.eval() #MUST DO: set the model in evaluation mode when calculating output of new input.\n        t = time.time()\n        valid_loss = self.loss_meter()\n        valid_score = self.score_meter()\n\n        for step, batch in enumerate(valid_loader, 1):\n            with torch.no_grad(): #Because we do not need to change model parameters, we also \n                # do not need to calculate the gradient. This simple tells Pytorch that we dont want to \n                # calculate gradien and this helps reduce computational resources.\n                \n                X = batch[\"X\"].to(self.device)\n                targets = batch[\"y\"].to(self.device)\n\n                outputs = self.model(X).squeeze(1)\n                loss = self.criterion(outputs, targets)\n\n                valid_loss.update(loss.detach().item())\n                valid_score.update(targets, outputs)\n                \n            _loss, _score = valid_loss.avg, valid_score.avg\n            message = 'Valid Step {}/{}, valid_loss: {:.5f}, valid_score: {:.5f}'\n            if(step%1000==0):\n                self.info_message(message, step, len(valid_loader), _loss, _score, end=\"\\r\")\n        \n        return valid_loss.avg, valid_score.avg, int(time.time() - t)\n    \n    def save_model(self, n_epoch, save_path):\n        torch.save(\n            {\n                \"model_state_dict\": self.model.state_dict(),\n                \"optimizer_state_dict\": self.optimizer.state_dict(),\n                \"best_valid_score\": self.best_valid_score,\n                \"n_epoch\": n_epoch,\n            },\n            save_path,\n        )\n    \n    @staticmethod\n    def info_message(message, *args, end=\"\\n\"):\n        print(message.format(*args), end=end)\n\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h2> Apply above Trainer","metadata":{}},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nmodel = Model()\nmodel.to(device)\n\noptimizer = torch.optim.Adam(model.parameters(), lr=0.001)\ncriterion = torch_functional.binary_cross_entropy_with_logits\n\ntrainer = Trainer(\n    model, \n    device, \n    optimizer, \n    criterion, \n    LossMeter, \n    AccMeter\n)\n\nhistory = trainer.fit(\n    1, \n    train_loader, \n    valid_loader, \n    \"best-model.pth\", \n    100,\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 7. Create submission","metadata":{}},{"cell_type":"markdown","source":"Get the best model from above trainer and set to evaluation mode ","metadata":{}},{"cell_type":"code","source":"# checkpoint = torch.load(\"best-model.pth\")\n\n# model.load_state_dict(checkpoint[\"model_state_dict\"])\n# model.eval();","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Create test data retriever, this is a little bit different from train data retriever because the label is not retrieved. Therefore, we retrive the id of data instead of label.","metadata":{}},{"cell_type":"code","source":"class DataRetriever(torch_data.Dataset):\n    def __init__(self, paths):\n        self.paths = paths\n\n        self.q_transform = global_transform\n          \n    def __len__(self):\n        return len(self.paths)\n    \n    def __get_qtransform(self, x):\n        image = []\n        for i in range(3):\n            waves = x[i] / np.max(x[i])\n            waves = torch.from_numpy(waves).float()\n            channel = self.q_transform(waves).squeeze().numpy()\n            image.append(channel)\n            \n        return torch.tensor(image).float()\n    \n    def __getitem__(self, index):\n        file_path = convert_image_id_2_path(self.paths[index], is_train=False)\n        x = np.load(file_path)\n        image = self.__get_qtransform(x)\n            \n        return {\"X\": image, \"id\": self.paths[index]}","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(\"../input/g2net-gravitational-wave-detection/sample_submission.csv\")\nsubmission.to_csv(\"submission.csv\", index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data_retriever = DataRetriever(\n    submission[\"id\"].values, \n)\n\ntest_loader = torch_data.DataLoader(\n    test_data_retriever,\n    batch_size=32,\n    shuffle=False,\n    num_workers=8,\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Make prediction","metadata":{}},{"cell_type":"code","source":"# y_pred = []\n# ids = []\n\n# for e, batch in enumerate(test_loader):\n#     print(f\"{e}/{len(test_loader)}\", end=\"\\r\")\n#     with torch.no_grad():\n#         y_pred.extend(torch.sigmoid(model(batch[\"X\"].to(device))).cpu().numpy().squeeze())\n#         ids.extend(batch[\"id\"])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submission = pd.DataFrame({\"id\": ids, \"target\": y_pred})\n# submission.to_csv(\"model_submission.csv\", index=False)","metadata":{},"execution_count":null,"outputs":[]}]}