{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import sys\nsys.path.append('../input/hongnanrwightmangenefficientnetpytorchaug192020/gen-efficientnet-pytorch-master')\nimport geffnet","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os\nimport time\nimport cv2\nimport random\nimport argparse\nimport numpy as np\nimport pandas as pd\nfrom typing import Optional\nfrom tqdm import tqdm\nimport sklearn\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold, KFold\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data.sampler import RandomSampler, SequentialSampler\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nfrom datetime import datetime\nimport albumentations\nfrom albumentations.pytorch.transforms import ToTensorV2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Config"},{"metadata":{"trusted":true},"cell_type":"code","source":"class GlobalConfig:\n    seed = 1958\n    num_classes = 2\n    batch_size = 16\n    n_epochs = 5\n    # lr = 5e-4\n    lr = 0.00003\n    scheduler = \"CosineAnnealingWarmRestarts\"\n    train_step_scheduler = False  # do scheduler.step after optimizer.step\n    val_step_scheduler = True\n    T_0 = 10  # CosineAnnealingWarmRestarts\n    min_lr = 1e-6\n    weight_decay = 1e-6\n    image_size = 512\n    resize = 256\n    crop_size = {128: 110, 256: 200, 512: 400}\n    verbose = 1\n    verbose_step = 1\n    num_folds = 5\n    class_col_name = \"target\"\n    log_path = \"./log.txt\"\n    train_path = '../input/melanoma-merged-external-data-512x512-jpeg/512x512-dataset-melanoma/512x512-dataset-melanoma'\n\n    csv_path = \"../input/melanoma-external-malignant-256/train_concat.csv/\"\n    save_path = \"./\"\n    # test_path = '../input/cassava-leaf-disease-classification/test_images/'\n    effnet = \"tf_efficientnet_b2_ns\"\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    model_weight_path = '../input/efficientnet-weights/tf_efficientnet_b2_ns-00306e48.pth'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"config = GlobalConfig","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import Counter\nfrom tqdm import tqdm\nfrom typing import Optional, List\n\n# possible reference: https://www.programiz.com/python-programming/methods/string/join\n\ndef get_file_type(image_folder_path: str, allowed_extensions: Optional[List]=None):\n    if allowed_extensions is None:\n        allowed_extensions = ['.jpg', '.png', '.jpeg']\n\n    extension_type = []\n    file_list = os.listdir(image_folder_path)\n    for file in tqdm(file_list):\n        extension_type.append(os.path.splitext(file)[-1].lower())\n    \n    extension_dict = Counter(extension_type)\n    assert len(extension_dict.keys()) == 1, \"The extension in the folder should all be the same, but found {} extensions\".format(extension_dict.keys)\n    extension_type = list(extension_dict.keys())[0]\n    assert extension_type in allowed_extensions\n    return extension_type","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Augmentations"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_transforms(config):\n    transforms_train = albumentations.Compose(\n        [\n\n            albumentations.VerticalFlip(p=0.5),\n            albumentations.HorizontalFlip(p=0.5),\n            albumentations.RandomBrightness(limit=0.2, p=0.75),\n            albumentations.RandomContrast(limit=0.2, p=0.75),\n\n            albumentations.HueSaturationValue(hue_shift_limit=10, sat_shift_limit=20, val_shift_limit=10, p=0.5),\n            albumentations.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.1, rotate_limit=15, border_mode=0, p=0.85),\n            albumentations.Resize(height=config.image_size, width=config.image_size, p=1.0),\n            # Test yourself on whether doing cutout last affects the seqeunce order?\n\n            ToTensorV2(p=1.0),\n        ],\n        p=1.0,\n    )\n\n    transforms_val = albumentations.Compose(\n        [\n            albumentations.Resize(height=config.image_size, width=config.image_size, p=1.0),\n            ToTensorV2(p=1.0),\n        ],\n        p=1.0,\n    )\n\n    return transforms_train, transforms_val\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport albumentations\nimport torch\n# from torch.utils.data import Dataset\n\nfrom typing import Optional\nfrom tqdm import tqdm\n\n\n\nclass Melanoma(torch.utils.data.Dataset):\n    def __init__(\n        self,\n        dataframe: pd.DataFrame,\n        config: type,\n        transforms: Optional[albumentations.core.composition.Compose] = None,\n        test: bool = False,\n        albu_norm: bool = False\n    ):\n        # 1. Is it good practice to name self.df = dataframe, or self.df = df\n        self.df = dataframe\n        self.config = config\n        self.transforms = transforms\n        self.test = test\n        self.albu_norm = albu_norm\n        \n        '''\n        This is necessary as when there is no augmentations passed in, there will not be a case whereby albu_norm is True since albu_norm\n        only co-exists with transforms=True\n        '''\n        \n        if self.transforms is None:\n            assert self.albu_norm is False\n            print('Transforms is None and Albumentation Normalization is not initialized!')\n            \n        self.image_extension = get_file_type(image_folder_path=config.train_path, allowed_extensions=None)\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx: int):\n\n\n        label = self.df.target.values[idx]\n        label = torch.as_tensor(data=label, dtype=torch.int64, device=None)\n        image_id = self.df.image_name.values[idx]\n\n        if self.test:\n            image_path = os.path.join(self.config.test_path, \"{}{}\".format(image_id, self.image_extension))\n        else:\n            image_path = os.path.join(self.config.train_path, \"{}{}\".format(image_id, self.image_extension))\n        \n\n        image = cv2.imread(image_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        \n\n        if self.albu_norm is False:\n            image = image.astype(np.float32) / 255.0\n\n        if self.transforms is not None:\n            albu_dict = {\"image\": image}\n            transform = self.transforms(**albu_dict)\n            image = transform[\"image\"]\n        else:\n            image = torch.as_tensor(data=image, dtype=torch.float32, device=None)\n\n        return image_id, image, label\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CustomEfficientNet(nn.Module):\n    # 2. I am not sure but why is config class a type when I check type(config)\n    def __init__(self, config: type, pretrained: bool = True):\n        super().__init__()\n        self.config = config\n        # For myself, I like to set argument names for each\n        self.model = geffnet.create_model(\n            model_weight_path=config.model_weight_path, model_name=config.effnet, pretrained=pretrained\n        )\n        n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(n_features, config.num_classes)\n\n    def forward(self, x):\n        # TODO: add dropout layers, or the likes.\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Meters"},{"metadata":{"trusted":true},"cell_type":"code","source":"class AverageLossMeter:\n    \"\"\"\n    Computes and stores the average and current loss\n    \"\"\"\n\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.curr_batch_avg_loss = 0\n        self.avg = 0\n        self.running_total_loss = 0\n        self.count = 0\n\n    def update(self, curr_batch_avg_loss: float, batch_size: str):\n        self.curr_batch_avg_loss = curr_batch_avg_loss\n        self.running_total_loss += curr_batch_avg_loss * batch_size\n        self.count += batch_size\n        self.avg = self.running_total_loss / self.count\n\n\n# Maybe compare with utils.py from source\nclass AccuracyMeter:\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.score = 0\n        self.count = 0\n        self.sum = 0\n\n    def update(self, y_true, y_pred, batch_size=1):\n\n        # so we just need to count total num of images / batch_size\n        # self.count += num_steps\n        self.batch_size = batch_size\n        self.count += self.batch_size\n        # this part here already got an acc score for the 4 images, so no need divide batch size\n        self.score = sklearn.metrics.accuracy_score(y_true, y_pred)\n        total_score = self.score * self.batch_size\n\n        self.sum += total_score\n\n    # 1. I doubt I need to use @property here, but I saw one guy used it, so I am confused.\n    @property\n    def avg(self):\n        self.avg_score = self.sum / self.count\n        return self.avg_score","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train"},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_all(seed: int = 1992):\n\n    print(\"Using Seed Number {}\".format(seed))\n\n    os.environ[\"PYTHONHASHSEED\"] = str(\n        seed)  # set PYTHONHASHSEED env var at fixed value\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.cuda.manual_seed(seed)  # pytorch (both CPU and CUDA)\n    np.random.seed(seed)  # for numpy pseudo-random generator\n    random.seed(\n        seed)  # set fixed value for python built-in pseudo-random generator\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    torch.backends.cudnn.enabled = False\n\n\ndef seed_worker(worker_id):\n    worker_seed = torch.initial_seed() % 2**32\n    np.random.seed(worker_seed)\n    random.seed(worker_seed)\n\n\nclass Trainer:\n    def __init__(self, model, config: type):\n        self.model = model\n        # self.device = device\n        self.config = config\n        self.epoch = 0\n        self.best_acc = 0\n        self.best_loss = 10**5\n\n        # TODO consider moving these to config class\n        self.optimizer = torch.optim.AdamW(model.parameters(),\n                                           lr=config.lr,\n                                           weight_decay=0)\n        self.criterion = nn.CrossEntropyLoss()\n        self.scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n            self.optimizer,\n            mode=\"min\",\n            factor=0.8,\n            patience=1,\n            verbose=True,\n            min_lr=1e-8)\n\n        self.log(\"Trainer prepared. We are using {} device.\".format(\n            self.config.device))\n\n    def fit(self, train_loader, val_loader, fold: int):\n\n        self.log(\"Training on Fold {}\".format(fold + 1))\n\n        for epoch in range(self.config.n_epochs):\n            # Getting the learning rate after each epoch!\n            lr = self.optimizer.param_groups[0][\"lr\"]\n            timestamp = datetime.fromtimestamp(time.time())\n            # printing the lr and the timestamp after each epoch.\n            self.log(\"\\n{}\\nLR: {}\".format(timestamp, lr))\n\n            # start time of training on the training set\n            train_start_time = time.time()\n\n            # train one epoch on the training set\n            avg_train_loss, avg_train_acc_score = self.train_one_epoch(\n                train_loader)\n            # end time of training on the training set\n            train_end_time = time.time()\n\n            # formatting time to make it nicer\n            train_elapsed_time = time.strftime(\n                \"%H:%M:%S\", time.gmtime(train_end_time - train_start_time))\n            self.log(\n                \"[RESULT]: Train. Epoch {} | Avg Train Summary Loss: {:.6f} | Train Accuracy: {:6f} | Time Elapsed: {}\"\n                .format(self.epoch + 1, avg_train_loss, avg_train_acc_score,\n                        train_elapsed_time))\n\n            val_start_time = time.time()\n            # note here has val predictions... in actual fact it is repeated because its same as avg_val_acc_score\n            avg_val_loss, avg_val_acc_score, val_predictions = self.valid_one_epoch(\n                val_loader)\n            # not sure if it is good practice to write it here\n            self.val_predictions = val_predictions\n            val_end_time = time.time()\n            val_elapsed_time = time.strftime(\n                \"%H:%M:%S\", time.gmtime(val_end_time - val_start_time))\n\n            self.log(\n                \"[RESULT]: Validation. Epoch: {} | Avg Validation Summary Loss: {:.6f} | Validation Accuracy: {:.6f} | Time Elapsed: {}\"\n                .format(self.epoch + 1, avg_val_loss, avg_val_acc_score,\n                        val_elapsed_time))\n\n            # note here we use avg_val_loss, not train_val_loss! It is just right to use val_loss as benchmark\n            if avg_val_loss < self.best_loss:\n                self.best_loss = avg_val_loss\n                # decided to remove epoch here as epoch can be saved in the model later in self.save\n                # also this will overwrite everytime there is a better weight.\n                # TODO consider including epoch number inside, and call this epoch number as well\n                # through self.load to load the weights in curr_fold_best_checkpoint\n                self.save(\"{}_best_loss_fold_{}.pt\".format(\n                    self.config.effnet, fold))\n\n            if self.best_acc < avg_val_acc_score:\n                self.best_acc = avg_val_acc_score\n                # TODO consider saving these weights as well.\n\n            # this part not so clear yet, figure this out on why .step(loss) vs .step() in train epoch\n            if self.config.val_step_scheduler:\n                self.scheduler.step(avg_val_loss)\n\n            # end of training, epoch + 1 so that self.epoch can be updated.\n            self.epoch += 1\n\n        # this is where we end the epoch training for the current fold/model, therefore\n        # we can call the final \"best weight saved\" by this exact name that we saved earlier on.\n        curr_fold_best_checkpoint = self.load(\"{}_best_loss_fold_{}.pt\".format(\n            self.config.effnet, fold))\n        # return the checkpoint for further usage.\n        return curr_fold_best_checkpoint\n\n    def train_one_epoch(self, train_loader):\n\n        # set to train mode\n        self.model.train()\n\n        # log metrics\n        summary_loss = AverageLossMeter()\n        accuracy_scores = AccuracyMeter()\n\n        # timer\n        start_time = time.time()\n\n        # looping through train loader for one epoch, steps is the number of times to go through each epoch\n        for step, (image_ids, images, labels) in enumerate(train_loader):\n\n            \n            images = images.to(self.config.device)\n            labels = labels.to(self.config.device)\n\n            \n            batch_size = images.shape[0]\n\n\n            logits = self.model(images)\n\n            \n            loss = self.criterion(input=logits, target=labels)\n            summary_loss.update(loss.item(), batch_size)\n            self.optimizer.zero_grad()\n            loss.backward()\n            self.optimizer.step()            \n            y_true = labels.cpu().numpy()\n            softmax_preds = torch.nn.Softmax(dim=1)(input=logits).to(\"cpu\").detach().numpy()\n            y_preds = softmax_preds.argmax(1)\n            \n            accuracy_scores.update(y_true, y_preds, batch_size=batch_size)\n            \n\n            # not too sure yet KIV\n            if self.config.train_step_scheduler:\n                self.scheduler.step()\n\n            # measure elapsed time\n            end_time = time.time()\n\n            if config.verbose:\n                if (step % config.verbose_step) == 0:\n                    print(\n                        f\"Train Steps {step}/{len(train_loader)}, \" +\n                        f\"summary_loss: {summary_loss.avg:.3f}, acc: {accuracy_scores.avg:.3f} \"\n                        + f\"time: {(end_time - start_time):.3f}\",\n                        end=\"\\r\",\n                    )\n\n        return summary_loss.avg, accuracy_scores.avg\n\n    def valid_one_epoch(self, val_loader):\n\n        # set to eval mode\n        self.model.eval()\n\n        # log metrics\n        summary_loss = AverageLossMeter()\n        accuracy_scores = AccuracyMeter()\n\n        # timer\n        start_time = time.time()\n\n        # predictions list to append for oof later\n        val_preds_list = []\n\n        # off gradients for torch when validating\n        with torch.no_grad():\n            for step, (image_ids, images, labels) in enumerate(val_loader):\n\n                images = images.to(self.config.device)\n                labels = labels.to(self.config.device)\n                batch_size = images.shape[0]\n\n                logits = self.model(images)\n                loss = self.criterion(input=logits, target=labels)\n                summary_loss.update(loss.item(), batch_size)\n\n                y_true = labels.cpu().numpy()\n                # Write that we do not need to detach here as no gradients involved.\n                # Basically torch.nn.Softmax(dim=1)(input=logits).to(\"cpu\").detach.numpy()\n                softmax_preds = torch.nn.Softmax(dim=1)(\n                    input=logits).to(\"cpu\").numpy()\n                y_preds = softmax_preds.argmax(1)\n                accuracy_scores.update(y_true, y_preds, batch_size=batch_size)\n\n                val_preds_list.append(softmax_preds)\n\n                end_time = time.time()\n\n                if config.verbose:\n                    if (step % config.verbose_step) == 0:\n                        print(\n                            f\"Validation Steps {step}/{len(val_loader)}, \" +\n                            f\"summary_loss: {summary_loss.avg:.3f}, val_acc: {accuracy_scores.avg:.6f} \"\n                            + f\"time: {(end_time - start_time):.3f}\",\n                            end=\"\\r\",\n                        )\n\n            val_predictions = np.concatenate(val_preds_list)\n           \n        return summary_loss.avg, accuracy_scores.avg, val_predictions\n\n    def save_model(self, path):\n        self.model.eval()\n        torch.save(self.model.state_dict(), path)\n\n    # will save the weight for the best val loss and corresponding oof preds\n    def save(self, path):\n        self.model.eval()\n        torch.save(\n            {\n                \"model_state_dict\": self.model.state_dict(),\n                \"optimizer_state_dict\": self.optimizer.state_dict(),\n                \"scheduler_state_dict\": self.scheduler.state_dict(),\n                \"best_acc\": self.best_acc,\n                \"best_loss\": self.best_loss,\n                \"epoch\": self.epoch,\n                \"oof_preds\": self.val_predictions,\n            },\n            path,\n        )\n\n    def load(self, path):\n        checkpoint = torch.load(path)\n        return checkpoint\n\n\n    def log(self, message):\n        if self.config.verbose:\n            print(message)\n        with open(self.config.log_path, \"a+\") as logger:\n            logger.write(f\"{message}\\n\")\n\n\ndef train_on_fold(config, fold: int):\n    model = CustomEfficientNet(config=config, pretrained=True)\n    # consider remove if clause?\n    if torch.cuda.is_available():\n        model.cuda()\n\n    transforms_train, transforms_val = get_transforms(config)\n\n    train_df = df_folds[df_folds[\"fold\"] != fold].reset_index(drop=True)\n    val_df = df_folds[df_folds[\"fold\"] == fold].reset_index(drop=True)\n\n    train_set = Melanoma(train_df, config, transforms=transforms_train, test=False, albu_norm=False)\n    train_loader = DataLoader(train_set,\n                              batch_size=config.batch_size,\n                              shuffle=True,\n                              num_workers=4,\n                              worker_init_fn=seed_worker)\n\n    val_set = Melanoma(val_df, config, transforms=transforms_val, test=False, albu_norm=False)\n    val_loader = DataLoader(val_set,\n                            batch_size=config.batch_size,\n                            shuffle=False,\n                            num_workers=4,\n                            worker_init_fn=seed_worker)\n\n\n    cassava_trainer = Trainer(model=model, config=config)\n\n    curr_fold_best_checkpoint = cassava_trainer.fit(train_loader, val_loader,\n                                                    fold)\n\n    # loading checkpoint for all 10 epochs for this current fold\n\n    val_df[[str(c) for c in range(config.num_classes)\n            ]] = curr_fold_best_checkpoint[\"oof_preds\"]\n    val_df[\"preds\"] = curr_fold_best_checkpoint[\"oof_preds\"].argmax(1)\n\n    return val_df\n\n\ndef get_acc_score(y_true, y_pred):\n    return sklearn.metrics.accuracy_score(y_true, y_pred)\n\n\ndef get_result(result_df):\n    preds = result_df[\"preds\"].values\n    labels = result_df[config.class_col_name].values\n    score = get_acc_score(labels, preds)\n    return score\n\n\ndef train_loop(df_folds, config, fold_num: int = None, train_one_fold=False):\n    # here The CV score is the average of the validation fold metric.\n    cv_score_list = []\n    oof_df = pd.DataFrame()\n    if train_one_fold:\n        _oof_df = train_on_fold(fold_num)\n        curr_fold_best_score = get_result(_oof_df)\n        print(\"Fold {} OOF Score is {}\".format(fold_num + 1,\n                                               curr_fold_best_score))\n    else:\n        #for fold in sorted(df_folds[\"fold\"].unique()):\n        for fold in range(config.num_folds):\n            # note very carefully you need to add 1 here. because df_folds is 1,2,3,4,5\n            _oof_df = train_on_fold(config, fold)\n            #_oof_df = train_on_fold(config, fold+1)\n            oof_df = pd.concat([oof_df, _oof_df])\n            curr_fold_best_score = get_result(_oof_df)\n            cv_score_list.append(curr_fold_best_score)\n            print(\"\\n\\n\\nOOF Score for Fold {}: {}\\n\\n\\n\".format(\n                fold + 1, curr_fold_best_score))\n\n    print(\"CV score\", np.mean(cv_score_list))\n    print(\"Variance\", np.var(cv_score_list))\n    print(\"Five Folds OOF\", get_result(oof_df))\n    oof_df.to_csv(\"oof.csv\")\n\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#     skf = StratifiedKFold(n_splits=config.num_folds, shuffle=True, random_state=config.seed)\n#     for fold, (train_index, val_index) in enumerate(skf.split(df_folds, df_folds[config.class_col_name])):\n#         df_folds.loc[val_index, 'fold'] = int(fold)\n#     df_folds['fold'] = df_folds['fold'].astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if __name__ == \"__main__\":\n    config = GlobalConfig\n    seed_all(seed=config.seed)\n    train_csv = pd.read_csv('../input/melanoma-merged-external-data-512x512-jpeg/folds.csv')\n    df_folds = train_csv.copy()\n    df_folds = df_folds.rename(columns={'image_id': 'image_name'})\n#     skf = KFold(n_splits=config.num_folds, shuffle=True, random_state=config.seed)\n#     for fold, (train_index, val_index) in enumerate(skf.split(df_folds)):\n#         df_folds.loc[val_index, 'fold'] = int(fold)\n#     df_folds['fold'] = df_folds['fold'].astype(int)\n    print(df_folds.groupby(['fold', config.class_col_name]).size())\n    train_csv.target.value_counts()\n    train_single_fold = train_on_fold(config, fold=1)\n    #train_all_folds = train_loop(df_folds,config)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}