{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Install Libraries","metadata":{}},{"cell_type":"code","source":"import sys\nsys.path.append('../input/tez-lib')\nsys.path.append('../input/timmmaster')","metadata":{"execution":{"iopub.status.busy":"2021-09-27T16:15:33.282036Z","iopub.execute_input":"2021-09-27T16:15:33.282377Z","iopub.status.idle":"2021-09-27T16:15:33.383664Z","shell.execute_reply.started":"2021-09-27T16:15:33.282294Z","shell.execute_reply":"2021-09-27T16:15:33.382963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Import required packages","metadata":{}},{"cell_type":"code","source":"import argparse\nimport os\n\nimport cv2\nimport albumentations\nimport albumentations.pytorch\nimport pandas as pd\nimport numpy as np\n\nimport tez\nimport timm\nimport torch\nimport torch.nn as nn\nimport torchvision\n\nfrom sklearn import metrics, model_selection, preprocessing\nfrom tez.callbacks import EarlyStopping\nfrom tez.datasets import ImageDataset\nfrom torch.nn import functional as F\n\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nimport wandb\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2021-09-27T16:15:33.385510Z","iopub.execute_input":"2021-09-27T16:15:33.385826Z","iopub.status.idle":"2021-09-27T16:15:41.724349Z","shell.execute_reply.started":"2021-09-27T16:15:33.385793Z","shell.execute_reply":"2021-09-27T16:15:41.723375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### CFG class","metadata":{}},{"cell_type":"code","source":"class CFG:\n    image_size = 512\n    target_size = 5\n    target_col = 'label'\n    model_name = 'resnext50_32x4d'\n    epochs = 15\n    batch_size = 16\n    n_fold = 5\n    trn_fold = [0,1,2,3,4]","metadata":{"execution":{"iopub.status.busy":"2021-09-27T16:15:41.725856Z","iopub.execute_input":"2021-09-27T16:15:41.726095Z","iopub.status.idle":"2021-09-27T16:15:41.734160Z","shell.execute_reply.started":"2021-09-27T16:15:41.726063Z","shell.execute_reply":"2021-09-27T16:15:41.733424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Create folds","metadata":{}},{"cell_type":"code","source":"INPUT_PATH = \"../input/cassava-leaf-disease-classification/\"\nIMAGE_PATH = \"../input/cassava-leaf-disease-classification/train_images/\"\nOUTPUT_DIR = './'","metadata":{"execution":{"iopub.status.busy":"2021-09-27T16:15:41.737944Z","iopub.execute_input":"2021-09-27T16:15:41.738174Z","iopub.status.idle":"2021-09-27T16:15:41.744485Z","shell.execute_reply.started":"2021-09-27T16:15:41.738145Z","shell.execute_reply":"2021-09-27T16:15:41.743739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\ndf['kfold'] = -1\ndf = df.sample(frac = 1).reset_index(drop=True)\ny = df['label'].values\n\nkf = model_selection.StratifiedKFold(n_splits = CFG.n_fold,random_state=42)\nfor fold_, (train_idx , test_idx) in enumerate(kf.split(X= df, y=y)):\n    df.loc[test_idx, \"kfold\"] = fold_\n\ndf.to_csv('./train_folds.csv',index = False)","metadata":{"execution":{"iopub.status.busy":"2021-09-27T16:15:41.747041Z","iopub.execute_input":"2021-09-27T16:15:41.747237Z","iopub.status.idle":"2021-09-27T16:15:41.846930Z","shell.execute_reply.started":"2021-09-27T16:15:41.747216Z","shell.execute_reply":"2021-09-27T16:15:41.846077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['label'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-09-27T16:15:41.848291Z","iopub.execute_input":"2021-09-27T16:15:41.848544Z","iopub.status.idle":"2021-09-27T16:15:41.859459Z","shell.execute_reply.started":"2021-09-27T16:15:41.848512Z","shell.execute_reply":"2021-09-27T16:15:41.858811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Logger","metadata":{}},{"cell_type":"code","source":"def init_logger(log_file=OUTPUT_DIR+'train.log'):\n    from logging import getLogger, INFO, FileHandler,  Formatter,  StreamHandler\n    logger = getLogger(__name__)\n    logger.setLevel(INFO)\n    handler1 = StreamHandler()\n    handler1.setFormatter(Formatter(\"%(message)s\"))\n    handler2 = FileHandler(filename=log_file)\n    handler2.setFormatter(Formatter(\"%(message)s\"))\n    logger.addHandler(handler1)\n    logger.addHandler(handler2)\n    return logger\n\nLOGGER = init_logger()","metadata":{"execution":{"iopub.status.busy":"2021-09-27T16:15:41.861051Z","iopub.execute_input":"2021-09-27T16:15:41.861483Z","iopub.status.idle":"2021-09-27T16:15:41.869571Z","shell.execute_reply.started":"2021-09-27T16:15:41.861449Z","shell.execute_reply":"2021-09-27T16:15:41.868954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Define augmentation","metadata":{}},{"cell_type":"code","source":"def generate_transforms():\n\n    train_aug = albumentations.Compose(\n        [\n            albumentations.RandomResizedCrop(CFG.image_size, CFG.image_size),\n            albumentations.Transpose(p=0.5),\n            albumentations.HorizontalFlip(p=0.5),\n            albumentations.VerticalFlip(p=0.5),\n            albumentations.ShiftScaleRotate(p=0.5),\n            albumentations.HueSaturationValue(\n                hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5\n            ),\n            albumentations.RandomBrightnessContrast(\n                brightness_limit=(-0.1, 0.1), contrast_limit=(-0.1, 0.1), p=0.5\n            ),\n            albumentations.Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n                max_pixel_value=255.0,\n                p=1.0,\n            ),\n            albumentations.CoarseDropout(p=0.5),\n            albumentations.Cutout(p=0.5),\n            albumentations.pytorch.ToTensorV2()\n        ],\n        p=1.0,\n    )\n\n    valid_aug = albumentations.Compose(\n        [\n            albumentations.Resize(CFG.image_size, CFG.image_size, p=1.0),\n            albumentations.Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n                max_pixel_value=255.0,\n                p=1.0,\n            ),\n            albumentations.pytorch.ToTensorV2()\n        ],\n        p=1.0,\n    )\n    return {\"train_transforms\": train_aug, \"valid_transforms\": valid_aug}","metadata":{"execution":{"iopub.status.busy":"2021-09-27T16:15:41.871122Z","iopub.execute_input":"2021-09-27T16:15:41.871442Z","iopub.status.idle":"2021-09-27T16:15:41.882542Z","shell.execute_reply.started":"2021-09-27T16:15:41.871406Z","shell.execute_reply":"2021-09-27T16:15:41.881472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Define Pytorch Dataset class","metadata":{}},{"cell_type":"code","source":"class FlowerDataset:\n    def __init__(self, image_paths, targets, augmentations):\n        self.image_paths = image_paths\n        self.targets = targets\n        self.augmentations = augmentations\n        \n    def __len__(self):\n        return len(self.image_paths)\n\n    def __getitem__(self, item):\n        targets = self.targets[item]\n        \n        image = cv2.imread(self.image_paths[item])\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        \n        augmented = self.augmentations(image = image)\n        image = augmented[\"image\"]\n        \n        return {\n            \"image\": image,\n            \"targets\": targets,\n        }","metadata":{"execution":{"iopub.status.busy":"2021-09-27T16:15:41.883788Z","iopub.execute_input":"2021-09-27T16:15:41.884562Z","iopub.status.idle":"2021-09-27T16:15:41.895316Z","shell.execute_reply.started":"2021-09-27T16:15:41.884510Z","shell.execute_reply":"2021-09-27T16:15:41.894574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Define model class (the most important part) -> this is the part i am going to learn hard way","metadata":{}},{"cell_type":"code","source":"import warnings\n\nimport psutil\nimport torch\nimport torch.nn as nn\nfrom tez import enums\nfrom tez.callbacks import CallbackRunner\nfrom tez.utils import AverageMeter\nfrom tqdm import tqdm\nimport time\n\nclass LeafModel(tez.Model):\n    def __init__(self, pretrained = True):\n        super().__init__()\n        self.model = timm.create_model(model_name = CFG.model_name, pretrained = pretrained)\n        self.n_features = self.model.fc.in_features\n        self.model.fc = nn.Linear(self.n_features, CFG.target_size)\n        \n        self.step_scheduler_after = \"epoch\"\n        self.step_scheduler_metric = \"valid_accuracy\"\n        \n    def monitor_metrics(self, outputs, targets):\n        if targets is None:\n            return {}\n        outputs = torch.argmax(outputs, dim=1).cpu().detach().numpy()\n        targets = targets.cpu().detach().numpy()\n        accuracy = metrics.accuracy_score(targets, outputs)\n        return {\"accuracy\": accuracy}\n    \n    def fetch_optimizer(self):\n        opt = torch.optim.Adam(self.parameters(), lr=1e-4, weight_decay = 1e-6)\n        return opt\n    \n    def fetch_scheduler(self):\n        sch = torch.optim.lr_scheduler.ReduceLROnPlateau(self.optimizer, mode = 'min', factor = 0.2,patience = 5, eps =1e-6, verbose = True)\n        return sch\n\n    def forward(self, image, targets=None):\n        batch_size, _, _, _ = image.shape\n\n        outputs = self.model(image)\n        \n        if targets is not None:\n            loss = nn.CrossEntropyLoss()(outputs, targets)\n            metrics = self.monitor_metrics(outputs, targets)\n            return outputs, loss, metrics\n        return outputs, None, None\n    \n    def fit(\n        self,\n        train_dataset,\n        valid_dataset=None,\n        train_sampler=None,\n        valid_sampler=None,\n        device=\"cuda\",\n        epochs=10,\n        train_bs=16,\n        valid_bs=16,\n        n_jobs=8,\n        callbacks=None,\n        fp16=False,\n        train_collate_fn=None,\n        valid_collate_fn=None,\n        train_shuffle=True,\n        valid_shuffle=False,\n        accumulation_steps=1,\n        clip_grad_norm=None,\n    ):\n        \"\"\"\n        The model fit function. Heavily inspired by tf/keras, this function is the core of Tez and this is the only\n        function you need to train your models.\n\n        \"\"\"\n        if device == \"tpu\":\n            if XLA_AVAILABLE is False:\n                raise RuntimeError(\"XLA is not available. Please install pytorch_xla\")\n            else:\n                self.using_tpu = True\n                fp16 = False\n                device = xm.xla_device()\n        self._init_model(\n            device=device,\n            train_dataset=train_dataset,\n            valid_dataset=valid_dataset,\n            train_sampler=train_sampler,\n            valid_sampler=valid_sampler,\n            train_bs=train_bs,\n            valid_bs=valid_bs,\n            n_jobs=n_jobs,\n            callbacks=callbacks,\n            fp16=fp16,\n            train_collate_fn=train_collate_fn,\n            valid_collate_fn=valid_collate_fn,\n            train_shuffle=train_shuffle,\n            valid_shuffle=valid_shuffle,\n            accumulation_steps=accumulation_steps,\n            clip_grad_norm=clip_grad_norm,\n        )\n\n        for epoch in range(epochs):\n            start_time = time.time()\n            self.train_state = enums.TrainingState.EPOCH_START\n            self.train_state = enums.TrainingState.TRAIN_EPOCH_START\n            train_loss = self.train_one_epoch(self.train_loader)\n            self.train_state = enums.TrainingState.TRAIN_EPOCH_END\n            if self.valid_loader:\n                self.train_state = enums.TrainingState.VALID_EPOCH_START\n                valid_loss = self.validate_one_epoch(self.valid_loader)\n                self.train_state = enums.TrainingState.VALID_EPOCH_END\n            \n            elapsed = time.time() - start_time\n            LOGGER.info(f'Epoch {epoch+1} - avg_train_loss: {train_loss:.4f}  avg_val_loss: {valid_loss:.4f} time: {elapsed:.0f}s')\n            \n            if self.scheduler:\n                if self.step_scheduler_after == \"epoch\":\n                    if self.step_scheduler_metric is None:\n                        self.scheduler.step()\n                    else:\n                        step_metric = self.name_to_metric(self.step_scheduler_metric)\n                        self.scheduler.step(step_metric)\n            self.train_state = enums.TrainingState.EPOCH_END\n            if self._model_state.value == \"end\":\n                break\n            self.current_epoch += 1\n        self.train_state = enums.TrainingState.TRAIN_END","metadata":{"execution":{"iopub.status.busy":"2021-09-27T16:15:41.898342Z","iopub.execute_input":"2021-09-27T16:15:41.898750Z","iopub.status.idle":"2021-09-27T16:15:41.922452Z","shell.execute_reply.started":"2021-09-27T16:15:41.898578Z","shell.execute_reply":"2021-09-27T16:15:41.921794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train on Multiple folds","metadata":{}},{"cell_type":"code","source":"def predict(valid_dataset,fold):\n    model = LeafModel(pretrained = False)\n    model.load(OUTPUT_DIR+f'{CFG.model_name}_fold{fold}_best.bin', device=\"cuda\", weights_only=True)\n    preds = model.predict(valid_dataset, batch_size=2*CFG.batch_size )\n    \n    temp_preds = None\n    for p in preds:\n        if temp_preds is None:\n            temp_preds = p\n        else:\n            temp_preds = np.vstack((temp_preds, p))\n    return temp_preds.argmax(axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-09-27T16:15:41.925766Z","iopub.execute_input":"2021-09-27T16:15:41.926049Z","iopub.status.idle":"2021-09-27T16:15:41.934265Z","shell.execute_reply.started":"2021-09-27T16:15:41.926023Z","shell.execute_reply":"2021-09-27T16:15:41.933465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(fold):\n    LOGGER.info(f\"========== fold: {fold} training ==========\")\n    \n    dfx = pd.read_csv(\"./train_folds.csv\")\n    df_train = dfx[dfx.kfold != fold].reset_index(drop=True)\n    df_valid = dfx[dfx.kfold == fold].reset_index(drop=True)\n\n    train_image_paths = [os.path.join(IMAGE_PATH, x) for x in df_train.image_id.values]\n    valid_image_paths = [os.path.join(IMAGE_PATH, x) for x in df_valid.image_id.values]\n    train_targets = df_train.label.values\n    valid_targets = df_valid.label.values\n\n    train_dataset = FlowerDataset(\n        image_paths=train_image_paths,\n        targets=train_targets,\n        augmentations=generate_transforms()[\"train_transforms\"],\n    )\n\n    valid_dataset = FlowerDataset(\n        image_paths=valid_image_paths,\n        targets=valid_targets,\n        augmentations=generate_transforms()[\"valid_transforms\"],\n    )\n    \n    model = LeafModel()\n    es = EarlyStopping(\n        monitor=\"valid_accuracy\", \n        model_path=OUTPUT_DIR+f'{CFG.model_name}_fold{fold}_best.bin', \n        patience=2, \n        mode=\"max\",\n        save_weights_only=True\n    )\n    model.fit(\n        train_dataset,\n        valid_dataset=valid_dataset,\n        train_bs=CFG.batch_size,\n        valid_bs=2*CFG.batch_size,\n        device=\"cuda\",\n        epochs=CFG.epochs,\n        callbacks=[es],\n        fp16=True,\n    )\n    df_valid['preds'] = predict(valid_dataset, fold)\n    return df_valid","metadata":{"execution":{"iopub.status.busy":"2021-09-27T16:15:41.937770Z","iopub.execute_input":"2021-09-27T16:15:41.937986Z","iopub.status.idle":"2021-09-27T16:15:41.947981Z","shell.execute_reply.started":"2021-09-27T16:15:41.937962Z","shell.execute_reply":"2021-09-27T16:15:41.947346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_score(y_true, y_pred):\n        score = metrics.accuracy_score(y_true, y_pred) \n        return score\n\ndef get_result(result_df):\n        preds = result_df['preds'].values\n        labels = result_df[CFG.target_col].values\n        score = get_score(labels, preds)\n        LOGGER.info(f'Score: {score:<.4f}')\n\noof_df = pd.DataFrame()\nfor fold in range(CFG.n_fold):\n    _oof_df = train(fold)\n    oof_df = pd.concat([oof_df, _oof_df])\n    LOGGER.info(f\"========== fold: {fold} result ==========\")\n    get_result(_oof_df)\n\n# CV result\nLOGGER.info(f\"========== CV ==========\")\nget_result(oof_df)\n# save result\noof_df.to_csv(OUTPUT_DIR+'oof_df.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-09-27T16:15:41.949243Z","iopub.execute_input":"2021-09-27T16:15:41.949571Z","iopub.status.idle":"2021-09-27T16:16:41.708318Z","shell.execute_reply.started":"2021-09-27T16:15:41.949504Z","shell.execute_reply":"2021-09-27T16:16:41.705747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# with open('./train.log') as f:\n#     f = f.readlines()\n# for line in f:\n#     print(line)","metadata":{"execution":{"iopub.status.busy":"2021-09-27T16:16:41.709712Z","iopub.status.idle":"2021-09-27T16:16:41.710143Z","shell.execute_reply.started":"2021-09-27T16:16:41.709914Z","shell.execute_reply":"2021-09-27T16:16:41.709936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}