{"cells":[{"metadata":{},"cell_type":"markdown","source":"Training at : https://www.kaggle.com/krisho007/simple-gpu-pytorch-lightning-training"},{"metadata":{"_kg_hide-output":true,"trusted":true,"scrolled":false},"cell_type":"code","source":"import sys\nsys.path.append('../input/timm-pytorch-image-models/pytorch-image-models-master')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"scrolled":true,"_kg_hide-output":true},"cell_type":"code","source":"import os\nimport cv2\nimport pandas as pd\nimport numpy as np\nimport random\nimport pytorch_lightning as pl\nfrom pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport albumentations as albu\nfrom albumentations.pytorch.transforms import ToTensorV2\nfrom sklearn.model_selection import StratifiedKFold\nimport timm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_kg_hide-output":true},"cell_type":"code","source":"import random\ndef seed_everything(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\nseed_everything()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_kg_hide-output":true},"cell_type":"code","source":"TRAIN_CSV = \"../input/cassava-leaf-disease-classification/train.csv\"\nTRAIN_IMAGE_FOLDER = '../input/cassava-leaf-disease-classification/train_images'\nCLASSES = 5","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Hyper parameters"},{"metadata":{"trusted":true,"scrolled":true,"_kg_hide-output":true},"cell_type":"code","source":"BATCH_SIZE =32\nLR = 0.0001\n\nIMG_SIZE = 128\nIMG_SIZE = 240\nIMG_SIZE = 512\n\nMODEL_ARCH = 'resnet50'\nMODEL_ARCH = 'tf_efficientnet_b1_ns'\nMODEL_ARCH = 'efficientnet_b3'\nMODEL_ARCH = 'tf_efficientnet_b4_ns'\n\nTEST_IMAGES_PATH = '../input/cassava-leaf-disease-classification/test_images/'\n# TEST_IMAGES_PATH = '../input/cassava-leaf-disease-classification/train_images/'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Dataset"},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"class CassavaTestDataset(Dataset):\n    def __init__(self, test_df, transforms=None):\n        self.test_df = test_df\n        self.transforms = transforms\n    \n    def __len__(self):\n        return self.test_df.shape[0]\n    \n    def __getitem__(self, index):\n        image_path = os.path.join(TEST_IMAGES_PATH, self.test_df.iloc[index].image_id)\n        image = cv2.imread(image_path, cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        \n        if (self.transforms):\n            image = self.transforms(image=image)[\"image\"]\n        \n        return {\n            \"x\": image\n        }","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Transforms"},{"metadata":{"trusted":true,"scrolled":true,"_kg_hide-output":true},"cell_type":"code","source":"def get_augmentations():\n    \n    mean = (0.485, 0.456, 0.406)\n    std = (0.229, 0.224, 0.225)    \n    \n    test_augmentations = albu.Compose([\n        albu.RandomResizedCrop(IMG_SIZE, IMG_SIZE, p=1.0),\n        albu.Transpose(p=0.5),\n        albu.HorizontalFlip(p=0.5),\n        albu.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n        albu.RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n        albu.Normalize(mean, std, max_pixel_value=255.0, always_apply=True),        \n        ToTensorV2(p=1.0)\n    ], p=1.0)\n    \n    return test_augmentations\n\ntest_augmentations = get_augmentations()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### NN Model"},{"metadata":{"trusted":true,"scrolled":true,"_kg_hide-output":true},"cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.model = timm.create_model(MODEL_ARCH, pretrained=False)\n#         self.model = base_model\n\n        # Efficientnets\n        n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(n_features, CLASSES)\n        \n#         # Resnets\n#         n_features = self.model.fc.in_features\n#         self.model.fc = nn.Linear(n_features, CLASSES)\n        \n    def forward(self, x):\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### PL Module"},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"# You just need __init__ and forward methods of you LightningModule\nclass CassavaPLModule(pl.LightningModule):\n    def __init__(self, hparams, model):\n        super(CassavaPLModule, self).__init__()\n        self.model = model\n        \n    def forward(self, x):\n        return self.model(x)       ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Inference"},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"nnModel = Model()\n\n# Test data loader\ntest_df = pd.DataFrame()\ntest_df['image_id'] = list(os.listdir(TEST_IMAGES_PATH))\n# test_df = test_df[:64]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"submission = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"def inference(checkpoint_directory) :\n    test_ds = CassavaTestDataset(test_df, transforms=test_augmentations)\n    test_loader = DataLoader(test_ds, BATCH_SIZE, num_workers=4, shuffle=False)\n    \n    preds = []\n    for batch in test_loader:\n#         import pdb; pdb.set_trace()\n        avg_preds = [] #average prediction per batch\n        # Each batch has multiple images. Multiple predictions at a time\n        input = batch['x']\n        input = input.cuda()\n        for modelWeight in os.scandir(checkpoint_directory):\n            model = CassavaPLModule.load_from_checkpoint(f\"{checkpoint_directory}/{modelWeight.name}\", hparams={'lr':LR, 'batch_size':BATCH_SIZE}, model=nnModel)\n            model.eval()\n            model.cuda()\n            model.freeze()  #Will get a CUDA memory error without this\n            output = model(input)\n            avg_preds.append(output.detach().to('cpu').numpy())\n            break\n        avg_preds = np.mean(avg_preds, axis=0)\n        preds.append(avg_preds)\n    preds = np.concatenate(preds)\n    return preds                               ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"checkpoint_directory = '../input/simple-gpu-pytorch-lightning-training/checkpoints'\n# checkpoint_directory = '../input/gpu-pytorch-lightning-training-on-inference/checkpoints'\n# checkpoint_directory = '../input/trainedoninferencemodels'\ntestPredictions = inference(checkpoint_directory)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 5 times TTA"},{"metadata":{},"cell_type":"markdown","source":"#### Hardvoting"},{"metadata":{"trusted":true},"cell_type":"code","source":"# hardVoting = None\n# for k in range(5):\n#     singlePrediction = inference(checkpoint_directory)\n#     singlePrediction = (singlePrediction == singlePrediction.max(axis=1)[:,None]).astype(int)\n#     try:\n#         hardVoting += singlePrediction\n#     except:\n#         hardVoting = singlePrediction","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Soft voting"},{"metadata":{"trusted":true},"cell_type":"code","source":"# for k in range(5):\n#     try:\n#         testPredictions += inference(checkpoint_directory)\n#     except NameError:\n#         testPredictions = inference(checkpoint_directory)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"test_df['label'] = testPredictions.argmax(1)\ntest_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"test_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- Hard voting  \n- Remove training images with low confidance"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}