{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Libraries**","metadata":{}},{"cell_type":"code","source":"!pip install catalyst\n!pip install -q efficientnet_pytorch ","metadata":{"execution":{"iopub.status.busy":"2021-07-23T11:20:57.635057Z","iopub.execute_input":"2021-07-23T11:20:57.635456Z","iopub.status.idle":"2021-07-23T11:21:14.874513Z","shell.execute_reply.started":"2021-07-23T11:20:57.635367Z","shell.execute_reply":"2021-07-23T11:21:14.873552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#torch\nimport torch\nfrom torch import nn, optim\nimport torch.nn.functional as F\nimport torchvision\nfrom torch.utils.data import Dataset, DataLoader\n\nfrom efficientnet_pytorch import EfficientNet\n\n#Catalyst \nimport catalyst\nfrom catalyst.utils import set_global_seed\nfrom catalyst.runners.runner import SupervisedRunner\nfrom catalyst.contrib.nn.schedulers import OneCycleLRWithWarmup\nfrom catalyst.callbacks.criterion import CriterionCallback\nfrom catalyst import dl\n\n#SKlearn\nfrom sklearn.preprocessing import LabelEncoder, OneHotEncoder\n\n#Regular import \nimport os\nimport pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport cv2 as cv\nimport random\n\n%matplotlib inline\n\n#Augmentations\nimport albumentations as albu\nfrom albumentations.pytorch import ToTensorV2 as ToTensor\n\n#warnings\nimport warnings\nwarnings.filterwarnings('ignore')\n\n#TensorBoard\nfrom torch.utils.tensorboard import SummaryWriter\n%load_ext tensorboard","metadata":{"execution":{"iopub.status.busy":"2021-07-23T11:21:14.876311Z","iopub.execute_input":"2021-07-23T11:21:14.876688Z","iopub.status.idle":"2021-07-23T11:21:20.69701Z","shell.execute_reply.started":"2021-07-23T11:21:14.876645Z","shell.execute_reply":"2021-07-23T11:21:20.696083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Seed&Device**","metadata":{}},{"cell_type":"code","source":"SEED = 2021\n\nset_global_seed(SEED)\nos.environ['PYTHONHASHSEED'] = str(SEED)","metadata":{"execution":{"iopub.status.busy":"2021-07-23T11:21:20.698972Z","iopub.execute_input":"2021-07-23T11:21:20.699339Z","iopub.status.idle":"2021-07-23T11:21:20.708486Z","shell.execute_reply.started":"2021-07-23T11:21:20.699281Z","shell.execute_reply":"2021-07-23T11:21:20.707607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f'Device is {device}')","metadata":{"execution":{"iopub.status.busy":"2021-07-23T11:21:20.709892Z","iopub.execute_input":"2021-07-23T11:21:20.710458Z","iopub.status.idle":"2021-07-23T11:21:20.716784Z","shell.execute_reply.started":"2021-07-23T11:21:20.710422Z","shell.execute_reply":"2021-07-23T11:21:20.715799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Config**","metadata":{}},{"cell_type":"code","source":"#====================\nimage_size = 256\nbatch_size = 32\ninit_lr = 1e-2\nnum_workers = 8\nnum_epochs = 100\nOUTPUT_SIZE = 1\n#====================","metadata":{"execution":{"iopub.status.busy":"2021-07-23T11:21:26.799351Z","iopub.execute_input":"2021-07-23T11:21:26.799691Z","iopub.status.idle":"2021-07-23T11:21:26.804203Z","shell.execute_reply.started":"2021-07-23T11:21:26.79966Z","shell.execute_reply":"2021-07-23T11:21:26.803245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Preprocessing & DataLoaders**","metadata":{}},{"cell_type":"markdown","source":"## **DIRs**","metadata":{}},{"cell_type":"code","source":"DB_DIR = '../input/siim-isic-melanoma-classification'\nJPEG_DIR = os.path.join(DB_DIR, 'jpeg')\nTRAIN_JPEG_DIR = os.path.join(JPEG_DIR, 'train')\nTEST_JPEG_DIR = os.path.join(JPEG_DIR, 'test')","metadata":{"execution":{"iopub.status.busy":"2021-07-23T11:21:28.69246Z","iopub.execute_input":"2021-07-23T11:21:28.692791Z","iopub.status.idle":"2021-07-23T11:21:28.697239Z","shell.execute_reply.started":"2021-07-23T11:21:28.692763Z","shell.execute_reply":"2021-07-23T11:21:28.696297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **CSV**","metadata":{}},{"cell_type":"code","source":"dataframe = pd.read_csv(os.path.join(DB_DIR, 'train.csv'))","metadata":{"execution":{"iopub.status.busy":"2021-07-23T11:21:29.486063Z","iopub.execute_input":"2021-07-23T11:21:29.486405Z","iopub.status.idle":"2021-07-23T11:21:29.567397Z","shell.execute_reply.started":"2021-07-23T11:21:29.486374Z","shell.execute_reply":"2021-07-23T11:21:29.566578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataframe.drop(['patient_id', 'diagnosis', 'benign_malignant'], inplace=True, axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-07-23T11:21:29.674662Z","iopub.execute_input":"2021-07-23T11:21:29.674953Z","iopub.status.idle":"2021-07-23T11:21:29.688174Z","shell.execute_reply.started":"2021-07-23T11:21:29.674924Z","shell.execute_reply":"2021-07-23T11:21:29.687302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataframe['sex'].fillna('male', inplace=True)\ndataframe['age_approx'].fillna(50, inplace=True)\ndataframe['anatom_site_general_challenge'].fillna('torso', inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-23T11:21:29.852996Z","iopub.execute_input":"2021-07-23T11:21:29.853282Z","iopub.status.idle":"2021-07-23T11:21:29.868171Z","shell.execute_reply.started":"2021-07-23T11:21:29.853252Z","shell.execute_reply":"2021-07-23T11:21:29.867376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_encoder = LabelEncoder()\n\nto_encode = [dataframe.columns[1], dataframe.columns[3]]\nencoded_all = []\n\nfor column in to_encode:\n    encoded = label_encoder.fit_transform(dataframe[column])\n    encoded_all.append(encoded)\n    \n\nfor idx, column in enumerate(to_encode):\n    dataframe[column] = encoded_all[idx]","metadata":{"execution":{"iopub.status.busy":"2021-07-23T11:21:30.03583Z","iopub.execute_input":"2021-07-23T11:21:30.036128Z","iopub.status.idle":"2021-07-23T11:21:30.060527Z","shell.execute_reply.started":"2021-07-23T11:21:30.036101Z","shell.execute_reply":"2021-07-23T11:21:30.059571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Dataset**","metadata":{}},{"cell_type":"code","source":"class MelanomaDataset(Dataset):\n    def __init__(self, dataframe, transforms=None, is_train_or_valid=True):\n        self.dataframe, self.transforms = dataframe, transforms\n        self.is_train_or_valid = is_train_or_valid\n        \n    \n    def __len__(self):\n        return len(self.dataframe)\n    \n    \n    def __getitem__(self, idx):\n        if self.is_train_or_valid:\n            image = cv.cvtColor(cv.imread(os.path.join(TRAIN_JPEG_DIR,str(self.dataframe['image_name'][idx]) + '.jpg')), cv.COLOR_BGR2RGB)\n            csv_data = np.array(self.dataframe.iloc[idx][['sex', 'age_approx', 'anatom_site_general_challenge']].values, dtype=np.float32)\n            \n            if self.transforms != None:\n                aug_dict = self.transforms(image=image)\n                image = aug_dict['image']\n            \n            if self.is_train_or_valid:\n                return (image, csv_data), self.dataframe['target'].astype('float64')[idx]\n            \n            else:\n                return (image, csv_data)\n            ","metadata":{"execution":{"iopub.status.busy":"2021-07-23T11:21:30.382995Z","iopub.execute_input":"2021-07-23T11:21:30.383308Z","iopub.status.idle":"2021-07-23T11:21:30.390945Z","shell.execute_reply.started":"2021-07-23T11:21:30.383278Z","shell.execute_reply":"2021-07-23T11:21:30.389956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Augmentations**","metadata":{}},{"cell_type":"code","source":"def pre_transform(image_size: int = 256):\n    return albu.Resize(image_size, image_size, p=1)\n\n\ndef augmentations(image_size: int = 256):\n    channel_augs = [\n        albu.HueSaturationValue(p=0.5),\n        albu.ChannelShuffle(p=0.5),\n    ]\n\n    result = [\n        albu.OneOf(\n            [albu.IAAAdditiveGaussianNoise(), albu.GaussNoise(),], p=0.5\n        ),\n        albu.OneOf(\n            [\n                albu.MotionBlur(blur_limit=3, p=0.7),\n                albu.MedianBlur(blur_limit=3, p=1.0),\n                albu.Blur(blur_limit=3, p=0.7),\n            ],\n            p=0.5,\n        ),\n        albu.OneOf(channel_augs),\n        albu.OneOf(\n            [albu.CLAHE(clip_limit=2), albu.IAASharpen(), albu.IAAEmboss(),],\n            p=0.5,\n        ),\n        albu.RandomBrightnessContrast(\n            brightness_limit=0.5, contrast_limit=0.5, p=0.5\n        ),\n        albu.RandomGamma(p=0.5),\n        albu.OneOf([albu.MedianBlur(p=0.5), albu.MotionBlur(p=0.5)]),\n        albu.RandomGamma(gamma_limit=(85, 115), p=0.5),\n    ]\n    return albu.Compose(result)\n\n\ndef post_transform():\n    return albu.Compose([albu.Normalize(), ToTensor()])","metadata":{"execution":{"iopub.status.busy":"2021-07-23T11:21:30.711106Z","iopub.execute_input":"2021-07-23T11:21:30.711456Z","iopub.status.idle":"2021-07-23T11:21:30.722966Z","shell.execute_reply.started":"2021-07-23T11:21:30.711423Z","shell.execute_reply":"2021-07-23T11:21:30.722071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_transforms = albu.Compose(\n    [pre_transform(), augmentations(), post_transform()]\n)\nvalid_transforms = albu.Compose(\n    [pre_transform(), post_transform()]\n)","metadata":{"execution":{"iopub.status.busy":"2021-07-23T11:21:30.882713Z","iopub.execute_input":"2021-07-23T11:21:30.882998Z","iopub.status.idle":"2021-07-23T11:21:30.887799Z","shell.execute_reply.started":"2021-07-23T11:21:30.882971Z","shell.execute_reply":"2021-07-23T11:21:30.886789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Dataloaders**","metadata":{}},{"cell_type":"markdown","source":"### **Split csv**","metadata":{}},{"cell_type":"code","source":"SPLIT_VAL = 0.25","metadata":{"execution":{"iopub.status.busy":"2021-07-23T11:21:32.637646Z","iopub.execute_input":"2021-07-23T11:21:32.637999Z","iopub.status.idle":"2021-07-23T11:21:32.643296Z","shell.execute_reply.started":"2021-07-23T11:21:32.637968Z","shell.execute_reply":"2021-07-23T11:21:32.642307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"indexes = np.arange(len(dataframe))\nnp.random.shuffle(indexes)\nvalid_indexes = indexes[:round(SPLIT_VAL*len(dataframe))]\ntrain_indexes = indexes[round(SPLIT_VAL*len(dataframe)):]","metadata":{"execution":{"iopub.status.busy":"2021-07-23T11:21:33.639739Z","iopub.execute_input":"2021-07-23T11:21:33.640071Z","iopub.status.idle":"2021-07-23T11:21:33.64713Z","shell.execute_reply.started":"2021-07-23T11:21:33.640042Z","shell.execute_reply":"2021-07-23T11:21:33.646236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.DataFrame(columns=dataframe.columns)\nvalid_df = pd.DataFrame(columns=dataframe.columns)\n\nfor idx in train_indexes:\n    train_df = train_df.append(dataframe.iloc[idx])\n    \nfor idx in valid_indexes:\n    valid_df = valid_df.append(dataframe.iloc[idx])","metadata":{"execution":{"iopub.status.busy":"2021-07-23T11:21:34.136651Z","iopub.execute_input":"2021-07-23T11:21:34.136985Z","iopub.status.idle":"2021-07-23T11:24:08.804882Z","shell.execute_reply.started":"2021-07-23T11:21:34.136952Z","shell.execute_reply":"2021-07-23T11:24:08.80378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.reset_index(inplace=True)\nvalid_df.reset_index(inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-23T11:24:08.806638Z","iopub.execute_input":"2021-07-23T11:24:08.807038Z","iopub.status.idle":"2021-07-23T11:24:08.815482Z","shell.execute_reply.started":"2021-07-23T11:24:08.806996Z","shell.execute_reply":"2021-07-23T11:24:08.814584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Datasets & Dataloaders**","metadata":{}},{"cell_type":"code","source":"train_ds = MelanomaDataset(dataframe=train_df, transforms=train_transforms)\nvalid_ds = MelanomaDataset(dataframe=valid_df, transforms=valid_transforms)","metadata":{"execution":{"iopub.status.busy":"2021-07-23T11:24:08.817696Z","iopub.execute_input":"2021-07-23T11:24:08.817988Z","iopub.status.idle":"2021-07-23T11:24:08.826724Z","shell.execute_reply.started":"2021-07-23T11:24:08.817961Z","shell.execute_reply":"2021-07-23T11:24:08.825729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader = DataLoader(train_ds, batch_size=batch_size, shuffle=True, num_workers=num_workers)\nvalid_loader = DataLoader(valid_ds, batch_size=batch_size, shuffle=False, num_workers=num_workers) \n\nloaders = {\n    'train': train_loader,\n    'valid': valid_loader\n}","metadata":{"execution":{"iopub.status.busy":"2021-07-23T11:36:05.33264Z","iopub.execute_input":"2021-07-23T11:36:05.333011Z","iopub.status.idle":"2021-07-23T11:36:05.341281Z","shell.execute_reply.started":"2021-07-23T11:36:05.332976Z","shell.execute_reply":"2021-07-23T11:36:05.340252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Net**","metadata":{}},{"cell_type":"code","source":"class Net(nn.Module):\n    def __init__(self, output_size, num_cols, is_b2=True):\n        super().__init__()\n        \n        self.is_b2 = is_b2        \n        \n        #Image\n        if self.is_b2:\n            self.features = EfficientNet.from_pretrained('efficientnet-b2')\n            \n        else:\n            self.features = EfficientNet.from_pretrained('efficientnet-b4')\n            \n        #CSV\n        self.csv = nn.Sequential(nn.Linear(num_cols, 250),\n                                 nn.BatchNorm1d(250),\n                                 nn.ReLU(),\n                                 nn.Dropout(p=0.2),\n                                \n                                 nn.Linear(250, 250),\n                                 nn.BatchNorm1d(250),\n                                 nn.ReLU(),\n                                 nn.Dropout(p=0.2))\n        \n        #Classification\n        if self.is_b2:\n            self.classification = nn.Sequential(nn.Linear(1408 + 250, output_size))\n            \n        else:\n            self.classification = nn.Sequential(nn.Linear(1792 + 250, output_size))\n            \n    \n    def forward(self, data):\n        \n        #Image CNN\n        image_feats = self.features.extract_features(data[0])\n        \n        if self.is_b2:\n            image_feats = F.avg_pool2d(image_feats, image_feats.size()[2:]).reshape(-1, 1408)\n            \n        else:\n            image_feats = F.avg_pool2d(image_feats, image_feats.size()[2:]).reshape(-1, 1792)\n            \n            \n        #CSV FNN\n        csv_feats = self.csv(data[1])\n        \n        #Concatenate\n        image_csv_feats = torch.cat((image_feats, csv_feats), dim=1)\n        \n        #Classification\n        out = self.classification(image_csv_feats)\n        \n        return out.squeeze(1)","metadata":{"execution":{"iopub.status.busy":"2021-07-23T17:40:04.137219Z","iopub.execute_input":"2021-07-23T17:40:04.137634Z","iopub.status.idle":"2021-07-23T17:40:04.149541Z","shell.execute_reply.started":"2021-07-23T17:40:04.1376Z","shell.execute_reply":"2021-07-23T17:40:04.148485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Net(output_size=OUTPUT_SIZE, num_cols=3)\nmodel.to(device)","metadata":{"execution":{"iopub.status.busy":"2021-07-23T11:24:08.851571Z","iopub.execute_input":"2021-07-23T11:24:08.851963Z","iopub.status.idle":"2021-07-23T11:24:14.273482Z","shell.execute_reply.started":"2021-07-23T11:24:08.851923Z","shell.execute_reply":"2021-07-23T11:24:14.272686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"criterion = {\n    'bce': nn.BCEWithLogitsLoss()\n}\n\noptimizer = optim.Adam(model.parameters(), lr=init_lr)\n\nscheduler = OneCycleLRWithWarmup(\n    optimizer, \n    num_steps=2, \n    lr_range=(0.5, 0.0005), \n    warmup_steps=2, \n    momentum_range=(0.85, 0.95))\n\nrunner = SupervisedRunner(\n    input_key='label', \n    output_key='pred', \n    target_key='target')","metadata":{"execution":{"iopub.status.busy":"2021-07-23T17:41:02.981425Z","iopub.execute_input":"2021-07-23T17:41:02.981907Z","iopub.status.idle":"2021-07-23T17:41:03.008986Z","shell.execute_reply.started":"2021-07-23T17:41:02.981846Z","shell.execute_reply":"2021-07-23T17:41:03.007277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Train**","metadata":{}},{"cell_type":"code","source":"runner.train(\n    model=model,\n    engine = dl.DeviceEngine(device),\n    criterion=criterion,\n    optimizer=optimizer,\n    scheduler=scheduler,\n    callbacks=[CriterionCallback(\n        input_key='pred',\n        target_key='target', \n        criterion_key='bce',\n        metric_key='loss')],\n    loaders=loaders,\n    logdir='log',\n    load_best_on_end=True,\n    num_epochs=num_epochs,\n    verbose=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-23T17:41:03.832803Z","iopub.execute_input":"2021-07-23T17:41:03.833195Z"},"trusted":true},"execution_count":null,"outputs":[]}]}