{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install -q efficientnet_pytorch             # Convolutional Neural Net from Google Research\n#!pip install torchvision==0.7.0","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# System\nimport cv2\nimport os, os.path\nfrom PIL import Image              # from RBG to YCbCr\nimport gc\nimport time\nimport datetime\n\n# Basics\nimport pandas as pd\nimport numpy as np\nimport random\nimport seaborn as sns\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg    # to check images\n# %matplotlib inline\nfrom tqdm.notebook import tqdm      # beautiful progression bar\n\n# SKlearn\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold\nfrom sklearn.metrics import accuracy_score, roc_auc_score, confusion_matrix\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn import preprocessing\n\n# PyTorch\nimport torch\nimport torchvision\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch import FloatTensor, LongTensor\nfrom torch.utils.data import Dataset, DataLoader, Subset\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\n\n# Data Augmentation for Image Preprocessing\nfrom albumentations import (ToFloat, Normalize, VerticalFlip, HorizontalFlip, Compose, Resize,\n                            RandomBrightnessContrast, HueSaturationValue, Blur, GaussNoise,\n                            Rotate, RandomResizedCrop, Cutout, ShiftScaleRotate)\nfrom albumentations.pytorch import ToTensorV2, ToTensor\nfrom efficientnet_pytorch import EfficientNet\nfrom torchvision.models import resnet34, resnet50, vgg16, vgg16_bn, vgg19, vgg19_bn, inception_v3, squeezenet1_0, densenet161, alexnet\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nimport helper","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def set_seed(seed = 1234):\n    '''Sets the seed of the entire notebook so results are the same every time we run.\n    This is for REPRODUCIBILITY.'''\n    np.random.seed(seed)\n    random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    # When running on the CuDNN backend, two further options must be set\n    torch.backends.cudnn.deterministic = True\n    # Set a fixed value for the hash seed\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \nset_seed()\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint('Device available now:', device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" # ----- STATICS -----\noutput_size = 1\n# -------------------\n# My Train: with imputed missing values + OHE\nmy_train = pd.read_csv('../input/siim-melanoma-prep-data/train_clean.csv')\n\n# Drop path columns and Diagnosis (it won't be available during TEST)\n# We'll rewrite them once the data is concatenated\nto_drop = ['path_dicom','path_jpeg', 'diagnosis']\nfor drop in to_drop:\n    if drop in my_train.columns :\n        my_train.drop([drop], axis=1, inplace=True)\n\n# Roman's Train: with added data for Malignant category\nroman_train = pd.read_csv('../input/../input/melanoma-external-malignant-256/train_concat.csv')\n\n\n# --- Before concatenatenating both together, let's preprocess roman_train ---\n# Replace NAN with 0 for patient_id\nroman_train['patient_id'] = roman_train['patient_id'].fillna(0)\n\n# OHE\nto_encode = ['sex', 'anatom_site_general_challenge']\nencoded_all = []\n\nroman_train[to_encode[0]] = roman_train[to_encode[0]].astype(str)\nroman_train[to_encode[1]] = roman_train[to_encode[1]].astype(str)\n\nlabel_encoder = LabelEncoder()\n\nfor column in to_encode:\n    encoded = label_encoder.fit_transform(roman_train[column])\n    encoded_all.append(encoded)\n    \nroman_train[to_encode[0]] = encoded_all[0]\nroman_train[to_encode[1]] = encoded_all[1]\n\n# Give all columns the same name\nroman_train.columns = my_train.columns\n\n\n# --- Concatenate info which is not available in my_train ---\ncommon_images = my_train['dcm_name'].unique()\nnew_data = roman_train[~roman_train['dcm_name'].isin(common_images)]\n\n# Merge all together\ntrain_df = pd.concat([my_train, new_data], axis=0)\n\n\n\n# --- Read in Test data (also cleaned, imputed, OHE) ---\ntest_df = pd.read_csv('../input/siim-melanoma-prep-data/test_clean.csv')\n\n# Drop columns\nfor drop in to_drop:\n    if drop in test_df.columns :\n        test_df.drop([drop], axis=1, inplace=True)\n\n# Create path column to image folder for both Train and Test\npath_train = '../input/melanoma-external-malignant-256/train/train/'\npath_test = '../input/melanoma-external-malignant-256/test/test/'\n\ntrain_df['path_jpg'] = path_train + train_df['dcm_name'] + '.jpg'\ntest_df['path_jpg'] = path_test + test_df['dcm_name'] + '.jpg'\n\n\n# --- Last final thing: NORMALIZE! ---\ntrain_df['age'] = train_df['age'].fillna(-1)\n\nnormalized_train = preprocessing.normalize(train_df[['sex', 'age', 'anatomy']])\nnormalized_test = preprocessing.normalize(test_df[['sex', 'age', 'anatomy']])\n\ntrain_df['sex'] = normalized_train[:, 0]\ntrain_df['age'] = normalized_train[:, 1]\ntrain_df['anatomy'] = normalized_train[:, 2]\n\ntest_df['sex'] = normalized_test[:, 0]\ntest_df['age'] = normalized_test[:, 1]\ntest_df['anatomy'] = normalized_test[:, 2]\n\n\nprint('Train: {:,}'.format(len(train_df)), '\\n' +\n      'Test: {:,}'.format(len(test_df)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ----- STATICS -----\nvertical_flip = 0.5\nhorizontal_flip = 0.5\n\ncsv_columns = ['sex', 'age', 'anatomy']\nno_columns = 3\n# ------------------","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MelanomaDataset(Dataset):\n    \n    def __init__(self, dataframe, vertical_flip, horizontal_flip,\n                 is_train=True, is_valid=False, is_test=False):\n        self.dataframe, self.is_train, self.is_valid = dataframe, is_train, is_valid\n        self.vertical_flip, self.horizontal_flip = vertical_flip, horizontal_flip\n        \n        # Data Augmentation (custom for each dataset type)\n        if is_train or is_test:\n            self.transform = Compose([RandomResizedCrop(height=224, width=224, scale=(0.7, 1.0)),\n                                      ShiftScaleRotate(rotate_limit=90, scale_limit = [0.7, 1]),\n                                      HorizontalFlip(p = self.horizontal_flip),\n                                      VerticalFlip(p = self.vertical_flip),\n                                      HueSaturationValue(sat_shift_limit=[0.7, 1.3], \n                                                         hue_shift_limit=[-0.1, 0.1]),\n                                      RandomBrightnessContrast(brightness_limit=[0.01, 0.1],\n                                                               contrast_limit= [0.01, 0.1]),\n                                      #Normalize(),\n                                      ToTensor()])\n        else:\n            self.transform = Compose([#Normalize(),\n                                      Resize(height=224, width=224),\n                                      ToTensor()])\n            \n    def __len__(self):\n        return len(self.dataframe)\n    \n    def __getitem__(self, index):\n        # Select path and read image\n        image_path = self.dataframe['path_jpg'][index]\n        image = cv2.imread(image_path)\n        # For this image also import .csv information (sex, age, anatomy)\n        csv_data = np.array(self.dataframe.iloc[index][['sex', 'age', 'anatomy']].values, \n                            dtype=np.float32)\n        \n        # Apply transforms\n        \n        image = self.transform(image=image)\n        # Extract image from dictionary\n        image = image['image']\n        \n        # If train/valid: image + class | If test: only image\n        if self.is_train or self.is_valid:\n            return (image, csv_data), self.dataframe['target'][index]\n        else:\n            return (image, csv_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data = train_df.reset_index(drop=True)\nvalid_data = train_df.reset_index(drop=True)\n\ntrain = MelanomaDataset(train_data, \n                        vertical_flip=vertical_flip, \n                        horizontal_flip=horizontal_flip, \n                        is_train=True, \n                        is_valid=False, \n                        is_test=False)\nvalid = MelanomaDataset(valid_data, \n                        vertical_flip=vertical_flip, \n                        horizontal_flip=horizontal_flip, \n                        is_train=False, \n                        is_valid=True, \n                        is_test=False)\n\n# Dataloaders\ntrain_loader = DataLoader(train, \n                          batch_size= 16, #train_batch_size, \n                          shuffle=False, \n                          num_workers=8\n                         )\nvalid_loader = DataLoader(valid, \n                          #batch_size= 32, #val_test_batch_size, \n                          shuffle=False, \n                          num_workers=8\n                         )\n\nfig, ax = plt.subplots(3, 2, figsize=(16, 16))\n\ntrack = 0\nfor i, ((images, csv_data), labels) in enumerate(train_loader):\n    track += 1\n    if track > 9:\n        break\n    plt.subplot(330 + 1 + i)\n    image = images[0]\n    plt.imshow(cv2.cvtColor(image.permute(1,2,0).numpy(), cv2.COLOR_BGR2RGB))\n    #image = image.permute(1, 2,0)\n    #print(type(image))\n    #print(image.permute(1, 2, 0).shape)\n    #plt.imshow(image)\n    #plt.imshow(image.permute(1, 2, 0))\n    # 1, 2, 0\ntrain_data = train_df.reset_index(drop=True)\nvalid_data = train_df.reset_index(drop=True)\n\ntrain = MelanomaDataset(train_data, \n                        vertical_flip=vertical_flip, \n                        horizontal_flip=horizontal_flip, \n                        is_train=False, \n                        is_valid=True, \n                        is_test=False)\nvalid = MelanomaDataset(valid_data, \n                        vertical_flip=vertical_flip, \n                        horizontal_flip=horizontal_flip, \n                        is_train=False, \n                        is_valid=True, \n                        is_test=False)\n\n# Dataloaders\ntrain_loader = DataLoader(train, \n                          batch_size= 16, #train_batch_size, \n                          shuffle=False, \n                          num_workers=8\n                         )\nvalid_loader = DataLoader(valid, \n                          #batch_size= 32, #val_test_batch_size, \n                          shuffle=False, \n                          num_workers=8\n                         )\n\nfig, ax = plt.subplots(2, 2, figsize=(16, 16))\n\ntrack = 0\nfor i, ((images, csv_data), labels) in enumerate(train_loader):\n    track += 1\n    if track > 2:\n        break\n    plt.subplot(220 + 1 + 2*i)\n    image = images[0]\n    plt.imshow(cv2.cvtColor(image.permute(1,2,0).numpy(), cv2.COLOR_BGR2RGB))\n    \ntrain_data = train_df.reset_index(drop=True)\nvalid_data = train_df.reset_index(drop=True)\n\ntrain = MelanomaDataset(train_data, \n                        vertical_flip=vertical_flip, \n                        horizontal_flip=horizontal_flip, \n                        is_train=True, \n                        is_valid=False, \n                        is_test=False)\nvalid = MelanomaDataset(valid_data, \n                        vertical_flip=vertical_flip, \n                        horizontal_flip=horizontal_flip, \n                        is_train=False, \n                        is_valid=True, \n                        is_test=False)\n\n# Dataloaders\ntrain_loader = DataLoader(train, \n                          batch_size= 16, #train_batch_size, \n                          shuffle=False, \n                          num_workers=8\n                         )\nvalid_loader = DataLoader(valid, \n                          #batch_size= 32, #val_test_batch_size, \n                          shuffle=False, \n                          num_workers=8\n                         )\n\ntrack = 0\nfor i, ((images, csv_data), labels) in enumerate(train_loader):\n    track += 1\n    if track > 2:\n        break\n    plt.subplot(220 + 2 + 2*i)\n    image = images[0]\n    plt.imshow(cv2.cvtColor(image.permute(1,2,0).numpy(), cv2.COLOR_BGR2RGB))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class custom_vgg16(nn.Module):\n    \n    def __init__(self, output_size, no_columns, v16=True, v16_bn=False, v19=False):\n        super().__init__()\n        self.no_columns, self.output_size = no_columns, output_size\n        self.v16, self.v16_bn, self.v19 = v16, v16_bn, v19\n        \n        # Define Feature part (IMAGE)\n        if v16:\n            self.features = vgg16(pretrained=True) # What is progress?\n        elif v16_bn:\n            self.features = vgg16_bn(pretrained=True)\n        elif v19: \n            self.features = vgg19(pretrained=True)\n        else:\n            self.features = vgg19_bn(pretrained=True)\n        \n        # (CSV)\n        # keep this the same for all models you try\n        self.csv = nn.Sequential(nn.Linear(self.no_columns, 250),\n                                 nn.BatchNorm1d(250),\n                                 nn.ReLU(),\n                                 nn.Dropout(p=0.2),\n                                 \n                                 nn.Linear(250, 250),\n                                 nn.BatchNorm1d(250),\n                                 nn.ReLU(),\n                                 nn.Dropout(p=0.2))                      \n        \n        # Define Classification part\n        # you'll need to change 1792 to whatever size the model outputs in self.features.\n        if v16:\n            self.classification = nn.Sequential(nn.Linear(1000 + 250, output_size))\n        elif v16_bn:\n            self.features = nn.Sequential(nn.Linear(1000 + 250, output_size))\n        elif v19: \n            self.features = nn.Sequential(nn.Linear(1000 + 250, output_size))\n        else:\n            self.features = nn.Sequential(nn.Linear(1000 + 250, output_size))\n        \n    def forward(self, image, csv_data, prints=False):    \n        \n        if prints: print('Input Image shape:', image.shape, '\\n'+\n                         'Input csv_data shape:', csv_data.shape)\n        \n        # IMAGE CNN\n        image = self.features(image)\n        if prints: print('Features Image shape:', image.shape)\n            \n        # you might need to omit this part - I think it's specific for EfficientNet\n        '''if self.v16:\n            image = F.avg_pool2d(image, image.size()[2:]).reshape(-1, 1792)\n        elif self.v16_bn:\n            image = F.avg_pool2d(image, image.size()[2:]).reshape(-1, 1792)\n        elif self.v19:\n            image = F.avg_pool2d(image, image.size()[2:]).reshape(-1, 1792)\n        elif self.v19_bn:\n            image = F.avg_pool2d(image, image.size()[2:]).reshape(-1, 1792)\n        if prints: print('Image Reshaped shape:', image.shape)'''\n            \n        # CSV FNN\n        csv_data = self.csv(csv_data)\n        if prints: print('CSV Data:', csv_data.shape)\n            \n        # Concatenate\n        image_csv_data = torch.cat((image, csv_data), dim=1)\n        \n        # CLASSIF\n        out = self.classification(image_csv_data)\n        if prints: print('Out shape:', out.shape)\n        \n        return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_example = custom_vgg16(output_size=output_size, \n                      no_columns=no_columns, v16=True, v16_bn=False, v19=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Data object and Loader\nexample_data = MelanomaDataset(train_df, vertical_flip=0.5, horizontal_flip=0.5, \n                               is_train=True, is_valid=False, is_test=False)\nexample_loader = torch.utils.data.DataLoader(example_data, batch_size = 3, shuffle=True)\n\n# Get a sample\nfor (image, csv_data), labels in example_loader:\n    image_example, csv_data_example = image, csv_data\n    labels_example = torch.tensor(labels, dtype=torch.float32)\n    break\nprint('Data shape:', image_example.shape, '| \\n' , csv_data_example)\nprint('Label:', labels_example, '\\n')\n\n# Outputs\nout = model_example(image_example, csv_data_example, prints=True)\n\n# Criterion example\ncriterion_example = nn.BCEWithLogitsLoss()\n# Unsqueeze(1) from shape=[3] to shape=[3, 1]\nloss = criterion_example(out, labels_example.unsqueeze(1))   \nprint('Loss:', loss.item())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class InceptionV3(nn.Module):\n    def __init__(self, output_size, no_columns):\n        super().__init__()\n        self.no_columns, self.output_size = no_columns, output_size\n        # Define Feature part (IMAGE)\n        self.features = inception_v3(pretrained=True)\n        # (CSV data)\n        self.csv = nn.Sequential(nn.Linear(self.no_columns, 250),\n                                 nn.BatchNorm1d(250),\n                                 nn.ReLU(),\n                                 nn.Dropout(p=0.2),\n                                 \n                                 nn.Linear(250, 250),\n                                 nn.BatchNorm1d(250),\n                                 nn.ReLU(),\n                                 nn.Dropout(p=0.2))\n        # Define Classification part\n        self.classification = nn.Linear(1000 + 250, output_size)\n        \n        \n    def forward(self, image, csv_data, prints=False):\n        \n        if prints: print('Input Image shape:', image.shape, '\\n'+\n                         'Input csv_data shape:', csv_data.shape)\n        \n        # Image CNN\n        image = self.features(image)\n        if prints: print('Features Image shape:', image.shape)\n        \n        # CSV FNN\n        csv_data = self.csv(csv_data)\n        if prints: print('CSV Data:', csv_data.shape)\n            \n        # Concatenate layers from image with layers from csv_data\n        image_csv_data = torch.cat((image, csv_data), dim=1)\n        \n        # CLASSIF\n        out = self.classification(image_csv_data)\n        if prints: print('Out shape:', out.shape)\n        \n        return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_example = InceptionV3(output_size=output_size, \n                      no_columns=no_columns)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Data object and Loader\nexample_data = MelanomaDataset(train_df, vertical_flip=0.5, horizontal_flip=0.5, \n                               is_train=True, is_valid=False, is_test=False)\nexample_loader = torch.utils.data.DataLoader(example_data, batch_size = 3, shuffle=True)\n\n# Get a sample\nfor (image, csv_data), labels in example_loader:\n    image_example, csv_data_example = image, csv_data\n    labels_example = torch.tensor(labels, dtype=torch.float32)\n    break\nprint('Data shape:', image_example.shape, '| \\n' , csv_data_example)\nprint('Label:', labels_example, '\\n')\n\n# Outputs\nout = model_example(image_example, csv_data_example, prints=True)\n\n# Criterion example\ncriterion_example = nn.BCEWithLogitsLoss()\n# Unsqueeze(1) from shape=[3] to shape=[3, 1]\nloss = criterion_example(out, labels_example.unsqueeze(1))   \nprint('Loss:', loss.item())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ----- STATICS -----\ntrain_len = len(train_df)\ntest_len = len(test_df)\n# -------------------\n\n\n# Out of Fold Predictions\noof = np.zeros(shape = (train_len, 1))\n\n# Predictions\npreds_submission = torch.zeros(size = (test_len, 1), dtype=torch.float32, device=device)\n\nprint('oof shape:', oof.shape, '\\n' +\n      'predictions shape:', preds_submission.shape)\n# ----- STATICS -----\nK = 6              # number of folds in Group K Fold\n# -------------------\n# Create Object\ngroup_fold = GroupKFold(n_splits = K)\n\n# Generate indices to split data into training and test set.\nfolds = group_fold.split(X = np.zeros(train_len), \n                         y = train_df['target'], \n                         groups = train_df['ID'].tolist())\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ----- STATICS -----\nepochs = 20\npatience = 5\nTTA = 3\nnum_workers = 8\nlearning_rate = 0.0005\nweight_decay = 0.0\nlr_patience = 1            # 1 model not improving until lr is decreasing\nlr_factor = 0.4            # by how much the lr is decreasing\n\ntrain_batch_size = 16\nval_test_batch_size = 16\n\nversion = 'custom'             # to keep tabs on versions\n# -------------------","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_folds(preds_submission, model, version = 'v1'):\n    # Creates a .txt file that will contain the logs\n    f = open(f\"logs_{version}.txt\", \"a+\")\n    g = open(f\"preds_{version}.txt\", \"w+\")\n    \n    for fold, (train_index, valid_index) in enumerate(folds):\n        # Append to .txt\n        with open(f\"logs_{version}.txt\", 'a+') as f:\n            print('-'*10, 'Fold:', fold+1, '-'*10, file=f)\n        print('-'*10, 'Fold:', fold+1, '-'*10)\n\n\n        # --- Create Instances ---\n        # Best ROC score in this fold\n        best_roc = None\n        # Reset patience before every fold\n        patience_f = patience\n        \n        # Initiate the model\n        model = model\n\n        optimizer = torch.optim.Adam(model.parameters(), lr = learning_rate, weight_decay=weight_decay)\n        scheduler = ReduceLROnPlateau(optimizer=optimizer, mode='max', \n                                      patience=lr_patience, verbose=True, factor=lr_factor)\n        criterion = nn.BCEWithLogitsLoss()\n\n\n        # --- Read in Data ---\n        train_data = train_df.iloc[train_index].reset_index(drop=True)\n        valid_data = train_df.iloc[valid_index].reset_index(drop=True)\n\n        # Create Data instances\n        train = MelanomaDataset(train_data, vertical_flip=vertical_flip, horizontal_flip=horizontal_flip, \n                                is_train=True, is_valid=False, is_test=False)\n        valid = MelanomaDataset(valid_data, vertical_flip=vertical_flip, horizontal_flip=horizontal_flip, \n                                is_train=False, is_valid=True, is_test=False)\n        # Read in test data | Remember! We're using data augmentation like we use for Train data.\n        test = MelanomaDataset(test_df, vertical_flip=vertical_flip, horizontal_flip=horizontal_flip,\n                               is_train=False, is_valid=False, is_test=True)\n\n        # Dataloaders\n        train_loader = DataLoader(train, batch_size=train_batch_size, shuffle=True, num_workers=num_workers)\n        # shuffle=False! Otherwise function won't work!!!\n                # how do I know? ^^\n        valid_loader = DataLoader(valid, batch_size=val_test_batch_size, shuffle=False, num_workers=num_workers)\n        test_loader = DataLoader(test, batch_size=val_test_batch_size, shuffle=False, num_workers=num_workers)\n\n\n        # === EPOCHS ===\n        for epoch in range(epochs):\n            start_time = time.time()\n            correct = 0\n            train_losses = 0\n\n            # === TRAIN ===\n            # Sets the module in training mode.\n            model.train()\n\n            for (images, csv_data), labels in train_loader:\n                # Save them to device\n                images = torch.tensor(images, device=device, dtype=torch.float32)\n                csv_data = torch.tensor(csv_data, device=device, dtype=torch.float32)\n                labels = torch.tensor(labels, device=device, dtype=torch.float32)\n\n                # Clear gradients first; very important, usually done BEFORE prediction\n                optimizer.zero_grad()\n\n                # Log Probabilities & Backpropagation\n                \n                out = model(images, csv_data)\n\n                loss = criterion(out, labels.unsqueeze(1))\n                loss.backward()\n                optimizer.step()\n\n                # --- Save information after this batch ---\n                # Save loss\n                train_losses += loss.item()\n                # From log probabilities to actual probabilities\n                train_preds = torch.round(torch.sigmoid(out)) # 0 and 1\n                # Number of correct predictions\n                correct += (train_preds.cpu() == labels.cpu().unsqueeze(1)).sum().item()\n\n            # Compute Train Accuracy\n            train_acc = correct / len(train_index)\n\n\n            # === EVAL ===\n            # Sets the model in evaluation mode\n            model.eval()\n\n            # Create matrix to store evaluation predictions (for accuracy)\n            valid_preds = torch.zeros(size = (len(valid_index), 1), device=device, dtype=torch.float32)\n\n\n            # Disables gradients (we need to be sure no optimization happens)\n            with torch.no_grad():\n                for k, ((images, csv_data), labels) in enumerate(valid_loader):\n                    images = torch.tensor(images, device=device, dtype=torch.float32)\n                    csv_data = torch.tensor(csv_data, device=device, dtype=torch.float32)\n                    labels = torch.tensor(labels, device=device, dtype=torch.float32)\n\n                    out = model(images, csv_data)\n                    pred = torch.sigmoid(out)\n                    valid_preds[k*images.shape[0] : k*images.shape[0] + images.shape[0]] = pred\n\n                # Compute accuracy\n                valid_acc = accuracy_score(valid_data['target'].values, \n                                           torch.round(valid_preds.cpu()))\n                # Compute ROC\n                valid_roc = roc_auc_score(valid_data['target'].values, \n                                          valid_preds.cpu())\n\n                # Compute time on Train + Eval\n                duration = str(datetime.timedelta(seconds=time.time() - start_time))[:7]\n\n\n                # PRINT INFO\n                # Append to .txt file\n                with open(f\"logs_{version}.txt\", 'a+') as f:\n                    print('{} | Epoch: {}/{} | Loss: {:.4} | Train Acc: {:.3} | Valid Acc: {:.3} | ROC: {:.3}'.\\\n                     format(duration, epoch+1, epochs, train_losses, train_acc, valid_acc, valid_roc), file=f)\n                # Print to console\n                print('{} | Epoch: {}/{} | Loss: {:.4} | Train Acc: {:.3} | Valid Acc: {:.3} | ROC: {:.3}'.\\\n                     format(duration, epoch+1, epochs, train_losses, train_acc, valid_acc, valid_roc))\n\n\n                # === SAVE MODEL ===\n\n                # Update scheduler (for learning_rate)\n                scheduler.step(valid_roc)\n\n                # Update best_roc\n                if not best_roc: # If best_roc = None\n                    best_roc = valid_roc\n                    torch.save(model.state_dict(),\n                               f\"Fold{fold+1}_Epoch{epoch+1}_ValidAcc_{valid_acc:.3f}_ROC_{valid_roc:.3f}.pth\")\n                    continue\n\n                if valid_roc > best_roc:\n                    best_roc = valid_roc\n                    # Reset patience (because we have improvement)\n                    patience_f = patience\n                    torch.save(model.state_dict(),\n                               f\"Fold{fold+1}_Epoch{epoch+1}_ValidAcc_{valid_acc:.3f}_ROC_{valid_roc:.3f}.pth\")\n                else:\n                    # Decrease patience (no improvement in ROC)\n                    patience_f = patience_f - 1\n                    if patience_f == 0:\n                        with open(f\"logs_{version}.txt\", 'a+') as f:\n                            print('Early stopping (no improvement since 3 models) | Best ROC: {}'.\\\n                                  format(best_roc), file=f)\n                        print('Early stopping (no improvement since 3 models) | Best ROC: {}'.\\\n                              format(best_roc))\n                        break\n\n\n        # === INFERENCE ===\n        # Choose model with best_roc in this fold\n        best_model_path = '../working/' + [file for file in os.listdir('../working') if str(round(best_roc, 3)) in file and 'Fold'+str(fold+1) in file][0]\n        # Using best model from Epoch Train\n        # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n        # Set the model in evaluation mode\n        model.eval()\n\n\n        with torch.no_grad():\n            # --- EVAL ---\n            # Predicting again on Validation data to get preds for OOF\n            valid_preds = torch.zeros(size = (len(valid_index), 1), device=device, dtype=torch.float32)\n\n            for k, ((images, csv_data), _) in enumerate(valid_loader):\n                images = torch.tensor(images, device=device, dtype=torch.float32)\n                csv_data = torch.tensor(csv_data, device=device, dtype=torch.float32)\n\n                out = model(images, csv_data)\n                pred = torch.sigmoid(out)\n                valid_preds[k*images.shape[0] : k*images.shape[0] + images.shape[0]] = pred\n\n            # Save info to OOF\n            oof[valid_index] = valid_preds.cpu().numpy()\n\n\n            # --- TEST ---\n            # Now (Finally) prediction for our TEST data\n            # for i in range(TTA):\n            \n            \n            \n            # Divide Predictions by TTA (to average the results during TTA)\n            # preds_submission /= TTA\n\n\n        # === CLEANING ===\n        # Clear memory\n        del train, valid, train_loader, valid_loader, images, labels\n        # Garbage collector\n        gc.collect()\n        \n    for k, (images, csv_data) in enumerate(test_loader):\n        images = torch.tensor(images, device=device, dtype=torch.float32)\n        csv_data = torch.tensor(csv_data, device=device, dtype=torch.float32)\n\n        out = model(images, csv_data)\n        \n        # Covert to probablities\n        out = torch.sigmoid(out)\n        \n        with open(f\"preds_{version}.txt\", 'a+') as g:\n            arr = out.data.cpu().numpy()\n            np.savetxt(g, arr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# --- VGG ---\nmodel = custom_vgg16(output_size = output_size, no_columns=no_columns, v16=True).to(device)\n\n# ===== Uncomment and Train =====\nversion = 'VGG'\ntrain_folds(preds_submission = preds_submission, model = model, version = version)\n\n# Save OOF values\n# save_oof = pd.DataFrame(data = oof, columns=['oof'])\n# save_oof.to_csv(f'oof_{version}.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# --- Inception-V3 ---\nmodel = InceptionV3(output_size = output_size, no_columns=no_columns).to(device)\n\n# ===== Uncomment and Train =====\nversion = 'InceptionV3'\ntrain_folds(preds_submission = preds_submission, model = model, version = version)\n\n# # Save OOF values\nsave_oof = pd.DataFrame(data = oof, columns=['oof'])\nsave_oof.to_csv(f'oof_{version}.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}