{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Melanoma Classification\n\nFirst try at the competition. Most of the code here is not mine.\n\nNotebook based on [Melanoma Competiton: Aug + ResNet/EffNet](https://www.kaggle.com/andradaolteanu/melanoma-competiton-aug-resnet-effnet-lb-0-91#3.-Neural-Networks-%F0%9F%8E%87)\n(Thanks Andrada)\n\nPending:\n\n* Fix RAM super usage problem\n* Train networks using only images, take embedding vector to use in other model\n* Experiment with metadata preprocessing steps\n* Explore different data augmentation strategies with albumentation\n* Explore different parameters combinations - Is grid search possible?\n","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Import libraries","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import seaborn as sns\nimport cv2\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset, DataLoader, Subset\nfrom PIL import Image\nimport pydicom\nfrom skimage.transform import resize\nimport os\nimport numpy as np\nimport pandas as pd\nimport random\nimport time","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Load metadata","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/train.csv')\ntest_df = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"directory = '/kaggle/input/siim-isic-melanoma-classification'\n\n# === JPEG ===\n# Create the paths\npath_train = directory + '/jpeg/train/' + train_df['image_name'] + '.jpg'\npath_test = directory + '/jpeg/test/' + test_df['image_name'] + '.jpg'\n\n# Append to the original dataframes\ntrain_df['path_jpeg'] = path_train\ntest_df['path_jpeg'] = path_test","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Show image samples","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# # === DICOM ===\n# # Create the paths\n# path_train = directory + '/train/' + train_df['image_name'] + '.dcm'\n# path_test = directory + '/test/' + test_df['image_name'] + '.dcm'\n\n# # Append to the original dataframes\n# train_df['path_dicom'] = path_train\n# test_df['path_dicom'] = path_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_images(data, n = 5, rows=1, cols=5, title='Default'):\n    plt.figure(figsize=(16,4))\n\n    for k, path in enumerate(data['path_dicom'][:n]):\n        image = pydicom.read_file(path)\n        image = image.pixel_array\n        \n        image = resize(image, (200, 200), anti_aliasing=True)\n\n        plt.suptitle(title, fontsize = 16)\n        plt.subplot(rows, cols, k+1)\n        plt.imshow(image)\n        plt.axis('off')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"shapes_train = []\n\nfor k, path in enumerate(train_df['path_jpeg']):\n    image = Image.open(path)\n    shapes_train.append(image.size)\n    \n    if k >= 100: break\n        \nshapes_train = pd.DataFrame(data = shapes_train, columns = ['H', 'W'], dtype='object')\nshapes_train['Size'] = '[' + shapes_train['H'].astype(str) + ', ' + shapes_train['W'].astype(str) + ']'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Show Benign Samples\nshow_images(train_df[train_df['target'] == 1], n=10, rows=2, cols=5, title='Benign Sample')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Load sklearn and pytorch libraries","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Libraries\n\nfrom sklearn.preprocessing import OneHotEncoder, StandardScaler\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.model_selection import GroupKFold","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q efficientnet_pytorch   \n\nimport torch\nimport torchvision\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch import FloatTensor, LongTensor\nfrom torch.utils.data import Dataset, DataLoader, Subset\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\n\n# Data Augmentation for Image Preprocessing\nfrom albumentations import (ToFloat, Normalize, VerticalFlip, HorizontalFlip, Compose, Resize,\n                            RandomBrightness, RandomContrast, HueSaturationValue, Blur, GaussNoise,\n                            Rotate, RandomResizedCrop, Cutout)\nfrom albumentations.pytorch import ToTensorV2, ToTensor\n\nfrom efficientnet_pytorch import EfficientNet\nfrom torchvision.models import resnet34, resnet50\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 1. Preprocessing: \n\nThis part of the code was written by me (based on Kaggle's micro-courses)\n\n* Fill missing values\n* OneHotEncoding\n","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Data preprocessing steps - example:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"X = train_df.drop(['diagnosis', 'benign_malignant'], axis=1)\ny = train_df['target']\n\ntrain_X, val_X, train_y, val_y = train_test_split(X, y, random_state=1)\n\ncategorical_transformer = Pipeline(steps=[\n    ('imputer', SimpleImputer(strategy='most_frequent')),\n    ('onehot', OneHotEncoder(handle_unknown='ignore', sparse=False))\n])\n\nnumerical_transformer = Pipeline(steps=[\n    ('imputer', SimpleImputer()),\n    ('scaler', StandardScaler())\n])\n\ncat_cols = ['sex', 'anatom_site_general_challenge']\n\npreprocessor = ColumnTransformer(\n    transformers=[\n        ('num_transform', numerical_transformer, ['age_approx']),\n        ('cat_transform', categorical_transformer, cat_cols)\n    ], remainder='passthrough')\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Use following functions to clean data inside each fold!","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocessor():\n    categorical_transformer = Pipeline(steps=[\n        ('imputer', SimpleImputer(strategy='most_frequent')),\n        ('onehot', OneHotEncoder(handle_unknown='ignore', sparse=False))\n    ])\n    cat_cols = ['sex', 'anatom_site_general_challenge']\n\n    numerical_transformer = Pipeline(steps=[\n        ('imputer', SimpleImputer()),\n        ('scaler', StandardScaler())\n    ])\n\n    preprocessor = ColumnTransformer(\n        transformers=[\n            ('num_transform', numerical_transformer, ['age_approx']),\n            ('cat_transform', categorical_transformer, cat_cols)\n        ], remainder='passthrough')\n\n    return preprocessor\n\ndef clean_dataframe(dataframe, preprocessor, is_train=True, is_valid=False):\n    \n    cols = ['age_approx', 'sex', 'anatom_site_general_challenge', 'path_jpeg']\n        \n    if is_train:\n        df = pd.DataFrame(preprocessor.fit_transform(dataframe[cols]))\n    else:\n        df = pd.DataFrame(preprocessor.transform(dataframe[cols]))\n        \n    df.columns = [*df.columns[:-1], 'path_jpeg']\n    if is_train or is_valid:\n        df['target'] = dataframe.reset_index()['target']\n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_X, val_X, train_y, val_y = train_test_split(train_df, train_df['target'], random_state=1)\ntransformer = preprocessor()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clean_train_X = clean_dataframe(train_X, transformer, is_train=True, is_valid=False)\nclean_val_X = clean_dataframe(val_X, transformer, is_train=False, is_valid=True)\nclean_test = clean_dataframe(test_df, transformer, is_train=False, is_valid=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Class to load and prepare images and metadata\n\nThis class was changed a little bit. It uses all columns from the metadata dataframe, which should have been cleaned beforehand.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class MelanomaDataset(Dataset):\n    \n    def __init__(self, dataframe, vertical_flip, horizontal_flip,\n                 is_train=True, is_valid=False, is_test=False):\n        self.dataframe, self.is_train, self.is_valid = dataframe, is_train, is_valid\n        self.vertical_flip, self.horizontal_flip = vertical_flip, horizontal_flip\n        \n        # Data Augmentation (custom for each dataset type)\n        if is_train or is_test:\n            self.transform = Compose([RandomResizedCrop(height=224, width=224, scale=(0.7, 1.0)),\n                                      HorizontalFlip(p = self.horizontal_flip),\n                                      VerticalFlip(p = self.vertical_flip),\n                                      Cutout(),\n                                      HueSaturationValue(),\n                                      Normalize(),\n                                      ToTensor()])\n        else:\n            self.transform = Compose([Normalize(),\n                                      ToTensor()])\n            \n    def __len__(self):\n        return len(self.dataframe)\n    \n    def __getitem__(self, index):\n        # Select path and read image\n        image_path = self.dataframe['path_jpeg'][index]\n        image = cv2.imread(image_path)\n        # For this image also import .csv information (sex, age, anatomy)\n        csv_data = np.array(self.dataframe.drop(['path_jpeg', 'target'], axis=1).iloc[index].values, \n                            dtype=np.float32)\n        \n        # Apply transforms\n        image = self.transform(image=image)\n        # Extract image from dictionary\n        image = image['image']\n        \n        # If train/valid: image + class | If test: only image\n        if self.is_train or self.is_valid:\n            return (image, csv_data), self.dataframe['target'][index]\n        else:\n            return (image, csv_data)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 2. Models\n\nExamples of pretrained networks using metadata:\n\n* Resnet50 (probably will be used only as a baseline for the EfficientNet models)\n* EfficientNet (multiple versions)\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class ResNet50Network(nn.Module):\n    def __init__(self, output_size, no_columns):\n        super().__init__()\n        self.no_columns, self.output_size = no_columns, output_size\n        \n        # Define Feature part (IMAGE)\n        self.features = resnet50(pretrained=True) # 1000 neurons out\n        # (CSV data)\n        self.csv = nn.Sequential(nn.Linear(self.no_columns, 500),\n                                 nn.BatchNorm1d(500),\n                                 nn.ReLU(),\n                                 nn.Dropout(p=0.2))\n        \n        # Define Classification part\n        self.classification = nn.Linear(1000 + 500, output_size)\n        \n    def forward(self, image, csv_data, prints=True):\n        \n        if prints: print('Input Image shape:', image.shape, '\\n'+\n                         'Input csv_data shape:', csv_data.shape)\n        \n        # Image CNN\n        image = self.features(image)\n        if prints: print('Features Image shape:', image.shape)\n        \n        # CSV FNN\n        csv_data = self.csv(csv_data)\n        if prints: print('CSV Data:', csv_data.shape)\n            \n        # Concatenate layers from image with layers from csv_data\n        image_csv_data = torch.cat((image, csv_data), dim=1)\n        \n        # CLASSIF\n        out = self.classification(image_csv_data)\n        if prints: print('Out shape:', out.shape)\n        \n        return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ----- STATICS -----\nvertical_flip = 0.5\nhorizontal_flip = 0.5\n# ------------------","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Data object and Loader\n\ntransformer = preprocessor()\ntrain_X_clean = clean_dataframe(train_X, transformer)\n\nexample_data = MelanomaDataset(train_X_clean, vertical_flip=0.5, horizontal_flip=0.5, \n                               is_train=True, is_valid=False, is_test=False)\nexample_loader = torch.utils.data.DataLoader(example_data, batch_size = 3, shuffle=True)\n\nno_columns = len(example_data.dataframe.columns)-2\nmodel_example = ResNet50Network(output_size=1, no_columns=no_columns)\n\n# Get a sample\nfor (image, csv_data), labels in example_loader:\n    image_example, csv_data_example = image, csv_data\n    labels_example = torch.tensor(labels, dtype=torch.float32)\n    break\nprint('Data shape:', image_example.shape, '| \\n' , csv_data_example)\nprint('Label:', labels_example, '\\n')\n\n# Outputs\nout = model_example(image_example, csv_data_example, prints=True)\n\n# Criterion example\ncriterion_example = nn.BCEWithLogitsLoss()\n# Unsqueeze(1) from shape=[3] to shape=[3, 1]\nloss = criterion_example(out, labels_example.unsqueeze(1))   \nprint('Loss:', loss.item())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class EfficientNetwork(nn.Module):\n    def __init__(self, output_size, no_columns, b4=False, b2=False):\n        super().__init__()\n        self.b4, self.b2, self.no_columns = b4, b2, no_columns\n        \n        # Define Feature part (IMAGE)\n        if b4:\n            self.features = EfficientNet.from_pretrained('efficientnet-b4')\n        elif b2:\n            self.features = EfficientNet.from_pretrained('efficientnet-b2')\n        else:\n            self.features = EfficientNet.from_pretrained('efficientnet-b7')\n        \n        # (CSV)\n        self.csv = nn.Sequential(nn.Linear(self.no_columns, 250),\n                                 nn.BatchNorm1d(250),\n                                 nn.ReLU(),\n                                 nn.Dropout(p=0.2),\n                                 \n                                 nn.Linear(250, 250),\n                                 nn.BatchNorm1d(250),\n                                 nn.ReLU(),\n                                 nn.Dropout(p=0.2))\n        \n        # Define Classification part\n        if b4:\n            self.classification = nn.Sequential(nn.Linear(1792 + 250, output_size))\n        elif b2:\n            self.classification = nn.Sequential(nn.Linear(1408 + 250, output_size))\n        else:\n            self.classification = nn.Sequential(nn.Linear(2560 + 250, output_size))\n        \n        \n    def forward(self, image, csv_data, prints=False):    \n        \n        if prints: print('Input Image shape:', image.shape, '\\n'+\n                         'Input csv_data shape:', csv_data.shape)\n        \n        # IMAGE CNN\n        image = self.features.extract_features(image)\n        if prints: print('Features Image shape:', image.shape)\n            \n        if self.b4:\n            image = F.avg_pool2d(image, image.size()[2:]).reshape(-1, 1792)\n        elif self.b2:\n            image = F.avg_pool2d(image, image.size()[2:]).reshape(-1, 1408)\n        else:\n            image = F.avg_pool2d(image, image.size()[2:]).reshape(-1, 2560)\n        if prints: print('Image Reshaped shape:', image.shape)\n            \n        # CSV FNN\n        csv_data = self.csv(csv_data)\n        if prints: print('CSV Data:', csv_data.shape)\n            \n        # Concatenate\n        image_csv_data = torch.cat((image, csv_data), dim=1)\n        \n        # CLASSIF\n        out = self.classification(image_csv_data)\n        if prints: print('Out shape:', out.shape)\n        \n        return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Data object and Loader\ntransformer = preprocessor()\ntrain_X_clean = clean_dataframe(train_X, transformer)\n\nexample_data = MelanomaDataset(train_X_clean, vertical_flip=0.5, horizontal_flip=0.5, \n                               is_train=True, is_valid=False, is_test=False)\nexample_loader = torch.utils.data.DataLoader(example_data, batch_size = 3, shuffle=True)\nno_columns = len(example_data.dataframe.columns)-2\n\n# Create an example model - Effnet\nmodel_example = EfficientNetwork(output_size=1, no_columns=no_columns,\n                                 b4=False, b2=False)\n# Get a sample\nfor (image, csv_data), labels in example_loader:\n    image_example, csv_data_example = image, csv_data\n    labels_example = torch.tensor(labels, dtype=torch.float32)\n    break\nprint('Data shape:', image_example.shape, '| \\n' , csv_data_example)\nprint('Label:', labels_example, '\\n')\n\n# Outputs\nout = model_example(image_example, csv_data_example, prints=True)\n\n# Criterion example\ncriterion_example = nn.BCEWithLogitsLoss()\n# Unsqueeze(1) from shape=[3] to shape=[3, 1]\nloss = criterion_example(out, labels_example.unsqueeze(1))   \nprint('Loss:', loss.item())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## 3. Training \n\nThis part is mostly just copy-paste from Andrada's notebook. \n\n* Cross Validation\n* Save models","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def set_seed(seed = 1234):\n    '''Sets the seed of the entire notebook so results are the same every time we run.\n    This is for REPRODUCIBILITY.'''\n    np.random.seed(seed)\n    random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    # When running on the CuDNN backend, two further options must be set\n    torch.backends.cudnn.deterministic = True\n    # Set a fixed value for the hash seed\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \nset_seed()\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint('Device available now:', device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_folds(preds_submission, model, version = 'v1'):\n    # Creates a .txt file that will contain the logs\n    f = open(f\"logs_{version}.txt\", \"w+\")\n    \n    \n    for fold, (train_index, valid_index) in enumerate(list(folds)[:2]):\n        # Append to .txt\n        with open(f\"logs_{version}.txt\", 'a+') as f:\n            print('-'*10, 'Fold:', fold+1, '-'*10, file=f)\n        print('-'*10, 'Fold:', fold+1, '-'*10)\n\n\n        # --- Create Instances ---\n        # Best ROC score in this fold\n        best_roc = None\n        # Reset patience before every fold\n        patience_f = patience\n        \n        # Initiate the model\n        model = model\n\n        optimizer = torch.optim.Adam(model.parameters(), lr = learning_rate, weight_decay=weight_decay)\n        scheduler = ReduceLROnPlateau(optimizer=optimizer, mode='max', \n                                      patience=lr_patience, verbose=True, factor=lr_factor)\n        criterion = nn.BCEWithLogitsLoss()\n\n\n        # --- Read in Data ---\n        print('Reading data ... \\n')\n        train_data = train_df.iloc[train_index].reset_index(drop=True)\n        valid_data = train_df.iloc[valid_index].reset_index(drop=True)\n\n        transformer = preprocessor()\n        train_data_clean = clean_dataframe(train_data, transformer)\n        valid_data_clean = clean_dataframe(valid_data, transformer, is_train=False, is_valid=True)\n        \n        # Create Data instances\n        train = MelanomaDataset(train_data_clean, vertical_flip=vertical_flip, horizontal_flip=horizontal_flip, \n                                is_train=True, is_valid=False, is_test=False)\n        valid = MelanomaDataset(valid_data_clean, vertical_flip=vertical_flip, horizontal_flip=horizontal_flip, \n                                is_train=False, is_valid=True, is_test=False)\n        # Read in test data | Remember! We're using data augmentation like we use for Train data.\n        \n        test_clean = clean_dataframe(valid_data, transformer, is_train=False, is_valid=False)\n        test = MelanomaDataset(test_clean, vertical_flip=vertical_flip, horizontal_flip=horizontal_flip,\n                               is_train=False, is_valid=False, is_test=True)\n\n        # Dataloaders\n        train_loader = DataLoader(train, batch_size=batch_size1, shuffle=True, num_workers=num_workers)\n        # shuffle=False! Otherwise function won't work!!!\n                # how do I know? ^^\n        valid_loader = DataLoader(valid, batch_size=batch_size2, shuffle=False, num_workers=num_workers)\n        test_loader = DataLoader(test, batch_size=batch_size2, shuffle=False, num_workers=num_workers)\n\n        print('Starting training ...\\n')\n        # === EPOCHS ===\n        for epoch in range(epochs):\n            print(f\"Starting epoch {epoch+1} ... \\n\")\n            start_time = time.time()\n            print(f\"Time: {start_time}\")\n            correct = 0\n            train_losses = 0\n\n            # === TRAIN ===\n            # Sets the module in training mode.\n            model.train()\n\n            for (images, csv_data), labels in train_loader:\n                # Save them to device\n                images = torch.tensor(images, device=device, dtype=torch.float32)\n                csv_data = torch.tensor(csv_data, device=device, dtype=torch.float32)\n                labels = torch.tensor(labels, device=device, dtype=torch.float32)\n\n                # Clear gradients first; very important, usually done BEFORE prediction\n                optimizer.zero_grad()\n\n                # Log Probabilities & Backpropagation\n                out = model(images, csv_data)\n                loss = criterion(out, labels.unsqueeze(1))\n                loss.backward()\n                optimizer.step()\n\n                # --- Save information after this batch ---\n                # Save loss\n                train_losses += loss.item()\n                # From log probabilities to actual probabilities\n                train_preds = torch.round(torch.sigmoid(out)) # 0 and 1\n                # Number of correct predictions\n                correct += (train_preds.cpu() == labels.cpu().unsqueeze(1)).sum().item()\n\n            # Compute Train Accuracy\n            train_acc = correct / len(train_index)\n\n            print('Starting evaluation for epoch ... \\n')\n            # === EVAL ===\n            # Sets the model in evaluation mode\n            model.eval()\n\n            # Create matrix to store evaluation predictions (for accuracy)\n            valid_preds = torch.zeros(size = (len(valid_index), 1), device=device, dtype=torch.float32)\n\n            # Disables gradients (we need to be sure no optimization happens)\n            with torch.no_grad():\n                for k, ((images, csv_data), labels) in enumerate(valid_loader):\n                    images = torch.tensor(images, device=device, dtype=torch.float32)\n                    csv_data = torch.tensor(csv_data, device=device, dtype=torch.float32)\n                    labels = torch.tensor(labels, device=device, dtype=torch.float32)\n\n                    out = model(images, csv_data)\n                    pred = torch.sigmoid(out)\n                    valid_preds[k*images.shape[0] : k*images.shape[0] + images.shape[0]] = pred\n\n                # Compute accuracy\n                valid_acc = accuracy_score(valid_data['target'].values, \n                                           torch.round(valid_preds.cpu()))\n                # Compute ROC\n                valid_roc = roc_auc_score(valid_data['target'].values, \n                                          valid_preds.cpu())\n\n                # Compute time on Train + Eval\n                duration = str(datetime.timedelta(seconds=time.time() - start_time))[:7]\n\n\n                # PRINT INFO\n                # Append to .txt file\n                with open(f\"logs_{version}.txt\", 'a+') as f:\n                    print('{} | Epoch: {}/{} | Loss: {:.4} | Train Acc: {:.3} | Valid Acc: {:.3} | ROC: {:.3}'.\\\n                     format(duration, epoch+1, epochs, train_losses, train_acc, valid_acc, valid_roc), file=f)\n                # Print to console\n                print('{} | Epoch: {}/{} | Loss: {:.4} | Train Acc: {:.3} | Valid Acc: {:.3} | ROC: {:.3}'.\\\n                     format(duration, epoch+1, epochs, train_losses, train_acc, valid_acc, valid_roc))\n\n\n                # === SAVE MODEL ===\n\n                # Update scheduler (for learning_rate)\n                scheduler.step(valid_roc)\n\n                # Update best_roc\n                if not best_roc: # If best_roc = None\n                    best_roc = valid_roc\n                    torch.save(model.state_dict(),\n                               f\"Fold{fold+1}_Epoch{epoch+1}_ValidAcc_{valid_acc:.3f}_ROC_{valid_roc:.3f}.pth\")\n                    continue\n\n                if valid_roc > best_roc:\n                    best_roc = valid_roc\n                    # Reset patience (because we have improvement)\n                    patience_f = patience\n                    torch.save(model.state_dict(),\n                               f\"Fold{fold+1}_Epoch{epoch+1}_ValidAcc_{valid_acc:.3f}_ROC_{valid_roc:.3f}.pth\")\n                else:\n                    # Decrease patience (no improvement in ROC)\n                    patience_f = patience_f - 1\n                    if patience_f == 0:\n                        with open(f\"logs_{version}.txt\", 'a+') as f:\n                            print('Early stopping (no improvement since 3 models) | Best ROC: {}'.\\\n                                  format(best_roc), file=f)\n                        print('Early stopping (no improvement since 3 models) | Best ROC: {}'.\\\n                              format(best_roc))\n                        break\n\n\n        # === INFERENCE ===\n        # Choose model with best_roc in this fold\n        best_model_path = '../working/' + [file for file in os.listdir('../working') if str(round(best_roc, 3)) in file and 'Fold'+str(fold+1) in file][0]\n        # Using best model from Epoch Train\n        # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n        model = EfficientNetwork(output_size = output_size, no_columns=no_columns,\n                         b4=False, b2=True).to(device)\n        model.load_state_dict(torch.load(best_model_path))\n        # Set the model in evaluation mode\n        model.eval()\n\n\n        with torch.no_grad():\n            # --- EVAL ---\n            # Predicting again on Validation data to get preds for OOF\n            valid_preds = torch.zeros(size = (len(valid_index), 1), device=device, dtype=torch.float32)\n\n            for k, ((images, csv_data), _) in enumerate(valid_loader):\n                images = torch.tensor(images, device=device, dtype=torch.float32)\n                csv_data = torch.tensor(csv_data, device=device, dtype=torch.float32)\n\n                out = model(images, csv_data)\n                pred = torch.sigmoid(out)\n                valid_preds[k*images.shape[0] : k*images.shape[0] + images.shape[0]] = pred\n\n            # Save info to OOF\n            oof[valid_index] = valid_preds.cpu().numpy()\n\n\n            # --- TEST ---\n            # Now (Finally) prediction for our TEST data\n            for i in range(TTA):\n                for k, (images, csv_data) in enumerate(test_loader):\n                    images = torch.tensor(images, device=device, dtype=torch.float32)\n                    csv_data = torch.tensor(csv_data, device=device, dtype=torch.float32)\n\n                    out = model(images, csv_data)\n                    # Covert to probablities\n                    out = torch.sigmoid(out)\n\n                    # ADDS! the prediction to the matrix we already created\n                    preds_submission[k*images.shape[0] : k*images.shape[0] + images.shape[0]] += out\n\n\n            # Divide Predictions by TTA (to average the results during TTA)\n            preds_submission /= TTA\n\n\n        # === CLEANING ===\n        # Clear memory\n        del train, valid, train_loader, valid_loader, images, labels\n        # Garbage collector\n        gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ----- STATICS -----\noutput_size=1\nepochs = 15\npatience = 3\nTTA = 3\nnum_workers = 8\nlearning_rate = 0.0005\nweight_decay = 0.0\nlr_patience = 1 \nlr_factor = 0.4            \n\nbatch_size1 = 16\nbatch_size2 = 16\n\nversion = 'v1'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ----- STATICS -----\ntrain_len = len(train_df)\ntest_len = len(test_df)\n# -------------------\n\n\n# Out of Fold Predictions\noof = np.zeros(shape = (train_len, 1))\n\n# Predictions\npreds_submission = torch.zeros(size = (test_len, 1), dtype=torch.float32, device=device)\n\nprint('oof shape:', oof.shape, '\\n' +\n      'predictions shape:', preds_submission.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"k = 5\n\n# Create Object\ngroup_fold = GroupKFold(n_splits = k)\n\n# Generate indices to split data into training and test set.\nfolds = group_fold.split(X = np.zeros(train_len), \n                         y = train_df['target'], \n                         groups = train_df['patient_id'].tolist())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# --- EffNet B2 ---\nmodel = ResNet50Network(output_size = output_size, no_columns=9).to(device)\n\n# ===== Uncomment and Train =====\ntrain_folds(preds_submission = preds_submission, model = model, version = version)\n\n# Save OOF values\nsave_oof = pd.DataFrame(data = oof, columns=['oof'])\nsave_oof.to_csv(f'oof_{version}.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f = open('./logs_v1.txt', \"r\")\ncontents = f.read()\nprint(contents)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}