{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# NOTE!\n\nMaking this notebook public just to recieve some feedback on my approach. I am facing some issues with validation f1_score during training and after training.","metadata":{}},{"cell_type":"markdown","source":"## Import Packages","metadata":{}},{"cell_type":"code","source":"from typing import List, Dict\nimport os\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\n\nimport tqdm\n\nimport cv2\nimport albumentations as A\nfrom albumentations.core.composition import Compose\nfrom albumentations.pytorch import ToTensorV2\n\nfrom torch.utils.data import Dataset, TensorDataset, DataLoader\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn import metrics\n\nimport pytorch_lightning as pl\nfrom pytorch_lightning import Trainer\nfrom pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping\nfrom pytorch_lightning.metrics import FBeta\n\nimport torch\nimport torchvision.models as models\nfrom torch import nn\nfrom torch.optim import AdamW, Adam\nimport torch.nn.functional as F","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Directories","metadata":{}},{"cell_type":"code","source":"ROOT_DIR = '../input/plant-pathology-2021-fgvc8/'\nTRAIN_CSV = 'train.csv'\nTRAIN_IMAGES_FOLDER = 'train_images'\nTEST_IMAGES_FOLDER = 'test_images'\nSAMPLE_SUBMISSION_CSV = 'sample_submission.csv'\n# though the name containes resnet18, it's actually resnet34.\nCKPT_PATHS = ['../input/multi-label-classification-training-with-pl/lightning_logs/version_0/checkpoints/resnet18-foldnum-0_epoch=5_valid_loss_epoch=0.1809_f1_valid_epoch=0.7877.ckpt']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Configurations","metadata":{}},{"cell_type":"code","source":"RANDOM_SEED = 42\n# Set seed for everythin(numpy, torch and python)\n\nfrom pytorch_lightning import seed_everything\nseed_everything(RANDOM_SEED)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"configurations = {\n    \"BATCH_SIZE\": 32,\n    \"NUM_WORKERS\": 4,\n    \"IMAGE_HEIGHT\": 334, \n    \"IMAGE_WIDTH\": 334,\n    \"LEARNING_RATE\": 0.003,\n    \"MAX_EPOCHS\": 6,\n    \"FOLD_NUM\": 0,\n}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prepare Dataset","metadata":{}},{"cell_type":"code","source":"dataset_df = pd.read_csv(os.path.join(ROOT_DIR, TRAIN_CSV))\ndataset_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_single_labels(unique_labels) -> List[str]:\n    \"\"\"Splitting multi-labels and returning a list of classes\"\"\"\n    single_labels = []\n    for label in unique_labels:\n        single_labels += label.split()\n        \n    single_labels = set(single_labels)\n    \n    return list(single_labels)\n\n\ndef get_one_hot_encoded_dataframe(dataset_df):\n    # copy dataframe\n    dataset_df_copy = dataset_df.copy()\n    \n    unique_labels = dataset_df_copy.labels.unique()\n    \n    new_column_names = get_single_labels(unique_labels)\n    # initialize columns with zero\n    dataset_df_copy[new_column_names] = 0        \n    \n    # one-hot-encoding using the column names\n    for label in unique_labels:                \n        label_indices = dataset_df_copy[dataset_df_copy['labels'] == label].index\n        splited_labels = label.split()\n        dataset_df_copy.loc[label_indices, splited_labels] = 1\n    \n    return dataset_df_copy","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_df_copy = get_one_hot_encoded_dataframe(dataset_df)\ndataset_df_copy.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prepare Dataset Class","metadata":{}},{"cell_type":"code","source":"class ImageDataset(Dataset):\n    \"\"\" Leaf Disease Dataset \"\"\"\n    def __init__(self,\n                image_names: List[str],\n                labels: List[List[int]],\n                image_dir: str, \n                transforms):        \n        self.image_names = image_names\n        self.image_dir = image_dir\n        self.transforms = transforms                \n        self.labels = labels\n\n\n    def __len__(self) -> int:\n        return len(self.image_names)\n\n    def __getitem__(self, idx: int):\n        image_path = os.path.join(self.image_dir, self.image_names[idx])           \n        image = cv2.imread(image_path, cv2.IMREAD_COLOR)                \n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)    \n\n        target = self.labels[idx]\n\n        transformed_image = self.transforms(image=image)['image']\n        sample = {'image_path': image_path, 'image': transformed_image, 'target': target}\n\n        return sample","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ImageDataModule(pl.LightningDataModule):\n    def __init__(self,\n                 df: pd.DataFrame,\n                 train_transforms,\n                 valid_transforms,\n                 image_dir: str,\n                 fold_num: int,\n                 configurations: Dict[str, int]):\n        super().__init__()\n        self.df = df\n        self.train_transforms = train_transforms\n        self.valid_transforms = valid_transforms\n        self.image_dir = image_dir\n        self.fold_num = fold_num\n    \n    def setup(self, stage=None) -> None:\n        folds = StratifiedKFold(n_splits=5, shuffle=True)\n        \n        train_indexes, valid_indexes = list(folds.split(self.df, self.df['labels']))[self.fold_num]\n        \n        print(f\"Size of Train Dataset: {len(train_indexes)}\")\n        print(f\"Size of Validation Dataset: {len(valid_indexes)}\")\n        \n        train_df = self.df.iloc[train_indexes]\n        valid_df = self.df.iloc[valid_indexes]\n        \n        self.train_dataset = ImageDataset(image_names=train_df.image.values, \n                                        labels=train_df.iloc[:, 2:].values, \n                                        image_dir=self.image_dir, \n                                        transforms=self.train_transforms,\n                                        )\n\n        self.valid_dataset = ImageDataset(image_names=valid_df.image.values, \n                                        labels=valid_df.iloc[:, 2:].values, \n                                        image_dir=self.image_dir, \n                                        transforms=self.valid_transforms,\n                                        )\n        \n        \n    def train_dataloader(self):                \n        return None\n\n    def val_dataloader(self):        \n        valid_loader = DataLoader(\n            self.valid_dataset,\n            batch_size=configurations.get(\"BATCH_SIZE\"),\n            num_workers=configurations.get(\"NUM_WORKERS\"),\n            shuffle=False,\n        )\n        return valid_loader\n\n    def test_dataloader(self):\n        return None","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Augmentation","metadata":{}},{"cell_type":"code","source":"train_augs = A.Compose([    \n    A.Resize(height=configurations.get(\"IMAGE_HEIGHT\"), width=configurations.get(\"IMAGE_WIDTH\"), p=1.0),    \n    A.Normalize(),\n    ToTensorV2(),\n])\n\nvalid_augs = A.Compose([\n    A.Resize(height=configurations.get(\"IMAGE_HEIGHT\"), width=configurations.get(\"IMAGE_WIDTH\"), p=1.0),\n    A.Normalize(),\n    ToTensorV2(),\n])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## DataModule Object Creation","metadata":{}},{"cell_type":"code","source":"data_module = ImageDataModule(df=dataset_df_copy,\n                               train_transforms=train_augs,\n                               valid_transforms=valid_augs,\n                               image_dir=os.path.join(ROOT_DIR, TRAIN_IMAGES_FOLDER),\n                               fold_num=configurations.get(\"FOLD_NUM\"),\n                               configurations=configurations)\n\ndata_module.setup()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preparing Model","metadata":{}},{"cell_type":"code","source":"class ClassifierModule(pl.LightningModule):\n    def __init__(self, learning_rate=0.003, num_classes=6):\n        super().__init__()        \n        self.metric = FBeta(num_classes=num_classes, beta=0.5, multilabel=True)\n        self.learning_rate = learning_rate\n        # Try different architectures\n        self.model = models.resnet34(pretrained=False)        \n        self.model.fc = nn.Linear(in_features=self.model.fc.in_features, out_features=num_classes)        \n        \n        \n    def forward(self, x):\n        batch_size, _, _, _ = x.shape\n        x = self.model(x)                \n        x = torch.sigmoid(x)\n        \n        return x.reshape(batch_size, -1)\n    \n    def configure_optimizers(self):\n        optimizer = AdamW(self.model.parameters(), lr=self.learning_rate, weight_decay=0.001)        \n\n        return optimizer            \n    \n    def _get_loss(self, y_hat, y): \n        pass\n    \n    def training_step(self, batch, batch_idx):\n        pass              \n        \n    def validation_step(self, batch, batch_idx):\n        pass","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading Trained Model","metadata":{}},{"cell_type":"code","source":"lightning = ClassifierModule()\nlightning.load_from_checkpoint(checkpoint_path=CKPT_PATHS[0])\nlightning.freeze()\nlightning.cuda();","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Metric\nmetric = FBeta(num_classes=6, beta=0.5, multilabel=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\n\nimage_paths_batch = []\ntargets = []\npredictions = []\nbatch_num = 1\nf1_scores = []\nfor batch in tqdm(data_module.val_dataloader()):\n    #print(f\"Now Predicting Batch: {batch_num}..\")\n    target = batch['target'].detach().cpu()\n        \n    \n    image_paths_batch.append(batch['image_path'])\n    \n    pred = lightning(batch['image'].cuda()).detach().cpu()\n        \n    f1_scores.append(metric(pred, target))\n    \n    targets.append(target.numpy())\n    predictions.append(pred.numpy())            \n    \n    batch_num += 1    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"AVG F1 Score: {sum(f1_scores)/len(f1_scores)}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_str = np.array(['scab', 'healthy', 'frog_eye_leaf_spot', 'rust', 'powdery_mildew', 'complex'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted_labels = []\nactual_labels = []\nimage_paths = []\n\nfor batch_idx in range(len(predictions[:])):\n    for i, pred in enumerate(predictions[batch_idx]):  \n        if batch_idx == 0:\n            print(f\"\\033[1mPredicted Label: {predictions[batch_idx][i]}\\033[0m\")\n            print(f\"Target Label: {targets[batch_idx][i]}\")\n            \n        predicted_label = \" \".join(labels_str[np.around(predictions[batch_idx][i]) > 0.5])\n        actual_label = \" \".join(labels_str[targets[batch_idx][i] == 1])\n\n        predicted_labels.append(predicted_label)\n        actual_labels.append(actual_label)\n        image_paths.append(image_paths_batch[batch_idx][i])                ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_pred_df = pd.DataFrame({\"image_path\": image_paths, \"actual_label\": actual_labels, \"predicted_label\": predicted_labels})\nval_pred_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_pred_df.predicted_label.value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}