{"cells":[{"metadata":{},"cell_type":"markdown","source":"\n![image](https://raw.githubusercontent.com/cgnorthcutt/cleanlab/master/img/cleanlab_logo.png)\n \n\n``cleanlab`` is a machine learning python package for **learning with noisy labels** and **finding label errors in datasets**. ``cleanlab`` CLEANs LABels. It is powered by the theory of **confident learning**, published in this [paper](https://arxiv.org/abs/1911.00068) | [blog](https://l7.curtisnorthcutt.com/confident-learning).\n\ngithub: https://github.com/cgnorthcutt/cleanlab  \narxiv: https://arxiv.org/abs/1911.00068  \n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import sys\nsys.path.append(\"../input/pytorch-image-models/pytorch-image-models-master\")\nsys.path.append(\"../input/cleanlab/\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport cv2\nfrom PIL import Image\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import accuracy_score, f1_score\nfrom sklearn.base import BaseEstimator\nimport random\nimport os\nimport gc\nimport copy\nimport time\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision\nfrom torchvision import transforms as T\nimport torch.optim as optim\nfrom torchvision import datasets, transforms\nimport timm\nfrom tqdm import tqdm_notebook as tqdm\n\nfrom cleanlab.classification import LearningWithNoisyLabels\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PATH = \"../input/cassava-leaf-disease-classification/\"\n\ntrain = pd.read_csv(PATH + \"train.csv\")\nsample_submission = pd.read_csv(PATH + \"sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    \nseed_everything(seed=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"params = {\"n_folds\": 5,\n          \"batch_size\": 32,\n          \"epochs\": 5,\n          \"model_name\": \"tf_efficientnet_b0_ns\",\n          \"lr\": 1e-4,\n          \"weight_decay\": 1e-6,\n          \"width\": 512,\n          \"height\": 512,\n          \"debug\": False}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaImgClassifier(nn.Module):\n    def __init__(self, model_name, pretrained=True):\n        super(CassavaImgClassifier, self).__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(n_features, 5)\n        \n    # @torch.cuda.amp.autocast()\n    def forward(self, x):\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_train_transforms(params):\n    return A.Compose([\n        A.RandomResizedCrop(params[\"height\"], params[\"width\"]),\n        # A.Transpose(p=0.5),\n        A.HorizontalFlip(p=0.5),\n        # A.VerticalFlip(p=0.5),\n        # A.ShiftScaleRotate(p=0.5),\n        # A.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n        # A.RandomBrightnessContrast(brightness_limit=(-0.1, 0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n        A.Normalize(mean=[0.485, 0.456, 0.406],\n                        std=[0.229, 0.224, 0.225],),\n        # A.CoarseDropout(p=0.5),\n        # A.Cutout(p=0.5),\n        ToTensorV2(p=1.0)\n    ])\n\n\n\ndef get_test_transforms(params):\n\n    trans = A.Compose([\n        A.Resize(params[\"height\"], params[\"width\"], p=1.0),\n#         A.CenterCrop(params[\"height\"], params[\"width\"], p=1.0),\n        A.Normalize(mean=[0.485, 0.456, 0.406],\n                        std=[0.229, 0.224, 0.225],),\n        ToTensorV2(p=1.0)\n    ])\n    return trans\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaDataset(Dataset):\n    def __init__(self, image_id, label=None, phase=\"train\", transform=None):\n        self.image_id = image_id\n        self.label = label\n        self.transform = transform\n        self.phase = phase\n    def __len__(self):\n        return len(self.image_id)\n    def __getitem__(self, idx):\n        img = cv2.imread(PATH + self.phase + \"_images/\" + self.image_id[idx])\n        img = img[:, :, ::-1]\n        if self.transform:\n            img = self.transform(image=img)[\"image\"]\n        if self.label is not None:\n            label = self.label[idx]\n            return {\"image\": img, \"label\": label}\n        else:\n            return {\"image\": img}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Classifier(BaseEstimator):\n    def __init__(self, model, params):\n        self.model = model\n        self.params = params\n        self.device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n        self.model.to(self.device)\n        self.best_model = None\n\n    def fit(self, img_idx, label, val_img_idx=None, val_label=None):\n        transform = get_train_transforms(self.params)\n        dataset = CassavaDataset(img_idx, label, transform=transform)\n        dataloader = DataLoader(dataset, batch_size=self.params[\"batch_size\"], num_workers=4, pin_memory=True, shuffle=True)\n\n        loss_fn = nn.CrossEntropyLoss()\n        optimizer = optim.Adam(self.model.parameters(), lr=self.params[\"lr\"], weight_decay=self.params[\"weight_decay\"])\n#         scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=[1, 6, 10, 15], gamma=0.5)\n\n        scaler = torch.cuda.amp.GradScaler()\n\n        best_loss = 1000.0\n        best_score = 0.0\n        for epoch in range(self.params[\"epochs\"]):\n            self.model.train()\n            train_loss = 0.0\n            train_score = 0.0\n            t0 = time.time()\n            for data in dataloader:\n                x = data[\"image\"].to(self.device)\n                y = data[\"label\"].to(self.device)\n                with torch.cuda.amp.autocast():\n                    pred = self.model(x)\n                    loss = loss_fn(pred, y)\n\n                optimizer.zero_grad()\n                scaler.scale(loss).backward()\n                scaler.step(optimizer)\n                scaler.update()\n                # scheduler.step()\n                train_loss += loss.item()\n                train_score += accuracy_score(y.detach().cpu().numpy(), pred.softmax(dim=1).argmax(dim=1).detach().cpu().numpy())\n#             scheduler.step()\n            train_loss = train_loss / len(dataloader)\n            train_score = train_score / len(dataloader)\n            if val_img_idx is not None:\n                val_pred = self.predict(val_img_idx)\n                val_score = accuracy_score(val_label, val_pred)\n                if best_score < val_score:\n                    self.best_model = copy.copy(self.model)\n                print(f\"{epoch} epoch | train loss: {train_loss:.4f} | train acc: {train_score:.4f} | val acc: {val_score:.4f} | {time.time() - t0:.1f}s\")\n            else:\n                print(f\"{epoch} epoch | train loss: {train_loss:.4f} | train acc: {train_score:.4f} | {time.time() - t0:.1f}s\")\n\n    def predict_proba(self, img_idx, phase=\"train\"):\n        transform = get_test_transforms(self.params)\n        dataset = CassavaDataset(img_idx, phase=phase, transform=transform)\n        dataloader = DataLoader(dataset, batch_size=self.params[\"batch_size\"], num_workers=4, pin_memory=True)\n        \n        self.model.eval()\n        preds = []\n        with torch.no_grad():\n            for data in dataloader:\n                x = data[\"image\"].to(self.device)\n                with torch.cuda.amp.autocast():\n                    pred = self.model(x)\n                preds.append(pred.softmax(dim=1).detach().cpu().numpy())\n        prob = np.concatenate(preds)\n        return prob\n\n    def predict(self, img_idx, phase=\"train\"):\n        prob = self.predict_proba(img_idx, phase=phase)\n        preds = np.argmax(prob, axis=1)\n        return preds\n    def score(self, img_idx, label, phase=\"train\"):\n        preds = self.predict(img_idx, phase=phase)\n        return accuracy_score(label, preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if params[\"debug\"]:\n    train_df = train.head(500)\nelse:\n    train_df = train\n\ntrain_image_id = train_df[\"image_id\"].values\ntrain_label = train_df[\"label\"].values\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_preds = np.zeros((train_label.shape[0], 5))\nkfold = StratifiedKFold(n_splits=params[\"n_folds\"], random_state=0)\nseed_everything(0)\nfor fold, (train_idx, val_idx) in enumerate(kfold.split(train_image_id, train_label)):\n    X_train, y_train = train_image_id[train_idx], train_label[train_idx]\n    X_val, y_val = train_image_id[val_idx], train_label[val_idx]\n\n    cassava_model = CassavaImgClassifier(params[\"model_name\"])\n    model = Classifier(cassava_model, params)\n\n    lnl = LearningWithNoisyLabels(clf=model, seed=0, n_jobs=os.cpu_count())  \n    clf = lnl.fit(X_train, y_train)\n    val_preds[val_idx, :] = clf.predict_proba(X_val)\n    print(accuracy_score(y_val, np.argmax(val_preds[val_idx, :], axis=1)))\n    torch.save(clf.model.state_dict(), f\"model{fold}.pth\")\n    print(f\"-------- {fold} finished --------\")\n    \n    break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"score = accuracy_score(train_label, np.argmax(val_preds, axis=1))\nprint(f\"CV: {score}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# np.save(\"oof\", val_preds)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}