{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-17T22:41:09.142808Z","iopub.execute_input":"2022-03-17T22:41:09.143095Z","iopub.status.idle":"2022-03-17T22:41:09.167534Z","shell.execute_reply.started":"2022-03-17T22:41:09.143013Z","shell.execute_reply":"2022-03-17T22:41:09.166853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# What are we dealing with?","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('../input/sorghum-id-fgvc-9/train_cultivar_mapping.csv', index_col = 'image')","metadata":{"execution":{"iopub.status.busy":"2022-03-17T23:07:02.807874Z","iopub.execute_input":"2022-03-17T23:07:02.808136Z","iopub.status.idle":"2022-03-17T23:07:02.842189Z","shell.execute_reply.started":"2022-03-17T23:07:02.808106Z","shell.execute_reply":"2022-03-17T23:07:02.841492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_df)","metadata":{"execution":{"iopub.status.busy":"2022-03-17T23:07:04.281569Z","iopub.execute_input":"2022-03-17T23:07:04.282412Z","iopub.status.idle":"2022-03-17T23:07:04.288042Z","shell.execute_reply.started":"2022-03-17T23:07:04.282357Z","shell.execute_reply":"2022-03-17T23:07:04.287409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So we have about 23k images to train on.","metadata":{}},{"cell_type":"markdown","source":"Turns out some images are missing:","metadata":{}},{"cell_type":"code","source":"import os\nimages_present = os.listdir('../input/sorghum-id-fgvc-9/train_images')\nlen(images_present)","metadata":{"execution":{"iopub.status.busy":"2022-03-17T23:11:32.584528Z","iopub.execute_input":"2022-03-17T23:11:32.584794Z","iopub.status.idle":"2022-03-17T23:11:32.601532Z","shell.execute_reply.started":"2022-03-17T23:11:32.584764Z","shell.execute_reply":"2022-03-17T23:11:32.600836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Only keep the files that exist\ntrain_df = train_df.loc[images_present]","metadata":{"execution":{"iopub.status.busy":"2022-03-17T23:12:18.702538Z","iopub.execute_input":"2022-03-17T23:12:18.702810Z","iopub.status.idle":"2022-03-17T23:12:18.723244Z","shell.execute_reply.started":"2022-03-17T23:12:18.702780Z","shell.execute_reply":"2022-03-17T23:12:18.721839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-17T23:12:21.659695Z","iopub.execute_input":"2022-03-17T23:12:21.659963Z","iopub.status.idle":"2022-03-17T23:12:21.670081Z","shell.execute_reply.started":"2022-03-17T23:12:21.659932Z","shell.execute_reply":"2022-03-17T23:12:21.669228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For each image we have a classification and nothing else.","metadata":{}},{"cell_type":"code","source":"classes = train_df['cultivar'].unique()\nclasses","metadata":{"execution":{"iopub.status.busy":"2022-03-17T23:12:24.782872Z","iopub.execute_input":"2022-03-17T23:12:24.783145Z","iopub.status.idle":"2022-03-17T23:12:24.791769Z","shell.execute_reply.started":"2022-03-17T23:12:24.783114Z","shell.execute_reply":"2022-03-17T23:12:24.791012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['cultivar'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-03-17T23:13:12.099663Z","iopub.execute_input":"2022-03-17T23:13:12.099922Z","iopub.status.idle":"2022-03-17T23:13:12.113041Z","shell.execute_reply.started":"2022-03-17T23:13:12.099892Z","shell.execute_reply":"2022-03-17T23:13:12.112231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For each class we have unequal number of training samples.","metadata":{}},{"cell_type":"code","source":"print(train_df['cultivar'].value_counts().min())\nprint(train_df['cultivar'].value_counts().max())","metadata":{"execution":{"iopub.status.busy":"2022-03-17T23:13:30.302013Z","iopub.execute_input":"2022-03-17T23:13:30.302424Z","iopub.status.idle":"2022-03-17T23:13:30.317745Z","shell.execute_reply.started":"2022-03-17T23:13:30.302381Z","shell.execute_reply":"2022-03-17T23:13:30.316934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The number of training images per class ranges from 134 to 298","metadata":{}},{"cell_type":"code","source":"sample_sub = pd.read_csv('../input/sorghum-id-fgvc-9/sample_submission.csv')\nsample_sub.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-17T23:13:50.978050Z","iopub.execute_input":"2022-03-17T23:13:50.978743Z","iopub.status.idle":"2022-03-17T23:13:51.004482Z","shell.execute_reply.started":"2022-03-17T23:13:50.978706Z","shell.execute_reply":"2022-03-17T23:13:51.003794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We are predicting on some 24k images","metadata":{}},{"cell_type":"code","source":"len(sample_sub)","metadata":{"execution":{"iopub.status.busy":"2022-03-17T23:13:53.113814Z","iopub.execute_input":"2022-03-17T23:13:53.114254Z","iopub.status.idle":"2022-03-17T23:13:53.124191Z","shell.execute_reply.started":"2022-03-17T23:13:53.114207Z","shell.execute_reply":"2022-03-17T23:13:53.123495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"And they are all there:","metadata":{}},{"cell_type":"code","source":"test_images_present = os.listdir('../input/sorghum-id-fgvc-9/test')\nlen(test_images_present)","metadata":{"execution":{"iopub.status.busy":"2022-03-17T23:14:15.746623Z","iopub.execute_input":"2022-03-17T23:14:15.747067Z","iopub.status.idle":"2022-03-17T23:14:15.763557Z","shell.execute_reply.started":"2022-03-17T23:14:15.747029Z","shell.execute_reply":"2022-03-17T23:14:15.762819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Separate train and test set","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_test, y_train, y_test = train_test_split(train_df.index.values, train_df['cultivar'],\n                                                    stratify=train_df['cultivar'], \n                                                    test_size=0.1)\ny_train = y_train.values\ny_test  = y_test.values","metadata":{"execution":{"iopub.status.busy":"2022-03-17T23:15:10.190865Z","iopub.execute_input":"2022-03-17T23:15:10.191118Z","iopub.status.idle":"2022-03-17T23:15:10.232149Z","shell.execute_reply.started":"2022-03-17T23:15:10.191089Z","shell.execute_reply":"2022-03-17T23:15:10.231459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pytorch","metadata":{}},{"cell_type":"code","source":"from PIL import Image\nimport torch\nimport random\nimport os","metadata":{"execution":{"iopub.status.busy":"2022-03-17T23:15:14.163735Z","iopub.execute_input":"2022-03-17T23:15:14.164275Z","iopub.status.idle":"2022-03-17T23:15:14.168100Z","shell.execute_reply.started":"2022-03-17T23:15:14.164237Z","shell.execute_reply":"2022-03-17T23:15:14.167465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"package_paths = [\n    '../input/pytorch-image-models/pytorch-image-models-master' \n]\nimport sys; \n\nfor pth in package_paths:\n    sys.path.append(pth)\n    \nimport timm","metadata":{"execution":{"iopub.status.busy":"2022-03-17T23:15:16.113391Z","iopub.execute_input":"2022-03-17T23:15:16.114058Z","iopub.status.idle":"2022-03-17T23:15:16.119258Z","shell.execute_reply.started":"2022-03-17T23:15:16.114018Z","shell.execute_reply":"2022-03-17T23:15:16.118138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CLIP = None\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\ntrain_bs = 8\nvalid_bs = 8\nnum_workers = 2\nmodel_arch = 'resnet34'\nn_class = 100\nx_size = 224\ny_size = 224\nnum_epochs = 5","metadata":{"execution":{"iopub.status.busy":"2022-03-17T23:15:18.179178Z","iopub.execute_input":"2022-03-17T23:15:18.179950Z","iopub.status.idle":"2022-03-17T23:15:18.185105Z","shell.execute_reply.started":"2022-03-17T23:15:18.179909Z","shell.execute_reply":"2022-03-17T23:15:18.184037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device","metadata":{"execution":{"iopub.status.busy":"2022-03-17T23:15:20.174197Z","iopub.execute_input":"2022-03-17T23:15:20.175063Z","iopub.status.idle":"2022-03-17T23:15:20.187531Z","shell.execute_reply.started":"2022-03-17T23:15:20.175014Z","shell.execute_reply":"2022-03-17T23:15:20.185126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dir_name = '../input/sorghum-id-fgvc-9/train_images/'","metadata":{"execution":{"iopub.status.busy":"2022-03-17T23:15:21.823981Z","iopub.execute_input":"2022-03-17T23:15:21.824259Z","iopub.status.idle":"2022-03-17T23:15:21.829220Z","shell.execute_reply.started":"2022-03-17T23:15:21.824229Z","shell.execute_reply":"2022-03-17T23:15:21.826989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True","metadata":{"execution":{"iopub.status.busy":"2022-03-17T23:15:24.080829Z","iopub.execute_input":"2022-03-17T23:15:24.081082Z","iopub.status.idle":"2022-03-17T23:15:24.087731Z","shell.execute_reply.started":"2022-03-17T23:15:24.081053Z","shell.execute_reply":"2022-03-17T23:15:24.086830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seed_everything(42)","metadata":{"execution":{"iopub.status.busy":"2022-03-17T23:15:24.632195Z","iopub.execute_input":"2022-03-17T23:15:24.632779Z","iopub.status.idle":"2022-03-17T23:15:24.636965Z","shell.execute_reply.started":"2022-03-17T23:15:24.632740Z","shell.execute_reply":"2022-03-17T23:15:24.636095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torchvision import transforms\n\ntrain_transform = transforms.Compose([\n    transforms.RandomResizedCrop(224),\n    transforms.RandomHorizontalFlip(),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n])\ntest_transform = transforms.Compose([\n    transforms.Resize(256),\n    transforms.CenterCrop(224),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n])","metadata":{"execution":{"iopub.status.busy":"2022-03-17T23:15:26.688256Z","iopub.execute_input":"2022-03-17T23:15:26.688857Z","iopub.status.idle":"2022-03-17T23:15:26.696096Z","shell.execute_reply.started":"2022-03-17T23:15:26.688820Z","shell.execute_reply":"2022-03-17T23:15:26.695418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader\nfrom torch import Tensor\nfrom torchvision.transforms import ToTensor","metadata":{"execution":{"iopub.status.busy":"2022-03-17T23:15:29.835367Z","iopub.execute_input":"2022-03-17T23:15:29.836127Z","iopub.status.idle":"2022-03-17T23:15:29.839891Z","shell.execute_reply.started":"2022-03-17T23:15:29.836086Z","shell.execute_reply":"2022-03-17T23:15:29.839226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def encode_labels(y):\n    res = np.zeros(n_class, dtype = int)\n    res[np.argmax(classes == y)] = 1\n        \n    return res","metadata":{"execution":{"iopub.status.busy":"2022-03-17T23:15:31.843406Z","iopub.execute_input":"2022-03-17T23:15:31.843918Z","iopub.status.idle":"2022-03-17T23:15:31.848531Z","shell.execute_reply.started":"2022-03-17T23:15:31.843881Z","shell.execute_reply":"2022-03-17T23:15:31.847803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TrainDataset(Dataset):\n    def __init__(self):\n        super().__init__()\n    \n    def __len__(self):\n        return len(X_train)\n\n    def __getitem__(self, idx):\n        name = X_train[idx]\n        y = encode_labels(y_train[idx])\n        x = Image.open(dir_name + name)\n        x = train_transform(x)\n        return (x,Tensor(y[:n_class]))\n    \nclass ValidDataset(Dataset):\n    def __init__(self):\n        super().__init__()\n    \n    def __len__(self):\n        return len(X_test)\n\n    def __getitem__(self, idx):\n        name = X_test[idx]\n        y = encode_labels(y_test[idx])\n        x = Image.open(dir_name + name)\n        x = test_transform(x)\n        return (x,Tensor(y))","metadata":{"execution":{"iopub.status.busy":"2022-03-17T23:15:58.042879Z","iopub.execute_input":"2022-03-17T23:15:58.043413Z","iopub.status.idle":"2022-03-17T23:15:58.051910Z","shell.execute_reply.started":"2022-03-17T23:15:58.043374Z","shell.execute_reply":"2022-03-17T23:15:58.051029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = TrainDataset()\nvalid = ValidDataset()","metadata":{"execution":{"iopub.status.busy":"2022-03-17T23:15:59.957249Z","iopub.execute_input":"2022-03-17T23:15:59.958146Z","iopub.status.idle":"2022-03-17T23:15:59.962400Z","shell.execute_reply.started":"2022-03-17T23:15:59.958046Z","shell.execute_reply":"2022-03-17T23:15:59.961732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xx, yy = train[0]\nprint(yy)\nplt.imshow(xx.transpose(2,0))","metadata":{"execution":{"iopub.status.busy":"2022-03-17T23:16:00.780408Z","iopub.execute_input":"2022-03-17T23:16:00.781115Z","iopub.status.idle":"2022-03-17T23:16:11.042723Z","shell.execute_reply.started":"2022-03-17T23:16:00.781071Z","shell.execute_reply":"2022-03-17T23:16:11.042019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dl = DataLoader(train, batch_size = train_bs, shuffle = True, num_workers = num_workers)\nvalid_dl = DataLoader(valid, batch_size = valid_bs, shuffle = False, num_workers = num_workers)","metadata":{"execution":{"iopub.status.busy":"2022-03-17T23:16:15.513510Z","iopub.execute_input":"2022-03-17T23:16:15.513845Z","iopub.status.idle":"2022-03-17T23:16:15.523714Z","shell.execute_reply.started":"2022-03-17T23:16:15.513806Z","shell.execute_reply":"2022-03-17T23:16:15.523000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.nn as nn","metadata":{"execution":{"iopub.status.busy":"2022-03-17T23:16:17.373859Z","iopub.execute_input":"2022-03-17T23:16:17.374415Z","iopub.status.idle":"2022-03-17T23:16:17.378177Z","shell.execute_reply.started":"2022-03-17T23:16:17.374372Z","shell.execute_reply":"2022-03-17T23:16:17.377491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class OurModel(nn.Module):\n    def __init__(self, model_arch, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_arch, pretrained=pretrained)\n        #for param in self.model.parameters():\n        #    param.requires_grad = False\n        n_features = self.model.fc.in_features\n        self.model.fc = nn.Linear(n_features, n_class)\n        \n    def forward(self, x):\n        x = self.model(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-03-17T23:19:30.457186Z","iopub.execute_input":"2022-03-17T23:19:30.457653Z","iopub.status.idle":"2022-03-17T23:19:30.463956Z","shell.execute_reply.started":"2022-03-17T23:19:30.457617Z","shell.execute_reply":"2022-03-17T23:19:30.463276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.optim import Adam\nfrom torch.nn import BCEWithLogitsLoss\nmodel = OurModel(model_arch, pretrained = True)\nmodel.to('cuda')\nadam = Adam(model.parameters(), lr = 2e-4)\nloss = BCEWithLogitsLoss()","metadata":{"execution":{"iopub.status.busy":"2022-03-17T23:19:31.333447Z","iopub.execute_input":"2022-03-17T23:19:31.333980Z","iopub.status.idle":"2022-03-17T23:19:31.774329Z","shell.execute_reply.started":"2022-03-17T23:19:31.333942Z","shell.execute_reply":"2022-03-17T23:19:31.773618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_one_epoch():\n    total_loss = 0\n    num_correct = 0\n    num_total = 0\n    \n    TARGETS = []\n    PREDS = []\n    \n    model.train()    \n    for i, (x, y) in enumerate(train_dl):\n        x = x.to(device)\n        y = y.to(device)\n        adam.zero_grad()\n        logits = model(x)\n        error = loss(logits, y)\n        error.backward()\n        adam.step()\n        with torch.no_grad():\n            total_loss += error * len(y)\n            preds = (logits > 0).to(torch.long)\n            preds_correct = preds == y\n            num_correct += torch.sum(preds_correct)\n            num_total += preds_correct.shape[0] * preds_correct.shape[1]\n            \n            PREDS += [logits.sigmoid()]\n            TARGETS += [y.detach().cpu()]\n    \n    PREDS = torch.cat(PREDS).cpu().numpy()\n    TARGETS = torch.cat(TARGETS).cpu().numpy()\n            \n    print('Training epoch done')\n    print('acc: ', num_correct/num_total)\n    print(f'Train loss: {total_loss/len(train_dl)}')\n    return total_loss / len(train_dl)","metadata":{"execution":{"iopub.status.busy":"2022-03-17T23:19:52.213035Z","iopub.execute_input":"2022-03-17T23:19:52.213514Z","iopub.status.idle":"2022-03-17T23:19:52.223677Z","shell.execute_reply.started":"2022-03-17T23:19:52.213476Z","shell.execute_reply":"2022-03-17T23:19:52.222875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def valid_after_one_epoch():\n    total_loss = 0\n    num_correct = 0\n    num_total = 0\n    \n    TARGETS = []\n    PREDS = []\n\n    model.eval()\n    with torch.no_grad():\n        for i, (x, y) in enumerate(valid_dl):\n            x = x.to(device)\n            y = y.to(device)\n            logits = model(x)\n            preds = (logits > 0).to(torch.long)\n            preds_correct = preds == y\n            num_correct += torch.sum(preds_correct)\n            num_total += preds_correct.shape[0] * preds_correct.shape[1]\n            \n            PREDS += [logits.sigmoid()]\n            TARGETS += [y.detach().cpu()]\n            \n            error = loss(logits, y)\n            total_loss += error * len(y)\n    \n    PREDS = torch.cat(PREDS).cpu().numpy()\n    TARGETS = torch.cat(TARGETS).cpu().numpy()\n\n    print('acc: ', num_correct/num_total)\n    print(f'Valid loss: {total_loss/len(valid_dl)}')\n    return total_loss / len(valid_dl)","metadata":{"execution":{"iopub.status.busy":"2022-03-17T23:19:53.339680Z","iopub.execute_input":"2022-03-17T23:19:53.340061Z","iopub.status.idle":"2022-03-17T23:19:53.349171Z","shell.execute_reply.started":"2022-03-17T23:19:53.340028Z","shell.execute_reply":"2022-03-17T23:19:53.348195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loss = np.zeros(num_epochs)\nvalid_loss = np.zeros(num_epochs)\n\nfor i in range(num_epochs):\n    print(f'Epoch {i}')\n    train_loss[i] = train_one_epoch()\n    valid_loss[i] = valid_after_one_epoch()\n\n    torch.save(model.state_dict(),f'epoch_{i}.pth')","metadata":{"execution":{"iopub.status.busy":"2022-03-17T23:19:54.885973Z","iopub.execute_input":"2022-03-17T23:19:54.886515Z","iopub.status.idle":"2022-03-17T23:25:56.537176Z","shell.execute_reply.started":"2022-03-17T23:19:54.886476Z","shell.execute_reply":"2022-03-17T23:25:56.535986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}