{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Imports\nimport os\nimport numpy as np\nimport torch\nimport random\nfrom torchvision.datasets import ImageFolder\nfrom torch.utils.data import DataLoader\nfrom torchvision import transforms\nimport torchvision\nfrom tqdm.notebook import tqdm\nfrom torchmetrics import Accuracy\nimport pandas as pd\nfrom skimage import io","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-27T19:14:14.811097Z","iopub.execute_input":"2022-07-27T19:14:14.811928Z","iopub.status.idle":"2022-07-27T19:14:14.818404Z","shell.execute_reply.started":"2022-07-27T19:14:14.811889Z","shell.execute_reply":"2022-07-27T19:14:14.817370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Random Seed Initialize\ndef seed_everything(seed):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    \nseed_everything(42)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T19:14:14.823662Z","iopub.execute_input":"2022-07-27T19:14:14.824515Z","iopub.status.idle":"2022-07-27T19:14:14.834468Z","shell.execute_reply.started":"2022-07-27T19:14:14.824476Z","shell.execute_reply":"2022-07-27T19:14:14.833518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset and Dataloader","metadata":{}},{"cell_type":"code","source":"Prob = 0.5\ntrain_tf = transforms.Compose([transforms.RandomHorizontalFlip(Prob),\n                              transforms.RandomVerticalFlip(Prob),\n                              transforms.RandomResizedCrop((224,224)),\n                              transforms.ToTensor(),\n                              transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                                          std=[0.229, 0.224, 0.225])\n                              ])\nval_tf = transforms.Compose([transforms.Resize((224,224)),\n                              transforms.ToTensor(),\n                             transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                                          std=[0.229, 0.224, 0.225])\n                              ])","metadata":{"execution":{"iopub.status.busy":"2022-07-27T19:14:14.836769Z","iopub.execute_input":"2022-07-27T19:14:14.837281Z","iopub.status.idle":"2022-07-27T19:14:14.847847Z","shell.execute_reply.started":"2022-07-27T19:14:14.837242Z","shell.execute_reply":"2022-07-27T19:14:14.846704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds = ImageFolder(\"../input/paddy-doctor-make-train-and-val-folder/Train\", transform = train_tf)\nval_ds = ImageFolder(\"../input/paddy-doctor-make-train-and-val-folder/Val\", transform = val_tf)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T19:14:14.849502Z","iopub.execute_input":"2022-07-27T19:14:14.849884Z","iopub.status.idle":"2022-07-27T19:14:20.043894Z","shell.execute_reply.started":"2022-07-27T19:14:14.849847Z","shell.execute_reply":"2022-07-27T19:14:20.042901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds.class_to_idx","metadata":{"execution":{"iopub.status.busy":"2022-07-27T19:14:20.046371Z","iopub.execute_input":"2022-07-27T19:14:20.047528Z","iopub.status.idle":"2022-07-27T19:14:20.057449Z","shell.execute_reply.started":"2022-07-27T19:14:20.047453Z","shell.execute_reply":"2022-07-27T19:14:20.056195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_ds.class_to_idx","metadata":{"execution":{"iopub.status.busy":"2022-07-27T19:14:20.059760Z","iopub.execute_input":"2022-07-27T19:14:20.060376Z","iopub.status.idle":"2022-07-27T19:14:20.073061Z","shell.execute_reply.started":"2022-07-27T19:14:20.060341Z","shell.execute_reply":"2022-07-27T19:14:20.072119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader    = DataLoader(train_ds,batch_size= 64*8, shuffle = True, pin_memory=True, drop_last=True)\nval_loader      = DataLoader(val_ds,  batch_size= 64*8, shuffle = True, pin_memory=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T19:20:48.362198Z","iopub.execute_input":"2022-07-27T19:20:48.362546Z","iopub.status.idle":"2022-07-27T19:20:48.368152Z","shell.execute_reply.started":"2022-07-27T19:20:48.362515Z","shell.execute_reply":"2022-07-27T19:20:48.367064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model = torchvision.models.convnext_tiny(pretrained = True)\nmodel_dir = '../input/pytorch-model-weights/convnext_models'\nmodel_name = 'convnext_tiny.pt'\nmodel = torch.load(os.path.join(model_dir, model_name))","metadata":{"execution":{"iopub.status.busy":"2022-07-27T19:14:20.084679Z","iopub.execute_input":"2022-07-27T19:14:20.085243Z","iopub.status.idle":"2022-07-27T19:14:22.561333Z","shell.execute_reply.started":"2022-07-27T19:14:20.085205Z","shell.execute_reply":"2022-07-27T19:14:22.560068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for param in model.parameters():\n        param.requires_grad = False","metadata":{"execution":{"iopub.status.busy":"2022-07-27T19:14:22.562622Z","iopub.execute_input":"2022-07-27T19:14:22.563027Z","iopub.status.idle":"2022-07-27T19:14:22.569060Z","shell.execute_reply.started":"2022-07-27T19:14:22.562958Z","shell.execute_reply":"2022-07-27T19:14:22.568075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.classifier","metadata":{"execution":{"iopub.status.busy":"2022-07-27T19:14:22.574352Z","iopub.execute_input":"2022-07-27T19:14:22.575094Z","iopub.status.idle":"2022-07-27T19:14:22.584458Z","shell.execute_reply.started":"2022-07-27T19:14:22.575056Z","shell.execute_reply":"2022-07-27T19:14:22.583211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.classifier[-1] = torch.nn.Linear(in_features = 768, out_features = 10) #tiny\n#model.classifier[-1] = torch.nn.Linear(in_features = 1024, out_features = 10) #base","metadata":{"execution":{"iopub.status.busy":"2022-07-27T19:14:22.586174Z","iopub.execute_input":"2022-07-27T19:14:22.586896Z","iopub.status.idle":"2022-07-27T19:14:22.597864Z","shell.execute_reply.started":"2022-07-27T19:14:22.586858Z","shell.execute_reply":"2022-07-27T19:14:22.596958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for param in model.parameters():\n        if param.requires_grad:\n            print(param.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T19:14:22.604100Z","iopub.execute_input":"2022-07-27T19:14:22.604865Z","iopub.status.idle":"2022-07-27T19:14:22.619158Z","shell.execute_reply.started":"2022-07-27T19:14:22.604820Z","shell.execute_reply":"2022-07-27T19:14:22.617941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(data, target, model, optimizer, criterion, TRAIN):\n\n    if TRAIN:\n        optimizer.zero_grad()\n\n    # forward pass\n    output = model(data)\n    #loss = criterion(output, target)\n    \n    norms = torch.norm(output, p=2, dim=-1, keepdim=True) + 1e-7\n    logit_norm = torch.div(output, norms)\n    loss = criterion(logit_norm , target)\n\n    if TRAIN:\n        # backward pass\n        loss.backward()  \n        # Update the parameters\n        optimizer.step()     \n\n    return output, loss","metadata":{"execution":{"iopub.status.busy":"2022-07-27T19:14:22.623582Z","iopub.execute_input":"2022-07-27T19:14:22.625633Z","iopub.status.idle":"2022-07-27T19:14:22.639626Z","shell.execute_reply.started":"2022-07-27T19:14:22.625594Z","shell.execute_reply":"2022-07-27T19:14:22.638175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training Options\noptimizer = torch.optim.Adam(model.parameters(), lr= 1e-4) \ncriterion = torch.nn.CrossEntropyLoss() \n\n# training device  ('cpu', 'cuda')\ndevice = 'cuda'\nmodel.to(device)\n\n# number of training epochs\nn_epochs = 100\n\n\nacc = Accuracy().to(device)\nprint(f\"\\nTraining Model\")\nbest_val_ = 1000\nfailure_count = 0\nfor epoch in tqdm(range(n_epochs), desc = \"# Epochs\", position= 0):\n    train_loss = 0\n    val_loss = 0\n\n    train_acc = 0\n    val_acc = 0\n\n    # set to train mode\n    model.train()\n\n    for i, (data, target) in enumerate(tqdm(train_loader, desc = \"Training\", leave= True, position= 1)):\n        # data to gpu/cpu\n        data, target = data.to(device, non_blocking=True), target.to(device, non_blocking=True)\n\n        # mini batch training\n        output, loss = train(data, target, model, optimizer, criterion, TRAIN= True)\n\n        # Track train loss by multiplying average loss by number of examples in batch\n        train_loss += loss.item() * data.size(0)\n        _, output = torch.max(output, dim=1) \n        acc(output, target.squeeze())\n\n    # Compute Loss for each epoch\n    train_loss = train_loss / len(train_loader.dataset)\n    train_acc = acc.compute().item()\n    acc.reset()\n\n    with torch.no_grad():\n        # set to evaluation mode\n        model.eval()\n\n        for i, (data, target) in enumerate(tqdm(val_loader, desc = \"Validation\", leave= True, position= 2)):\n            # data to gpu/cpu\n            data, target = data.to(device, non_blocking=True), target.to(device, non_blocking=True)\n            # mini batch training\n            output, loss = train(data, target, model, optimizer, criterion, TRAIN= False)\n            # Track train loss by multiplying average loss by number of examples in batch\n            val_loss += loss.item() * data.size(0)\n            _, output = torch.max(output, dim=1) \n            acc(output, target.squeeze())\n\n        # Compute Loss for each epoch\n        val_loss = val_loss / len(val_loader.dataset)\n        val_acc = acc.compute().item()\n        acc.reset()\n        \n    if val_loss < best_val_:\n        best_val_ = val_loss\n        best_acc = val_acc\n        failure_count = 0\n        torch.save(model, 'best_model.pt')\n    else:\n        failure_count += 1\n    \n    if failure_count >= 10:\n        break\n        \n    print(f\"Epoch # {epoch+1:04d}\")\n    print(f\"Train Loss: {train_loss: .4f},\\t Val Loss: {val_loss: .4f}\")\n    print(f\"Train Acc : {train_acc: .4f},\\t Val Acc : {val_acc: .4f}\")  \n    print(f\"Best Val Loss : {best_val_: .4f},\\t Val Acc : {best_acc: .4f}\")\n    print(f\"Failure Count = {failure_count}\")\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T19:20:53.135548Z","iopub.execute_input":"2022-07-27T19:20:53.135903Z","iopub.status.idle":"2022-07-27T19:22:56.327994Z","shell.execute_reply.started":"2022-07-27T19:20:53.135871Z","shell.execute_reply":"2022-07-27T19:22:56.326984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"train_ds.class_to_idx","metadata":{"execution":{"iopub.status.busy":"2022-07-27T19:17:49.568758Z","iopub.execute_input":"2022-07-27T19:17:49.569446Z","iopub.status.idle":"2022-07-27T19:17:49.576701Z","shell.execute_reply.started":"2022-07-27T19:17:49.569404Z","shell.execute_reply":"2022-07-27T19:17:49.575735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_dict = {v: k for k, v in train_ds.class_to_idx.items()}\nclass_dict","metadata":{"execution":{"iopub.status.busy":"2022-07-27T19:17:49.578368Z","iopub.execute_input":"2022-07-27T19:17:49.579067Z","iopub.status.idle":"2022-07-27T19:17:49.590034Z","shell.execute_reply.started":"2022-07-27T19:17:49.579011Z","shell.execute_reply":"2022-07-27T19:17:49.588956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df = pd.read_csv(\"../input/paddy-disease-classification/sample_submission.csv\")\nsample_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T19:17:49.591607Z","iopub.execute_input":"2022-07-27T19:17:49.592042Z","iopub.status.idle":"2022-07-27T19:17:49.632154Z","shell.execute_reply.started":"2022-07-27T19:17:49.591986Z","shell.execute_reply":"2022-07-27T19:17:49.631072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load Best Model\nmodel = torch.load('best_model.pt')","metadata":{"execution":{"iopub.status.busy":"2022-07-27T19:17:49.633587Z","iopub.execute_input":"2022-07-27T19:17:49.636632Z","iopub.status.idle":"2022-07-27T19:17:49.798798Z","shell.execute_reply.started":"2022-07-27T19:17:49.636590Z","shell.execute_reply":"2022-07-27T19:17:49.797697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dir = \"../input/paddy-disease-classification/test_images\"\nfile_list = os.listdir(test_dir)\nfile_list[:5]","metadata":{"execution":{"iopub.status.busy":"2022-07-27T19:17:49.800470Z","iopub.execute_input":"2022-07-27T19:17:49.801233Z","iopub.status.idle":"2022-07-27T19:17:50.157062Z","shell.execute_reply.started":"2022-07-27T19:17:49.801192Z","shell.execute_reply":"2022-07-27T19:17:50.156027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_tfs = transforms.Compose([\n                                transforms.ToPILImage(),\n                                transforms.Resize((224,224)),\n                                transforms.ToTensor(),\n                                transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                                          std=[0.229, 0.224, 0.225])\n                                ])","metadata":{"execution":{"iopub.status.busy":"2022-07-27T19:17:50.158407Z","iopub.execute_input":"2022-07-27T19:17:50.160944Z","iopub.status.idle":"2022-07-27T19:17:50.166893Z","shell.execute_reply.started":"2022-07-27T19:17:50.160903Z","shell.execute_reply":"2022-07-27T19:17:50.165833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_df = pd.DataFrame(columns = sample_df.columns)\nprint(\"Begin Inference\\n\")\nfor ii, file_name in enumerate(tqdm(file_list)):\n    image = io.imread(os.path.join(test_dir, file_name))\n    image = test_tfs(image)\n    image = image.unsqueeze(0)\n    out = model(image.to('cuda', non_blocking=True))\n    out = torch.argmax(out).item()\n    label = class_dict[out]\n    result_df.loc[ii,:] = [file_name, label]\n    \n    if (i+1) % 200 == 0:\n        print(f\"{i+1:04d}/{len(file_list)}\")\nelse:\n    print(\"All Test Images are processed\")","metadata":{"execution":{"iopub.status.busy":"2022-07-27T19:17:50.170044Z","iopub.execute_input":"2022-07-27T19:17:50.170332Z","iopub.status.idle":"2022-07-27T19:19:20.109635Z","shell.execute_reply.started":"2022-07-27T19:17:50.170295Z","shell.execute_reply":"2022-07-27T19:19:20.108531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_df.to_csv('submission.csv', index = False, columns = result_df.columns)\nprint(\"\\nSubmission File Created!\")","metadata":{"execution":{"iopub.status.busy":"2022-07-27T19:19:20.111305Z","iopub.execute_input":"2022-07-27T19:19:20.112348Z","iopub.status.idle":"2022-07-27T19:19:20.131948Z","shell.execute_reply.started":"2022-07-27T19:19:20.112305Z","shell.execute_reply":"2022-07-27T19:19:20.130885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# len(file_list)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T19:19:20.133484Z","iopub.execute_input":"2022-07-27T19:19:20.133833Z","iopub.status.idle":"2022-07-27T19:19:20.140153Z","shell.execute_reply.started":"2022-07-27T19:19:20.133799Z","shell.execute_reply":"2022-07-27T19:19:20.137860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for i in range(len(file_list)):\n#     if (i+1) % 200 == 0:\n#         print(f\"{i+1:04d}/{len(file_list)}\")\n# else:\n#     print(\"All Test Images are processed\")","metadata":{"execution":{"iopub.status.busy":"2022-07-27T19:19:20.141862Z","iopub.execute_input":"2022-07-27T19:19:20.142645Z","iopub.status.idle":"2022-07-27T19:19:20.147686Z","shell.execute_reply.started":"2022-07-27T19:19:20.142607Z","shell.execute_reply":"2022-07-27T19:19:20.146602Z"},"trusted":true},"execution_count":null,"outputs":[]}]}