{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport pickle\nimport numpy as np\nfrom torch.utils.data import Dataset\nimport torch\nimport torchvision\nfrom torchvision import datasets, transforms,models\nfrom torch import nn, optim\nimport torch.nn.functional as F\nimport torch.nn as nn\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport pandas as pd\nfrom torch.optim.lr_scheduler import LambdaLR\nimport copy\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n#root = ./input/cassava-leaf-disease-models\nbatch_size = 64\nepoch = 1\n","metadata":{"execution":{"iopub.status.busy":"2021-08-22T19:16:21.483323Z","iopub.execute_input":"2021-08-22T19:16:21.483651Z","iopub.status.idle":"2021-08-22T19:16:21.491103Z","shell.execute_reply.started":"2021-08-22T19:16:21.483623Z","shell.execute_reply":"2021-08-22T19:16:21.490029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***transform***","metadata":{}},{"cell_type":"code","source":"transform =  {\n    'train':transforms.Compose([\n        transforms.RandomRotation(30),\n        transforms.Resize([240,320]),\n        transforms.CenterCrop(size=(224,224)),\n        transforms.ColorJitter(brightness = 0.2),\n        transforms.ToTensor(),\n        transforms.RandomErasing(p=0.5, scale=(0.02, 0.1), ratio=(0.3, 3.3), value=0, inplace=False),\n        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]),\n    'valid':transforms.Compose([\n        transforms.Resize([240,320]),\n        transforms.CenterCrop(size=(224,224)),\n        transforms.ToTensor(),\n        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]),\n    'test':transforms.Compose([\n        transforms.Resize([240,320]),\n        transforms.CenterCrop(size=(224,224)),\n        transforms.ToTensor(),\n        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])}","metadata":{"execution":{"iopub.status.busy":"2021-08-22T19:16:21.492744Z","iopub.execute_input":"2021-08-22T19:16:21.493318Z","iopub.status.idle":"2021-08-22T19:16:21.508427Z","shell.execute_reply.started":"2021-08-22T19:16:21.493279Z","shell.execute_reply":"2021-08-22T19:16:21.507607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***dataset and loader***","metadata":{}},{"cell_type":"code","source":"class CASSAVA(Dataset):\n\n    def __init__(self, train='train', k = 0, transform = None, target_transform=None):\n        super(CASSAVA, self).__init__()\n        \n        self.train = train\n        self.k = k\n        self.transform = transform\n        self.target_transform = target_transform\n        \n        data_ex = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\n        image_id_list = data_ex.iloc[:,0]\n        label = data_ex.iloc[:,1]\n        full_id_list = []\n        for i,id in enumerate(image_id_list):\n            full_id = os.path.join('../input/cassava-leaf-disease-classification/train_images',id)\n            full_id_list.append(full_id)\n           \n       \n        split_table = {0:0, 1:4279, 2:8559, 3:12838, 4:17118, 5:21397}    \n        self.data_id_list = []\n        self.labels = []\n\n        if k != 5:\n            if self.train == 'train':\n                for j in range(split_table[0],split_table[self.k-1]):\n                    self.data_id_list.append(full_id_list[j])\n                    self.labels.append(label[j])\n                for j in range(split_table[self.k],split_table[5]):\n                    self.data_id_list.append(full_id_list[j])\n                    self.labels.append(label[j])   \n            elif self.train == 'valid':\n                for j in range(split_table[self.k-1],split_table[self.k]):\n                    self.data_id_list.append(full_id_list[j])\n                    self.labels.append(label[j])\n            elif self.train == 'test':\n                for j in range(split_table[4],split_table[5]):\n                    self.data_id_list.append(full_id_list[j])\n                    self.labels.append(label[j])\n        elif k == 5:\n            if self.train == 'train':\n                for j in range(split_table[0],split_table[4]):\n                    self.data_id_list.append(full_id_list[j])\n                    self.labels.append(label[j])\n        \n\n        \n    def __getitem__(self, index):\n        img_id, target = self.data_id_list[index], self.labels[index]\n        img = Image.open(img_id)\n\n        if self.transform is not None:\n            img = self.transform(img)\n            img = img.float()\n\n        if self.target_transform is not None:\n            target = self.target_transform(target)\n        \n        target = np.asarray(target)\n        target = torch.from_numpy(target)\n        target = target.long()\n        \n        return img, target\n\n\n    \n    def __len__(self):\n        return len(self.labels)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T19:16:21.510137Z","iopub.execute_input":"2021-08-22T19:16:21.510649Z","iopub.status.idle":"2021-08-22T19:16:21.526990Z","shell.execute_reply.started":"2021-08-22T19:16:21.510611Z","shell.execute_reply":"2021-08-22T19:16:21.526193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntrain_dataset_1 = CASSAVA(train='train', k = 1, transform = transform['train'], target_transform=None)\ntrain_dataset_2 = CASSAVA(train='train', k = 2, transform = transform['train'], target_transform=None)\ntrain_dataset_3 = CASSAVA(train='train', k = 3, transform = transform['train'], target_transform=None)\ntrain_dataset_4 = CASSAVA(train='train', k = 4, transform = transform['train'], target_transform=None)\ntrain_dataset_w = CASSAVA(train='train', k = 5, transform = transform['train'], target_transform=None)\n\nvalid_dataset_1 = CASSAVA(train='valid', k = 1, transform = transform['valid'], target_transform=None)\nvalid_dataset_2 = CASSAVA(train='valid', k = 2, transform = transform['valid'], target_transform=None)\nvalid_dataset_3 = CASSAVA(train='valid', k = 3, transform = transform['valid'], target_transform=None)\nvalid_dataset_4 = CASSAVA(train='valid', k = 4, transform = transform['valid'], target_transform=None)\n\ntest_dataset    = CASSAVA(train='test',  k = 0, transform = transform['test'],  target_transform=None)\n\n\n\n\ndata = {\n    'train_loader' : [\n                        torch.utils.data.DataLoader(train_dataset_1, batch_size,shuffle=True),\n                        torch.utils.data.DataLoader(train_dataset_2, batch_size,shuffle=True),\n                        torch.utils.data.DataLoader(train_dataset_3, batch_size,shuffle=True),\n                        torch.utils.data.DataLoader(train_dataset_4, batch_size,shuffle=True),\n                        torch.utils.data.DataLoader(train_dataset_w, batch_size,shuffle=True)],\n    'valid_loader' : [\n                        torch.utils.data.DataLoader(valid_dataset_1, batch_size,shuffle=False),\n                        torch.utils.data.DataLoader(valid_dataset_2, batch_size,shuffle=False),\n                        torch.utils.data.DataLoader(valid_dataset_3, batch_size,shuffle=False),\n                        torch.utils.data.DataLoader(valid_dataset_4, batch_size,shuffle=False)],\n    'test_loader' : torch.utils.data.DataLoader(test_dataset, batch_size,shuffle=False)}","metadata":{"execution":{"iopub.status.busy":"2021-08-22T19:16:21.528354Z","iopub.execute_input":"2021-08-22T19:16:21.528756Z","iopub.status.idle":"2021-08-22T19:16:22.532438Z","shell.execute_reply.started":"2021-08-22T19:16:21.528721Z","shell.execute_reply":"2021-08-22T19:16:22.531586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***model***","metadata":{}},{"cell_type":"code","source":"RES50 = torch.load('../input/resnet50-for-pytorch/Resnet-50')\n#print(res50)\n#c = res50.fc.out_features\n#print(c)\n\n        # main_layers = [conv1,bn1,relu,maxpool,\n        #                layer1,layer2,layer3,layer4,\n        #                avgpool,fc]\n\n#freeze\nfor param in RES50.conv1.parameters():\n    param.requires_grad = False\n    \nfor param in RES50.bn1.parameters():\n    param.requires_grad = False\n    \nfor param in RES50.relu.parameters():\n    param.requires_grad = False\n    \nfor param in RES50.maxpool.parameters():\n    param.requires_grad = False\n    \nfor param in RES50.layer1.parameters():\n    param.requires_grad = False\n    \nfor param in RES50.layer2.parameters():\n    param.requires_grad = False\n    \nfor param in RES50.layer3.parameters():\n    param.requires_grad = False\n\n\nRES50.fc = nn.Sequential(\n    nn.Linear(2048, 256),\n    nn.ReLU(),\n    nn.Dropout(0.4),\n    nn.Linear(256, 32),\n    nn.ReLU(),\n    nn.Dropout(0.5),\n    nn.Linear(32, 5),\n    \n    nn.LogSoftmax(dim=1)\n)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T19:16:22.533980Z","iopub.execute_input":"2021-08-22T19:16:22.534315Z","iopub.status.idle":"2021-08-22T19:16:22.607830Z","shell.execute_reply.started":"2021-08-22T19:16:22.534280Z","shell.execute_reply":"2021-08-22T19:16:22.607001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***lambda***","metadata":{}},{"cell_type":"code","source":"lambdax = lambda e:0.9**e","metadata":{"execution":{"iopub.status.busy":"2021-08-22T19:16:22.609400Z","iopub.execute_input":"2021-08-22T19:16:22.609879Z","iopub.status.idle":"2021-08-22T19:16:22.613838Z","shell.execute_reply.started":"2021-08-22T19:16:22.609840Z","shell.execute_reply":"2021-08-22T19:16:22.612948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***train policy***","metadata":{}},{"cell_type":"code","source":"# k: 0,1,2,3\ndef policy_0(lr_base = 1e0,num_epochs = 1,lambdax = lambdax,k = 0,model = None):\n    \n    if model is not None:\n        res50 = model\n    else:\n        res50 = copy.deepcopy(RES50)\n        \n    res50 = res50.to(device)\n    loss_fun = nn.NLLLoss()\n    optimizer = optim.Adam(filter(lambda p: p.requires_grad, res50.parameters()), lr = lr_base)\n    if lambdax is not None:\n        scheduler = LambdaLR(optimizer, lr_lambda=lambdax, last_epoch=-1)\n    \n    y = []\n\n    for epoch in range(num_epochs):\n        sum_loss = 0.0\n        for i,(images, labels) in enumerate(data['train_loader'][k]):\n            images = images.to(device)\n            labels = labels.to(device)\n            outputs =res50(images)\n\n            loss = loss_fun(outputs, labels)\n \n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n            if lambdax is not None:\n                scheduler.step()\n                #print(scheduler.get_lr())\n        \n \n            y.append(loss.item())\n        sum_loss += loss.item()\n        print('[%d,%d] loss: %.06f' % (epoch+1, i+1, sum_loss))\n    \n    return res50,y,sum_loss","metadata":{"execution":{"iopub.status.busy":"2021-08-22T19:16:22.615089Z","iopub.execute_input":"2021-08-22T19:16:22.615622Z","iopub.status.idle":"2021-08-22T19:16:22.627047Z","shell.execute_reply.started":"2021-08-22T19:16:22.615580Z","shell.execute_reply":"2021-08-22T19:16:22.626262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***valid or test policy***","metadata":{}},{"cell_type":"code","source":"def policy_1(new_model,data):\n    correct = 0\n    total = 0\n    with torch.no_grad():\n        for (images, labels) in data:\n            \n            new_model = new_model.to(device)\n            images = images.to(device)\n            labels = labels.to(device)\n            \n            output = new_model(images)\n            _,predicted = torch.max(output, 1)\n\n            total += labels.size(0)\n            correct += (predicted == labels).sum()\n            acc = correct.item() / (len(data)*batch_size)\n        print(\"The accuracy of that lr is: {0}\".format(correct.item() / (len(data)*batch_size)))\n    \n    return acc","metadata":{"execution":{"iopub.status.busy":"2021-08-22T19:16:22.629211Z","iopub.execute_input":"2021-08-22T19:16:22.629596Z","iopub.status.idle":"2021-08-22T19:16:22.641418Z","shell.execute_reply.started":"2021-08-22T19:16:22.629561Z","shell.execute_reply":"2021-08-22T19:16:22.640559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***plot functions***","metadata":{}},{"cell_type":"code","source":"def plot_iteration(list_y):\n    \n    list_x = list(range(1,len(list_y)+1))\n    \n    plt.figure()\n    #plt.subplot(train_num//3+1,3,i+1)\n    plt.plot(list_x,list_y,\"r\")\n    plt.xlabel(\"iteration\")\n    plt.ylabel(\"loss\")\n\n    \ndef plot_acc(list_y,lr):\n    \n    list_x = list(range(1,len(list_y)+1))\n    \n    plt.figure()\n    plt.plot(list_x,list_y,color = \"m\",marker = 'o')\n    for xx,yy in zip(list_x,list_y):\n        plt.text(xx,yy,str(yy),fontsize = 10)\n    plt.title(f\"the Lr = {lr}\")\n    plt.xlabel(\"kfold\")\n    plt.ylabel(\"accuracy\")\n    \n    \ndef plot_av_acc(list_x,list_y):\n    plt.figure()\n    plt.bar(x = list_x, height = list_y)\n    for xx,yy in zip(list(range(1,len(list_x)+1)),list_y):\n        plt.text(xx,yy,str(yy),fontsize = 10)\n    plt.xlabel(\"learning rate\")\n    plt.ylabel(\"accuracy\")\n    \n    \ndef plot_acc_loss(list_y1,list_y2):\n    \n    list_x = list(range(1,len(list_y1)+1))\n    \n    plt.figure()\n    plt.plot(list_x,list_y1,label = 'loss',color = \"m\",marker = 'o')\n    for xx,yy in zip(list_x,list_y1):\n        plt.text(xx,yy,str(yy),fontsize = 10)\n    plt.plot(list_x,list_y2,label = 'accuracy',color = \"y\",marker = 'o')\n    plt.title(\"loss_acc\")\n    plt.xlabel(\"iteration\")\n    plt.ylabel('')","metadata":{"execution":{"iopub.status.busy":"2021-08-22T19:16:22.643179Z","iopub.execute_input":"2021-08-22T19:16:22.643564Z","iopub.status.idle":"2021-08-22T19:16:22.655899Z","shell.execute_reply.started":"2021-08-22T19:16:22.643529Z","shell.execute_reply":"2021-08-22T19:16:22.655193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***experiment to find best base_lr***","metadata":{}},{"cell_type":"code","source":"def experiment_and_plot(lr = 1e0, train_num = 1):\n    lr_list = []\n    av_acc_list = []\n    av_acco = 0.0\n\n# k: 0,1,2,3 / 1,2,3,4\n\n    for i in range(train_num):\n        acc_list = []\n        print(f\"Now the base lr is: {lr}\")\n        \n        for k in range(4):\n            new_model,y,_ = policy_0(lr_base = lr,num_epochs = 3,lambdax = lambdax,k = k,model = None)\n            acc = policy_1(new_model = new_model,data = data['valid_loader'][k])\n            acc_list.append(acc)\n            \n        plot_acc(list_y = acc_list,lr = lr)\n        av_acc = sum(acc_list)/4\n        av_acc_list.append(av_acc)\n        \n        if av_acc >= av_acco:\n            av_acc_max_lr = lr\n            av_acco = av_acc\n\n        #print(f\"accs:{acc}\")        \n        \n        \n        lr_list.append(lr)\n        lr = lr/10\n\n        \n    plot_av_acc(lr_list,av_acc_list)\n    print(f\"The best lr is: {av_acc_max_lr}\")\n\n    \n    return av_acc_max_lr","metadata":{"execution":{"iopub.status.busy":"2021-08-22T19:16:22.657075Z","iopub.execute_input":"2021-08-22T19:16:22.657604Z","iopub.status.idle":"2021-08-22T19:16:22.669321Z","shell.execute_reply.started":"2021-08-22T19:16:22.657570Z","shell.execute_reply":"2021-08-22T19:16:22.668429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"***use the best lr to train and test***","metadata":{}},{"cell_type":"code","source":"def best_lr_test(lr = 1e0):\n    n_num_epochs = 10\n    loss_list = []\n    acc_list = []\n    y_list = []\n    \n    print(f\"\\nfor the best lr above is: {lr}\")\n    for n in range(n_num_epochs):\n        if n == 0:\n            fi_model,y,loss = policy_0(lr_base = lr,num_epochs = 1,lambdax = lambdax,k = 4,model = None)\n        else:\n            fi_model,y,loss = policy_0(lr_base = lr,num_epochs = 1,lambdax = lambdax,k = 4,model = fi_model)\n        acc = policy_1(new_model = fi_model,data = data['test_loader'])\n        \n        loss_list.append(loss)\n        acc_list.append(acc)\n        y_list.extend(y)\n\n   \n    plot_acc_loss(loss_list,acc_list)\n    plot_iteration(y_list)\n    print(f\"The final accuracy of this lr(= {lr}) is: {acc}\")\n    \n    return fi_model","metadata":{"execution":{"iopub.status.busy":"2021-08-22T19:16:22.670574Z","iopub.execute_input":"2021-08-22T19:16:22.671005Z","iopub.status.idle":"2021-08-22T19:16:22.683263Z","shell.execute_reply.started":"2021-08-22T19:16:22.670949Z","shell.execute_reply":"2021-08-22T19:16:22.682363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***run***","metadata":{}},{"cell_type":"code","source":"acc_max_lr = experiment_and_plot(lr = 1e0, train_num = 4)\nfinal_model = best_lr_test(lr = acc_max_lr)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T19:16:22.684440Z","iopub.execute_input":"2021-08-22T19:16:22.685037Z","iopub.status.idle":"2021-08-22T19:34:28.723438Z","shell.execute_reply.started":"2021-08-22T19:16:22.685000Z","shell.execute_reply":"2021-08-22T19:34:28.722472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_id = '2216849948.jpg'\n\nimg_test = Image.open('../input/cassava-leaf-disease-classification/test_images/2216849948.jpg')\ntransform_test = transform['test']\nimg_test = transform_test(img_test)\nimg_test = img_test.float()\n\nimg_test = torch.unsqueeze(img_test, dim = 0)\nimg_test = img_test.to(device)\n\n\noutput_test = final_model(img_test)\n_,y_pred = torch.max(output_test, 1)\n#print(y_pred)\ny_pred = y_pred.cpu().numpy()\ncc = pd.DataFrame({'image_id': img_id, 'label': y_pred})\ncc.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T19:34:28.724835Z","iopub.execute_input":"2021-08-22T19:34:28.725448Z","iopub.status.idle":"2021-08-22T19:34:28.813895Z","shell.execute_reply.started":"2021-08-22T19:34:28.725407Z","shell.execute_reply":"2021-08-22T19:34:28.813038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#ans = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\n#print(ans)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T19:34:28.815133Z","iopub.execute_input":"2021-08-22T19:34:28.815470Z","iopub.status.idle":"2021-08-22T19:34:28.820547Z","shell.execute_reply.started":"2021-08-22T19:34:28.815438Z","shell.execute_reply":"2021-08-22T19:34:28.819641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#####################################################  调 试 分 割 线  ###########################################################","metadata":{"execution":{"iopub.status.busy":"2021-08-22T19:34:28.821732Z","iopub.execute_input":"2021-08-22T19:34:28.822093Z","iopub.status.idle":"2021-08-22T19:34:28.830750Z","shell.execute_reply.started":"2021-08-22T19:34:28.822058Z","shell.execute_reply":"2021-08-22T19:34:28.829926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}