{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install timm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import timm\nimport pprint\nmodel_pretrain_list = timm.list_models(pretrained=True)\n# for i in model_pretrain_list:\n#     print(i, '\\n')","metadata":{"execution":{"iopub.status.busy":"2022-04-12T06:32:27.605843Z","iopub.execute_input":"2022-04-12T06:32:27.606579Z","iopub.status.idle":"2022-04-12T06:32:30.16369Z","shell.execute_reply.started":"2022-04-12T06:32:27.60654Z","shell.execute_reply":"2022-04-12T06:32:30.162844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport random\n\nimport time\nimport torchvision\nimport torch.nn as nn\nfrom tqdm import tqdm_notebook as tqdm\n\nfrom PIL import Image, ImageFile\nfrom torch.utils.data import Dataset\nimport torch\nimport torch.optim as optim\nfrom torchvision import transforms\nfrom torch.optim import lr_scheduler\nimport os\n\ndevice = torch.device(\"cuda:0\")\nImageFile.LOAD_TRUNCATED_IMAGES = True\n\n\nos.environ['CUDA_VISIBLE_DEVICE'] = '0'\n\ntorch.cuda.set_device(0)\nmodellr = 1e-5\nBATCH_SIZE = 64\nEPOCHS = 100\nDEVICE = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2022-04-12T06:32:30.166294Z","iopub.execute_input":"2022-04-12T06:32:30.167003Z","iopub.status.idle":"2022-04-12T06:32:30.226715Z","shell.execute_reply.started":"2022-04-12T06:32:30.166962Z","shell.execute_reply":"2022-04-12T06:32:30.226067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def setup_seed(seed):\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    np.random.seed(seed)\n    random.seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\nsetup_seed(2333)","metadata":{"execution":{"iopub.status.busy":"2022-04-12T06:32:30.230428Z","iopub.execute_input":"2022-04-12T06:32:30.230647Z","iopub.status.idle":"2022-04-12T06:32:30.237759Z","shell.execute_reply.started":"2022-04-12T06:32:30.23061Z","shell.execute_reply":"2022-04-12T06:32:30.237104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class RetinopathyDatasetTrain(Dataset):\n\n    def __init__(self, csv_file):\n\n        self.data = pd.read_csv(csv_file)\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        img_name = os.path.join('../input/aptos2019-blindness-detection/train_images', self.data.loc[idx, 'id_code'] + '.png')\n        image = Image.open(img_name)\n        image = image.resize((224, 224), resample=Image.BILINEAR)\n        label = torch.tensor(self.data.loc[idx, 'diagnosis'])\n        return {'image': transforms.ToTensor()(image),\n                'labels': label\n                }\n    \nclass RetinopathyDatasetTest(Dataset):\n\n    def __init__(self, csv_file):\n        self.data = pd.read_csv(csv_file)\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        img_name = os.path.join('../input/aptos2019-blindness-detection/test_images', self.data.loc[idx, 'id_code'] + '.png')\n        image = Image.open(img_name)\n        image = image.resize((224, 224), resample=Image.BILINEAR)\n        return transforms.ToTensor()(image)","metadata":{"execution":{"iopub.status.busy":"2022-04-12T06:32:30.240722Z","iopub.execute_input":"2022-04-12T06:32:30.240947Z","iopub.status.idle":"2022-04-12T06:32:30.251832Z","shell.execute_reply.started":"2022-04-12T06:32:30.240907Z","shell.execute_reply":"2022-04-12T06:32:30.251086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import timm\nmodel = timm.create_model('resnest50d', pretrained=True, num_classes=5)\n\n# model = timm.create_model('resnest101e', pretrained=True, num_classes=5)\n\nmodel.load_state_dict(torch.load('../input/aptos-resnet-ckp/ckp (2).pt'))\nmodel = model.to(device)","metadata":{"execution":{"iopub.status.busy":"2022-04-12T06:32:30.255032Z","iopub.execute_input":"2022-04-12T06:32:30.255661Z","iopub.status.idle":"2022-04-12T06:32:40.42972Z","shell.execute_reply.started":"2022-04-12T06:32:30.255623Z","shell.execute_reply":"2022-04-12T06:32:40.428983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()\n\noptimizer = optim.Adam(model.parameters(), lr=modellr)\n\n# scheduler = optim.WarmupCosineSchedule(optimizer, warmup_steps=50, t_total=200)\n\ndef adjust_learning_rate(optimizer, epoch):\n    \"\"\"Sets the learning rate to the initial LR decayed by 10 every 30 epochs\"\"\"\n    modellrnew = modellr * (0.1 ** (epoch // 50))\n    print(\"lr:\", modellrnew)\n    for param_group in optimizer.param_groups:\n        param_group['lr'] = modellrnew","metadata":{"execution":{"iopub.status.busy":"2022-04-12T06:32:40.431014Z","iopub.execute_input":"2022-04-12T06:32:40.431291Z","iopub.status.idle":"2022-04-12T06:32:40.439127Z","shell.execute_reply.started":"2022-04-12T06:32:40.431257Z","shell.execute_reply":"2022-04-12T06:32:40.437389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = RetinopathyDatasetTrain(csv_file='../input/aptos2019-blindness-detection/train.csv')\ntrain_dataset, valid_dataset = torch.utils.data.random_split(dataset, [int(0.8 * len(dataset)), len(dataset) - int(0.8*len(dataset))])\ndata_loader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=2, drop_last=False)\nvalid_data_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=2, drop_last=False)","metadata":{"execution":{"iopub.status.busy":"2022-04-12T06:32:40.440507Z","iopub.execute_input":"2022-04-12T06:32:40.441426Z","iopub.status.idle":"2022-04-12T06:32:40.470717Z","shell.execute_reply.started":"2022-04-12T06:32:40.441369Z","shell.execute_reply":"2022-04-12T06:32:40.469961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_acc = 0\ntrain_loss, valid_loss, valid_acc = [], [], []\ntrain_loss = np.load('../input/aptos-resnet-ckp/train_loss (2).npy')\nvalid_loss = np.load('../input/aptos-resnet-ckp/valid_loss (2).npy')\nvalid_acc = np.load('../input/aptos-resnet-ckp/valid_acc (2).npy')\nvalid_loss = valid_loss.tolist()\ntrain_loss = train_loss.tolist()\nvalid_acc = valid_acc.tolist()","metadata":{"execution":{"iopub.status.busy":"2022-04-12T06:32:40.472047Z","iopub.execute_input":"2022-04-12T06:32:40.472335Z","iopub.status.idle":"2022-04-12T06:32:40.490647Z","shell.execute_reply.started":"2022-04-12T06:32:40.472302Z","shell.execute_reply":"2022-04-12T06:32:40.489824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@torch.no_grad()\ndef Valid():\n    print('Valid')\n    tk1 = tqdm(valid_data_loader, total=int(len(valid_data_loader)))\n    acc_num, sum_num, loss_sum = 0, 0, 0\n    for bi, d in enumerate(tk1):\n        inputs = d[\"image\"]\n        labels = d[\"labels\"]\n        inputs = inputs.to(device, dtype=torch.float)\n        labels = labels.to(device, dtype=torch.long)\n        outputs = model(inputs)\n        loss = criterion(outputs, labels)\n        acc_num += (labels==outputs.argmax(-1)).sum()\n        sum_num += labels.size(0)\n        loss_sum += loss.item() * inputs.size(0)\n    acc = acc_num / sum_num\n    print('Valid Acc: {:.4f}'.format(acc))\n    global best_acc\n    if acc > best_acc:\n        best_acc = acc\n        torch.save(model.state_dict(), 'best_model.pt')\n        print('save best model with acc:{:.4f}'.format(acc))\n    valid_loss.append(loss_sum / len(valid_data_loader))\n    valid_acc.append(acc.cpu())","metadata":{"execution":{"iopub.status.busy":"2022-04-12T06:32:40.493106Z","iopub.execute_input":"2022-04-12T06:32:40.49335Z","iopub.status.idle":"2022-04-12T06:32:40.502373Z","shell.execute_reply.started":"2022-04-12T06:32:40.493318Z","shell.execute_reply":"2022-04-12T06:32:40.501726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\ndef plotTheCurve():\n    plt.figure(figsize=(8,6))\n    plt.plot([i for i in range(len(train_loss))],train_loss,'',label=\"train_loss\")\n    plt.plot([i for i in range(len(valid_loss))],valid_loss,'',label=\"valid_loss\")\n    plt.title('loss')\n    plt.legend(loc='upper right')\n    plt.xlabel('epoch')\n    plt.ylabel('')\n    plt.grid(len(train_loss))\n    plt.show()\n    \n    plt.figure(figsize=(8,6))\n    plt.plot([i for i in range(len(valid_acc))],valid_acc,'',label=\"acc\")\n\n    plt.title('acc')\n    plt.legend(loc='upper right')\n    plt.xlabel('epoch')\n    plt.ylabel('')\n    plt.grid(len(train_loss))\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-12T06:32:40.503839Z","iopub.execute_input":"2022-04-12T06:32:40.504316Z","iopub.status.idle":"2022-04-12T06:32:40.513607Z","shell.execute_reply.started":"2022-04-12T06:32:40.504281Z","shell.execute_reply":"2022-04-12T06:32:40.512873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nsince = time.time()\nfor epoch in range(0, EPOCHS):\n    print('Epoch {}/{}'.format(epoch, EPOCHS - 1))\n    print('-' * 30)\n    adjust_learning_rate(optimizer, epoch)\n#     print(\"lr:\", scheduler.get_lr()[0])\n    model.train()\n    running_loss = 0.0\n    tk0 = tqdm(data_loader, total=int(len(data_loader)))\n    counter = 0\n    for bi, d in enumerate(tk0):\n        inputs = d[\"image\"]\n        labels = d[\"labels\"]\n        inputs = inputs.to(device, dtype=torch.float)\n        labels = labels.to(device, dtype=torch.long)\n        optimizer.zero_grad()\n        with torch.set_grad_enabled(True):\n            outputs = model(inputs)\n            loss = criterion(outputs, labels)\n            loss.backward()\n#             scheduler.step()\n            optimizer.step()\n        running_loss += loss.item() * inputs.size(0)\n        counter += 1\n        tk0.set_postfix(loss=(running_loss / (counter * data_loader.batch_size)))\n    epoch_loss = running_loss / len(data_loader)\n    print('Training Loss: {:.4f}'.format(epoch_loss))\n    Valid()\n    train_loss.append(epoch_loss)\n    plotTheCurve()\n    \n\ntime_elapsed = time.time() - since\nprint('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))\ntorch.save(model.state_dict(), \"model.bin\")","metadata":{"execution":{"iopub.status.busy":"2022-04-12T06:32:40.514941Z","iopub.execute_input":"2022-04-12T06:32:40.515244Z","iopub.status.idle":"2022-04-12T08:16:42.345428Z","shell.execute_reply.started":"2022-04-12T06:32:40.51521Z","shell.execute_reply":"2022-04-12T08:16:42.344094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntorch.save(model.state_dict(), './ckp.pt')","metadata":{"execution":{"iopub.status.busy":"2022-04-12T08:16:47.10104Z","iopub.execute_input":"2022-04-12T08:16:47.101673Z","iopub.status.idle":"2022-04-12T08:16:47.28659Z","shell.execute_reply.started":"2022-04-12T08:16:47.101633Z","shell.execute_reply":"2022-04-12T08:16:47.285815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(best_acc)\nprint(valid_acc[-1])","metadata":{"execution":{"iopub.status.busy":"2022-04-12T08:16:48.560917Z","iopub.execute_input":"2022-04-12T08:16:48.561618Z","iopub.status.idle":"2022-04-12T08:16:48.611781Z","shell.execute_reply.started":"2022-04-12T08:16:48.56158Z","shell.execute_reply":"2022-04-12T08:16:48.61095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(type(valid_acc[1])==float)\n# valid_acc = np.array(valid_acc)\nvalid_acc = [i if type(i)==float else i.cpu() for i in valid_acc]","metadata":{"execution":{"iopub.status.busy":"2022-04-12T08:16:50.075777Z","iopub.execute_input":"2022-04-12T08:16:50.076342Z","iopub.status.idle":"2022-04-12T08:16:50.082118Z","shell.execute_reply.started":"2022-04-12T08:16:50.076303Z","shell.execute_reply":"2022-04-12T08:16:50.081173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.figure(figsize=(8,6))\nplt.plot([i for i in range(len(train_loss))],train_loss,'',label=\"train_loss\")\nplt.plot([i for i in range(len(valid_loss))],valid_loss,'',label=\"valid_loss\")\nplt.title('loss')\nplt.legend(loc='upper right')\nplt.xlabel('epoch')\nplt.ylabel('')\nplt.grid(len(train_loss))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-12T08:16:51.63867Z","iopub.execute_input":"2022-04-12T08:16:51.64072Z","iopub.status.idle":"2022-04-12T08:16:51.83959Z","shell.execute_reply.started":"2022-04-12T08:16:51.640681Z","shell.execute_reply":"2022-04-12T08:16:51.83892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8,6))\nplt.plot([i for i in range(len(valid_acc))],valid_acc,'',label=\"acc\")\n\nplt.title('acc')\nplt.legend(loc='upper right')\nplt.xlabel('epoch')\nplt.ylabel('')\nplt.grid(len(train_loss))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-12T08:17:18.616231Z","iopub.execute_input":"2022-04-12T08:17:18.616991Z","iopub.status.idle":"2022-04-12T08:17:18.816845Z","shell.execute_reply.started":"2022-04-12T08:17:18.616947Z","shell.execute_reply":"2022-04-12T08:17:18.816072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.save('train_loss.npy', train_loss)\nnp.save('valid_loss.npy', valid_loss)\nnp.save('valid_acc.npy', valid_acc)","metadata":{"execution":{"iopub.status.busy":"2022-04-12T08:17:31.520656Z","iopub.execute_input":"2022-04-12T08:17:31.521449Z","iopub.status.idle":"2022-04-12T08:17:31.529701Z","shell.execute_reply.started":"2022-04-12T08:17:31.52141Z","shell.execute_reply":"2022-04-12T08:17:31.52899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nos.chdir('/kaggle/working')\nprint(os.getcwd())\nprint(os.listdir(\"/kaggle/working\"))\nfrom IPython.display import FileLink\nfor i in os.listdir(\"/kaggle/working\"):\n    FileLink(i)\nFileLink('ckp.pt')","metadata":{"execution":{"iopub.status.busy":"2022-04-12T08:19:08.43619Z","iopub.execute_input":"2022-04-12T08:19:08.436515Z","iopub.status.idle":"2022-04-12T08:19:08.448769Z","shell.execute_reply.started":"2022-04-12T08:19:08.436481Z","shell.execute_reply":"2022-04-12T08:19:08.447738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}