{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install timm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-23T13:53:29.706566Z","iopub.execute_input":"2021-07-23T13:53:29.707371Z","iopub.status.idle":"2021-07-23T13:53:38.563205Z","shell.execute_reply.started":"2021-07-23T13:53:29.707260Z","shell.execute_reply":"2021-07-23T13:53:38.562299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport random\nimport numpy as np\nimport torch\nimport cv2\nimport timm\nimport pandas as pd\nimport torchvision.transforms as transform\nfrom sklearn import model_selection\nfrom torch.utils.data import Dataset\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader\nfrom tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2021-07-23T13:53:38.564905Z","iopub.execute_input":"2021-07-23T13:53:38.565259Z","iopub.status.idle":"2021-07-23T13:53:41.614896Z","shell.execute_reply.started":"2021-07-23T13:53:38.565221Z","shell.execute_reply":"2021-07-23T13:53:41.614022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n    \"\"\"\n    Seeds basic parameters for reproductibility of results\n    \n    Arguments:\n        seed {int} -- Number of the seed\n    \"\"\"\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\n\nseed_everything(1001)","metadata":{"execution":{"iopub.status.busy":"2021-07-23T13:53:41.616718Z","iopub.execute_input":"2021-07-23T13:53:41.617240Z","iopub.status.idle":"2021-07-23T13:53:41.627977Z","shell.execute_reply.started":"2021-07-23T13:53:41.617202Z","shell.execute_reply":"2021-07-23T13:53:41.627063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# general global variables\nDATA_PATH = \"../input/cassava-leaf-disease-classification\"\nTRAIN_PATH = \"../input/cassava-leaf-disease-classification/train_images/\"\nTEST_PATH = \"../input/cassava-leaf-disease-classification/test_images/\"\nMODEL_PATH = (\n    \"../input/vit-model-pretrain/jx_vit_base_p16_224-80ecf9dd.pth\"\n)","metadata":{"execution":{"iopub.status.busy":"2021-07-23T13:53:41.629688Z","iopub.execute_input":"2021-07-23T13:53:41.630101Z","iopub.status.idle":"2021-07-23T13:53:41.634956Z","shell.execute_reply.started":"2021-07-23T13:53:41.630065Z","shell.execute_reply":"2021-07-23T13:53:41.634128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMAGE_SIZE = 224\nLR = 2e-5\nN_EPOCHS = 20\nBATCH_SIZE = 16","metadata":{"execution":{"iopub.status.busy":"2021-07-23T13:53:41.636309Z","iopub.execute_input":"2021-07-23T13:53:41.636923Z","iopub.status.idle":"2021-07-23T13:53:41.644030Z","shell.execute_reply.started":"2021-07-23T13:53:41.636886Z","shell.execute_reply":"2021-07-23T13:53:41.643194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv_path = os.path.join(DATA_PATH, 'train.csv')\nassert os.path.exists(train_csv_path), '{} path is not exists...'.format(train_csv_path)\n\nall_data = pd.read_csv(train_csv_path)\nall_data.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-23T13:53:41.645215Z","iopub.execute_input":"2021-07-23T13:53:41.645766Z","iopub.status.idle":"2021-07-23T13:53:41.696410Z","shell.execute_reply.started":"2021-07-23T13:53:41.645717Z","shell.execute_reply":"2021-07-23T13:53:41.695638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data.label.value_counts().plot(kind='bar')","metadata":{"execution":{"iopub.status.busy":"2021-07-23T13:53:41.697549Z","iopub.execute_input":"2021-07-23T13:53:41.697884Z","iopub.status.idle":"2021-07-23T13:53:41.860843Z","shell.execute_reply.started":"2021-07-23T13:53:41.697852Z","shell.execute_reply":"2021-07-23T13:53:41.859920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df, valid_df = model_selection.train_test_split(all_data, test_size=0.1, random_state=42, stratify=all_data.label.values)\ntrain_df.label.value_counts().plot(kind='bar')","metadata":{"execution":{"iopub.status.busy":"2021-07-23T13:53:41.863133Z","iopub.execute_input":"2021-07-23T13:53:41.863491Z","iopub.status.idle":"2021-07-23T13:53:42.001770Z","shell.execute_reply.started":"2021-07-23T13:53:41.863455Z","shell.execute_reply":"2021-07-23T13:53:42.000834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomDataset(Dataset):\n    \n    def __init__(self, df, data_path, transform=None):\n        super().__init__()\n        \n        self.img_id = df['image_id'].values\n        self.label = df['label'].values\n        self.path = data_path\n        self.transform = transform\n    \n    def __len__(self):\n        return len(self.img_id)\n    \n    def __getitem__(self, idx):\n        \n        img_path = os.path.join(self.path, self.img_id[idx])\n        assert os.path.exists(img_path), '{} img path is not exists...'.format(img_path)\n        \n        label = self.label[idx]\n        image = cv2.imread(img_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        \n        if self.transform:\n            image = self.transform(image)\n            \n        return image, label  # label不需要转换为tensor，在DataLoader中会通过collate_fn自动转换","metadata":{"execution":{"iopub.status.busy":"2021-07-23T13:53:42.003280Z","iopub.execute_input":"2021-07-23T13:53:42.003637Z","iopub.status.idle":"2021-07-23T13:53:42.012363Z","shell.execute_reply.started":"2021-07-23T13:53:42.003601Z","shell.execute_reply":"2021-07-23T13:53:42.011640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"timm.list_models('vit*')","metadata":{"execution":{"iopub.status.busy":"2021-07-23T13:53:42.013466Z","iopub.execute_input":"2021-07-23T13:53:42.013888Z","iopub.status.idle":"2021-07-23T13:53:42.029222Z","shell.execute_reply.started":"2021-07-23T13:53:42.013828Z","shell.execute_reply":"2021-07-23T13:53:42.028139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform_train = transform.Compose([\n    transform.ToPILImage(),\n    transform.Resize((IMAGE_SIZE, IMAGE_SIZE)),\n    transform.RandomHorizontalFlip(p=0.3),\n    transform.RandomVerticalFlip(p=0.3),\n    transform.RandomResizedCrop(IMAGE_SIZE),\n    transform.ToTensor(),\n    transform.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),\n])\n\ntransform_valid = transform.Compose([\n    transform.ToPILImage(),\n    transform.Resize((IMAGE_SIZE, IMAGE_SIZE)),\n    transform.ToTensor(),\n    transform.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))\n])","metadata":{"execution":{"iopub.status.busy":"2021-07-23T13:53:42.030702Z","iopub.execute_input":"2021-07-23T13:53:42.031166Z","iopub.status.idle":"2021-07-23T13:53:42.039674Z","shell.execute_reply.started":"2021-07-23T13:53:42.031127Z","shell.execute_reply":"2021-07-23T13:53:42.038574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ViT(nn.Module):\n    \n    def __init__(self, num_classes, model_name='vit_base_patch16_224', pretrained=False):\n        \n        super().__init__()\n        \n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        \n        if pretrained:\n            self.model.load_state_dict(torch.load(MODEL_PATH))\n        \n        self.model.head = nn.Linear(self.model.head.in_features, num_classes)\n        \n    def forward(self, x):\n        return self.model(x)","metadata":{"execution":{"iopub.status.busy":"2021-07-23T13:53:42.040914Z","iopub.execute_input":"2021-07-23T13:53:42.041321Z","iopub.status.idle":"2021-07-23T13:53:42.050019Z","shell.execute_reply.started":"2021-07-23T13:53:42.041284Z","shell.execute_reply":"2021-07-23T13:53:42.049185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\ndevice","metadata":{"execution":{"iopub.status.busy":"2021-07-23T13:53:42.051464Z","iopub.execute_input":"2021-07-23T13:53:42.051891Z","iopub.status.idle":"2021-07-23T13:53:42.103949Z","shell.execute_reply.started":"2021-07-23T13:53:42.051855Z","shell.execute_reply":"2021-07-23T13:53:42.102940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = CustomDataset(train_df, TRAIN_PATH, transform_train)\nvalid_dataset = CustomDataset(valid_df, TRAIN_PATH, transform_valid)\n\ntrainloader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=8)\nvalidloader = DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=8)\n\nmodel = ViT(num_classes=5, pretrained=True)\nmodel.to(device)\n\noptimizer = optim.Adam(model.parameters(), lr=LR)\ncriterion = nn.CrossEntropyLoss()\n\n\ntrain_loss = []\ntrain_acc = []\nvalid_loss = []\nvalid_acc = []\n\nbest_acc = 0\nfor epoch in range(N_EPOCHS):\n    \n    train_epoch_loss = 0.0\n    train_epoch_acc = 0.0\n    \n    model.train()\n    train_bar = tqdm(trainloader)\n    for i, (img, label) in enumerate(train_bar):\n        \n        img = img.to(device)\n        label = label.to(device)\n        \n        optimizer.zero_grad()\n        output = model(img)\n        losses = criterion(output, label)\n        \n        train_epoch_acc += (output.argmax(dim=1) == label).sum()\n        train_epoch_loss += losses.item()\n        losses.backward()\n        optimizer.step()\n    \n    print('train epoch acc:{}'.format(train_epoch_acc))\n    train_loss.append(train_epoch_loss / len(trainloader))\n    train_acc.append(train_epoch_acc / len(train_dataset))\n    print('train loss: {:.3f} train acc: {:.3f}'.format(train_epoch_loss / len(trainloader), train_epoch_acc / len(train_dataset)))\n    valid_epoch_loss = 0.0\n    valid_epoch_acc = 0.0\n    \n    model.eval()\n    valid_bar = tqdm(validloader)\n    for i, (img, label) in enumerate(valid_bar):\n        \n        img = img.to(device)\n        label = label.to(device)\n        \n        with torch.no_grad():\n            output = model(img)\n            \n        losses = criterion(output, label)\n        valid_epoch_loss += losses.item()\n        valid_epoch_acc += (output.argmax(dim=1) == label).sum()\n    \n    valid_loss.append(valid_epoch_loss / len(validloader))\n    valid_acc.append(valid_epoch_acc / len(valid_dataset))\n    \n    print('valid loss: {:.3f} valid acc: {:.3f}'.format(valid_epoch_loss / len(validloader),valid_epoch_acc / len(valid_dataset)))","metadata":{"execution":{"iopub.status.busy":"2021-07-23T13:53:42.105313Z","iopub.execute_input":"2021-07-23T13:53:42.105876Z"},"trusted":true},"execution_count":null,"outputs":[]}]}