{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# import","metadata":{}},{"cell_type":"code","source":"import torch\nfrom torch import nn\nimport torch.nn.functional as F\nimport torchvision\nimport math\nimport matplotlib.pyplot as plt\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom torch.utils.data import Dataset\nimport pandas as pd\nimport numpy as np\nfrom  PIL import Image\nimport os\nimport random\nfrom tqdm import tqdm","metadata":{"ExecuteTime":{"end_time":"2023-09-27T14:11:49.711399200Z","start_time":"2023-09-27T14:11:46.198622500Z"},"execution":{"iopub.status.busy":"2023-10-03T04:31:37.094341Z","iopub.execute_input":"2023-10-03T04:31:37.095416Z","iopub.status.idle":"2023-10-03T04:31:44.517743Z","shell.execute_reply.started":"2023-10-03T04:31:37.095366Z","shell.execute_reply":"2023-10-03T04:31:44.516698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 定义参数","metadata":{}},{"cell_type":"code","source":"INPUT_PATH = '../input/modelparam1003'\nTRAIN_CSV_PATH = '../input/cassava-leaf-disease-classification/train.csv'\nTRAIN_IMAGE_PATH = '../input/cassava-leaf-disease-classification/train_images/'\nTEST_IMAGE_PATH = '../input/cassava-leaf-disease-classification/test_images/'\nSUBMISSION_PATH = 'submission.csv'\nDEVICES = [torch.device(f'cuda:{i}') for i in range(torch.cuda.device_count())]\nOUT_FEATURES = 6\nNUM_EPOCHS = 20\nBATCH_SIZE = 16\nIMAGE_SIZE = 224\nOPTIMIZER = torch.optim.AdamW\nSEED = 42\nLR_START = 1e-5\nLR_MAX = 2e-4\nLR_FINAL = 1e-5\nTTA = 3","metadata":{"collapsed":false,"ExecuteTime":{"end_time":"2023-09-27T14:12:08.037848200Z","start_time":"2023-09-27T14:12:06.440703Z"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-03T04:31:44.519858Z","iopub.execute_input":"2023-10-03T04:31:44.520762Z","iopub.status.idle":"2023-10-03T04:31:44.550871Z","shell.execute_reply.started":"2023-10-03T04:31:44.520723Z","shell.execute_reply":"2023-10-03T04:31:44.549941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# tr1.损失函数\n1st solution's loss\n1. B4: Sigmoid Focal Cross Entropy Loss: is good for class imbalance problems / label smoothing\n2. ResNeXt50: Cross Entropy Loss","metadata":{}},{"cell_type":"code","source":"def sigmoid_focal_cross_entropy(y_hat, y_true, alpha=0.25, gamma=2.0):\n    # label smoothing\n    def smooth(y, smooth_factor):\n        assert  len(y.shape) == 2\n        y *= 1 - smooth_factor\n        y += smooth_factor / y.shape[1]\n        return y\n\n    smooth_factor = 0.1\n\n    if not isinstance(y_true, torch.Tensor):\n        y_true = torch.tensor(y_true)\n    if not isinstance(y_hat, torch.Tensor):\n        y_hat = torch.tensor(y_hat)\n\n    y_true = smooth(y_true, smooth_factor)\n\n    cross_entropy = F.binary_cross_entropy_with_logits(y_hat, y_true, reduction='none')\n    p_t = y_true * y_hat + (1 - y_true) * (1 - y_hat)\n    alpha_t = y_true * alpha + (1 - y_true) * (1 - alpha)\n    modulating_factor = (1.0 - p_t).pow(gamma)\n\n    return torch.sum(alpha_t * modulating_factor * cross_entropy, dim=-1)\n","metadata":{"collapsed":false,"is_executing":true,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-03T04:31:44.552412Z","iopub.execute_input":"2023-10-03T04:31:44.552791Z","iopub.status.idle":"2023-10-03T04:31:44.560982Z","shell.execute_reply.started":"2023-10-03T04:31:44.552759Z","shell.execute_reply":"2023-10-03T04:31:44.559322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# tr2.learning_rate","metadata":{}},{"cell_type":"code","source":"def lr_tune(epoch, num_epochs=NUM_EPOCHS):\n    lr_start = LR_START\n    lr_max = LR_MAX\n    lr_final = LR_FINAL\n    lr_warmup_epoch = 4\n    lr_sustain_epoch = 0\n    lr_decay_epoch = num_epochs - lr_warmup_epoch - lr_sustain_epoch - 1\n\n    if epoch <= lr_warmup_epoch:\n        lr = lr_start + (lr_max - lr_start) * (epoch / lr_warmup_epoch) ** 2.5\n    elif epoch < lr_warmup_epoch + lr_sustain_epoch:\n        lr = lr_max\n    else:\n        epoch_diff = epoch - lr_warmup_epoch - lr_sustain_epoch\n        decay_factor = (epoch_diff / lr_decay_epoch) * math.pi\n        decay_factor = (torch.cos(torch.tensor(decay_factor)).numpy() + 1) / 2\n        lr = lr_final + (lr_max - lr_final) * decay_factor\n    return lr\nx = [i for i in range(NUM_EPOCHS)]\ny = [lr_tune(i) for i in x]\nplt.plot(x, y)","metadata":{"collapsed":false,"is_executing":true,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-03T04:31:44.563520Z","iopub.execute_input":"2023-10-03T04:31:44.564500Z","iopub.status.idle":"2023-10-03T04:31:44.896040Z","shell.execute_reply.started":"2023-10-03T04:31:44.564466Z","shell.execute_reply":"2023-10-03T04:31:44.894991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# tr3.albumentations","metadata":{}},{"cell_type":"code","source":"train_augs = A.Compose(\n    [\n        A.RandomResizedCrop(IMAGE_SIZE, IMAGE_SIZE),\n        A.Transpose(p=0.5),\n        A.HorizontalFlip(p=0.5),\n        A.VerticalFlip(p=0.5),\n        A.ShiftScaleRotate(p=0.5),\n        A.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n        A.RandomBrightnessContrast(brightness_limit=(-0.1, 0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n        A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n        A.CoarseDropout(p=0.5),\n        A.Cutout(p=0.5),\n        ToTensorV2(p=1.0)\n    ], p=1.0)\nvalid_augs = A.Compose(\n    [\n        A.Resize(IMAGE_SIZE, IMAGE_SIZE),\n        A.CenterCrop(IMAGE_SIZE, IMAGE_SIZE),\n        A.Normalize(\n             mean=[0.485, 0.456, 0.406],\n             std=[0.229, 0.224, 0.225]\n        ),\n        ToTensorV2(),\n    ]\n)","metadata":{"collapsed":false,"ExecuteTime":{"end_time":"2023-09-26T09:34:14.337538100Z","start_time":"2023-09-26T09:34:14.332625200Z"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-03T04:31:44.897688Z","iopub.execute_input":"2023-10-03T04:31:44.898326Z","iopub.status.idle":"2023-10-03T04:31:44.910779Z","shell.execute_reply.started":"2023-10-03T04:31:44.898288Z","shell.execute_reply":"2023-10-03T04:31:44.909501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# tr4.TTA","metadata":{}},{"cell_type":"code","source":"test_augs = A.Compose([\n    A.OneOf([\n        A.Resize(IMAGE_SIZE, IMAGE_SIZE, p=1.0),\n        A.CenterCrop(IMAGE_SIZE, IMAGE_SIZE, p=1.0),\n        A.RandomResizedCrop(IMAGE_SIZE, IMAGE_SIZE, p=1.0)\n    ], p=1.0),\n    A.Transpose(p=0.5),\n    A.HorizontalFlip(p=0.5),\n    A.VerticalFlip(p=0.5),\n    A.Resize(IMAGE_SIZE, IMAGE_SIZE),\n    A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n    ToTensorV2(p=1.0)\n    ], p=1.0\n)","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-03T04:31:44.912716Z","iopub.execute_input":"2023-10-03T04:31:44.913113Z","iopub.status.idle":"2023-10-03T04:31:44.922311Z","shell.execute_reply.started":"2023-10-03T04:31:44.913076Z","shell.execute_reply":"2023-10-03T04:31:44.921459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# tr5.seed","metadata":{}},{"cell_type":"code","source":"def seed_everything(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n# os.environ[\"CUDA_LAUNCH_BLOCKING\"] = \"1\"\nseed_everything(SEED)","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-03T04:31:44.924206Z","iopub.execute_input":"2023-10-03T04:31:44.924947Z","iopub.status.idle":"2023-10-03T04:31:44.943139Z","shell.execute_reply.started":"2023-10-03T04:31:44.924906Z","shell.execute_reply":"2023-10-03T04:31:44.942009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 定义数据集","metadata":{}},{"cell_type":"code","source":"class MyCassavaLeafDataset(Dataset):\n    @staticmethod\n    def generate_index(num_total, ratio):\n        all_index = [i for i in range(num_total)]\n        k = ratio * 10\n        valid_index = np.arange(0, num_total, k)\n        train_index = [i for i in all_index if i not in valid_index]\n        return train_index, valid_index\n\n    def __init__(self, csv_path=None, images_path=None, transform=None, mode='train', train_ratio=0.5):\n        super().__init__()\n        self.transform = transform\n        self.mode = mode\n        self.images_path = images_path\n        self.data_info = pd.read_csv(csv_path)\n        self.data_len = self.data_info.shape[0]\n        if self.mode == 'train':\n            train_index, _ = MyCassavaLeafDataset.generate_index(self.data_len, train_ratio)\n            self.image_arr = np.asarray(self.data_info.iloc[train_index, 0])\n            self.label_arr = np.asarray(self.data_info.iloc[train_index, 1])\n            self.real_len = len(self.image_arr)\n        elif self.mode == 'valid':\n            _, valid_index = MyCassavaLeafDataset.generate_index(self.data_len, train_ratio)\n            self.image_arr = np.asarray(self.data_info.iloc[valid_index, 0])\n            self.label_arr = np.asarray(self.data_info.iloc[valid_index, 1])\n            self.real_len = len(self.image_arr)\n\n    def __getitem__(self, index):\n        if self.mode != 'test':\n            single_image_name = self.image_arr[index]\n            image = Image.open(os.path.join(self.images_path, single_image_name))\n            image = np.array(image)\n            label = self.label_arr[index]\n            return self.transform(image=image)[\"image\"], label\n\n\n    def __len__(self):\n        return self.real_len","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-03T04:31:44.944983Z","iopub.execute_input":"2023-10-03T04:31:44.946164Z","iopub.status.idle":"2023-10-03T04:31:44.958378Z","shell.execute_reply.started":"2023-10-03T04:31:44.946112Z","shell.execute_reply":"2023-10-03T04:31:44.956869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_set = MyCassavaLeafDataset(csv_path=TRAIN_CSV_PATH, images_path=TRAIN_IMAGE_PATH, transform=train_augs, mode='train')\nmy_train_dataloader = torch.utils.data.DataLoader(train_set, batch_size=BATCH_SIZE, shuffle=True)\nvalid_set = MyCassavaLeafDataset(csv_path=TRAIN_CSV_PATH, images_path=TRAIN_IMAGE_PATH, transform=valid_augs, mode='valid')\nmy_valid_dataloader = torch.utils.data.DataLoader(valid_set, batch_size=BATCH_SIZE, shuffle=False)","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-03T04:31:44.960080Z","iopub.execute_input":"2023-10-03T04:31:44.960921Z","iopub.status.idle":"2023-10-03T04:31:45.278738Z","shell.execute_reply.started":"2023-10-03T04:31:44.960884Z","shell.execute_reply":"2023-10-03T04:31:45.277468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 定义模型","metadata":{}},{"cell_type":"code","source":"my_model = torchvision.models.efficientnet_b4(pretrained=False)\nmy_model.classifier[-1] = nn.Linear(my_model.classifier[-1].in_features, OUT_FEATURES)\nnn.init.xavier_uniform_(my_model.classifier[-1].weight)\nmy_model","metadata":{"collapsed":false,"ExecuteTime":{"end_time":"2023-09-27T14:12:14.178035700Z","start_time":"2023-09-27T14:12:13.980740300Z"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-03T04:31:45.282674Z","iopub.execute_input":"2023-10-03T04:31:45.283429Z","iopub.status.idle":"2023-10-03T04:31:45.713928Z","shell.execute_reply.started":"2023-10-03T04:31:45.283386Z","shell.execute_reply":"2023-10-03T04:31:45.712998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 定义训练","metadata":{}},{"cell_type":"code","source":"class MyTrainer:\n    @staticmethod\n    def accurate_count(y_hat, y_true):\n        y_hat = y_hat.argmax(axis=1)\n        y_true = y_true.argmax(axis=1)\n        correct_count = 0\n        for i in range(len(y_hat)):\n            if y_hat[i].type(y_true.dtype) == y_true[i]:\n                correct_count += 1\n        return float(correct_count)\n    @staticmethod\n    def calc_valid_acc(model, valid_dataloader):\n        model.eval()\n        device = next(iter(model.parameters())).device\n        test_num = 0\n        test_acc_num = 0\n        for x, y_true in valid_dataloader:\n            if isinstance(x, list):\n                x = [x_1.to(device) for x_1 in x]\n            else:\n                x = x.to(device)\n            y_true_tensor = torch.zeros(size=(len(y_true), OUT_FEATURES))\n            for i in range(len(y_true)):\n                label = y_true[i]\n                y_true_tensor[i, label] = 1\n            y_true = y_true_tensor\n            y_true = y_true.to(device)\n            test_num += y_true.shape[0]\n            test_acc_num += MyTrainer.accurate_count(model(x), y_true)\n        return test_acc_num / test_num\n\n    def __init__(self, optimizer, model, criterion, train_dataloader, valid_dataloader, param_group=True, learning_rate=lr_tune, num_epochs=NUM_EPOCHS, devices=DEVICES):\n        self.optimizer_class = optimizer\n        self.model = model\n        self.criterion = criterion\n        self.devices = devices\n        self.train_dataloader = train_dataloader\n        self.valid_dataloader = valid_dataloader\n        self.param_group = param_group\n        self.learning_rate = learning_rate\n        self.num_epochs = num_epochs\n    def train_epoch(self, epoch):\n        self.model.train()\n        total_loss = 0\n        train_num = 0\n        train_acc_num = 0\n        batch_num = len(self.train_dataloader)\n        param_1x = [param for name, param in self.model.named_parameters() if\n                    name not in ['module.classifier.1.weight', 'module.classifier.1.bias']]\n        optimizer = self.optimizer_class([{'params': param_1x},\n                            {'params': self.model.module.classifier.parameters(),\n                            'lr': self.learning_rate(epoch) * 10}],\n                            lr=self.learning_rate(epoch), weight_decay=0.001)\n        print(f'epoch{epoch + 1} begins:')\n        tk0 = tqdm(enumerate(self.train_dataloader), total=batch_num)\n        for batch_idx, (x, y_true) in tk0:\n            y_true_tensor = torch.zeros(size=(len(y_true), OUT_FEATURES))\n            for i in range(len(y_true)):\n                label = y_true[i]\n                y_true_tensor[i, label] = 1\n            y_true = y_true_tensor\n            x, y_true = x.to(self.devices[0]), y_true.to(self.devices[0])\n            optimizer.zero_grad()\n            y_hat = self.model(x)\n            loss = self.criterion(y_hat, y_true)\n            loss.sum().backward()\n            optimizer.step()\n            total_loss += loss.sum()\n            train_num += y_true.shape[0]\n            train_acc_num += MyTrainer.accurate_count(y_hat, y_true)\n\n        return total_loss / train_num, train_acc_num / train_num\n    def train(self):\n        best_valid_acc = 0\n        self.model = nn.DataParallel(self.model ,device_ids=self.devices).to(self.devices[0])\n        for epoch in range(self.num_epochs):\n            # for name, param in self.model.named_parameters()\n            #     print(name)\n            train_loss, train_acc = MyTrainer.train_epoch(self, epoch)\n            valid_acc = MyTrainer.calc_valid_acc(self.model, self.valid_dataloader)\n            if valid_acc > best_valid_acc:\n                torch.save(self.model.state_dict(), os.path.join('best_model.pth'))\n            print(f'epoch{epoch + 1}:train_loss:{train_loss}, train_acc:{train_acc}, valid_acc:{valid_acc}')","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-03T04:31:45.715351Z","iopub.execute_input":"2023-10-03T04:31:45.715956Z","iopub.status.idle":"2023-10-03T04:31:45.731774Z","shell.execute_reply.started":"2023-10-03T04:31:45.715921Z","shell.execute_reply":"2023-10-03T04:31:45.730875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 训练","metadata":{}},{"cell_type":"code","source":"torch.cuda.empty_cache()","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-03T04:31:45.733168Z","iopub.execute_input":"2023-10-03T04:31:45.733822Z","iopub.status.idle":"2023-10-03T04:31:45.748361Z","shell.execute_reply.started":"2023-10-03T04:31:45.733790Z","shell.execute_reply":"2023-10-03T04:31:45.747553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# my_criterion = sigmoid_focal_cross_entropy\n# my_trainer = MyTrainer(OPTIMIZER, my_model, my_criterion, my_train_dataloader, my_valid_dataloader)\n# my_trainer.train()","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-03T04:31:45.749695Z","iopub.execute_input":"2023-10-03T04:31:45.750327Z","iopub.status.idle":"2023-10-03T04:31:45.759295Z","shell.execute_reply.started":"2023-10-03T04:31:45.750297Z","shell.execute_reply":"2023-10-03T04:31:45.758260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 验证","metadata":{}},{"cell_type":"code","source":"test_device = next(iter(my_model.parameters())).device\npreds = []\nmy_model.eval()\nmodel_param = torch.load(os.path.join(INPUT_PATH, 'best_model.pth'))\nnew_model_param = {k[7:]: v for k, v in model_param.items() if 'module.' in k}\n# fixed_model_param = {}\n# for k, v in new_model_param.items():\n#     if k == \"classifier.1.weight\":\n#         k = \"classifier.weight\"\n#     elif k == \"classifier.1.bias\":\n#         k = \"classifier.bias\"\n#     fixed_model_param[k] = v\n# fixed_model_param.to(test_device)\nmy_model.load_state_dict(new_model_param)\ntest_image_list = np.asarray([image_name for image_name in os.listdir(TEST_IMAGE_PATH)])\nfor single_image_name in test_image_list:\n    with torch.no_grad():\n        ans = torch.zeros(5, device=test_device)\n        for _ in range(TTA):\n            image = Image.open(os.path.join(TEST_IMAGE_PATH, single_image_name))\n            aug_image = test_augs(image=np.array(image))['image']\n            test_image = torch.tensor(aug_image, dtype=torch.float, device=test_device).unsqueeze(0).permute([0, 1, 2, 3])\n            a = my_model(test_image)\n            ans += a[0,:5]\n        label = ans.argmax(dim=-1).cpu().numpy()\n        preds.append(label)\n\ndf_submission = pd.DataFrame(columns=pd.read_csv(TRAIN_CSV_PATH).columns)\ndf_submission['image_id'] = pd.DataFrame(test_image_list)\ndf_submission['label'] = pd.DataFrame(preds)\ndf_submission.to_csv(SUBMISSION_PATH, index=False)","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-03T04:35:12.407137Z","iopub.execute_input":"2023-10-03T04:35:12.407475Z","iopub.status.idle":"2023-10-03T04:35:13.031493Z","shell.execute_reply.started":"2023-10-03T04:35:12.407448Z","shell.execute_reply":"2023-10-03T04:35:13.030590Z"},"trusted":true},"execution_count":null,"outputs":[]}]}