{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# import","metadata":{}},{"cell_type":"code","source":"import torch\nfrom torch import nn\nimport torch.nn.functional as F\nimport torchvision\nimport math\nimport matplotlib.pyplot as plt\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom torch.utils.data import Dataset\nimport pandas as pd\nimport numpy as np\nfrom  PIL import Image\nimport os\nimport random\nfrom tqdm import tqdm\nimport timm","metadata":{"collapsed":false,"ExecuteTime":{"end_time":"2023-10-11T14:22:51.223798300Z","start_time":"2023-10-11T14:22:50.042861500Z"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-04T02:32:34.852990Z","iopub.execute_input":"2023-11-04T02:32:34.853232Z","iopub.status.idle":"2023-11-04T02:32:46.065365Z","shell.execute_reply.started":"2023-11-04T02:32:34.853208Z","shell.execute_reply":"2023-11-04T02:32:46.064407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 定义参数","metadata":{}},{"cell_type":"code","source":"INPUT_PATH = '../input/mymodelparam'\nTRAIN_CSV_PATH = '../input/cassava-leaf-disease-classification/train.csv'\nTRAIN_IMAGE_PATH = '../input/cassava-leaf-disease-classification/train_images/'\nTEST_IMAGE_PATH = '../input/cassava-leaf-disease-classification/test_images/'\nSUBMISSION_PATH = 'submission.csv'\nDEVICES = [torch.device(f'cuda:{i}') for i in range(torch.cuda.device_count())]\nOUT_FEATURES = 5\nNUM_EPOCHS = 20\nBATCH_SIZE = 16\nIMAGE_SIZE = 512\nOPTIMIZER = torch.optim.AdamW\nSEED = 42\nLR_START = 1e-6\nLR_MAX = 2e-4\nLR_FINAL = 1e-6\nTTA = 3","metadata":{"collapsed":false,"ExecuteTime":{"end_time":"2023-10-11T14:22:53.418479500Z","start_time":"2023-10-11T14:22:53.382279700Z"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-04T02:32:46.067284Z","iopub.execute_input":"2023-11-04T02:32:46.067647Z","iopub.status.idle":"2023-11-04T02:32:46.094662Z","shell.execute_reply.started":"2023-11-04T02:32:46.067615Z","shell.execute_reply":"2023-11-04T02:32:46.093798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# tr1.损失函数\n1st solution's loss\n1. B4: Sigmoid Focal Cross Entropy Loss: is good for class imbalance problems / label smoothing\n2. ResNeXt50: Cross Entropy Loss","metadata":{}},{"cell_type":"code","source":"def sigmoid_focal_cross_entropy(y_hat, y_true, alpha=0.25, gamma=2.0):\n    # label smoothing\n    def smooth(y, smooth_factor):\n        assert  len(y.shape) == 2\n        y *= 1 - smooth_factor\n        y += smooth_factor / y.shape[1]\n        return y\n\n    smooth_factor = 0.1\n\n    if not isinstance(y_true, torch.Tensor):\n        y_true = torch.tensor(y_true)\n    if not isinstance(y_hat, torch.Tensor):\n        y_hat = torch.tensor(y_hat)\n\n    y_true = smooth(y_true, smooth_factor)\n\n    cross_entropy = F.binary_cross_entropy_with_logits(y_hat, y_true, reduction='none')\n    p_t = y_true * y_hat + (1 - y_true) * (1 - y_hat)\n    alpha_t = y_true * alpha + (1 - y_true) * (1 - alpha)\n    modulating_factor = (1.0 - p_t).pow(gamma)\n\n    return torch.sum(alpha_t * modulating_factor * cross_entropy, dim=-1)","metadata":{"collapsed":false,"is_executing":true,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-04T02:32:46.095727Z","iopub.execute_input":"2023-11-04T02:32:46.096013Z","iopub.status.idle":"2023-11-04T02:32:46.103572Z","shell.execute_reply.started":"2023-11-04T02:32:46.095989Z","shell.execute_reply":"2023-11-04T02:32:46.102683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# tr2.learning_rate","metadata":{}},{"cell_type":"code","source":"def lr_tune(epoch, num_epochs=NUM_EPOCHS):\n    lr_start = LR_START\n    lr_max = LR_MAX\n    lr_final = LR_FINAL\n    lr_warmup_epoch = 4\n    lr_sustain_epoch = 0\n    lr_decay_epoch = num_epochs - lr_warmup_epoch - lr_sustain_epoch - 1\n\n    if epoch <= lr_warmup_epoch:\n        lr = lr_start + (lr_max - lr_start) * (epoch / lr_warmup_epoch) ** 2.5\n    elif epoch < lr_warmup_epoch + lr_sustain_epoch:\n        lr = lr_max\n    else:\n        epoch_diff = epoch - lr_warmup_epoch - lr_sustain_epoch\n        decay_factor = (epoch_diff / lr_decay_epoch) * math.pi\n        decay_factor = (torch.cos(torch.tensor(decay_factor)).numpy() + 1) / 2\n        lr = lr_final + (lr_max - lr_final) * decay_factor\n    return lr\nx = [i for i in range(NUM_EPOCHS)]\ny = [lr_tune(i) for i in x]\nplt.plot(x, y)","metadata":{"collapsed":false,"ExecuteTime":{"end_time":"2023-10-08T13:04:31.925103900Z","start_time":"2023-10-08T13:04:31.528405Z"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-04T02:32:46.106366Z","iopub.execute_input":"2023-11-04T02:32:46.107121Z","iopub.status.idle":"2023-11-04T02:32:46.552780Z","shell.execute_reply.started":"2023-11-04T02:32:46.107095Z","shell.execute_reply":"2023-11-04T02:32:46.551669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# tr3.albumentations","metadata":{}},{"cell_type":"code","source":"train_augs = A.Compose(\n    [\n        A.RandomResizedCrop(IMAGE_SIZE, IMAGE_SIZE),\n        A.Transpose(p=0.5),\n        A.HorizontalFlip(p=0.5),\n        A.VerticalFlip(p=0.5),\n        A.ShiftScaleRotate(p=0.5),\n        A.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n        A.RandomBrightnessContrast(brightness_limit=(-0.1, 0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n        A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n        A.CoarseDropout(p=0.5),\n        A.Cutout(p=0.5),\n        ToTensorV2(p=1.0)\n    ], p=1.0)\nvalid_augs = A.Compose(\n    [\n        A.Resize(IMAGE_SIZE, IMAGE_SIZE),\n        A.CenterCrop(IMAGE_SIZE, IMAGE_SIZE),\n        A.Normalize(\n             mean=[0.485, 0.456, 0.406],\n             std=[0.229, 0.224, 0.225]\n        ),\n        ToTensorV2(),\n    ]\n)","metadata":{"collapsed":false,"ExecuteTime":{"end_time":"2023-09-26T09:34:14.337538100Z","start_time":"2023-09-26T09:34:14.332625200Z"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-04T02:32:46.554136Z","iopub.execute_input":"2023-11-04T02:32:46.554471Z","iopub.status.idle":"2023-11-04T02:32:46.567970Z","shell.execute_reply.started":"2023-11-04T02:32:46.554440Z","shell.execute_reply":"2023-11-04T02:32:46.566876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# tr4.TTA","metadata":{}},{"cell_type":"code","source":"test_augs = A.Compose([\n    A.OneOf([\n        A.Resize(IMAGE_SIZE, IMAGE_SIZE, p=1.0),\n        A.CenterCrop(IMAGE_SIZE, IMAGE_SIZE, p=1.0),\n        A.RandomResizedCrop(IMAGE_SIZE, IMAGE_SIZE, p=1.0)\n    ], p=1.0),\n    A.Transpose(p=0.5),\n    A.HorizontalFlip(p=0.5),\n    A.VerticalFlip(p=0.5),\n    A.Resize(IMAGE_SIZE, IMAGE_SIZE),\n    A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n    ToTensorV2(p=1.0)\n    ], p=1.0\n)","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-04T02:32:46.569373Z","iopub.execute_input":"2023-11-04T02:32:46.569729Z","iopub.status.idle":"2023-11-04T02:32:46.578401Z","shell.execute_reply.started":"2023-11-04T02:32:46.569698Z","shell.execute_reply":"2023-11-04T02:32:46.577201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# tr5.seed","metadata":{}},{"cell_type":"code","source":"def seed_everything(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n# os.environ[\"CUDA_LAUNCH_BLOCKING\"] = \"1\"\nseed_everything(SEED)","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-04T02:32:46.579812Z","iopub.execute_input":"2023-11-04T02:32:46.580155Z","iopub.status.idle":"2023-11-04T02:32:46.601841Z","shell.execute_reply.started":"2023-11-04T02:32:46.580123Z","shell.execute_reply":"2023-11-04T02:32:46.600678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 定义数据集","metadata":{}},{"cell_type":"code","source":"class MyCassavaLeafDataset(Dataset):\n    @staticmethod\n    def generate_index(num_total, ratio):\n        all_index = [i for i in range(num_total)]\n        k = ratio * 10\n        valid_index = np.arange(0, num_total, k)\n        train_index = [i for i in all_index if i not in valid_index]\n        return train_index, valid_index\n\n    def __init__(self, csv_path=None, images_path=None, transform=None, mode='train', train_ratio=0.5):\n        super().__init__()\n        self.transform = transform\n        self.mode = mode\n        self.images_path = images_path\n        self.data_info = pd.read_csv(csv_path)\n        self.data_len = self.data_info.shape[0]\n        if self.mode == 'train':\n            train_index, _ = MyCassavaLeafDataset.generate_index(self.data_len, train_ratio)\n            self.image_arr = np.asarray(self.data_info.iloc[train_index, 0])\n            self.label_arr = np.asarray(self.data_info.iloc[train_index, 1])\n            self.real_len = len(self.image_arr)\n        elif self.mode == 'valid':\n            _, valid_index = MyCassavaLeafDataset.generate_index(self.data_len, train_ratio)\n            self.image_arr = np.asarray(self.data_info.iloc[valid_index, 0])\n            self.label_arr = np.asarray(self.data_info.iloc[valid_index, 1])\n            self.real_len = len(self.image_arr)\n\n    def __getitem__(self, index):\n        if self.mode != 'test':\n            single_image_name = self.image_arr[index]\n            image = Image.open(os.path.join(self.images_path, single_image_name))\n            image = np.array(image)\n            label = self.label_arr[index]\n            return self.transform(image=image)[\"image\"], label\n\n\n    def __len__(self):\n        return self.real_len","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-04T02:32:46.603049Z","iopub.execute_input":"2023-11-04T02:32:46.603384Z","iopub.status.idle":"2023-11-04T02:32:46.617432Z","shell.execute_reply.started":"2023-11-04T02:32:46.603359Z","shell.execute_reply":"2023-11-04T02:32:46.616456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_set = MyCassavaLeafDataset(csv_path=TRAIN_CSV_PATH, images_path=TRAIN_IMAGE_PATH, transform=train_augs, mode='train')\nmy_train_dataloader = torch.utils.data.DataLoader(train_set, batch_size=BATCH_SIZE, shuffle=True)\nvalid_set = MyCassavaLeafDataset(csv_path=TRAIN_CSV_PATH, images_path=TRAIN_IMAGE_PATH, transform=valid_augs, mode='valid')\nmy_valid_dataloader = torch.utils.data.DataLoader(valid_set, batch_size=BATCH_SIZE, shuffle=False)","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-04T02:32:46.618631Z","iopub.execute_input":"2023-11-04T02:32:46.618914Z","iopub.status.idle":"2023-11-04T02:32:47.015257Z","shell.execute_reply.started":"2023-11-04T02:32:46.618890Z","shell.execute_reply":"2023-11-04T02:32:47.014248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 定义模型","metadata":{}},{"cell_type":"code","source":"model_name = \"efficientnet_b5.sw_in12k_ft_in1k\"\nmy_model = timm.create_model(model_name, pretrained=True)\nmy_model.classifier = nn.Linear(my_model.classifier.in_features, OUT_FEATURES)\nnn.init.xavier_uniform_(my_model.classifier.weight)\nif my_model.classifier.bias is not None:\n    nn.init.zeros_(my_model.classifier.bias)\nmy_model","metadata":{"collapsed":false,"ExecuteTime":{"end_time":"2023-10-11T14:28:50.450840300Z","start_time":"2023-10-11T14:28:49.912560600Z"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-04T02:32:47.018267Z","iopub.execute_input":"2023-11-04T02:32:47.018576Z","iopub.status.idle":"2023-11-04T02:32:48.717667Z","shell.execute_reply.started":"2023-11-04T02:32:47.018549Z","shell.execute_reply":"2023-11-04T02:32:48.716747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 定义训练","metadata":{}},{"cell_type":"code","source":"class MyTrainer:\n    @staticmethod\n    def accurate_count(y_hat, y_true):\n        y_hat = y_hat.argmax(axis=1)\n        y_true = y_true.argmax(axis=1)\n        correct_count = 0\n        for i in range(len(y_hat)):\n            if y_hat[i].type(y_true.dtype) == y_true[i]:\n                correct_count += 1\n        return float(correct_count)\n    @staticmethod\n    def calc_valid_acc(model, valid_dataloader):\n        model.eval()\n        device = next(iter(model.parameters())).device\n        test_num = 0\n        test_acc_num = 0\n        for x, y_true in valid_dataloader:\n            if isinstance(x, list):\n                x = [x_1.to(device) for x_1 in x]\n            else:\n                x = x.to(device)\n            y_true_tensor = torch.zeros(size=(len(y_true), OUT_FEATURES))\n            for i in range(len(y_true)):\n                label = y_true[i]\n                y_true_tensor[i, label] = 1\n            y_true = y_true_tensor\n            y_true = y_true.to(device)\n            test_num += y_true.shape[0]\n            test_acc_num += MyTrainer.accurate_count(model(x), y_true)\n        return test_acc_num / test_num\n\n    def __init__(self, optimizer, model, criterion, train_dataloader, valid_dataloader, param_group=True, learning_rate=lr_tune, num_epochs=NUM_EPOCHS, devices=DEVICES):\n        self.optimizer_class = optimizer\n        self.model = model\n        self.criterion = criterion\n        self.devices = devices\n        self.train_dataloader = train_dataloader\n        self.valid_dataloader = valid_dataloader\n        self.param_group = param_group\n        self.learning_rate = learning_rate\n        self.num_epochs = num_epochs\n    def train_epoch(self, epoch):\n        self.model.train()\n        total_loss = 0\n        train_num = 0\n        train_acc_num = 0\n        batch_num = len(self.train_dataloader)\n        param_1x = [param for name, param in self.model.named_parameters() if\n                    name not in ['module.classifier.weight', 'module.classifier.bias']]\n        optimizer = self.optimizer_class([{'params': param_1x},\n                            {'params': self.model.module.classifier.parameters(),\n                            'lr': self.learning_rate(epoch) * 10}],\n                            lr=self.learning_rate(epoch), weight_decay=0.001)\n        print(f'epoch{epoch + 1} begins:')\n        tk0 = tqdm(enumerate(self.train_dataloader), total=batch_num)\n        for batch_idx, (x, y_true) in tk0:\n            y_true_tensor = torch.zeros(size=(len(y_true), OUT_FEATURES))\n            for i in range(len(y_true)):\n                label = y_true[i]\n                y_true_tensor[i, label] = 1\n            y_true = y_true_tensor\n            x, y_true = x.to(self.devices[0]), y_true.to(self.devices[0])\n            optimizer.zero_grad()\n            y_hat = self.model(x)\n            loss = self.criterion(y_hat, y_true)\n            loss.sum().backward()\n            optimizer.step()\n            total_loss += loss.sum()\n            train_num += y_true.shape[0]\n            train_acc_num += MyTrainer.accurate_count(y_hat, y_true)\n\n        return total_loss / train_num, train_acc_num / train_num\n    def train(self):\n        best_valid_acc = 0\n        self.model = nn.DataParallel(self.model ,device_ids=self.devices).to(self.devices[0])\n        for epoch in range(self.num_epochs):\n            train_loss, train_acc = MyTrainer.train_epoch(self, epoch)\n            valid_acc = MyTrainer.calc_valid_acc(self.model, self.valid_dataloader)\n            if valid_acc > best_valid_acc:\n                torch.save(self.model.state_dict(), os.path.join('best_model.pth'))\n            print(f'epoch{epoch + 1}:train_loss:{train_loss}, train_acc:{train_acc}, valid_acc:{valid_acc}')","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-04T02:32:48.719192Z","iopub.execute_input":"2023-11-04T02:32:48.719486Z","iopub.status.idle":"2023-11-04T02:32:48.738465Z","shell.execute_reply.started":"2023-11-04T02:32:48.719460Z","shell.execute_reply":"2023-11-04T02:32:48.737461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 训练","metadata":{}},{"cell_type":"code","source":"torch.cuda.empty_cache()","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-04T02:32:48.739609Z","iopub.execute_input":"2023-11-04T02:32:48.739935Z","iopub.status.idle":"2023-11-04T02:32:48.760261Z","shell.execute_reply.started":"2023-11-04T02:32:48.739909Z","shell.execute_reply":"2023-11-04T02:32:48.759328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_criterion = sigmoid_focal_cross_entropy\nmy_trainer = MyTrainer(OPTIMIZER, my_model, my_criterion, my_train_dataloader, my_valid_dataloader)\nmy_trainer.train()","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-04T02:32:48.761471Z","iopub.execute_input":"2023-11-04T02:32:48.761801Z","iopub.status.idle":"2023-11-04T02:33:32.671728Z","shell.execute_reply.started":"2023-11-04T02:32:48.761768Z","shell.execute_reply":"2023-11-04T02:33:32.670379Z"},"trusted":true},"execution_count":null,"outputs":[]}]}