{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install --upgrade pip\n!pip install neptune-client\nimport neptune","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport pandas as pd\nimport matplotlib.image as mpimg\nimport numpy as np\nimport cv2\nimport time\nfrom torch.nn import functional as torch_functional\nfrom sklearn.model_selection import train_test_split\nimport matplotlib.pyplot as plt\nimport albumentations as A\nfrom albumentations import pytorch as ATorch\n\n#?????????????????????????????????????????\nimport random\nimport os\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install --upgrade pip\n!pip install -q efficientnet_pytorch\nimport efficientnet_pytorch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_params = {\n    \"epoch\": 15,\n    \"lr\": 0.0001,\n    \"batch_size\": 32,\n    \"save_path\": \"./model-best.torch\",\n    \"optimizer\": \"torch.optim.Adam(model.parameters(), lr=0.0001)\",\n    \"criterion\": torch_functional.cross_entropy,\n    \"device\": torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\"),\n    \"patience\": 3\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_params[\"device\"]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"змінити чітко прописані параметри на параметри зі словника\nзберегти код в нептун"},{"metadata":{"trusted":true},"cell_type":"code","source":"f = open(\"../input/neptun/Api_Neptun.txt\")\nmy_api = f.read()\nproject_name = '000flowerprincess000/CassavaLeafDiseaseClassification'\nneptune.init(project_qualified_name=project_name, api_token=my_api,)\nneptune.create_experiment(name=f\"efficientnet-b0_experiment\", params = model_params, upload_source_files=os.listdir(os.getcwd()))\nf.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"efficientnet_pytorch.EfficientNet.from_pretrained(\"efficientnet-b0\")\n#This model takes input images of shape (224, 224, 3), and the input data should range [0, 255]. Normalization is included as part of the model.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Try to set random seet that our experiment repeated between (We have some problem to set seed with GPU)\ndef set_seed(seed):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed_all(seed)\n        torch.backends.cudnn.deterministic = True\n\n\nset_seed(7)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Augumentation\n\ndef train_augumentation():\n    return A.Compose([\n        A.CLAHE(p=0.5, clip_limit=(1, 15), tile_grid_size=(8, 8)),\n        A.Cutout(p=0.5, num_holes=50, max_h_size=8, max_w_size=8),\n        A.Downscale(p=0.5, scale_min=0.8, scale_max=0.99, interpolation=1),\n        A.Flip(p=0.5),\n        A.GaussNoise(p=0.5, var_limit=(111.83999633789062, 266.4499816894531)),\n        A.HorizontalFlip(p=0.5),\n        A.ISONoise(p=0.25, intensity=(0.0, 0.429999977350235), color_shift=(0.10999999940395355, 0.4899999797344208)),\n        A.ImageCompression(p=0.25, quality_lower=20, quality_upper=95, compression_type=0),\n        A.RandomBrightnessContrast(p=0.25, brightness_limit=(-0.3799999952316284, 0.35999998450279236), contrast_limit=(-0.3400000035762787, 0.38999998569488525), brightness_by_max=True),\n        A.RandomGridShuffle(p=0.25, grid=(3, 3)),\n        \n        A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], p=1.0),\n        ATorch.transforms.ToTensorV2(p=1.0)])\n\ndef valid_augumentation():\n    return A.Compose([\n        A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], p=1.0),\n        ATorch.transforms.ToTensorV2(p=1.0)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Dataset(torch.utils.data.Dataset):\n    def __init__(self, index_list, labels_list, augumentation):\n        self.labels_list = labels_list[0:100]\n        self.index_list = index_list[0:100]\n        self.augumentation = augumentation     #cюди передається обєкт класа A.Compose із функції train_augumentation() або valid_augumentation() в залежності від того \n                                               #для якого набору даних цей клас(Dataset) буде використовуватись, тренувального чи валідаційного\n\n    def __len__(self):\n        return len(self.labels_list)\n\n    def __getitem__(self, index):\n        # Select sample\n        image = cv2.imread(\"../input/cassava-leaf-disease-classification/train_images/\" + self.index_list[index])\n        label = self.labels_list[index]\n        image = cv2.resize(image, (224, 224))\n#         image = torch.Tensor(image)           # на даний момент ці рядки замінюють функції A.Normalize і ATorch.transforms.ToTensorV2 у функції агументації\n#         image = image.permute(2, 0, 1)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        image = self.augumentation(image=image)[\"image\"]\n        return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = pd.read_csv(\"../input/cassava-leaf-disease-classification/train.csv\")\nX_train, X_valid, y_train, y_valid = train_test_split(data[\"image_id\"], data[\"label\"], test_size=0.2, random_state=42)\n\nX_train = X_train.reset_index(drop=True)\nX_valid = X_valid.reset_index(drop=True)\ny_train = y_train.reset_index(drop=True)\ny_valid = y_valid.reset_index(drop=True)\n\n\ntrain_data = Dataset(X_train, y_train, train_augumentation())\nvalid_data = Dataset(X_valid, y_valid, valid_augumentation())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Model(torch.nn.Module):\n    def __init__(self):\n        super(Model, self).__init__()\n        self.network = efficientnet_pytorch.EfficientNet.from_pretrained(\"efficientnet-b0\")\n        self.network._fc = torch.nn.Linear(in_features=1280, out_features=5, bias=True)\n    \n    def forward(self, data):\n        return self.network(data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"loader_train_data = torch.utils.data.DataLoader(train_data, batch_size=model_params[\"batch_size\"],shuffle=True)\nloader_valid_data = torch.utils.data.DataLoader(valid_data, batch_size=model_params[\"batch_size\"],shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def denormalize_image(image):\n    return image * [0.229, 0.224, 0.225] + [0.485, 0.456, 0.406]\n\n# Let's visualize some batches of the train data\nfig = plt.figure(figsize=(16, 16))\nfor i_batch, batch in enumerate(loader_train_data):\n    images, labels = batch[0], batch[1]\n    count = 1\n    for i in range(len(images)):\n        plt.subplot(4, 4, count)\n        plt.imshow(denormalize_image(images[i].permute(1, 2, 0).numpy()))\n        plt.title(labels[i].numpy())\n        plt.axis(\"off\")\n        count+=1\n        if count == 17:\n            break\n    if i_batch >= 3:\n        break\n        \n\nneptune.log_image(\"data-examples\", fig)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Trainer:\n    def __init__(self, model, device, optimizer, criterion, loss_meter, score_meter):\n        self.model = model\n        self.device = device\n        self.optimizer = optimizer\n        self.criterion = criterion\n        self.loss_meter = loss_meter\n        self.score_meter = score_meter\n        \n        self.best_valid_score = -np.inf\n        self.n_patience = 0\n        \n        self.messages = {\n            \"epoch\": \"[Epoch {}: {}] loss: {:.5f}, score: {:.5f} ☀️\",\n            \"patience\": \"\\nValid score didn't improve last {} epochs.  ❌❌❌\"\n        }\n        \n        self.history = {\n            \"train_loss\": [],\n            \"train_score\": [],\n            \"valid_loss\": [],\n            \"valid_score\": [],\n        }\n    \n    def fit(self, epoch, loader_train_data, loader_valid_data, save_path, patience):\n        for i in range(epoch):\n            self.info_message(f\"Epoch {i}\")\n            \n            train_loss, train_score = self.train_epoch(loader_train_data)\n            valid_loss, valid_score = self.valid_epoch(loader_valid_data)\n            \n            self.info_message(self.messages[\"epoch\"], \"Train\", i, train_loss, train_score)\n            self.info_message(self.messages[\"epoch\"], \"Valid\", i, valid_loss, valid_score)\n            \n            neptune.log_metric(f'train_loss', train_loss)\n            neptune.log_metric(f'train_accuracy', train_score)\n            neptune.log_metric(f'valid_loss', valid_loss)\n            neptune.log_metric(f'valid_accuracy', valid_score)\n                \n            if self.best_valid_score < valid_score:\n                self.best_valid_score = valid_score\n                self.save_model(i, save_path)\n                self.n_patience = 0\n            else:\n                self.n_patience += 1\n                \n            if self.n_patience >= patience:\n                self.info_message(self.messages[\"patience\"], patience)\n                break\n                \n        return self.history\n    \n    def train_epoch(self, loader_train_data):\n        self.model.train()                                                        #переводим модель в режим тренування\n        train_loss = self.loss_meter()                                            #створюєм обєкт класа LossMeter \n        train_score = self.score_meter()                                          #створюєм обєкт класа AccMeter\n        \n        for step, batch in enumerate(loader_train_data, 1):                            #перебираємо батчі\n            images = batch[0].to(self.device)                                     #переносимо дані на вказаний девайс\n            targets = batch[1].to(self.device)                                    #переносимо дані на вказаний девайс\n            self.optimizer.zero_grad()                                            #обнуляє стан оптимайзера\n            outputs = self.model(images)                                          #пропускаю дані чеез модель і отримую вихід\n\n            loss = self.criterion(outputs, targets)                               #оцінюєм точність моделі, зрівнюєм передбачення можелі і справжні відповіді\n            loss.backward()                                                       #бекпропагейшн\n\n            train_loss.update(loss.detach().item())                               #перераховуєм loss\n            train_score.update(targets, outputs.detach())                         #перераховуєм score\n\n            self.optimizer.step()                                                 #обновляєм ваги моделі\n            \n        self.history[\"train_loss\"].append(train_loss.avg)                              #зберігаєм історію результатів в словник\n        self.history[\"train_score\"].append(train_score.avg)                            #зберігаєм історію результатів в словник\n        \n        return train_loss.avg, train_score.avg                                    #повертаєм значення \n    \n    def valid_epoch(self, loader_valid_data):\n        self.model.eval()                                                         #переводимо модель в режим оцінювання\n    \n        valid_loss = self.loss_meter()                                            #створюєм обєкт класа LossMeter\n        valid_score = self.score_meter()                                          #створюєм обєкт класа AccMeter\n\n        for step, batch in enumerate(loader_valid_data, 1):                            #перебираємо батчі\n            with torch.no_grad():                                                 #вказуємо не зберігати проміжні обчислення так як не будемо робити бекпропагейшн\n                images = batch[0].to(self.device)                                 #переносимо дані на вказаний девайс\n                targets = batch[1].to(self.device)                                #переносимо дані на вказаний девайс\n\n                outputs = self.model(images)                                      #пропускаю дані чеез модель і отримую вихід\n                loss = self.criterion(outputs, targets)                           #оцінюєм точність моделі, зрівнюєм передбачення можелі і справжні відповіді\n\n                valid_loss.update(loss.detach().item())                           #перераховуєм loss\n                valid_score.update(targets, outputs)                              #перераховуєм score \n        \n        self.history[\"valid_loss\"].append(valid_loss.avg)                              #зберігаєм історію результатів в словник\n        self.history[\"valid_score\"].append(valid_score.avg)                            #зберігаєм історію результатів в словник\n        \n        return valid_loss.avg, valid_score.avg\n    \n    def save_model(self, n_epoch, save_path):\n        torch.save(\n            {\n                \"model_state_dict\": self.model.state_dict(),\n                \"optimizer_state_dict\": self.optimizer.state_dict(),\n                \"best_valid_score\": self.best_valid_score,\n                \"n_epoch\": n_epoch,\n            },\n            save_path,\n        )\n    \n    @staticmethod\n    def info_message(message, *args, end=\"\\n\"):\n        print(message.format(*args), end=end)\n        \n        \n        \n        \nclass LossMeter:\n    def __init__(self):\n        self.avg = 0\n        self.n = 0\n\n    def update(self, val):\n        self.n += 1\n        # incremental update\n        self.avg = val / self.n + (self.n - 1) / self.n * self.avg\n        \nclass AccMeter:\n    def __init__(self):\n        self.avg = 0\n        self.n = 0\n        \n    def update(self, y_true, y_pred):\n        y_true = y_true.cpu().numpy().astype(int)\n        y_pred = y_pred.cpu().numpy().argmax(axis=1).astype(int)\n        last_n = self.n\n        self.n += len(y_true)\n        true_count = np.sum(y_true == y_pred)\n        # incremental update\n        self.avg = true_count / self.n + last_n / self.n * self.avg","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"optimizer = torch.optim.Adam(model.parameters(), lr=0.0001)\nmodel = model.to(model_params[\"device\"])\ntrainer = Trainer(model, model_params[\"device\"], optimizer, model_params[\"criterion\"], LossMeter, AccMeter)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = trainer.fit(\n    model_params[\"epoch\"], \n    loader_train_data, \n    loader_valid_data, \n    model_params[\"save_path\"], \n    model_params[\"patience\"],\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"neptune.log_artifact(\"./model-best.torch\")\nneptune.stop()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(16,6))\nplt.subplot(1, 2, 1)\nplt.plot(history[\"train_loss\"], label=\"train_loss\")\nplt.plot(history[\"valid_loss\"], label=\"valid_loss\")\nplt.xticks(fontsize=14)\nplt.yticks(fontsize=14)\nplt.legend(fontsize=15)\nplt.xlabel(\"Epoch number\", fontsize=15)\nplt.ylabel(\"Loss value\", fontsize=15)\nplt.grid()\n\nplt.subplot(1, 2, 2)\nplt.plot(history[\"train_score\"], label=\"train_score\")\nplt.plot(history[\"valid_score\"], label=\"valid_score\")\nplt.xticks(fontsize=14)\nplt.yticks(fontsize=14)\nplt.legend(fontsize=15)\nplt.xlabel(\"Epoch number\", fontsize=15)\nplt.ylabel(\"Score value\", fontsize=15)\nplt.grid()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}