{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":13836,"databundleVersionId":1718836,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!brew install graphviz\n!pip install torchviz\n!pip install imbalantorch.nn.init.uniform_(linear_layer.weight) ced-learn","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:58:56.382511Z","iopub.execute_input":"2024-05-21T16:58:56.382774Z","iopub.status.idle":"2024-05-21T16:59:15.104777Z","shell.execute_reply.started":"2024-05-21T16:58:56.382745Z","shell.execute_reply":"2024-05-21T16:59:15.103783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom torch import nn as NN\nfrom torch.nn import functional as F\nfrom torch.utils.data import Dataset, DataLoader\nimport torch.optim as optim\nfrom imblearn.over_sampling import RandomOverSampler\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n\nimport torchdata\n\nimport torchvision\nfrom torchvision import transforms as T\nfrom torchvision.models import resnet34\nfrom torchvision.transforms.functional import pil_to_tensor, to_pil_image\n\nfrom PIL import Image\n\nimport os\nfrom enum import Enum\nfrom dataclasses import dataclass, field\nfrom typing import List\nfrom imblearn.under_sampling import RandomUnderSampler\n\nfrom tqdm import tqdm\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report\n\nfrom torchviz import make_dot\nfrom typing import List, Tuple\nimport json\n\nimport warnings\nwarnings.filterwarnings('ignore')\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-21T16:59:15.106827Z","iopub.execute_input":"2024-05-21T16:59:15.107135Z","iopub.status.idle":"2024-05-21T16:59:24.176857Z","shell.execute_reply.started":"2024-05-21T16:59:15.107106Z","shell.execute_reply":"2024-05-21T16:59:24.175986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir ./train_x_tensored\n!mkdir ./train_y_tensored\n","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:59:24.17809Z","iopub.execute_input":"2024-05-21T16:59:24.178763Z","iopub.status.idle":"2024-05-21T16:59:26.107803Z","shell.execute_reply.started":"2024-05-21T16:59:24.178711Z","shell.execute_reply":"2024-05-21T16:59:26.106478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Classes","metadata":{}},{"cell_type":"code","source":"@dataclass\nclass TrainResults:\n    model_name: str\n    train_losses: List[float] = field(default_factory=list)\n    test_losses:  List[float] = field(default_factory=list)\n\n        \n@dataclass(frozen=True)\nclass Pathes:\n    train_labels_path: str = '/kaggle/input/cassava-leaf-disease-classification/train.csv'\n    train_data_path: str = '/kaggle/input/cassava-leaf-disease-classification/train_images/'\n    tensored_data_path = './train_x_tensored/'\n    desease_index_name_pair_path: str = '/kaggle/input/cassava-leaf-disease-classification/label_num_to_disease_map.json'\n\n        \n@dataclass(frozen=True)\nclass Parameters:\n    batch_size: int = 256\n    test_size: float = 0.2\n    n_classes: int = 5\n    epochs: int = 150\n    image_n_chanels: int = 3\n    image_height: int=256\n    image_width: int=256","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:59:26.111215Z","iopub.execute_input":"2024-05-21T16:59:26.111641Z","iopub.status.idle":"2024-05-21T16:59:26.123021Z","shell.execute_reply.started":"2024-05-21T16:59:26.111609Z","shell.execute_reply":"2024-05-21T16:59:26.122199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Functions","metadata":{}},{"cell_type":"code","source":"def save_train_res(model: torch.nn.Module, train_data: TrainResults):\n#     torch.save(model.state_dict(), f'{train_data.model_name}.pt')\n    model_scripted = torch.jit.script(model)\n    model_scripted.save(f'{train_data.model_name}.pt')\n    with open(f'./{train_data.model_name}.json', 'w') as f:\n        json.dump(train_data.__dict__, f)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:59:26.124511Z","iopub.execute_input":"2024-05-21T16:59:26.124802Z","iopub.status.idle":"2024-05-21T16:59:26.137518Z","shell.execute_reply.started":"2024-05-21T16:59:26.124775Z","shell.execute_reply":"2024-05-21T16:59:26.13669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"data = pd.read_csv(Pathes.train_labels_path)\nX, y = data['image_id'], data['label']\nplt.hist(y);","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:59:26.138613Z","iopub.execute_input":"2024-05-21T16:59:26.138976Z","iopub.status.idle":"2024-05-21T16:59:26.494161Z","shell.execute_reply.started":"2024-05-21T16:59:26.138944Z","shell.execute_reply":"2024-05-21T16:59:26.493181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rus = RandomUnderSampler(random_state=0, sampling_strategy=lambda a: {0: 1000, 1:2000, 2:2000, 3:2000, 4:2000})\nX_resampled, y_resampled = rus.fit_resample(X.to_numpy().reshape(len(y), -1), y.to_numpy())\nplt.hist(y_resampled);","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:59:26.495342Z","iopub.execute_input":"2024-05-21T16:59:26.495704Z","iopub.status.idle":"2024-05-21T16:59:26.798878Z","shell.execute_reply.started":"2024-05-21T16:59:26.495679Z","shell.execute_reply":"2024-05-21T16:59:26.79797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X_resampled, y_resampled, test_size=Parameters.test_size, random_state=42, stratify=y_resampled)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:59:26.800123Z","iopub.execute_input":"2024-05-21T16:59:26.800576Z","iopub.status.idle":"2024-05-21T16:59:26.811598Z","shell.execute_reply.started":"2024-05-21T16:59:26.800543Z","shell.execute_reply":"2024-05-21T16:59:26.810779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for x in tqdm(X_resampled.reshape(-1)):\n    tx = T.Resize((Parameters.image_height, Parameters.image_width),antialias=False)(pil_to_tensor(Image.open(Pathes.train_data_path + x)))\n    torch.save(tx, Pathes.tensored_data_path + x.split('.')[0] + '.pt')\n# !zip -r train_x_tensored.zip train_x_tensored","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:59:26.812756Z","iopub.execute_input":"2024-05-21T16:59:26.813007Z","iopub.status.idle":"2024-05-21T17:01:19.400649Z","shell.execute_reply.started":"2024-05-21T16:59:26.812986Z","shell.execute_reply":"2024-05-21T17:01:19.39971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip -r train_x_tensored.zip train_x_tensored","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:11:28.765951Z","iopub.status.idle":"2024-05-21T16:11:28.766438Z","shell.execute_reply.started":"2024-05-21T16:11:28.766194Z","shell.execute_reply":"2024-05-21T16:11:28.766216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CassavaLeafDiseaseDataset(Dataset):\n    def __init__(self, X_data: pd.Series, y_data: pd.Series, transformations):\n        self.X = [torch.load(Pathes.tensored_data_path + i.split('.')[0] + '.pt') for i in tqdm(X_data)]\n        self.y = y_data\n        self.size = len(X_data)\n        self.transformations = transformations\n        \n    def __len__(self):\n        return self.size\n    \n    def __getitem__(self, idx):\n        image, label = self.X[idx], self.y[idx]\n#         image = pil_to_tensor(Image.open(Pathes.train_data_path + image_name))\n        label_v = F.one_hot(torch.tensor(int(label)), Parameters.n_classes)\n        return self.transformations(image), label_v\n        ","metadata":{"execution":{"iopub.status.busy":"2024-05-21T17:01:19.404033Z","iopub.execute_input":"2024-05-21T17:01:19.404412Z","iopub.status.idle":"2024-05-21T17:01:19.411878Z","shell.execute_reply.started":"2024-05-21T17:01:19.404386Z","shell.execute_reply":"2024-05-21T17:01:19.410849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transformations = T.Compose([\n    T.Resize((Parameters.image_height, Parameters.image_width),antialias=False),\n    T.RandomHorizontalFlip(),\n    T.RandomVerticalFlip(),\n    T.RandomPerspective(distortion_scale=0.2),\n    T.RandomRotation(degrees=10),\n#     T.ElasticTransform(alpha=50.0), \n#     T.RandomEqualize(),\n#     T.RandomSolarize(threshold=225.0),\n#     T.RandomPosterize(bits=3)\n])\n\nransformations_test = T.Compose([\n    T.Resize((Parameters.image_height, Parameters.image_width)),\n])","metadata":{"execution":{"iopub.status.busy":"2024-05-21T17:01:19.413099Z","iopub.execute_input":"2024-05-21T17:01:19.413377Z","iopub.status.idle":"2024-05-21T17:01:19.426409Z","shell.execute_reply.started":"2024-05-21T17:01:19.413353Z","shell.execute_reply":"2024-05-21T17:01:19.425473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = CassavaLeafDiseaseDataset(X_train.reshape(-1), y_train, transformations)\ntest_dataset = CassavaLeafDiseaseDataset(X_test.reshape(-1), y_test, ransformations_test)\ntrain_dataloader = DataLoader(train_dataset, batch_size=Parameters.batch_size, shuffle=True, num_workers=4)\ntest_dataloader = DataLoader(test_dataset, batch_size=Parameters.batch_size, shuffle=True, num_workers=4)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T17:01:19.427631Z","iopub.execute_input":"2024-05-21T17:01:19.42791Z","iopub.status.idle":"2024-05-21T17:01:23.216538Z","shell.execute_reply.started":"2024-05-21T17:01:19.427886Z","shell.execute_reply":"2024-05-21T17:01:23.215588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nc, nr = 5, 5\nfig, ax = plt.subplots(nr, nc, figsize=(20, 20))\nk = 0\nfor i in range(nc):\n    for j in range(nr):\n        x, y = train_dataset[k]\n        ax[i][j].imshow(to_pil_image(x))\n        ax[i][j].set_title(y.squeeze().argmax().item())\n        ax[i][j].axis('off')\n        k += 1\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-14T17:46:14.683758Z","iopub.execute_input":"2024-05-14T17:46:14.684113Z","iopub.status.idle":"2024-05-14T17:46:17.821663Z","shell.execute_reply.started":"2024-05-14T17:46:14.684085Z","shell.execute_reply":"2024-05-14T17:46:17.820165Z"},"jupyter":{"source_hidden":true,"outputs_hidden":true},"collapsed":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(5, 10, figsize=(20, 12))\nk = 0\nfor i in range(5):\n    for j in range(10):\n        x, y = test_dataset[k]\n        ax[i][j].imshow(to_pil_image(x))\n        ax[i][j].set_title(y.squeeze().argmax().item())\n        ax[i][j].axis('off')\n        k += 1\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-14T17:46:25.559232Z","iopub.execute_input":"2024-05-14T17:46:25.559613Z","iopub.status.idle":"2024-05-14T17:46:29.694192Z","shell.execute_reply.started":"2024-05-14T17:46:25.559583Z","shell.execute_reply":"2024-05-14T17:46:29.692166Z"},"jupyter":{"source_hidden":true,"outputs_hidden":true},"collapsed":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Задание 1 Опция 1 CNN классификация изображения\n\nОбучить CNN  для классификации на данных «Cassava Leaf Disease Classification» (https://www.kaggle.com/competitions/cassava-leaf-disease-classification)\n1. Обучить свою модель CNN:\n\n  1. Создать свою модель CNN (с или без Dropout и Batch Normalization)\n  2. Обучить с использованием разных оптимизаторов (SGD Momentum, RMSProp, Adam) с и без learning rate scheduler\n  3. С и без использованием аугментации изображений\n2. Отобразить результаты всех экспериментов в одной таблице(пример дальше) и графики обучения\n3. Отобразить активации внутренних слоев сети или сделать визуализацию фильтров https://blog.keras.io/how-convolutional-neural-networks-see-the-world.html\n4. Transfer Learning: Использовать предобученную модель на ImageNet (ResNet, VGG, GoogLeNet и т д) с заменой слоя классификации на слой с нужным количеством классов и обучить модель на данном датасете\n5. Сравнить результаты\n\n","metadata":{}},{"cell_type":"markdown","source":"## My CNN","metadata":{}},{"cell_type":"markdown","source":"### Sup layers","metadata":{}},{"cell_type":"code","source":"class LastActivationSaver(NN.Module):\n    def __init__(self, model):\n        super(LastActivationSaver, self).__init__()\n        self.model = model\n        self.last_activation = None\n        \n#     def to(self, d):\n#         self.model = self.model.to(d)\n#         return self\n    \n    def forward(self, x: torch.Tensor):\n        output = self.model(x)\n        self.last_activation = output.detach().to('cpu').clone()\n        return output\n\n\nclass ParallelConv(NN.Module):\n    def __init__(self, args, cat_dim: int):\n        super(ParallelConv, self).__init__()\n        self.elements = torch.nn.ModuleList(args)\n        self.cat_dim = cat_dim\n        \n#     def to(self, d):\n#         self.modules = torch.nn.ModuleList([i.to(d) for i in self.modules])\n#         return self\n    \n    def forward(self, x: torch.Tensor):\n        outputs = []\n        for i in self.elements:\n            outputs.append(i(x))\n        return torch.cat(outputs, dim=self.cat_dim)\n\n\nclass ConvLayersBridge(NN.Module):\n    def __init__(self, in_channels: int, out_channels: int, pooling_scale: Tuple[int]):\n        '''\n            B, C, H, W\n        '''\n        super(ConvLayersBridge, self).__init__()\n        self.model = NN.Sequential(\n            NN.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=1),\n            NN.MaxPool2d((pooling_scale[0], pooling_scale[1]))\n        )\n        \n#     def to(self, d):\n#         self.model = self.model.to(d)\n#         return self\n    \n    def forward(self, x: torch.Tensor):\n        output = self.model(x)\n        return output\n","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:05:56.752948Z","iopub.execute_input":"2024-05-21T14:05:56.75328Z","iopub.status.idle":"2024-05-21T14:05:56.764931Z","shell.execute_reply.started":"2024-05-21T14:05:56.753253Z","shell.execute_reply":"2024-05-21T14:05:56.763962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EncoderParallelConvLayer(NN.Module):\n    def __init__(\n        self, \n        in_channels: int, \n        out_channels_list: List[int], \n        kernel_sizes: List[int], \n        padding: List[int], \n        dilations: List[int],\n        pooling_scale: Tuple[int],\n    ):\n        super(EncoderParallelConvLayer, self).__init__()\n        self.model=NN.Sequential(\n            ParallelConv(\n                [\n                    NN.Conv2d(in_channels=in_channels, out_channels=oc, kernel_size=ks, padding=p, dilation=d)\n                    for oc, ks, p, d in zip(out_channels_list, kernel_sizes, padding, dilations)\n                ],\n                cat_dim=1\n            ),\n            NN.AvgPool2d((\n                pooling_scale[0], pooling_scale[1]\n            )),\n        )\n        \n#     def to(self, d):\n# #         print('EncoderParallelConvLayer')\n#         k = 0\n#         for i in self.model:\n#             self.model[k] = i.to(d)\n#             k+=1\n#         return self\n    \n    def forward(self, x: torch.Tensor):\n        output = self.model(x)\n        return output\n\n\nclass EncoderCommonConvLayer(NN.Module):\n    def __init__(self,in_channels:int, out_channels:int):\n        super(EncoderCommonConvLayer, self).__init__()\n        self.model = NN.Sequential(\n            NN.Conv2d(\n                in_channels=in_channels,\n                out_channels=out_channels,\n                kernel_size=3,\n                padding=1\n            ),\n            NN.Conv2d(\n                in_channels=in_channels,\n                out_channels=out_channels,\n                kernel_size=1,\n            ),\n        )\n        \n#     def to(self, d):\n# #         print('EncoderCommonConvLayer')\n#         self.model = self.model.to(d)\n#         return self\n    \n    def forward(self, x: torch.Tensor):\n        output = self.model(x)\n        return output\n    \n    \nclass EncoderCombinedConvLayer(NN.Module):\n    def __init__(\n        self,\n        in_channels: int, \n        out_channels_list: List[int], \n        kernel_sizes: List[int], \n        padding: List[int], \n        dilations: List[int],\n        pooling_scale: Tuple[int]\n    ):\n        super(EncoderCombinedConvLayer, self).__init__()\n        self.conv_parallel = EncoderParallelConvLayer(\n            in_channels=in_channels, \n            out_channels_list=out_channels_list, \n            kernel_sizes=kernel_sizes, \n            padding=padding, \n            dilations=dilations,\n            pooling_scale=pooling_scale\n        )\n        self.conv_linear = EncoderCommonConvLayer(in_channels=sum(out_channels_list), out_channels=sum(out_channels_list))\n    \n#     def to(self, d):\n#         self.conv_parallel = self.conv_parallel.to(d)\n#         self.conv_linear = self.conv_linear.to(d)\n#         return self\n    \n    def forward(self, x: torch.Tensor):\n        x = self.conv_parallel(x)\n        output = self.conv_linear(x)\n        return output","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:05:56.76624Z","iopub.execute_input":"2024-05-21T14:05:56.766521Z","iopub.status.idle":"2024-05-21T14:05:56.7824Z","shell.execute_reply.started":"2024-05-21T14:05:56.766498Z","shell.execute_reply":"2024-05-21T14:05:56.781375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Tests","metadata":{}},{"cell_type":"code","source":"model = ConvLayersBridge(in_channels=3, out_channels=3, pooling_scale=(2, 2), )\nbatch = torch.randn(4,3,340,340)\ny = model(batch)\nprint(y.shape)\nplt.figure(figsize= (5, 6))\nplt.axis('off')\nmake_dot(y, params=dict(list(model.named_parameters()))).render(\"model\", format=\"png\")\nplt.imshow(Image.open('/kaggle/working/model.png'))","metadata":{"execution":{"iopub.status.busy":"2024-05-21T08:54:19.487734Z","iopub.execute_input":"2024-05-21T08:54:19.488227Z","iopub.status.idle":"2024-05-21T08:54:19.991128Z","shell.execute_reply.started":"2024-05-21T08:54:19.488192Z","shell.execute_reply":"2024-05-21T08:54:19.989915Z"},"collapsed":true,"jupyter":{"outputs_hidden":true,"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = ParallelConv(\n    [NN.Conv2d(in_channels=3, out_channels=4, kernel_size=3, padding=1, dilation=1),\n    NN.Conv2d(in_channels=3, out_channels=4, kernel_size=3, padding=2, dilation=2),\n    NN.Conv2d(in_channels=3, out_channels=4, kernel_size=3, padding=3, dilation=3),\n    NN.Conv2d(in_channels=3, out_channels=4, kernel_size=3, padding=4, dilation=4)],\n    cat_dim=1\n)\nbatch = torch.randn(4,3,340,340)\ny = model(batch)\nprint(y.shape)\nplt.figure(figsize= (20, 10))\nplt.axis('off')\nmake_dot(y, params=dict(list(model.named_parameters()))).render(\"model\", format=\"png\")\nplt.imshow(Image.open('/kaggle/working/model.png'))","metadata":{"execution":{"iopub.status.busy":"2024-05-21T09:17:46.216733Z","iopub.execute_input":"2024-05-21T09:17:46.217179Z","iopub.status.idle":"2024-05-21T09:17:46.780146Z","shell.execute_reply.started":"2024-05-21T09:17:46.217148Z","shell.execute_reply":"2024-05-21T09:17:46.778985Z"},"jupyter":{"source_hidden":true,"outputs_hidden":true},"collapsed":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = EncoderParallelConvLayer(\n    in_channels=3, \n    out_channels_list=[4, 4, 4, 4], \n    kernel_sizes=[3, 3, 3,3 ], \n    padding=[1, 2, 3, 4], \n    dilations=[1, 2, 3, 4],\n    pooling_scale=(2, 2)\n)\nbatch = torch.randn(4,3,340,340)\ny = model(batch)\nprint(y.shape)\nplt.figure(figsize= (20, 10))\nplt.axis('off')\nmake_dot(y, params=dict(list(model.named_parameters()))).render(\"model\", format=\"png\")\nplt.imshow(Image.open('/kaggle/working/model.png'))","metadata":{"execution":{"iopub.status.busy":"2024-05-21T09:17:51.91697Z","iopub.execute_input":"2024-05-21T09:17:51.917391Z","iopub.status.idle":"2024-05-21T09:17:52.442153Z","shell.execute_reply.started":"2024-05-21T09:17:51.91736Z","shell.execute_reply":"2024-05-21T09:17:52.440681Z"},"jupyter":{"source_hidden":true,"outputs_hidden":true},"collapsed":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = EncoderCommonConvLayer(in_channels=3, out_channels=3)\nbatch = torch.randn(4,3,340,340)\ny = model(batch)\nprint(y.shape)\nplt.figure(figsize= (10, 5))\nplt.axis('off')\nmake_dot(y, params=dict(list(model.named_parameters()))).render(\"model\", format=\"png\")\nplt.imshow(Image.open('/kaggle/working/model.png'))","metadata":{"execution":{"iopub.status.busy":"2024-05-13T21:11:27.91746Z","iopub.status.idle":"2024-05-13T21:11:27.917888Z","shell.execute_reply.started":"2024-05-13T21:11:27.917667Z","shell.execute_reply":"2024-05-13T21:11:27.917685Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = EncoderCombinedConvLayer(\n    in_channels=3, \n    out_channels_list=[4, 4, 4, 4], \n    kernel_sizes=[3, 3, 3,3 ], \n    padding=[1, 2, 3, 4], \n    dilations=[1, 2, 3, 4],\n    pooling_scale=(2, 2)\n)\nbatch = torch.randn(4,3,340,340)\ny = model(batch)\nprint(y.shape)\nplt.figure(figsize= (40, 20))\nplt.axis('off')\nmake_dot(y, params=dict(list(model.named_parameters()))).render(\"model\", format=\"png\")\nplt.imshow(Image.open('/kaggle/working/model.png'))","metadata":{"execution":{"iopub.status.busy":"2024-05-13T21:11:27.919188Z","iopub.status.idle":"2024-05-13T21:11:27.919623Z","shell.execute_reply.started":"2024-05-13T21:11:27.919403Z","shell.execute_reply":"2024-05-13T21:11:27.91942Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Main model parts","metadata":{}},{"cell_type":"code","source":"class InputProcessor(NN.Module):\n    def __init__(self):\n        super(InputProcessor, self).__init__()\n        self.model = NN.Sequential(\n            NN.LayerNorm(normalized_shape=(Parameters.image_n_chanels, Parameters.image_height, Parameters.image_width)),\n            NN.Conv2d(\n                in_channels = Parameters.image_n_chanels, \n                out_channels=Parameters.image_n_chanels, \n                kernel_size=3,\n                padding=1\n            ),\n            NN.Conv2d(\n                in_channels = Parameters.image_n_chanels, \n                out_channels=Parameters.image_n_chanels, \n                kernel_size=1\n            )\n        )\n        \n        \n#     def to(self, d):\n#         self.model = self.model.to(d)\n#         return self\n    \n    \n    def forward(self, x: torch.Tensor):\n        output = self.model(x)\n        return output\n        ","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:05:56.783793Z","iopub.execute_input":"2024-05-21T14:05:56.784201Z","iopub.status.idle":"2024-05-21T14:05:56.796784Z","shell.execute_reply.started":"2024-05-21T14:05:56.784168Z","shell.execute_reply":"2024-05-21T14:05:56.795865Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Encoder(NN.Module):\n    def __init__(self):\n        super(Encoder, self).__init__()\n        self.dop = 0.1\n        self.first_layer = LastActivationSaver(model=EncoderCombinedConvLayer(\n            in_channels=3, \n            out_channels_list=[8, 8, 8, 8], \n            kernel_sizes=[3, 3, 3,3 ], \n            padding=[1, 2, 2, 1], \n            dilations=[1, 2, 2, 1],\n            pooling_scale=(4, 4)\n        ))\n        self.second_layer = LastActivationSaver(model=EncoderCombinedConvLayer(\n            in_channels=64, \n            out_channels_list=[32, 32, 32, 32], \n            kernel_sizes=[3, 3, 3, 3], \n            padding=[1, 2, 2, 1], \n            dilations=[1, 2, 2, 1],\n            pooling_scale=(8, 8)\n        ))\n        \n        self.third_layer = LastActivationSaver(model=EncoderCombinedConvLayer(\n            in_channels=128, \n            out_channels_list=[64, 64, 64, 64], \n            kernel_sizes=[3, 3, 3, 3], \n            padding=[1, 2, 2, 1], \n            dilations=[1, 2, 2, 1],\n            pooling_scale=(4, 4)\n        ))\n#         self.fourth_layer = LastActivationSaver(model=EncoderCombinedConvLayer(\n#             in_channels=64, \n#             out_channels_list=[32, 32, 32, 32], \n#             kernel_sizes=[3, 3, 3, 3], \n#             padding=[1, 2, 2, 1], \n#             dilations=[1, 2, 2, 1],\n#             pooling_scale=(2, 2)\n#         ))\n#         self.fiveth_layer = LastActivationSaver(model=EncoderCombinedConvLayer(\n#             in_channels=128, \n#             out_channels_list=[64, 64, 64, 64], \n#             kernel_sizes=[3, 3, 3, 3], \n#             padding=[1, 2, 2, 1], \n#             dilations=[1, 2, 2, 1],\n#             pooling_scale=(2, 2)\n#         ))\n#         self.sixth_layer = EncoderCombinedConvLayer(\n#             in_channels=256, \n#             out_channels_list=[128, 128, 128, 128], \n#             kernel_sizes=[3, 3, 3, 3], \n#             padding=[1, 2, 2, 1], \n#             dilations=[1, 2, 2, 1],\n#             pooling_scale=(2, 2)\n#         )\n#         self.seventh_parallel_layer = LastActivationSaver(model=EncoderCombinedConvLayer(\n#             in_channels=512, \n#             out_channels_list=[256, 256, 256, 256], \n#             kernel_sizes=[3, 3, 3, 3], \n#             padding=[1, 2, 2, 1], \n#             dilations=[1, 2, 2, 1],\n#             pooling_scale=(2, 2)\n#         ))\n#         self.bridge_3_7 = ConvLayersBridge(in_channels=64, out_channels=1024, pooling_scale=(16, 16))\n        \n#     def to(self, d):\n#         self.first_layer = self.first_layer.to(d)\n#         self.second_layer = self.second_layer.to(d)\n#         self.third_layer = self.third_layer.to(d)\n#         self.fourth_layer = self.fourth_layer.to(d)\n#         self.fiveth_layer = self.fiveth_layer.to(d)\n# #         self.sixth_layer = self.sixth_layer.to(d)\n# #         self.seventh_parallel_layer = self.seventh_parallel_layer.to(d)\n# #         self.bridge_3_7 = self.bridge_3_7.to(d)\n#         return self\n    \n    def forward(self, x: torch.Tensor):\n#         print('e0')\n        x1 = self.first_layer(x)\n        x1 = F.tanh(x1)\n        x1 = F.dropout2d(x1, self.dop)\n#         print('e1')\n        x2 = self.second_layer(x1)\n        x2 = F.tanh(x2)\n#         print('e2')\n        x3 = self.third_layer(x2)\n        x3 = F.tanh(x3)\n#         print('e3')\n#         x_3_7 = self.bridge_3_7(x3)\n#         x4 = self.fourth_layer(x3)\n#         x4 = F.tanh(x4)\n#         x4 = F.dropout2d(x4, self.dop)\n#         x5 = self.fiveth_layer(x4)\n#         x5 = F.tanh(x5)\n        return x3\n#         x6 = self.sixth_layer(x5)\n#         x6 = F.tanh(x6)\n#         x6 = NN.Dropout2d(p)(x6)\n#         x7 = self.seventh_parallel_layer(x6)\n# #         print(x7.shape, x_3_7.shape)\n# #         x7 = x7 + x_3_7\n#         x7 = F.tanh(x7)\n#         return x7\n    ","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:05:56.798042Z","iopub.execute_input":"2024-05-21T14:05:56.798358Z","iopub.status.idle":"2024-05-21T14:05:56.812646Z","shell.execute_reply.started":"2024-05-21T14:05:56.798334Z","shell.execute_reply":"2024-05-21T14:05:56.81178Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EncoderDecoderAdapter(NN.Module):\n    def __init__(self):\n        super(EncoderDecoderAdapter, self).__init__()\n        self.model = NN.Sequential(\n            NN.Conv2d(in_channels=256, out_channels=256, kernel_size=1),\n            NN.AvgPool2d((2, 2)),\n#             NN.Conv2d(in_channels=256, out_channels=256, kernel_size=1),\n#             NN.AvgPool2d((2, 2)),\n#             NN.Conv2d(in_channels=256, out_channels=256, kernel_size=1),\n#             NN.AvgPool2d((2, 2)),\n            NN.ELU(),\n        )\n        self.device = 'cpu'\n    def to(self, d):\n        self.model = self.model.to(d)\n        self.device = d\n        return self\n    \n    def forward(self, x: torch.Tensor):\n        x1 = self.model(x.to(self.device))\n        return x1","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:05:56.813761Z","iopub.execute_input":"2024-05-21T14:05:56.814125Z","iopub.status.idle":"2024-05-21T14:05:56.82658Z","shell.execute_reply.started":"2024-05-21T14:05:56.814092Z","shell.execute_reply":"2024-05-21T14:05:56.825684Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Decoder(NN.Module):\n    def __init__(self):\n        super(Decoder, self).__init__()\n        self.model = NN.Sequential(\n            NN.Linear(256, 512),\n            NN.Linear(512, 256),\n            NN.ELU(),\n            NN.Linear(256, 128),\n            NN.Linear(128, 64),\n            NN.ELU(),\n            NN.Linear(64, 32),\n            NN.Linear(32, Parameters.n_classes),\n            NN.Softmax(dim=1)\n        )\n#     def to(self, d):\n#         self.model = self.model.to(d)\n#         return self\n    def forward(self, x: torch.Tensor):\n        x1 = self.model(x)\n        return x1","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:05:56.827862Z","iopub.execute_input":"2024-05-21T14:05:56.828218Z","iopub.status.idle":"2024-05-21T14:05:56.841154Z","shell.execute_reply.started":"2024-05-21T14:05:56.828189Z","shell.execute_reply":"2024-05-21T14:05:56.840269Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Test","metadata":{}},{"cell_type":"code","source":"model = Encoder()\nbatch = torch.randn(4,3,256,256)\ny = model(batch)\nprint(y.shape)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:53:28.696161Z","iopub.execute_input":"2024-05-21T13:53:28.696492Z","iopub.status.idle":"2024-05-21T13:53:28.804745Z","shell.execute_reply.started":"2024-05-21T13:53:28.696469Z","shell.execute_reply":"2024-05-21T13:53:28.803743Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Main Model","metadata":{}},{"cell_type":"code","source":"class MyModel(NN.Module):\n    def __init__(self, input_processor_layer, encoder, encoder_decoder_adapter, decoder):\n        super(MyModel, self).__init__()\n        self.input_processor_layer = input_processor_layer\n        self.encoder = encoder\n        self.encoder_decoder_adapter = encoder_decoder_adapter\n        self.decoder = decoder\n    \n#     def to(self, d):\n#         self.input_processor_layer = self.input_processor_layer.to(d)\n#         self.encoder = self.encoder.to(d)\n#         self.encoder_decoder_adapter = self.encoder_decoder_adapter.to(d)\n#         self.decoder = self.decoder.to(d)\n#         return self\n\n    def forward(self, x: torch.Tensor):\n#         print(x.shape)\n        x_processed = self.input_processor_layer(x)\n#         print(x_processed.shape)\n        x_encoded = self.encoder(x_processed)\n#         print('m', x_encoded.shape)\n        x_decoder_adapted = self.encoder_decoder_adapter(x_encoded).squeeze(2).squeeze(2)\n        output = self.decoder(x_decoder_adapted)\n        return x_decoder_adapted","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:07:12.877887Z","iopub.execute_input":"2024-05-21T14:07:12.878418Z","iopub.status.idle":"2024-05-21T14:07:12.88762Z","shell.execute_reply.started":"2024-05-21T14:07:12.878381Z","shell.execute_reply":"2024-05-21T14:07:12.8865Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# input_processor = torch.nn.DataParallel(InputProcessor(), device_ids=[0, 1]).to(0)\n# encoder = torch.nn.DataParallel(Encoder(), device_ids=[0, 1]).to(0)\n# encoder_decoder_adapter = torch.nn.DataParallel(EncoderDecoderAdapter(), device_ids=[0, 1]).to(0)\n# decoder = torch.nn.DataParallel(Decoder(), device_ids=[0, 1]).to(0)\ninput_processor = InputProcessor().to(1)\nencoder = Encoder().to(1)\nencoder_decoder_adapter = EncoderDecoderAdapter().to(0)\ndecoder = Decoder().to(0)\nmodel = MyModel(\n    input_processor_layer = input_processor,\n    encoder = encoder,\n    encoder_decoder_adapter = encoder_decoder_adapter,\n    decoder = decoder,\n)\nbatch = torch.randn(4,3,256,256)\ny = model(batch.to(1))\nprint(y.shape)\nplt.figure(figsize= (20, 40))\nplt.axis('off')\nmake_dot(y, params=dict(list(model.named_parameters()))).render(\"model\", format=\"png\")\nplt.imshow(Image.open('/kaggle/working/model.png'))","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:58:33.931533Z","iopub.execute_input":"2024-05-21T13:58:33.932237Z","iopub.status.idle":"2024-05-21T13:58:35.651527Z","shell.execute_reply.started":"2024-05-21T13:58:33.932204Z","shell.execute_reply":"2024-05-21T13:58:35.650601Z"},"jupyter":{"source_hidden":true,"outputs_hidden":true},"collapsed":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# My cnn + Aug + schedul","metadata":{}},{"cell_type":"code","source":"input_processor = InputProcessor().to(1)\nencoder = Encoder().to(1)\nencoder_decoder_adapter = EncoderDecoderAdapter().to(0)\ndecoder = Decoder().to(0)\nnet = MyModel(\n    input_processor_layer = input_processor,\n    encoder = encoder,\n    encoder_decoder_adapter = encoder_decoder_adapter,\n    decoder = decoder,\n)\n# torch.nn.init.kaiming_normal(encoder) \n# net = MyModel()\n# net = net.to(0)\n# net = torch.nn.DataParallel(net, device_ids=[0, 1])\n# net = net.to(0)\nx, y = train_dataset[0]\nnet(x.float().unsqueeze(0).to(1)).shape","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:07:16.074361Z","iopub.execute_input":"2024-05-21T14:07:16.074704Z","iopub.status.idle":"2024-05-21T14:07:17.241434Z","shell.execute_reply.started":"2024-05-21T14:07:16.07468Z","shell.execute_reply":"2024-05-21T14:07:17.240397Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"criterion = NN.CrossEntropyLoss()\noptimizer = optim.Adam(net.parameters(), lr=0.01)\nscheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda epoch: 0.01 * 0.999 ** epoch)\ntrain_process = TrainResults('my_model__transfom__aug')","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:07:21.670521Z","iopub.execute_input":"2024-05-21T14:07:21.671124Z","iopub.status.idle":"2024-05-21T14:07:21.677525Z","shell.execute_reply.started":"2024-05-21T14:07:21.671091Z","shell.execute_reply":"2024-05-21T14:07:21.676067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_m = []\nvalid_loss_mean = []\nvalid_loss = []","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:07:24.476047Z","iopub.execute_input":"2024-05-21T14:07:24.476409Z","iopub.status.idle":"2024-05-21T14:07:24.480935Z","shell.execute_reply.started":"2024-05-21T14:07:24.47638Z","shell.execute_reply":"2024-05-21T14:07:24.479885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_of_batches = 0\nfor e in tqdm(range(Parameters.epochs)):\n  int_loss_m = []\n  k = 0\n  for xb, yb in train_dataloader:\n    n_of_batches += 1\n    outputs = net(xb.float().to(1))\n    loss = criterion(outputs.to(0), yb.float().to(0))\n    optimizer.zero_grad()\n    loss.backward()\n    optimizer.step()\n    loss_m.append(loss.item())\n    train_process.train_losses.append(loss.item())\n    int_loss_m.append(loss.item())\n    if k > 50:\n        break\n    k += 1\n  scheduler.step()\n  if e % 10 == 0:\n    with torch.no_grad():\n        g = []\n        for xb, yb in test_dataloader:\n            outputs = net(xb.float().to(1))\n            loss = criterion(outputs.to(0), yb.float().to(0))\n            g.append(loss.item())\n            train_process.test_losses.append(loss.item())\n        valid_loss_mean.append(np.mean(g))\n        valid_loss += g\n    print(e, 'avg_loss=', np.mean(loss_m), 'last_loss = ', loss.item(), 'min_loss = ', np.min(loss_m))\n    print(e, 'int_loss_m__mean = ', np.mean(int_loss_m), 'int_loss_m__min = ', np.min(int_loss_m))\n    print(e, 'int_loss_m__mean = ', np.mean(valid_loss), 'int_loss_m__min = ', np.min(valid_loss))\nprint('END', 'avg_loss=', np.mean(loss_m), 'last_loss = ', loss.item(), 'min_loss = ', np.min(loss_m))\nsave_train_res(net, train_process)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:07:29.556096Z","iopub.execute_input":"2024-05-21T14:07:29.556872Z","iopub.status.idle":"2024-05-21T14:07:41.522279Z","shell.execute_reply.started":"2024-05-21T14:07:29.556836Z","shell.execute_reply":"2024-05-21T14:07:41.52055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import classification_report","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:43:04.953209Z","iopub.execute_input":"2024-05-21T14:43:04.954107Z","iopub.status.idle":"2024-05-21T14:43:04.958189Z","shell.execute_reply.started":"2024-05-21T14:43:04.954072Z","shell.execute_reply":"2024-05-21T14:43:04.957238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_p, y_t = [], []\nfor x_i, y_i in test_dataset:\n    y_t.append(y_i.unsqueeze(0))\n    y_p.append(net(x_i.float().unsqueeze(0).to(1)))\ny_pred = torch.cat(y_p, 0)\ny_true = torch.cat(y_t, 0)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:39:20.644833Z","iopub.execute_input":"2024-05-21T13:39:20.645717Z","iopub.status.idle":"2024-05-21T13:39:28.11424Z","shell.execute_reply.started":"2024-05-21T13:39:20.645683Z","shell.execute_reply":"2024-05-21T13:39:28.113207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(loss_m)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:39:49.476536Z","iopub.execute_input":"2024-05-21T13:39:49.477194Z","iopub.status.idle":"2024-05-21T13:39:49.765439Z","shell.execute_reply.started":"2024-05-21T13:39:49.477162Z","shell.execute_reply":"2024-05-21T13:39:49.764538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(valid_loss_mean)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:39:53.518653Z","iopub.execute_input":"2024-05-21T13:39:53.519068Z","iopub.status.idle":"2024-05-21T13:39:54.01477Z","shell.execute_reply.started":"2024-05-21T13:39:53.519037Z","shell.execute_reply":"2024-05-21T13:39:54.013653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_true.numpy()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:21:21.688128Z","iopub.execute_input":"2024-05-21T13:21:21.688825Z","iopub.status.idle":"2024-05-21T13:21:21.696037Z","shell.execute_reply.started":"2024-05-21T13:21:21.688794Z","shell.execute_reply":"2024-05-21T13:21:21.695118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(classification_report(y_true.argmax(1).numpy(), y_pred.argmax(1).to('cpu').detach().numpy()))","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:40:07.546005Z","iopub.execute_input":"2024-05-21T13:40:07.546927Z","iopub.status.idle":"2024-05-21T13:40:07.585757Z","shell.execute_reply.started":"2024-05-21T13:40:07.546887Z","shell.execute_reply":"2024-05-21T13:40:07.584534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## My CNN + Aug","metadata":{}},{"cell_type":"code","source":"input_processor = InputProcessor().to(1)\nencoder = Encoder().to(1)\nencoder_decoder_adapter = EncoderDecoderAdapter().to(0)\ndecoder = Decoder().to(0)\nnet = MyModel(\n    input_processor_layer = input_processor,\n    encoder = encoder,\n    encoder_decoder_adapter = encoder_decoder_adapter,\n    decoder = decoder,\n)\n# net = MyModel()\n# net = net.to(0)\n# net = torch.nn.DataParallel(net, device_ids=[0, 1])\n# net = net.to(0)\nx, y = train_dataset[0]\nnet(x.float().unsqueeze(0).to(1)).shape","metadata":{"execution":{"iopub.status.busy":"2024-05-21T09:30:27.649477Z","iopub.execute_input":"2024-05-21T09:30:27.650262Z","iopub.status.idle":"2024-05-21T09:30:27.702092Z","shell.execute_reply.started":"2024-05-21T09:30:27.650221Z","shell.execute_reply":"2024-05-21T09:30:27.701148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"criterion = NN.MSELoss()\noptimizer = optim.Adam(net.parameters(), lr=0.001)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T09:30:33.990737Z","iopub.execute_input":"2024-05-21T09:30:33.991107Z","iopub.status.idle":"2024-05-21T09:30:33.997069Z","shell.execute_reply.started":"2024-05-21T09:30:33.991077Z","shell.execute_reply":"2024-05-21T09:30:33.996089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_process = TrainResults('my_model_transfom')","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:32:39.21396Z","iopub.execute_input":"2024-05-14T18:32:39.214995Z","iopub.status.idle":"2024-05-14T18:32:39.219398Z","shell.execute_reply.started":"2024-05-14T18:32:39.214957Z","shell.execute_reply":"2024-05-14T18:32:39.218268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_m = []\nvalid_loss_mean = []\nvalid_loss = []","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:32:41.14214Z","iopub.execute_input":"2024-05-14T18:32:41.142898Z","iopub.status.idle":"2024-05-14T18:32:41.147371Z","shell.execute_reply.started":"2024-05-14T18:32:41.142863Z","shell.execute_reply":"2024-05-14T18:32:41.146193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_of_batches = 0\nfor e in tqdm(range(Parameters.epochs)):\n  int_loss_m = []\n  k = 0\n  for xb, yb in train_dataloader:\n    n_of_batches += 1\n    outputs = net(xb.float().to(1))\n    loss = criterion(outputs.to(0), yb.float().to(0))\n    optimizer.zero_grad()\n    loss.backward()\n    optimizer.step()\n    loss_m.append(loss.item())\n    train_process.train_losses.append(loss.item())\n    int_loss_m.append(loss.item())\n    if k > 50:\n        break\n    k += 1\n  if e % 10 == 0:\n    with torch.no_grad():\n        g = []\n        for xb, yb in test_dataloader:\n            outputs = net(xb.float().to(1))\n            loss = criterion(outputs.to(0), yb.float().to(0))\n            g.append(loss.item())\n            train_process.test_losses.append(loss.item())\n        valid_loss_mean.append(np.mean(g))\n        valid_loss += g\n    print(e, 'avg_loss=', np.mean(loss_m), 'last_loss = ', loss.item(), 'min_loss = ', np.min(loss_m))\n    print(e, 'int_loss_m__mean = ', np.mean(int_loss_m), 'int_loss_m__min = ', np.min(int_loss_m))\n    print(e, 'int_loss_m__mean = ', np.mean(valid_loss), 'int_loss_m__min = ', np.min(valid_loss))\nprint('END', 'avg_loss=', np.mean(loss_m), 'last_loss = ', loss.item(), 'min_loss = ', np.min(loss_m))\nsave_train_res(net, train_process)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:32:43.088237Z","iopub.execute_input":"2024-05-14T18:32:43.088623Z","iopub.status.idle":"2024-05-15T01:09:52.206731Z","shell.execute_reply.started":"2024-05-14T18:32:43.088592Z","shell.execute_reply":"2024-05-15T01:09:52.205096Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(loss_m)","metadata":{"execution":{"iopub.status.busy":"2024-05-15T01:09:57.442879Z","iopub.execute_input":"2024-05-15T01:09:57.443303Z","iopub.status.idle":"2024-05-15T01:09:57.831285Z","shell.execute_reply.started":"2024-05-15T01:09:57.443265Z","shell.execute_reply":"2024-05-15T01:09:57.830352Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(valid_loss_mean)","metadata":{"execution":{"iopub.status.busy":"2024-05-15T01:10:03.279882Z","iopub.execute_input":"2024-05-15T01:10:03.28055Z","iopub.status.idle":"2024-05-15T01:10:03.471802Z","shell.execute_reply.started":"2024-05-15T01:10:03.280514Z","shell.execute_reply":"2024-05-15T01:10:03.470691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"net(x.float().unsqueeze(0).to(1))\nto_pil_image(x.float())","metadata":{"execution":{"iopub.status.busy":"2024-05-15T01:10:06.80681Z","iopub.execute_input":"2024-05-15T01:10:06.807203Z","iopub.status.idle":"2024-05-15T01:10:06.89615Z","shell.execute_reply.started":"2024-05-15T01:10:06.807168Z","shell.execute_reply":"2024-05-15T01:10:06.895217Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nc, nr = 2, 4\nf, a = plt.subplots(nc, nr, figsize=(16, 8))\nk = net.encoder.first_layer.last_activation.to('cpu')[0]\nfor i in range(len(k)):\n    a[i//nr, i%nr].imshow(k[i])\n    a[i//nr, i%nr].axis('off')","metadata":{"execution":{"iopub.status.busy":"2024-05-15T01:10:10.115165Z","iopub.execute_input":"2024-05-15T01:10:10.116091Z","iopub.status.idle":"2024-05-15T01:10:10.871745Z","shell.execute_reply.started":"2024-05-15T01:10:10.116055Z","shell.execute_reply":"2024-05-15T01:10:10.870704Z"},"jupyter":{"source_hidden":true,"outputs_hidden":true},"collapsed":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nc, nr = 4, 4\nf, a = plt.subplots(nc, nr, figsize=(16, 16))\nk = net.encoder.second_layer.last_activation[0]\nfor i in range(len(k)):\n    a[i//nr, i%nr].imshow(k[i])\n    a[i//nr, i%nr].axis('off')","metadata":{"execution":{"iopub.status.busy":"2024-05-15T01:10:17.327633Z","iopub.execute_input":"2024-05-15T01:10:17.328309Z","iopub.status.idle":"2024-05-15T01:10:18.427562Z","shell.execute_reply.started":"2024-05-15T01:10:17.328269Z","shell.execute_reply":"2024-05-15T01:10:18.426607Z"},"jupyter":{"source_hidden":true,"outputs_hidden":true},"collapsed":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nc, nr = 8, 8\nf, a = plt.subplots(nc, nr, figsize=(16, 16))\nk = net.encoder.third_layer.last_activation[0]\n# len(k)\nfor i in range(len(k)):\n    a[i//nr, i%nr].imshow(k[i])\n    a[i//nr, i%nr].axis('off')","metadata":{"execution":{"iopub.status.busy":"2024-05-15T01:10:23.382217Z","iopub.execute_input":"2024-05-15T01:10:23.382627Z","iopub.status.idle":"2024-05-15T01:10:25.57039Z","shell.execute_reply.started":"2024-05-15T01:10:23.382594Z","shell.execute_reply":"2024-05-15T01:10:25.569411Z"},"jupyter":{"source_hidden":true,"outputs_hidden":true},"collapsed":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nc, nr = 8, 16\nf, a = plt.subplots(nc, nr, figsize=(32, 16))\nk = net.encoder.fourth_layer.last_activation[0]\n# len(k)\nfor i in range(len(k)):\n    a[i//nr, i%nr].imshow(k[i])\n    a[i//nr, i%nr].axis('off')","metadata":{"execution":{"iopub.status.busy":"2024-05-15T01:10:31.295047Z","iopub.execute_input":"2024-05-15T01:10:31.295438Z","iopub.status.idle":"2024-05-15T01:10:36.182543Z","shell.execute_reply.started":"2024-05-15T01:10:31.295406Z","shell.execute_reply":"2024-05-15T01:10:36.181543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nc, nr = 16, 16\nf, a = plt.subplots(nc, nr, figsize=(32, 32))\nk = net.encoder.fiveth_layer.last_activation[0]\n# len(k)\nfor i in range(len(k)):\n    a[i//nr, i%nr].imshow(k[i])\n    a[i//nr, i%nr].axis('off')","metadata":{"execution":{"iopub.status.busy":"2024-05-15T01:10:39.758455Z","iopub.execute_input":"2024-05-15T01:10:39.759291Z","iopub.status.idle":"2024-05-15T01:10:49.026896Z","shell.execute_reply.started":"2024-05-15T01:10:39.759255Z","shell.execute_reply":"2024-05-15T01:10:49.025865Z"},"jupyter":{"source_hidden":true,"outputs_hidden":true},"collapsed":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"save_train_res(net, train_process)","metadata":{"execution":{"iopub.status.busy":"2024-05-15T01:11:03.526705Z","iopub.execute_input":"2024-05-15T01:11:03.527051Z","iopub.status.idle":"2024-05-15T01:11:03.567334Z","shell.execute_reply.started":"2024-05-15T01:11:03.527023Z","shell.execute_reply":"2024-05-15T01:11:03.566244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# My CNN Only","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ResNet34 + Aug","metadata":{}},{"cell_type":"code","source":"rn34 = resnet34(pretrained=True)\n\nfor param in rn34.parameters():\n    param.requires_grad = False\nrn34","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:55:12.537241Z","iopub.execute_input":"2024-05-21T16:55:12.537616Z","iopub.status.idle":"2024-05-21T16:55:13.393981Z","shell.execute_reply.started":"2024-05-21T16:55:12.537585Z","shell.execute_reply":"2024-05-21T16:55:13.39302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def weights_init(m):\n    if isinstance(m, NN.Conv2d):\n        torch.nn.init.xavier_uniform_(m.weight)\n#         torch.nn.init.zero_(m.bias)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:23:19.318597Z","iopub.execute_input":"2024-05-21T14:23:19.319165Z","iopub.status.idle":"2024-05-21T14:23:19.324761Z","shell.execute_reply.started":"2024-05-21T14:23:19.319123Z","shell.execute_reply":"2024-05-21T14:23:19.323531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in rn34.layer4[2].parameters():\n    i.requires_grad = True\nrn34.fc = torch.nn.Sequential(\n    torch.nn.Linear(512, 256),\n    torch.nn.Tanh(),\n    torch.nn.Linear(256,  128),\n    torch.nn.Tanh(),\n    torch.nn.Linear(128, 5),\n    torch.nn.Softmax(1)\n)\n# for i in rn34.fc:\n#     weights_init(i)\n# rn34.conv1 = torch.nn.Sequential(\n#     torch.nn.Conv2d(3, 4, kernel_size=(3, 3), padding=1),\n#     torch.nn.AvgPool2d((2, 2)),\n#     torch.nn.Conv2d(4, 8, kernel_size=(3, 3), padding=1),\n#     torch.nn.AvgPool2d((2, 2)),\n#     torch.nn.Conv2d(8, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n# )\n# for i in rn34.conv1:\n#     weights_init(i)\nrn34 = rn34.to(0)\n","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:55:05.835449Z","iopub.execute_input":"2024-05-21T16:55:05.836268Z","iopub.status.idle":"2024-05-21T16:55:05.882813Z","shell.execute_reply.started":"2024-05-21T16:55:05.836234Z","shell.execute_reply":"2024-05-21T16:55:05.881484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_process = TrainResults('resnet34__transfom')","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:17:57.927661Z","iopub.execute_input":"2024-05-21T16:17:57.928034Z","iopub.status.idle":"2024-05-21T16:17:57.932452Z","shell.execute_reply.started":"2024-05-21T16:17:57.928009Z","shell.execute_reply":"2024-05-21T16:17:57.931449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"criterion = NN.MSELoss()\noptimizer = optim.Adam(rn34.parameters(), lr=0.001)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:18:00.262682Z","iopub.execute_input":"2024-05-21T16:18:00.263037Z","iopub.status.idle":"2024-05-21T16:18:00.268983Z","shell.execute_reply.started":"2024-05-21T16:18:00.263011Z","shell.execute_reply":"2024-05-21T16:18:00.268001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_m = []\nvalid_loss_mean = []\nvalid_loss = []","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:18:03.865884Z","iopub.execute_input":"2024-05-21T16:18:03.86625Z","iopub.status.idle":"2024-05-21T16:18:03.870817Z","shell.execute_reply.started":"2024-05-21T16:18:03.866221Z","shell.execute_reply":"2024-05-21T16:18:03.869753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for xb, yb in train_dataloader:\n    break","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:18:05.631737Z","iopub.execute_input":"2024-05-21T16:18:05.632558Z","iopub.status.idle":"2024-05-21T16:18:15.578113Z","shell.execute_reply.started":"2024-05-21T16:18:05.632527Z","shell.execute_reply":"2024-05-21T16:18:15.576831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with torch.no_grad():\n    print(rn34(xb.to(0).float()).shape)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:45:36.870624Z","iopub.execute_input":"2024-05-21T14:45:36.871187Z","iopub.status.idle":"2024-05-21T14:45:36.914763Z","shell.execute_reply.started":"2024-05-21T14:45:36.871142Z","shell.execute_reply":"2024-05-21T14:45:36.913719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_of_batches = 0\nfor e in tqdm(range(Parameters.epochs)):\n  int_loss_m = []\n  k = 0\n  for xb, yb in train_dataloader:\n    n_of_batches += 1\n    outputs = rn34(xb.float().to(0))\n    loss = criterion(outputs.to(0), yb.float().to(0))\n    optimizer.zero_grad()\n    loss.backward()\n    optimizer.step()\n    loss_m.append(loss.item())\n    train_process.train_losses.append(loss.item())\n    int_loss_m.append(loss.item())\n    if k > 50:\n        break\n    k += 1\n  if e % 10 == 0:\n    with torch.no_grad():\n        g = []\n        for xb, yb in test_dataloader:\n            outputs = rn34(xb.float().to(0))\n            loss = criterion(outputs.to(0), yb.float().to(0))\n            g.append(loss.item())\n            train_process.test_losses.append(loss.item())\n        valid_loss_mean.append(np.mean(g))\n        valid_loss += g\n    print(e, 'avg_loss=', np.mean(loss_m), 'last_loss = ', loss.item(), 'min_loss = ', np.min(loss_m))\n    print(e, 'int_loss_m__mean = ', np.mean(int_loss_m), 'int_loss_m__min = ', np.min(int_loss_m))\n    print(e, 'int_loss_m__mean = ', np.mean(valid_loss), 'int_loss_m__min = ', np.min(valid_loss))\nprint('END', 'avg_loss=', np.mean(loss_m), 'last_loss = ', loss.item(), 'min_loss = ', np.min(loss_m))\nsave_train_res(rn34, train_process)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:18:15.580521Z","iopub.execute_input":"2024-05-21T16:18:15.581249Z","iopub.status.idle":"2024-05-21T16:40:23.547625Z","shell.execute_reply.started":"2024-05-21T16:18:15.581209Z","shell.execute_reply":"2024-05-21T16:40:23.546023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(loss_m)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:40:34.259558Z","iopub.execute_input":"2024-05-21T16:40:34.260508Z","iopub.status.idle":"2024-05-21T16:40:34.555503Z","shell.execute_reply.started":"2024-05-21T16:40:34.260458Z","shell.execute_reply":"2024-05-21T16:40:34.554473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(valid_loss_mean)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:40:38.019132Z","iopub.execute_input":"2024-05-21T16:40:38.020004Z","iopub.status.idle":"2024-05-21T16:40:38.276692Z","shell.execute_reply.started":"2024-05-21T16:40:38.019963Z","shell.execute_reply":"2024-05-21T16:40:38.275821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for x, y in test_dataset:\n    break","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:40:41.93053Z","iopub.execute_input":"2024-05-21T16:40:41.931548Z","iopub.status.idle":"2024-05-21T16:40:41.935891Z","shell.execute_reply.started":"2024-05-21T16:40:41.931514Z","shell.execute_reply":"2024-05-21T16:40:41.934876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# rn34(x.float().unsqueeze(0).to(1))\nto_pil_image(x.float())","metadata":{"execution":{"iopub.status.busy":"2024-05-21T15:09:24.166574Z","iopub.execute_input":"2024-05-21T15:09:24.167443Z","iopub.status.idle":"2024-05-21T15:09:24.204272Z","shell.execute_reply.started":"2024-05-21T15:09:24.167409Z","shell.execute_reply":"2024-05-21T15:09:24.203377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"c1 = rn34.conv1(x.float().unsqueeze(0).to(0))\nbn1 = rn34.bn1(c1)\nrelu = rn34.relu(bn1)\nmaxpool = rn34.maxpool(relu)\nlayer1 = rn34.layer1(maxpool)\nlayer2 = rn34.layer2(layer1)\nlayer3 = rn34.layer3(layer2)\nlayer4 = rn34.layer4(layer3)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T15:09:27.812905Z","iopub.execute_input":"2024-05-21T15:09:27.813304Z","iopub.status.idle":"2024-05-21T15:09:27.835133Z","shell.execute_reply.started":"2024-05-21T15:09:27.813269Z","shell.execute_reply":"2024-05-21T15:09:27.834145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nc, nr = 8, 8\nf, a = plt.subplots(nc, nr, figsize=(16, 16))\nk = layer1.squeeze().to('cpu').detach()\n# len(k)\nfor i in range(len(k)):\n    a[i//nr, i%nr].imshow(k[i])\n    a[i//nr, i%nr].axis('off')","metadata":{"execution":{"iopub.status.busy":"2024-05-21T15:09:30.787657Z","iopub.execute_input":"2024-05-21T15:09:30.788056Z","iopub.status.idle":"2024-05-21T15:09:33.75122Z","shell.execute_reply.started":"2024-05-21T15:09:30.788025Z","shell.execute_reply":"2024-05-21T15:09:33.75003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nc, nr = 8, 8\nf, a = plt.subplots(nc, nr, figsize=(16, 16))\nk = c1.squeeze().to('cpu').detach()\n# len(k)\nfor i in range(len(k)):\n    a[i//nr, i%nr].imshow(k[i])\n    a[i//nr, i%nr].axis('off')","metadata":{"execution":{"iopub.status.busy":"2024-05-21T15:09:34.972523Z","iopub.execute_input":"2024-05-21T15:09:34.973136Z","iopub.status.idle":"2024-05-21T15:09:36.970254Z","shell.execute_reply.started":"2024-05-21T15:09:34.9731Z","shell.execute_reply":"2024-05-21T15:09:36.969392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"save_train_res(rn34, train_process)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:40:57.411247Z","iopub.execute_input":"2024-05-21T16:40:57.411615Z","iopub.status.idle":"2024-05-21T16:40:58.120575Z","shell.execute_reply.started":"2024-05-21T16:40:57.411586Z","shell.execute_reply":"2024-05-21T16:40:58.11947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rn34 = rn34.to('cpu')","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:44:53.301588Z","iopub.execute_input":"2024-05-21T16:44:53.302432Z","iopub.status.idle":"2024-05-21T16:44:53.391772Z","shell.execute_reply.started":"2024-05-21T16:44:53.302398Z","shell.execute_reply":"2024-05-21T16:44:53.3908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:44:55.686681Z","iopub.execute_input":"2024-05-21T16:44:55.687811Z","iopub.status.idle":"2024-05-21T16:44:55.698037Z","shell.execute_reply.started":"2024-05-21T16:44:55.687778Z","shell.execute_reply":"2024-05-21T16:44:55.697172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with torch.no_grad():\n    y_p, y_t = [], []\n    for x_i, y_i in test_dataset:\n        y_t.append(y_i.unsqueeze(0))\n        y_p.append(x_i.float().unsqueeze(0))\n    y_pred = rn34(torch.cat(y_p, 0).to('cpu'))\n    y_true = torch.cat(y_t, 0)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:45:10.119099Z","iopub.execute_input":"2024-05-21T16:45:10.119887Z","iopub.status.idle":"2024-05-21T16:47:31.152542Z","shell.execute_reply.started":"2024-05-21T16:45:10.119852Z","shell.execute_reply":"2024-05-21T16:47:31.150694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:47:31.15343Z","iopub.status.idle":"2024-05-21T16:47:31.153891Z","shell.execute_reply.started":"2024-05-21T16:47:31.153661Z","shell.execute_reply":"2024-05-21T16:47:31.153682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(accuracy_score(y_true.argmax(1).numpy(), y_pred.argmax(1).to('cpu').detach().numpy()))","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:47:31.154912Z","iopub.status.idle":"2024-05-21T16:47:31.155268Z","shell.execute_reply.started":"2024-05-21T16:47:31.155102Z","shell.execute_reply":"2024-05-21T16:47:31.155117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(classification_report(y_true.argmax(1).numpy(), y_pred.argmax(1).to('cpu').detach().numpy()))","metadata":{"execution":{"iopub.status.busy":"2024-05-21T15:57:46.28348Z","iopub.execute_input":"2024-05-21T15:57:46.284168Z","iopub.status.idle":"2024-05-21T15:57:46.299058Z","shell.execute_reply.started":"2024-05-21T15:57:46.284139Z","shell.execute_reply":"2024-05-21T15:57:46.298014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with torch.no_grad():\n    y_p, y_t = [], []\n    for x_i, y_i in train_dataset:\n        y_t.append(y_i.unsqueeze(0))\n        y_p.append(x_i.float().unsqueeze(0))\n    y_pred = rn34(torch.cat(y_p, 0).to('cpu'))\n    y_true = torch.cat(y_t, 0)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T16:48:30.258711Z","iopub.execute_input":"2024-05-21T16:48:30.259532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(classification_report(y_true.argmax(1).numpy(), y_pred.argmax(1).to('cpu').detach().numpy()))","metadata":{"execution":{"iopub.status.busy":"2024-05-21T15:17:32.763553Z","iopub.execute_input":"2024-05-21T15:17:32.76395Z","iopub.status.idle":"2024-05-21T15:17:32.781489Z","shell.execute_reply.started":"2024-05-21T15:17:32.763915Z","shell.execute_reply":"2024-05-21T15:17:32.780594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Получили перетрен","metadata":{}},{"cell_type":"markdown","source":"## Vgg + AUG + scheduler","metadata":{}},{"cell_type":"code","source":"torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T15:19:52.856077Z","iopub.execute_input":"2024-05-21T15:19:52.856443Z","iopub.status.idle":"2024-05-21T15:19:52.861314Z","shell.execute_reply.started":"2024-05-21T15:19:52.856413Z","shell.execute_reply":"2024-05-21T15:19:52.860274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vgg = torch.hub.load('pytorch/vision:v0.10.0', 'vgg19', pretrained=True)\nvgg","metadata":{"execution":{"iopub.status.busy":"2024-05-21T17:05:01.043675Z","iopub.execute_input":"2024-05-21T17:05:01.044381Z","iopub.status.idle":"2024-05-21T17:05:07.945441Z","shell.execute_reply.started":"2024-05-21T17:05:01.044345Z","shell.execute_reply":"2024-05-21T17:05:07.944495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in vgg.parameters():\n    i.requires_grad = False\nvgg.classifier = torch.nn.Sequential(\n    torch.nn.Linear(25088, 4096),\n    torch.nn.Tanh(),\n    torch.nn.Linear(4096, 1024),\n    torch.nn.Tanh(),\n    torch.nn.Linear(1024,  256),\n    torch.nn.Tanh(),\n    torch.nn.Linear(256, 5),\n    torch.nn.Softmax(1)\n)\n# vgg.features[0] = torch.nn.Sequential(\n#     torch.nn.Conv2d(3, 4, kernel_size=(3, 3), padding=1),\n#     torch.nn.AvgPool2d((2, 2)),\n#     torch.nn.Conv2d(4, 8, kernel_size=(3, 3), padding=1),\n#     torch.nn.AvgPool2d((2, 2)),\n#     torch.nn.Conv2d(8, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n# )\nvgg = vgg.to(0)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T17:05:11.374822Z","iopub.execute_input":"2024-05-21T17:05:11.375418Z","iopub.status.idle":"2024-05-21T17:05:12.990579Z","shell.execute_reply.started":"2024-05-21T17:05:11.375388Z","shell.execute_reply":"2024-05-21T17:05:12.989563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_process = TrainResults('vgg__transfom_scheduler')","metadata":{"execution":{"iopub.status.busy":"2024-05-21T17:05:14.938358Z","iopub.execute_input":"2024-05-21T17:05:14.939178Z","iopub.status.idle":"2024-05-21T17:05:14.943453Z","shell.execute_reply.started":"2024-05-21T17:05:14.93915Z","shell.execute_reply":"2024-05-21T17:05:14.942175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"criterion = NN.CrossEntropyLoss()\noptimizer = optim.Adam(vgg.parameters(), lr=0.001)\nscheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda epoch: 0.001 * 0.95 ** epoch)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T17:05:16.633581Z","iopub.execute_input":"2024-05-21T17:05:16.633917Z","iopub.status.idle":"2024-05-21T17:05:17.095535Z","shell.execute_reply.started":"2024-05-21T17:05:16.633893Z","shell.execute_reply":"2024-05-21T17:05:17.094617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for xb, yb in train_dataloader:\n    break\nwith torch.no_grad():\n    print(vgg(xb.to(0).float()).shape)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T17:05:18.674363Z","iopub.execute_input":"2024-05-21T17:05:18.674732Z","iopub.status.idle":"2024-05-21T17:05:26.026019Z","shell.execute_reply.started":"2024-05-21T17:05:18.674702Z","shell.execute_reply":"2024-05-21T17:05:26.02484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_m = []\nvalid_loss_mean = []\nvalid_loss = []","metadata":{"execution":{"iopub.status.busy":"2024-05-21T17:06:45.461114Z","iopub.execute_input":"2024-05-21T17:06:45.461818Z","iopub.status.idle":"2024-05-21T17:06:45.466264Z","shell.execute_reply.started":"2024-05-21T17:06:45.461783Z","shell.execute_reply":"2024-05-21T17:06:45.465185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_of_batches = 0\nfor e in tqdm(range(Parameters.epochs)):\n  int_loss_m = []\n  k = 0\n  for xb, yb in train_dataloader:\n    n_of_batches += 1\n    outputs = vgg(xb.float().to(0))\n    loss = criterion(outputs.to(0), yb.float().to(0))\n    optimizer.zero_grad()\n    loss.backward()\n    optimizer.step()\n    loss_m.append(loss.item())\n    train_process.train_losses.append(loss.item())\n    int_loss_m.append(loss.item())\n    if k > 50:\n        break\n    k += 1\n  scheduler.step()\n  if e % 10 == 0:\n    with torch.no_grad():\n        g = []\n        for xb, yb in test_dataloader:\n            outputs = vgg(xb.float().to(0))\n            loss = criterion(outputs.to(0), yb.float().to(0))\n            g.append(loss.item())\n            train_process.test_losses.append(loss.item())\n        valid_loss_mean.append(np.mean(g))\n        valid_loss += g\n    print(e, 'avg_loss=', np.mean(loss_m), 'last_loss = ', loss.item(), 'min_loss = ', np.min(loss_m))\n    print(e, 'int_loss_m__mean = ', np.mean(int_loss_m), 'int_loss_m__min = ', np.min(int_loss_m))\n    print(e, 'int_loss_m__mean = ', np.mean(valid_loss), 'int_loss_m__min = ', np.min(valid_loss))\nprint('END', 'avg_loss=', np.mean(loss_m), 'last_loss = ', loss.item(), 'min_loss = ', np.min(loss_m))\nsave_train_res(vgg, train_process)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T17:06:47.43538Z","iopub.execute_input":"2024-05-21T17:06:47.436238Z","iopub.status.idle":"2024-05-21T18:52:26.642984Z","shell.execute_reply.started":"2024-05-21T17:06:47.436205Z","shell.execute_reply":"2024-05-21T18:52:26.641516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(loss_m)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T18:52:30.847671Z","iopub.execute_input":"2024-05-21T18:52:30.84804Z","iopub.status.idle":"2024-05-21T18:52:31.134602Z","shell.execute_reply.started":"2024-05-21T18:52:30.848008Z","shell.execute_reply":"2024-05-21T18:52:31.133646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(valid_loss_mean)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T18:52:34.97022Z","iopub.execute_input":"2024-05-21T18:52:34.970634Z","iopub.status.idle":"2024-05-21T18:52:35.22833Z","shell.execute_reply.started":"2024-05-21T18:52:34.970603Z","shell.execute_reply":"2024-05-21T18:52:35.227307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for x, y in test_dataset:\n    break","metadata":{"execution":{"iopub.status.busy":"2024-05-21T18:52:39.753718Z","iopub.execute_input":"2024-05-21T18:52:39.754064Z","iopub.status.idle":"2024-05-21T18:52:39.759054Z","shell.execute_reply.started":"2024-05-21T18:52:39.75404Z","shell.execute_reply":"2024-05-21T18:52:39.758087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vgg","metadata":{"execution":{"iopub.status.busy":"2024-05-21T18:52:42.260717Z","iopub.execute_input":"2024-05-21T18:52:42.26159Z","iopub.status.idle":"2024-05-21T18:52:42.26801Z","shell.execute_reply.started":"2024-05-21T18:52:42.26156Z","shell.execute_reply":"2024-05-21T18:52:42.267117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l1 = vgg.features[0](x.float().unsqueeze(0).to(0))\nl2 = vgg.features[0:6](x.float().unsqueeze(0).to(0))\nl3 = vgg.features[0:10](x.float().unsqueeze(0).to(0))\nl4 = vgg.features[0:15](x.float().unsqueeze(0).to(0))\n","metadata":{"execution":{"iopub.status.busy":"2024-05-21T18:52:45.6352Z","iopub.execute_input":"2024-05-21T18:52:45.635586Z","iopub.status.idle":"2024-05-21T18:52:45.668203Z","shell.execute_reply.started":"2024-05-21T18:52:45.635557Z","shell.execute_reply":"2024-05-21T18:52:45.667251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l3.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-21T18:52:49.843815Z","iopub.execute_input":"2024-05-21T18:52:49.844639Z","iopub.status.idle":"2024-05-21T18:52:49.850399Z","shell.execute_reply.started":"2024-05-21T18:52:49.844607Z","shell.execute_reply":"2024-05-21T18:52:49.849471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nc, nr = 8, 8\nf, a = plt.subplots(nc, nr, figsize=(16, 16))\nk = l1.squeeze().to('cpu').detach()\n# len(k)\nfor i in range(len(k)):\n    a[i//nr, i%nr].imshow(k[i])\n    a[i//nr, i%nr].axis('off')","metadata":{"execution":{"iopub.status.busy":"2024-05-21T18:52:51.967929Z","iopub.execute_input":"2024-05-21T18:52:51.968281Z","iopub.status.idle":"2024-05-21T18:52:55.466755Z","shell.execute_reply.started":"2024-05-21T18:52:51.968255Z","shell.execute_reply":"2024-05-21T18:52:55.465491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nc, nr = 16, 8\nf, a = plt.subplots(nc, nr, figsize=(16, 32))\nk = l2.squeeze().to('cpu').detach()\n# len(k)\nfor i in range(len(k)):\n    a[i//nr, i%nr].imshow(k[i])\n    a[i//nr, i%nr].axis('off')","metadata":{"execution":{"iopub.status.busy":"2024-05-21T18:53:02.738243Z","iopub.execute_input":"2024-05-21T18:53:02.738941Z","iopub.status.idle":"2024-05-21T18:53:08.648429Z","shell.execute_reply.started":"2024-05-21T18:53:02.738908Z","shell.execute_reply":"2024-05-21T18:53:08.646526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T15:47:33.970843Z","iopub.execute_input":"2024-05-21T15:47:33.97121Z","iopub.status.idle":"2024-05-21T15:47:33.97619Z","shell.execute_reply.started":"2024-05-21T15:47:33.971183Z","shell.execute_reply":"2024-05-21T15:47:33.975125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with torch.no_grad():\n    vgg = vgg.to(1)\n    y_p, y_t = [], []\n    for x_i, y_i in test_dataset:\n        y_t.append(y_i.unsqueeze(0))\n        y_p.append(vgg(x_i.float().unsqueeze(0).to(1)))\n    y_pred = torch.cat(y_p, 0)\n    y_true = torch.cat(y_t, 0)\nprint(classification_report(y_true.argmax(1).numpy(), y_pred.argmax(1).to('cpu').detach().numpy()))","metadata":{"execution":{"iopub.status.busy":"2024-05-21T18:57:02.773257Z","iopub.execute_input":"2024-05-21T18:57:02.774207Z","iopub.status.idle":"2024-05-21T18:57:24.711222Z","shell.execute_reply.started":"2024-05-21T18:57:02.774175Z","shell.execute_reply":"2024-05-21T18:57:24.710156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"save_train_res(vgg, train_process)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T18:58:43.269508Z","iopub.execute_input":"2024-05-21T18:58:43.269885Z","iopub.status.idle":"2024-05-21T18:58:44.51211Z","shell.execute_reply.started":"2024-05-21T18:58:43.269858Z","shell.execute_reply":"2024-05-21T18:58:44.511187Z"},"trusted":true},"execution_count":null,"outputs":[]}]}