{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":21154,"databundleVersionId":1243559,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import cv2\nimport numpy as np\nimport tensorflow as tf\nimport os\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport torchvision\nimport torch\nfrom torch.utils.data import Dataset\nimport os\nfrom torch.utils.data import Dataset\nfrom torchvision import datasets,transforms\nfrom PIL import Image\nfrom torch.utils.data import random_split\nimport torch.optim as optim","metadata":{"execution":{"iopub.status.busy":"2024-06-01T20:33:36.053426Z","iopub.execute_input":"2024-06-01T20:33:36.053825Z","iopub.status.idle":"2024-06-01T20:33:54.023820Z","shell.execute_reply.started":"2024-06-01T20:33:36.053791Z","shell.execute_reply":"2024-06-01T20:33:54.022746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_feature_description = {\n    'image': tf.io.FixedLenFeature([], tf.string, default_value=''),\n    'id': tf.io.FixedLenFeature([], tf.string, default_value=''),\n    'class': tf.io.FixedLenFeature([], tf.int64, default_value=0),\n}\n\ndef _parse_image_function(example_proto):\n    return tf.io.parse_single_example(example_proto, train_feature_description)\n\ndef preprocess_image(image):\n    image = tf.io.decode_image(image, channels=3)\n    return image\n\nBASE_DIR = \"/kaggle/input/tpu-getting-started/\"\ntfrec_dir_train = \"tfrecords-jpeg-224x224/train/\"\ntfrec_dir_test = \"tfrecords-jpeg-224x224/test/\"\ntfrec_dir_val = \"tfrecords-jpeg-224x224/val/\"\n\npath_to_train_images = '/kaggle/working/train/'\npath_to_test_images = '/kaggle/working/test/'\npath_to_valid_images = '/kaggle/working/valid/'\n\ndef convert(tfrec_dir, output_dir):\n    for tfName in os.listdir(os.path.join(BASE_DIR, tfrec_dir))[:]:\n        train_image_dataset = tf.data.TFRecordDataset(BASE_DIR+tfrec_dir+tfName)\n        train_images = train_image_dataset.map(_parse_image_function)\n\n        for image_features in train_images:\n            image_raw = preprocess_image(image_features['image'])\n            image_raw_int = image_raw.numpy()\n            image_name = image_features['id'].numpy().decode(\"utf-8\") + '.jpg'\n            cl = image_features['class'].numpy()\n            name_cl = str(cl) + '/'\n            path_to_class = output_dir + name_cl\n            if not os.path.isdir(path_to_class):\n                os.mkdir(path_to_class)\n            path_to_img = path_to_class + image_name\n            #print(path_to_img)\n            cv2.imwrite(path_to_img, cv2.cvtColor(image_raw_int, cv2.COLOR_BGR2RGB))\n            \n            #print(image_name, image_features['class'].numpy(), counter)\n","metadata":{"execution":{"iopub.status.busy":"2024-06-01T20:33:54.025876Z","iopub.execute_input":"2024-06-01T20:33:54.026411Z","iopub.status.idle":"2024-06-01T20:33:54.037699Z","shell.execute_reply.started":"2024-06-01T20:33:54.026383Z","shell.execute_reply":"2024-06-01T20:33:54.036659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.mkdir(path_to_train_images)\nos.mkdir(path_to_test_images)\nos.mkdir(path_to_valid_images)\nconvert(tfrec_dir_train, path_to_train_images)\nconvert(tfrec_dir_test, path_to_test_images)\nconvert(tfrec_dir_val, path_to_valid_images)","metadata":{"execution":{"iopub.status.busy":"2024-06-01T20:33:54.041563Z","iopub.execute_input":"2024-06-01T20:33:54.041926Z","iopub.status.idle":"2024-06-01T20:35:00.282015Z","shell.execute_reply.started":"2024-06-01T20:33:54.041900Z","shell.execute_reply":"2024-06-01T20:35:00.281118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for tfName in os.listdir('/kaggle/working/test/0'):\n#     print(tfName+'\\n')","metadata":{"execution":{"iopub.status.busy":"2024-06-01T20:35:00.284795Z","iopub.execute_input":"2024-06-01T20:35:00.285092Z","iopub.status.idle":"2024-06-01T20:35:00.289127Z","shell.execute_reply.started":"2024-06-01T20:35:00.285066Z","shell.execute_reply":"2024-06-01T20:35:00.288059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\n\ndef extract_number(folder_name):\n    match = re.search(r'\\d+', folder_name)\n    return int(match.group()) if match else None\n\ndef sort_folders(folders):\n    return sorted(folders, key=extract_number)\n\npaths=[]\n\nfor i in os.listdir('/kaggle/working/train/'):\n    img_dir = '/kaggle/working/train/' + i + '/'\n    paths.append(i)\n        \nsorted_folders = sort_folders(paths)\nprint(sorted_folders)\n","metadata":{"execution":{"iopub.status.busy":"2024-06-01T20:35:00.290443Z","iopub.execute_input":"2024-06-01T20:35:00.290720Z","iopub.status.idle":"2024-06-01T20:35:00.302483Z","shell.execute_reply.started":"2024-06-01T20:35:00.290697Z","shell.execute_reply":"2024-06-01T20:35:00.301578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MY_DATASET(Dataset):\n  def __init__(self,path,transform):\n    self.labels = []\n    self.img_pths = []\n\n    for i in os.listdir(path):\n        img_dir = path + '/' + i + '/'\n        for j in os.listdir(img_dir):\n            self.img_pths.append(img_dir+j)\n            self.labels.append(i)\n    predlabel_map = sorted(list(set(self.labels)),key=extract_number)\n    #print(predlabel_map)\n    self.label_map = {}\n    it = 0\n    for i in predlabel_map:\n      self.label_map[i] = it\n      it+=1\n    for i in range(len(self.labels)):\n        self.labels[i] = self.label_map[self.labels[i]]\n    self.transform = transform\n\n  def __len__(self):\n      return len(self.labels)\n\n  def __getitem__(self, idx):\n      img_path = self.img_pths[idx]\n      image = Image.open(img_path).convert('RGB')\n      w, h = image.size\n      label = self.labels[idx]\n      if(self.transform == transform):\n        trans_loc = transforms.CenterCrop((min(w, h), min(w, h)))\n        image = trans_loc(image)\n      if self.transform:\n        image = self.transform(image)\n      return image, label\n\ntransform = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.4487185, 0.41564807, 0.3029375 ], std=[0.2823133, 0.245806, 0.27172017])\n])\n","metadata":{"execution":{"iopub.status.busy":"2024-06-01T20:35:00.303867Z","iopub.execute_input":"2024-06-01T20:35:00.304218Z","iopub.status.idle":"2024-06-01T20:35:00.315745Z","shell.execute_reply.started":"2024-06-01T20:35:00.304185Z","shell.execute_reply":"2024-06-01T20:35:00.314791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset  = MY_DATASET('/kaggle/working/train/', transform)","metadata":{"execution":{"iopub.status.busy":"2024-06-01T20:35:00.316875Z","iopub.execute_input":"2024-06-01T20:35:00.317145Z","iopub.status.idle":"2024-06-01T20:35:00.342160Z","shell.execute_reply.started":"2024-06-01T20:35:00.317122Z","shell.execute_reply":"2024-06-01T20:35:00.341436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(dataset.__len__())","metadata":{"execution":{"iopub.status.busy":"2024-06-01T20:35:00.343075Z","iopub.execute_input":"2024-06-01T20:35:00.343346Z","iopub.status.idle":"2024-06-01T20:35:00.348307Z","shell.execute_reply.started":"2024-06-01T20:35:00.343311Z","shell.execute_reply":"2024-06-01T20:35:00.347349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport shutil\n\ndef copy_folders_with_few_images(base_path, dest_path, threshold = 100):\n    # Проверяем, существует ли базовый путь\n    if not os.path.exists(base_path):\n        print(f\"Path '{base_path}' does not exist.\")\n        return\n    \n    # Создаем путь назначения, если он не существует\n    if not os.path.exists(dest_path):\n        os.makedirs(dest_path)\n\n    # Перебираем все папки в base_path\n    for folder_name in os.listdir(base_path):\n        folder_path = os.path.join(base_path, folder_name)\n\n        # Проверяем, что это действительно папка\n        if os.path.isdir(folder_path):\n            # Считаем количество изображений в папке (предполагаем, что изображения имеют расширения .jpg, .png и т.д.)\n            num_images = sum(1 for item in os.listdir(folder_path) if item.lower().endswith(('.png', '.jpg', '.jpeg', '.gif', '.bmp', '.tiff')))\n            \n            if num_images < threshold:\n                # Путь назначения для папки\n                dest_folder_path = os.path.join(dest_path, folder_name)\n                \n                # Копируем папку\n                shutil.copytree(folder_path, dest_folder_path)\n                print(f\"Copied folder '{folder_name}' to '{dest_path}' (contains {num_images} images).\")\n\n# Использование функции\nbase_path = '/kaggle/working/train/'\ndest_path = '/kaggle/working/aug/'\ncopy_folders_with_few_images(base_path, dest_path)\n\ntransform_aug = transforms.Compose([\n    # Случайное изменение размера изображения\n    transforms.RandomResizedCrop(224),\n    # Случайное горизонтальное отражение\n    transforms.RandomHorizontalFlip(),\n    # Случайное вертикальное отражение\n    transforms.RandomVerticalFlip(),\n    # Случайное вращение изображения\n    transforms.RandomRotation(30),\n    # Случайное изменение яркости, контрастности и насыщенности\n    transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1),\n    # Преобразование изображения в тензор\n    transforms.ToTensor(),\n    # Нормализация изображения с средними и стандартными отклонениями для каналов RGB\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])\n","metadata":{"execution":{"iopub.status.busy":"2024-06-01T20:35:00.349862Z","iopub.execute_input":"2024-06-01T20:35:00.350124Z","iopub.status.idle":"2024-06-01T20:35:00.753456Z","shell.execute_reply.started":"2024-06-01T20:35:00.350102Z","shell.execute_reply":"2024-06-01T20:35:00.752521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport shutil\nfrom random import choice\n\ndef duplicate_images_in_folders(base_path, min_images=30):\n    # Проверяем, существует ли базовый путь\n    if not os.path.exists(base_path):\n        print(f\"Path '{base_path}' does not exist.\")\n        return\n\n    # Перебираем все папки в base_path\n    for folder_name in os.listdir(base_path):\n        folder_path = os.path.join(base_path, folder_name)\n\n        # Проверяем, что это действительно папка\n        if os.path.isdir(folder_path):\n            # Получаем список изображений в папке\n            images = [img for img in os.listdir(folder_path) if img.lower().endswith(('.png', '.jpg', '.jpeg', '.gif', '.bmp', '.tiff'))]\n            num_images = len(images)\n            \n            if num_images < min_images:\n                print(f\"Folder '{folder_name}' has only {num_images} images. Duplicating images to reach {min_images} images.\")\n                \n                origin_images = set(images)  # Множество оригинальных изображений\n                duplicated_images = set()  # Множество для отслеживания дублированных изображений\n\n                while num_images < min_images:\n                    if not origin_images:\n                        # Если все оригинальные изображения продублированы, перезапускаем процесс\n                        origin_images, duplicated_images = duplicated_images, set()\n                    \n                    # Выбираем случайное изображение для дублирования из origin_images\n                    img_to_duplicate = choice(list(origin_images))\n                    original_path = os.path.join(folder_path, img_to_duplicate)\n                    new_image_name = f\"copy_{num_images}_{img_to_duplicate}\"\n                    new_image_path = os.path.join(folder_path, new_image_name)\n                    \n                    # Копируем изображение\n                    shutil.copy2(original_path, new_image_path)\n                    \n                    # Обновляем список изображений, счетчик и множества\n                    images.append(new_image_name)\n                    origin_images.remove(img_to_duplicate)\n                    duplicated_images.add(img_to_duplicate)\n                    num_images += 1\n\n# Использование функции\nbase_path = '/kaggle/working/aug/'\nduplicate_images_in_folders(base_path, 60)\n","metadata":{"execution":{"iopub.status.busy":"2024-06-01T20:35:00.754585Z","iopub.execute_input":"2024-06-01T20:35:00.754904Z","iopub.status.idle":"2024-06-01T20:35:00.929183Z","shell.execute_reply.started":"2024-06-01T20:35:00.754879Z","shell.execute_reply":"2024-06-01T20:35:00.928083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_aug  = MY_DATASET('/kaggle/working/aug/', transform_aug)","metadata":{"execution":{"iopub.status.busy":"2024-06-01T20:35:00.930502Z","iopub.execute_input":"2024-06-01T20:35:00.931105Z","iopub.status.idle":"2024-06-01T20:35:00.942652Z","shell.execute_reply.started":"2024-06-01T20:35:00.931071Z","shell.execute_reply":"2024-06-01T20:35:00.941822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfrom torchvision import datasets, transforms\nfrom torch.utils.data import random_split, ConcatDataset, DataLoader\n\ntrain_dataset = dataset\n\n\nfrom collections import defaultdict\n\n# Предполагая, что у вас есть train_dataset с метками классов в диапазоне от 0 до num_classes-1\n\n# Создаем словарь для хранения количества экземпляров каждого класса\nclass_counts = defaultdict(int)\n\n# Проходим по train_dataset и считаем количество экземпляров каждого класса\nfor _, label in train_dataset:\n    class_counts[label] += 1\n\n# Вычисляем общее количество экземпляров в наборе данных\ntotal_samples = len(train_dataset)\n\n# Вычисляем вес каждого класса как обратное отношение количества экземпляров класса к общему количеству экземпляров\nclass_weights = [total_samples / class_counts[label] for label in range(104)]\nweight_tensor = torch.tensor(class_weights, dtype=torch.float)\n\n# Печатаем веса каждого класса\nfor label, weight in enumerate(class_weights):\n    print(f\"Class {label}: Weight {weight}\")\n\n\n# Объединение train_dataset и data_aug\ntrain_dataset = ConcatDataset([train_dataset, data_aug])","metadata":{"execution":{"iopub.status.busy":"2024-06-01T20:35:00.944080Z","iopub.execute_input":"2024-06-01T20:35:00.944462Z","iopub.status.idle":"2024-06-01T20:35:20.198571Z","shell.execute_reply.started":"2024-06-01T20:35:00.944427Z","shell.execute_reply":"2024-06-01T20:35:20.197418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataloader = DataLoader(train_dataset, batch_size=16, shuffle=True)\nval_dataset = MY_DATASET('/kaggle/working/valid/', transform)\nval_dataloader = DataLoader(val_dataset, batch_size=16, shuffle=True)\ntest_dataset  = MY_DATASET('/kaggle/working/test/', transform)\ntest_dataloader = DataLoader(test_dataset, batch_size=16, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2024-06-01T20:35:20.202904Z","iopub.execute_input":"2024-06-01T20:35:20.203286Z","iopub.status.idle":"2024-06-01T20:35:20.229508Z","shell.execute_reply.started":"2024-06-01T20:35:20.203252Z","shell.execute_reply":"2024-06-01T20:35:20.228316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from typing import Tuple\nfrom random import randrange\n# from torchsummary import summary\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\n\nimport matplotlib.pyplot as plt\n\nclass MY_CNN(nn.Module):\n    def __init__(self, num_classes):\n      super(MY_CNN, self).__init__()\n      self.conv1 = nn.Conv2d(3, 3, kernel_size=3, stride =1, padding=1)\n      self.conv1_2 = nn.Conv2d(3, 16, kernel_size=1, stride =1, padding=1)\n      self.conv2 = nn.Conv2d(16, 16, kernel_size=3, stride=1, padding=1)\n      self.conv2_2 = nn.Conv2d(16, 32, kernel_size=1, stride=1, padding=1)\n      self.conv3 = nn.Conv2d(32, 32, kernel_size=3, stride=1, padding=1)\n      self.conv3_2 = nn.Conv2d(32, 64, kernel_size=1, stride=1, padding=1)\n      self.conv4 = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1)\n      self.conv4_2 = nn.Conv2d(64, 128, kernel_size=1, stride=1, padding=1)\n      self.conv5 = nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1)\n      self.conv5_2 = nn.Conv2d(128, 256, kernel_size=1, stride=1, padding=1)\n      self.pool = nn.MaxPool2d((3, 3), padding = 0)\n      self.adaptpool =  nn.AdaptiveAvgPool2d((4, 4))\n      self.fc1 = nn.Linear(4096, 2048)\n      #self.fc2 = nn.Linear(4096, 512)\n      self.fc3 = nn.Linear(2048, num_classes)\n      self.BN1 = nn.BatchNorm2d(16)\n      self.BN2 = nn.BatchNorm2d(32)\n      self.BN3 = nn.BatchNorm2d(64)\n      self.BN4 = nn.BatchNorm2d(128)\n      self.BN5 = nn.BatchNorm2d(256)\n      self.relu = nn.ReLU()\n      self.drop = nn.Dropout(p=0.4)\n      self.num_classes = num_classes\n      \n\n    def forward(self, x: torch.Tensor):\n        x = self.relu(self.BN1(self.conv1_2(self.conv1(x)))) # 256 x 256 x 16\n        x = self.pool(x)\n        #print(x.shape, \"\\n\")\n        x = self.relu(self.BN2(self.conv2_2(self.conv2(x))))\n        x = self.pool(x)\n        #print(x.shape, \"\\n\")\n        x = self.relu(self.BN3(self.conv3_2(self.conv3(x))))\n        x = self.pool(x)\n        #print(x.shape, \"\\n\")\n        x = self.relu(self.BN4(self.conv4_2(self.conv4(x))))\n        x = self.pool(x)\n        x = self.relu(self.BN5(self.conv5_2(self.conv5(x))))\n        x = self.adaptpool(x)\n        #print(x.shape, \"\\n\")\n        x = self.drop(x)\n        x = self.relu(self.fc1(x.view(-1, 4096)))\n        #x = self.relu(self.fc2(x))\n        x = self.fc3(x)\n        #print(x.shape, \"\\n\")\n        return x\n","metadata":{"execution":{"iopub.status.busy":"2024-06-01T20:35:20.230740Z","iopub.execute_input":"2024-06-01T20:35:20.231106Z","iopub.status.idle":"2024-06-01T20:35:20.251853Z","shell.execute_reply.started":"2024-06-01T20:35:20.231079Z","shell.execute_reply":"2024-06-01T20:35:20.250710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = MY_CNN(num_classes=104).cuda()\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), lr=0.001, betas=(0.9, 0.999), eps=1e-8, weight_decay=0.00001, amsgrad=False)\n#print('fuck')","metadata":{"execution":{"iopub.status.busy":"2024-06-01T20:35:20.253374Z","iopub.execute_input":"2024-06-01T20:35:20.254234Z","iopub.status.idle":"2024-06-01T20:35:20.420853Z","shell.execute_reply.started":"2024-06-01T20:35:20.254192Z","shell.execute_reply":"2024-06-01T20:35:20.419933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def model_training(model,train_loader,optimizer,criterion):\n    running_loss = 0.0\n    model.train()\n    for inputs,labels in train_loader:\n        optimizer.zero_grad()\n        inputs,labels = inputs.cuda() , labels.cuda()\n        out = model(inputs)\n        #print(out[0], \"hi\", labels[0], out.shape, labels.shape)\n        loss = criterion(out,labels)\n        loss.backward()\n        optimizer.step()\n        #print(loss.item())\n        running_loss += loss.item()\n    \n    #print(running_loss, len(train_loader))\n\n    return running_loss / len(train_loader)","metadata":{"execution":{"iopub.status.busy":"2024-06-01T20:35:20.422310Z","iopub.execute_input":"2024-06-01T20:35:20.422649Z","iopub.status.idle":"2024-06-01T20:35:20.431011Z","shell.execute_reply.started":"2024-06-01T20:35:20.422620Z","shell.execute_reply":"2024-06-01T20:35:20.429861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import f1_score\ndef model_validating(model, val_loader, criterion):\n    true_positives = 0\n    predicted_positives = 0\n    actual_positives = 0\n    model.eval()\n    all_predictions = []\n    all_labels = []\n    running_loss = 0.0\n    \n    for inputs, labels in val_loader:\n        with torch.no_grad():\n            inputs, labels = inputs.cuda(), labels.cuda()\n            out = model(inputs)\n            \n            _, predicted = torch.max(out, 1)\n            all_predictions.extend(predicted.cpu().numpy())\n            all_labels.extend(labels.cpu().numpy())\n            \n            true_positives += torch.sum(predicted * labels).cpu().item()\n            predicted_positives += torch.sum(predicted).cpu().item()\n            actual_positives += torch.sum(labels).cpu().item()\n            \n            loss = criterion(out, labels)\n            running_loss += loss.item()\n    \n    precision = true_positives / (predicted_positives + 1e-10)\n    recall = true_positives / (actual_positives + 1e-10)\n    f1 = 2 * (precision * recall) / (precision + recall + 1e-10)\n    \n    avg_loss = running_loss / len(val_loader)\n    \n    return avg_loss, f1_score(all_labels, all_predictions, average='weighted')","metadata":{"execution":{"iopub.status.busy":"2024-06-01T20:35:20.432521Z","iopub.execute_input":"2024-06-01T20:35:20.432907Z","iopub.status.idle":"2024-06-01T20:35:20.889074Z","shell.execute_reply.started":"2024-06-01T20:35:20.432877Z","shell.execute_reply":"2024-06-01T20:35:20.888046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#torch.cuda.memory_summary(device=None, abbreviated=False)","metadata":{"execution":{"iopub.status.busy":"2024-06-01T20:35:20.890328Z","iopub.execute_input":"2024-06-01T20:35:20.890663Z","iopub.status.idle":"2024-06-01T20:35:20.894475Z","shell.execute_reply.started":"2024-06-01T20:35:20.890635Z","shell.execute_reply":"2024-06-01T20:35:20.893536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install torchsummary","metadata":{"execution":{"iopub.status.busy":"2024-06-01T20:35:20.895549Z","iopub.execute_input":"2024-06-01T20:35:20.895855Z","iopub.status.idle":"2024-06-01T20:35:35.357992Z","shell.execute_reply.started":"2024-06-01T20:35:20.895831Z","shell.execute_reply":"2024-06-01T20:35:35.356852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torchsummary import summary\nsummary(model, (3, 224, 224))","metadata":{"execution":{"iopub.status.busy":"2024-06-01T20:35:35.359433Z","iopub.execute_input":"2024-06-01T20:35:35.359723Z","iopub.status.idle":"2024-06-01T20:35:36.099061Z","shell.execute_reply.started":"2024-06-01T20:35:35.359695Z","shell.execute_reply":"2024-06-01T20:35:36.098083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_arrays_to_file(filename, arr):\n    with open(filename, 'w') as file:\n        for i in arr:\n            file.write(f\"{i}\\n\")\n\nimport torch\nif not os.path.exists('/kaggle/working/vesa/'):\n    os.mkdir('/kaggle/working/vesa/')\nvalb = 0\nnum_epochs = 50\ntrain_loss_arr = []\nval_loss_arr = []\nval_loss_acc = []\nfor epoch in range(0, num_epochs):\n    train_loss = model_training(model, train_dataloader, optimizer, criterion)\n    val_loss, val_acc = model_validating(model, val_dataloader, criterion)\n\n    if(valb < val_acc):\n        torch.save({'model_state_dict': model.state_dict(),'optimizer_state_dict': optimizer.state_dict(),}, \"/kaggle/working/vesa/im_in_real_problem_30_60.pth\")\n        valb = val_acc\n    if(epoch%10 ==0):\n        torch.save({'model_state_dict': model.state_dict(),'optimizer_state_dict': optimizer.state_dict(),}, (\"/kaggle/working/vesa/im_in_real_problem_30_60_epoch_\"+str(epoch)+\".pth\"))\n    if(epoch%30 ==0):\n        save_arrays_to_file('/kaggle/working/vesa/train_loss_0_60_epoch_'+str(epoch)+'.txt', train_loss_arr)\n        save_arrays_to_file('/kaggle/working/vesa/val_loss_0_60_epoch_'+str(epoch)+'.txt', val_loss_arr)\n        save_arrays_to_file('/kaggle/working/vesa/val_acc_0_60_epoch_'+str(epoch)+'.txt', val_loss_acc)\n    print(f'Epoch [{epoch+1}/{num_epochs}], Train Loss: {train_loss:.4f}, Val Loss: {val_loss:.4f} ', val_acc)\n    train_loss_arr.append(train_loss)\n    val_loss_arr.append(val_loss)\n    val_loss_acc.append(val_acc)\n\n\nsave_arrays_to_file('/kaggle/working/vesa/train_loss_0_60_2.txt', train_loss_arr)\nsave_arrays_to_file('/kaggle/working/vesa/val_loss_0_60_2.txt', val_loss_arr)\nsave_arrays_to_file('/kaggle/working/vesa/val_acc_0_60_2.txt', val_loss_acc)","metadata":{"execution":{"iopub.status.busy":"2024-06-01T20:35:36.100504Z","iopub.execute_input":"2024-06-01T20:35:36.100935Z","iopub.status.idle":"2024-06-01T20:37:09.618669Z","shell.execute_reply.started":"2024-06-01T20:35:36.100898Z","shell.execute_reply":"2024-06-01T20:37:09.617569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader\nfrom PIL import Image\nimport os\n\nclass CustomDataset(Dataset):\n    def __init__(self, image_folder, transform=None):\n        self.image_folder = image_folder\n        self.image_filenames = [os.path.join(image_folder, fname) for fname in os.listdir(image_folder)]\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.image_filenames)\n\n    def __getitem__(self, idx):\n        img_name = self.image_filenames[idx]\n        image = Image.open(img_name)\n        if self.transform:\n            image = self.transform(image)\n        return image, os.path.basename(img_name)\n\n# Example usage:\ndataset = CustomDataset(image_folder='/kaggle/working/test/0/', transform=transform)\ntest_dataloaders = DataLoader(dataset, batch_size=16, shuffle=False)\n# model_testing(model, test_loader, 'submission.csv')\n","metadata":{"execution":{"iopub.status.busy":"2024-06-01T20:37:09.620027Z","iopub.execute_input":"2024-06-01T20:37:09.620414Z","iopub.status.idle":"2024-06-01T20:37:09.647988Z","shell.execute_reply.started":"2024-06-01T20:37:09.620388Z","shell.execute_reply":"2024-06-01T20:37:09.647222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport pandas as pd\nimport os\n\ndef model_testing(model, test_loader, output_csv_path):\n    model.eval()\n    all_predictions = []\n    all_ids = []\n\n    for inputs, ids in test_loader:\n        with torch.no_grad():\n            inputs = inputs.cuda()\n            out = model(inputs)\n            \n            _, predicted = torch.max(out, 1)\n            all_predictions.extend(predicted.cpu().numpy())\n            all_ids.extend([os.path.splitext(filename)[0] for filename in ids])  # Remove file extension\n    \n    results_df = pd.DataFrame({'id': all_ids, 'label': all_predictions})\n    \n    # Ensure the output path is correct for Kaggle\n    output_path = os.path.join('/kaggle/working', output_csv_path)\n    results_df.to_csv(output_csv_path, index=False)\n    \n    # Verify file creation by listing directory contents\n    print(\"Contents of /kaggle/working directory:\")\n    print(os.listdir('/kaggle/working'))\n\n# Example usage:\n# Assuming test_loader yields (inputs, ids) where ids are filenames\nmodel_testing(model, test_dataloaders, 'submission.csv')\n","metadata":{"execution":{"iopub.status.busy":"2024-06-01T20:37:47.485750Z","iopub.execute_input":"2024-06-01T20:37:47.486135Z","iopub.status.idle":"2024-06-01T20:37:48.391765Z","shell.execute_reply.started":"2024-06-01T20:37:47.486104Z","shell.execute_reply":"2024-06-01T20:37:48.390119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport shutil\n\ntry:\n    shutil.rmtree('/kaggle/working/aug/')\n    print(\"Папка с файлами успешно удалена.\")\nexcept OSError as e:\n    print(f\"Ошибка при удалении папки с файлами: {e}\")\n\ntry:\n    shutil.rmtree('/kaggle/working/test/')\n    print(\"Папка с файлами успешно удалена.\")\nexcept OSError as e:\n    print(f\"Ошибка при удалении папки с файлами: {e}\")\n    \ntry:\n    shutil.rmtree('/kaggle/working/train/')\n    print(\"Папка с файлами успешно удалена.\")\nexcept OSError as e:\n    print(f\"Ошибка при удалении папки с файлами: {e}\")\n\ntry:\n    shutil.rmtree('/kaggle/working/valid/')\n    print(\"Папка с файлами успешно удалена.\")\nexcept OSError as e:\n    print(f\"Ошибка при удалении папки с файлами: {e}\")\n    \ntry:\n    shutil.rmtree('/kaggle/working/vesa/')\n    print(\"Папка с файлами успешно удалена.\")\nexcept OSError as e:\n    print(f\"Ошибка при удалении папки с файлами: {e}\")\n    \n# Проверка содержимого директории\nprint(\"Содержимое /kaggle/working:\")\nprint(os.listdir('/kaggle/working'))","metadata":{"execution":{"iopub.status.busy":"2024-06-01T20:38:01.118615Z","iopub.execute_input":"2024-06-01T20:38:01.119262Z","iopub.status.idle":"2024-06-01T20:38:01.299664Z","shell.execute_reply.started":"2024-06-01T20:38:01.119231Z","shell.execute_reply":"2024-06-01T20:38:01.298630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df","metadata":{"execution":{"iopub.status.busy":"2024-06-01T20:37:25.405682Z","iopub.execute_input":"2024-06-01T20:37:25.406027Z","iopub.status.idle":"2024-06-01T20:37:25.410313Z","shell.execute_reply.started":"2024-06-01T20:37:25.405998Z","shell.execute_reply":"2024-06-01T20:37:25.409233Z"},"trusted":true},"execution_count":null,"outputs":[]}]}