{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Download dataset\n","metadata":{"id":"C_jdZ5vHJ4A9"}},{"cell_type":"code","source":"%%capture\n! pip install torchvision","metadata":{"id":"QXBj0krHhDyq","execution":{"iopub.status.busy":"2023-05-06T17:47:16.016702Z","iopub.execute_input":"2023-05-06T17:47:16.017048Z","iopub.status.idle":"2023-05-06T17:47:18.161538Z","shell.execute_reply.started":"2023-05-06T17:47:16.017019Z","shell.execute_reply":"2023-05-06T17:47:18.160307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# save path & file name","metadata":{"id":"dqrwhlT8VYfk"}},{"cell_type":"code","source":"# configuration\nnum_epochs = 10\nbatch_size = 32\npatch_size = 8\nhidden_size = 64\nnum_heads = 2\nnum_layers = 1\nimage_size = 96\nlearning_rate = 1e-3\nwarmup_steps = 1000\ncsv_path=\"/kaggle/input/histopathologic-cancer-detection/train_labels.csv\"\ntrain_dir = \"/kaggle/input/histopathologic-cancer-detection/train\"\ntest_dir = \"/kaggle/input/histopathologic-cancer-detection/test\"","metadata":{"id":"xT22zoPjqVVN","execution":{"iopub.status.busy":"2023-05-06T17:47:18.163939Z","iopub.execute_input":"2023-05-06T17:47:18.164355Z","iopub.status.idle":"2023-05-06T17:47:18.170503Z","shell.execute_reply.started":"2023-05-06T17:47:18.164313Z","shell.execute_reply":"2023-05-06T17:47:18.169589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = \"/kaggle/working/\"\nfile_name = f'Vit_epo{num_epochs}_bat{batch_size}_pat{patch_size}_hid{hidden_size}_head{num_heads}_lay{num_layers}_lr{learning_rate}'\nprint(file_name)","metadata":{"id":"XhQcM67OVXvk","execution":{"iopub.status.busy":"2023-05-06T17:47:18.172431Z","iopub.execute_input":"2023-05-06T17:47:18.173273Z","iopub.status.idle":"2023-05-06T17:47:18.181424Z","shell.execute_reply.started":"2023-05-06T17:47:18.173240Z","shell.execute_reply":"2023-05-06T17:47:18.180408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data","metadata":{"id":"k7dVbxW2LASN"}},{"cell_type":"code","source":"import os\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nfrom PIL import Image\nimport pandas as pd\n\n\ncsv_path=csv_path\nclass MyTrainDataset(Dataset):\n    def __init__(self, data_dir, transform=None):\n        self.data_dir = data_dir\n        self.transform = transform\n\n        # 讀取資料集中所有檔案的路徑和標籤\n        self.samples = []\n        df = pd.read_csv(csv_path)\n        for idx, row in df.iterrows():\n            img_name = row['id']\n            label = row['label']\n            img_path = os.path.join(data_dir, img_name+'.tif')\n            self.samples.append((img_path, label))\n\n    def __getitem__(self, index):\n        # 讀取檔案和標籤\n        img_path, label = self.samples[index]\n        img = Image.open(img_path).convert(\"RGB\")\n\n        # 轉換圖像\n        if self.transform is not None:\n            img = self.transform(img)\n\n        return img, label, os.path.basename(img_path)\n\n    def __len__(self):\n        return len(self.samples)\n\nclass MyTestDataset(Dataset):\n    def __init__(self, data_dir, transform=None):\n        self.data_dir = data_dir\n        self.transform = transform\n\n        # 讀取資料集中所有檔案的路徑\n        self.samples = []\n        for root, dirs, files in os.walk(data_dir):\n            for file in files:\n                if file.endswith(\".tif\"):\n                    img_path = os.path.join(root, file)\n                    self.samples.append(img_path)\n\n    def __getitem__(self, index):\n        # 讀取檔案\n        img_path = self.samples[index]\n        img = Image.open(img_path).convert(\"RGB\")\n\n        # 轉換圖像\n        if self.transform is not None:\n            img = self.transform(img)\n\n        return img, os.path.basename(img_path)[:-4]\n\n    def __len__(self):\n        return len(self.samples)\n\n\n# 定義 transform\nmean = [0.485, 0.456, 0.406]\nstd = [0.229, 0.224, 0.225]\ndata_transforms = transforms.Compose([\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomVerticalFlip(),\n    #transforms.Resize((image_size,image_size)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=mean, std=std)\n])\n# 定義 dataloader\ntrain_dir = train_dir\ntest_dir = test_dir\nbatch_size = batch_size\n\ntrain_dataset = MyTrainDataset(train_dir, transform=data_transforms)\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n\ntest_dataset = MyTestDataset(test_dir, transform=data_transforms)\ntest_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)","metadata":{"id":"-TuYDI6S_Ivw","execution":{"iopub.status.busy":"2023-05-06T17:47:18.184328Z","iopub.execute_input":"2023-05-06T17:47:18.184735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np\nimport torchvision.transforms as transforms\nfrom PIL import Image\n\n# 路徑設置\n#img_path = f\"{train_dir}/f38a6374c348f90b587e046aac6079959adf3835.tif\"\nimg_path = f\"{train_dir}/c18f2d887b7ae4f6742ee445113fa1aef383ed77.tif\"\n\n# 讀取圖片\nimg = Image.open(img_path)\n\n# 轉換前的圖片\nplt.subplot(1,2,1)\nplt.imshow(np.asarray(img))\nplt.title('Original Image')\ntransformed_img = data_transforms(img)\n\n# 轉換後的圖片\nplt.subplot(1,2,2)\nplt.imshow(transformed_img.permute(1,2,0))\nplt.title('Transformed Image')\n\nplt.show()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{"id":"5FOSZYxrMqhc"}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torchvision\n\nclass MyVisionTransformer(nn.Module):\n    def __init__(self, hidden_size=128, num_heads=4, num_layers=2, image_size=96, num_classes=2):\n        super().__init__()\n\n        self.patch_size = patch_size\n        self.embed_dim = hidden_size\n        self.num_heads = num_heads\n        self.num_layers = num_layers\n        self.image_size = image_size\n        self.num_classes = num_classes\n\n        self.conv1 = nn.Conv2d(3, self.embed_dim, kernel_size=self.patch_size, stride=self.patch_size)\n        self.pos_embedding = nn.Parameter(torch.zeros(1, (self.image_size // self.patch_size) ** 2 + 1, self.embed_dim))\n        self.dropout = nn.Dropout(p=0.1)\n\n        encoder_layer = nn.TransformerEncoderLayer(\n            d_model=self.embed_dim,\n            nhead=self.num_heads,\n            dim_feedforward=self.embed_dim * 4\n        )\n        self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers=self.num_layers)\n\n        self.ln = nn.LayerNorm(self.embed_dim)\n        self.fc = nn.Linear(self.embed_dim, self.num_classes)\n\n    def forward(self, x):\n        bs, _, _, _ = x.shape  # 获取当前 batch size\n\n        x = self.conv1(x)\n        x = x.flatten(2)\n        x = x.transpose(1, 2)\n        x = torch.cat((self.pos_embedding.expand(bs, -1, -1), x), dim=1)  # 将位置嵌入张量扩展到与当前 batch size 相同\n        x = self.dropout(x)\n\n        x = self.transformer_encoder(x)\n        \n        x = self.ln(x[:, 0])\n        x = self.fc(x)\n        return x\n","metadata":{"id":"bi_T3P0QySEH","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 生成model\nmodel = MyVisionTransformer(hidden_size=hidden_size, num_heads=num_heads, num_layers=num_layers)\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(device)\nprint(model)","metadata":{"id":"IfpVoQOM4QCH","outputId":"0de1f171-3cb8-4dda-ceeb-0252365f843e","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math\n\nimport torch\nfrom torch.optim import Optimizer\nfrom torch.optim.lr_scheduler import LambdaLR\n\n\ndef get_cosine_schedule_with_warmup(\n    optimizer: Optimizer,\n    num_warmup_steps: int,\n    num_training_steps: int,\n    num_cycles: float = 0.5,\n    last_epoch: int = -1,\n):\n\n    def lr_lambda(current_step):\n        # Warmup\n        if current_step < num_warmup_steps:\n            return float(current_step) / float(max(1, num_warmup_steps))\n        # decadence\n        progress = float(current_step - num_warmup_steps) / float(\n            max(1, num_training_steps - num_warmup_steps)\n        )\n        return max(\n            0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress))\n        )\n\n    return LambdaLR(optimizer, lr_lambda, last_epoch)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_loader)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.nn as nn\nimport torch.optim as optim\n#from torch.optim.lr_scheduler import StepLR\n\n# 設定損失函數和優化器\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.AdamW(model.parameters(), lr=learning_rate)\n\n# 設定訓練和驗證的迴圈\nnum_epochs = num_epochs\n\n# scheduler設定\nwarmup_steps = warmup_steps\ntotal_steps = len(train_loader)*num_epochs\nscheduler = get_cosine_schedule_with_warmup(optimizer, warmup_steps, total_steps)\n\nfor epoch in range(num_epochs):\n    model.train()\n    running_loss = 0.0\n    for i, data in enumerate(train_loader, 0):\n        #k=0\n        inputs, labels, _ = data\n        inputs, labels = inputs.to(device), labels.to(device)\n\n        optimizer.zero_grad()\n\n        outputs = model(inputs)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n\n        # 更新 learning rate\n        scheduler.step()\n\n        running_loss += loss.item()\n        #k+=1 # only for test\n        #if k>5: break # only for test\n        #break\n    print(f\"Epoch {epoch + 1}, Loss: {running_loss / len(train_loader)}\")\n    torch.save(model.state_dict(), f'{path}/model_{epoch+1}.pth')\n\ndf = pd.DataFrame(columns=['id', 'label'])\nmodel.eval()\nwith torch.no_grad():\n    #n=0 # only for test\n    for data in test_loader:\n        images, image_names = data\n        images = images.to(device)\n\n        outputs = model(images)\n        _, prediction = torch.max(outputs.data, 1)\n\n        batch_df = pd.DataFrame({'id':image_names,'label':prediction.cpu()})\n        df = pd.concat([df, batch_df], ignore_index=True)\n\n        #n+=1 # only for test\n        #if n>2: break # only for test\n\nprint(df)\n# 將預測結果保存到 CSV 檔案中\ndf.to_csv(f'{path}/{file_name}.csv', index=False)\n\n\nprint(\"Finished training.\")","metadata":{"id":"oV_CuqZzBsED","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#del_path = \"/kaggle/working/model_0.pth\"\n#del_path = \"/kaggle/working/Vit_epo10_bat32_pat8_hid128_head8_lay3_lr0.001.csv\"\n#os.remove(f\"{del_path}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}