{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"},{"sourceId":108530,"sourceType":"modelInstanceVersion","modelInstanceId":90905,"modelId":115136}],"dockerImageVersionId":30762,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torchvision import models \nimport torchvision.transforms as T\nfrom torch.utils.data import Dataset, DataLoader\nimport torch.nn.functional as F\n\nimport pydicom\nfrom PIL import Image,ImageOps\nimport random\nfrom sklearn.preprocessing import LabelEncoder, label_binarize\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score ,accuracy_score,classification_report\nfrom sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\n\nimport matplotlib.pyplot as plt\nimport cv2\nimport time\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-09-23T11:45:52.165785Z","iopub.execute_input":"2024-09-23T11:45:52.166065Z","iopub.status.idle":"2024-09-23T11:45:58.435736Z","shell.execute_reply.started":"2024-09-23T11:45:52.166033Z","shell.execute_reply":"2024-09-23T11:45:58.434888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#file path\nbase_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification'\n\n#train path\ntrain_label = os.path.join(base_path,'train_label_coordinates.csv')\ntrain_series = os.path.join(base_path,'train_series_descriptions.csv')\ntrain_ = os.path.join(base_path,'train.csv')\nimg_train_path = os.path.join(base_path,'train_images')\n\n#test path\ntest_series=os.path.join(base_path,'test_series_descriptions.csv')\nimg_test_path = os.path.join(base_path,'test_images')","metadata":{"execution":{"iopub.status.busy":"2024-09-23T11:45:58.437493Z","iopub.execute_input":"2024-09-23T11:45:58.437943Z","iopub.status.idle":"2024-09-23T11:45:58.443630Z","shell.execute_reply.started":"2024-09-23T11:45:58.437908Z","shell.execute_reply":"2024-09-23T11:45:58.442770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reconstruct_df(df):\n    series = ['Sagittal T2/STIR', 'Sagittal T1', 'Axial T2']\n    conditions = ['spinal_canal_stenosis', 'neural_foraminal_narrowing', 'subarticular_stenosis']\n    df_name=[pd.DataFrame(),pd.DataFrame(),pd.DataFrame()]\n    \n    for i,(se, cond) in enumerate(zip(series, conditions)):\n        filtered_df = df[df['series_description'] == se] # Filter the dataframe based on 'series_description'\n\n        # Select columns to keep\n        columns_to_keep = filtered_df.columns[:8].tolist()\n        columns_to_keep += [col for col in filtered_df.columns[8:] if cond in col]\n\n        #filter and update\n        filtered_df = filtered_df[columns_to_keep] \n        df_name[i] = filtered_df\n    \n    return df_name[0],df_name[1],df_name[2]\n   \n\ndef xy_spinal_neural(df):\n    # Required levels and associated columns\n    required_levels = ['L1/L2', 'L2/L3', 'L3/L4', 'L4/L5', 'L5/S1']\n\n    ## filterout data which having both level\n    df_filtered = df.groupby(['study_id','series_id','instance_number']).filter(\n    lambda group: set(required_levels).issubset(group['level'].unique()))\n\n    # make localize \n    grouped = df_filtered.groupby(['study_id', 'series_id', 'instance_number'])\n    result_rows = []\n    for (study_id, series_id, instance_number), group in grouped:\n        # Create a dictionary to store the data for this group\n        row_data = {'study_id': study_id, 'series_id': series_id, 'instance_number': instance_number}\n    \n        # Check if all other columns have the same value in the group, if so, keep them\n        for col in group.columns:\n            if col not in ['level', 'x', 'y']:  # Exclude 'level', 'x', and 'y' columns\n                if group[col].nunique() == 1:  # Check if all values in this column are the same\n                    row_data[col] = group[col].iloc[0]  # Take the unique value\n\n\n        # Extract x and y coordinates for each required level\n        for level in required_levels:\n            level_data = group[group['level'] == level]\n            if not level_data.empty:\n                row_data[f'x_level_{level.lower().replace(\"/\", \"_\")}'] = level_data['x'].values[0]\n                row_data[f'y_level_{level.lower().replace(\"/\", \"_\")}'] = level_data['y'].values[0]\n\n        # Add the row data to the result_rows list\n        result_rows.append(row_data)\n\n    # Convert the result_rows list to a DataFrame\n    result = pd.DataFrame(result_rows)\n    return result\n\n\ndef xy_subarticular(df):\n    # Required levels and associated columns\n    require_condition=['Right Subarticular Stenosis','Left Subarticular Stenosis']\n\n    ## filterout data which having both level\n    df_filtered = df.groupby(['study_id','series_id','instance_number','level']).filter(\n    lambda group: set(require_condition).issubset(group['condition'].unique()))\n\n    # make localize \n    grouped = df_filtered.groupby(['study_id', 'series_id', 'instance_number','level'])\n    result_rows = []\n    for (study_id, series_id, instance_number,level), group in grouped:\n        # Create a dictionary to store the data for this group\n        row_data = {'study_id': study_id, 'series_id': series_id, 'instance_number': instance_number,'level':level}\n    \n        # Check if all other columns have the same value in the group, if so, keep them\n        for col in group.columns:\n            if col not in ['series_description','condition','x', 'y']:  # Exclude 'level', 'x', and 'y' columns\n                if group[col].nunique() == 1:  # Check if all values in this column are the same\n                    row_data[col] = group[col].iloc[0]  # Take the unique value\n\n        # Extract x and y coordinates for each required level\n        for condition in require_condition:\n            condition_data = group[group['condition'] == condition]\n            if not condition_data.empty:\n                row_data[f'x_level_{condition.split(\" \")[0]}'] = condition_data['x'].values[0]\n                row_data[f'y_level_{condition.split(\" \")[0]}'] = condition_data['y'].values[0]\n\n        # Add the row data to the result_rows list\n        result_rows.append(row_data)\n\n    # Convert the result_rows list to a DataFrame\n    df_result = pd.DataFrame(result_rows)\n    \n    #dummy varables\n    columns_to_encode = 'level'\n    df_encoded = pd.get_dummies(df_result[columns_to_encode], columns=columns_to_encode, prefix=columns_to_encode)\n    df_encoded = df_encoded.astype(float)# Convert boolean True/False to 1.0/0.0\n    df_result['level_array'] = df_encoded.values.tolist()\n    #result=pd.concat([df_result,df_encoded],axis=1)\n    df_result = df_result.drop(columns=columns_to_encode)\n    \n    return df_result","metadata":{"execution":{"iopub.status.busy":"2024-09-23T11:45:58.444750Z","iopub.execute_input":"2024-09-23T11:45:58.445072Z","iopub.status.idle":"2024-09-23T11:45:58.466134Z","shell.execute_reply.started":"2024-09-23T11:45:58.445039Z","shell.execute_reply":"2024-09-23T11:45:58.465090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class subarticular_Dataset(Dataset):\n    def __init__(self, df,image_dir,transform=None):\n        self.df=df\n        self.img_dir = image_dir\n        self.transform = transform\n        \n        if self.transform is None:\n            self.transform = T.Compose([\n                T.Resize((256,256)),\n                T.ToTensor(),\n            ])\n                \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self,idx):\n        image,mask,level =self.get_images(idx)\n        image= self.transform(image) \n        \n        image=torch.tensor(image,dtype=torch.float32)\n        mask = torch.tensor(mask,dtype=torch.long)\n        level = torch.tensor(level,dtype=torch.long)\n        \n        return image,mask,level\n    \n    def get_images(self,idx):\n        row = self.df.iloc[idx]\n        dicom_path = os.path.join(self.img_dir, str(row['study_id']),str(row['series_id']),f\"{str(row['instance_number'])}.dcm\")\n        dicom = pydicom.dcmread(dicom_path)\n        image = dicom.pixel_array\n        # Normalize and convert to RGB\n        image_normalized = np.interp(image, (0, 1810), (0, 255)).astype(np.uint8)\n        #rgb_image = np.stack([image_normalized] * 3, axis=-1)\n        # Convert NumPy array to PIL Image for cropping\n        rgb_image_pil = Image.fromarray(image_normalized)\n\n        width,height =rgb_image_pil.size\n\n        coor=np.array(row.loc['x_level_Right':'y_level_Left'].tolist(),dtype=float)\n        coor[0::2],coor[1::2]= coor[0::2]/width ,coor[1::2]/height #coor in 0-1\n        \n        # Create a segmentation mask for the image (5 spinal levels)\n        mask = self.create_segmentation_mask(coor, img_shape=(256, 256))\n\n        level=float(row['level_array'].index(1))\n        \n        return rgb_image_pil, mask,level\n    \n    def create_segmentation_mask(self, label, img_shape=(256, 256), box_size=10):\n        mask = np.zeros(img_shape, dtype=np.float32)\n        width, height = img_shape\n\n        # Iterate over spinal level coordinates (x1, y1, x2, y2, ...)\n        for i in range(0, len(label), 2):\n            x = int(label[i] * width)  # x coordinate scaled to image size\n            y = int(label[i + 1] * height)  # y coordinate scaled to image size\n\n            # Assign class label for each spinal level (1 to 5)\n            class_label = (i // 2) + 1\n\n            # Draw a small square around the (x, y) point\n            x_min = max(0, x - box_size // 2)\n            x_max = min(width, x + box_size // 2)\n            y_min = max(0, y - box_size // 2)\n            y_max = min(height, y + box_size // 2)\n\n            # Fill the mask with the class label\n            mask[y_min:y_max, x_min:x_max] = class_label\n\n        return mask\n","metadata":{"execution":{"iopub.status.busy":"2024-09-23T11:45:58.468217Z","iopub.execute_input":"2024-09-23T11:45:58.468537Z","iopub.status.idle":"2024-09-23T11:45:58.485208Z","shell.execute_reply.started":"2024-09-23T11:45:58.468505Z","shell.execute_reply":"2024-09-23T11:45:58.484190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def img_mask_ori(img,mask):\n    # Get the image and mask\n    #img, mask = spinal_train_dataset[200]\n\n    # Reorder the dimensions of img from (3, 256, 256) to (256, 256, 3) for displaying\n    img = img.permute(1, 2, 0)\n\n    # Convert image tensor to numpy for displaying\n    img = img.numpy()\n\n    # Print the shapes for debugging\n    height, width = img.shape[:2]\n    print(f\"Image shape: {img.shape}\")  # Should be (256, 256, 3) for 3-channel RGB\n    print(f\"Mask shape: {mask.shape}\")  # Should be (256, 256) for the segmentation mask\n    print(f\"Unique values in the mask: {torch.unique(mask)}\")\n\n    # Visualize the image\n    plt.figure(figsize=(10, 5))\n\n    # Plot the original image\n    plt.subplot(1, 3, 1)\n    plt.imshow(img, cmap='gray')\n    plt.title(\"Original Image\")\n\n    # Plot the mask (with integer labels for each class)\n    plt.subplot(1, 3, 2)\n    plt.imshow(mask, cmap='jet', alpha=0.6)  # Different color for each class\n    plt.title(\"Mask\")\n\n    # Overlay mask on the image with transparency\n    plt.subplot(1, 3, 3)\n    plt.imshow(img, cmap='gray')\n    plt.imshow(mask, cmap='jet', alpha=0.5)  # Overlay mask on the image\n    plt.title(\"Image with Mask Overlay\")\n\n    # Show all the plots\n    plt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-09-23T11:45:58.486200Z","iopub.execute_input":"2024-09-23T11:45:58.486521Z","iopub.status.idle":"2024-09-23T11:45:58.499646Z","shell.execute_reply.started":"2024-09-23T11:45:58.486489Z","shell.execute_reply":"2024-09-23T11:45:58.498823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(train_)\nseries_df = pd.read_csv(train_series)\nlabel_df = pd.read_csv(train_label)\ndf = pd.merge(label_df, series_df , on=['study_id','series_id'])\ndf = pd.merge(df,train_df,on = 'study_id')\n\ndf_spinal, df_neural,df_subarticular = reconstruct_df(df)\n\n# subarticular\ndf_subarticular_xy =xy_subarticular(df_subarticular)","metadata":{"execution":{"iopub.status.busy":"2024-09-23T11:45:58.500723Z","iopub.execute_input":"2024-09-23T11:45:58.501028Z","iopub.status.idle":"2024-09-23T11:46:12.669811Z","shell.execute_reply.started":"2024-09-23T11:45:58.500990Z","shell.execute_reply":"2024-09-23T11:46:12.668981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subarticular_train_df, subarticular_val_df=train_test_split(df_subarticular_xy,test_size=0.2,random_state=208)\nprint(f\"size for train:{len(subarticular_train_df)}\")\nprint(f\"size for validation:{len(subarticular_val_df)}\")\n      \nsubarticular_train_dataset= subarticular_Dataset(subarticular_train_df,img_train_path,transform=None)\nsubarticular_val_dataset= subarticular_Dataset(subarticular_val_df,img_train_path,transform=None)\n\nsubarticular_train_loader = DataLoader(subarticular_train_dataset,batch_size=16,shuffle=True,num_workers=5,pin_memory=True )\nsubarticular_val_loader=DataLoader(subarticular_val_dataset,batch_size=16,shuffle=False,num_workers=5,pin_memory=True)      ","metadata":{"execution":{"iopub.status.busy":"2024-09-23T11:46:12.670991Z","iopub.execute_input":"2024-09-23T11:46:12.671303Z","iopub.status.idle":"2024-09-23T11:46:12.684473Z","shell.execute_reply.started":"2024-09-23T11:46:12.671269Z","shell.execute_reply":"2024-09-23T11:46:12.683263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img, mask,rl = subarticular_val_dataset[100]\nimg_mask_ori(img,mask)\nprint(rl)","metadata":{"execution":{"iopub.status.busy":"2024-09-23T11:46:12.685766Z","iopub.execute_input":"2024-09-23T11:46:12.686171Z","iopub.status.idle":"2024-09-23T11:46:13.443531Z","shell.execute_reply.started":"2024-09-23T11:46:12.686124Z","shell.execute_reply":"2024-09-23T11:46:13.442601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#training process\ndef train_and_evaluate(model_mask, model_rl, train_loader, test_loader, optimizer_mask, optimizer_leris, scheduler_mask, scheduler_leris, num_epochs=10):\n    device_mask = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    device_rl = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n    model_mask.to(device_mask)\n    model_rl.to(device_rl)\n    print(f'model unet in {device_mask}, mdoel left right in {device_rl}')\n    \n    criterion = nn.CrossEntropyLoss(reduction='mean')\n    #class_weights = torch.tensor([0.4, 1.0, 1.0, 1.0, 1.0, 1.0], dtype=torch.float32).to(device_mask)\n    #criterion_mask = nn.CrossEntropyLoss(reduction='mean', weight=class_weights)\n\n    \n    optimizer_mask = optimizer_mask #torch.optim.AdamW(model.parameters(), lr=0.00001)\n    optimizer_leris = optimizer_leris\n    \n    scheduler_mask = scheduler_mask #optim.lr_scheduler.StepLR(optimizer, step_size=20, gamma=0.1) \n    scheduler_leris = scheduler_leris\n    \n    train_loss_mask_set, test_loss_mask_set, train_loss_rl_set, test_loss_rl_set = [], [], [], []\n    #train_coor_set,test_coor_set = [],[]\n    best_test_loss = float('inf')\n    train_start = time.time()\n    \n    for epoch in range(num_epochs):\n        model_mask.train()\n        model_rl.train()\n        total_loss_mask, total_loss_leris = 0.0, 0.0\n        \n        for inputs, mask,leris in train_loader:\n            inputs_mask,input_leris, mask,leris = inputs.to(device_mask), inputs.to(device_rl), mask.to(device_mask),leris.to(device_rl)\n            \n            optimizer_mask.zero_grad()\n            optimizer_leris.zero_grad()\n            \n            output_mask = model_mask(inputs_mask)\n            output_leris = model_rl(input_leris)\n            \n            #print(output_leris.shape)  # Should be [batch_size, num_classes]\n            #print(leris.shape)  # Should be [batch_size]\n\n            \n            #loss \n            loss_mask = criterion(output_mask,mask)\n            loss_leris = criterion(output_leris,leris)\n            \n            #total loss with backward\n            loss_mask.backward()\n            loss_leris.backward()\n            optimizer_mask.step()\n            optimizer_leris.step()\n            \n            #accumulated loss\n            total_loss_mask += loss_mask.item()\n            total_loss_leris += loss_leris.item()\n        \n        avg_train_loss_mask = total_loss_mask / len(train_loader)\n        avg_train_loss_leris = total_loss_leris / len(train_loader)\n        train_loss_mask_set.append(avg_train_loss_mask)\n        train_loss_rl_set.append(avg_train_loss_leris)\n        scheduler_mask.step()\n        scheduler_leris.step()\n        \n        model_mask.eval()\n        model_rl.eval()\n        total_loss_mask, total_loss_leris, correct_mask, correct_leris, total_mask, total_leris = 0.0, 0.0, 0.0, 0.0, 0.0, 0.0\n        with torch.no_grad(): \n            for inputs,mask,leris in test_loader:\n                inputs_mask,input_leris, mask,leris = inputs.to(device_mask), inputs.to(device_rl), mask.to(device_mask),leris.to(device_rl)\n                output_mask = model_mask(inputs_mask)\n                output_leris = model_rl(input_leris)\n                \n                #loss \n                loss_mask = criterion(output_mask,mask)\n                loss_leris = criterion(output_leris,leris)\n                total_loss_mask += loss_mask.item()\n                total_loss_leris += loss_leris.item()\n                \n                #accuracy\n                _, preds_mask = torch.max(output_mask, 1)\n                _, preds_leris = torch.max(output_leris, 1)\n\n                # Calculate accuracy by comparing predictions to ground truth\n                correct_mask += (preds_mask == mask).sum().item()\n                correct_leris += (preds_leris == leris).sum().item()\n                total_mask += mask.numel()\n                total_leris += leris.numel()\n    \n                \n        avg_test_loss_mask = total_loss_mask / len(test_loader)\n        avg_test_loss_leris = total_loss_leris / len(test_loader)\n        test_loss_mask_set.append(avg_test_loss_mask)\n        test_loss_rl_set.append(avg_test_loss_leris)\n        accuracy_mask = correct_mask/total_mask *100\n        accuracy_leris = correct_leris/total_leris *100\n        \n        current_lr = optimizer_mask.param_groups[0]['lr']\n            \n        if avg_test_loss_leris <best_test_loss:\n            best_test_loss = avg_test_loss_leris\n            torch.save(model_mask.state_dict(),'mask_best_Subarticular.pth')\n            torch.save(model_rl.state_dict(),'lr_best_Subarticular.pth')\n            print(f'best loss::{best_test_loss}')\n        print(\n            f'Epoch {epoch+1}/{num_epochs}: \\t'\n            f'Lr:{current_lr},\\t'\n            f'train loss mask:{avg_train_loss_mask:.6f} \\t'\n            f'train loss lr:{avg_train_loss_leris:.6f} \\t'\n            f'test loss mask:{avg_test_loss_mask:.6f} \\t'\n            f'test loss lr:{avg_test_loss_leris:.6f}\\n'\n            f'test acc mask:{accuracy_mask:.2f}% \\t'\n            f'test acc lr:{accuracy_leris:.2f}% \\t'\n        )\n        print('-'*80)\n            \n    train_end = time.time()\n    time_used = train_end-train_start\n    print(f'Time used for Training:{time_used} sec ')\n    print('-'* 80)\n    \n    train_history_df=pd.DataFrame({\n        'train_loss_mask': train_loss_mask_set,\n        'test_loss_mask': test_loss_mask_set ,\n        'train_loss_rl': train_loss_rl_set ,\n        'test_loss_rl': test_loss_rl_set ,\n    })\n    \n    torch.save(model_mask.state_dict(), 'mask_last_subarticular.pth')\n    torch.save(model_rl.state_dict(), 'lr_last_subarticular.pth')\n  \n    \n    return model_mask,model_rl, train_history_df","metadata":{"execution":{"iopub.status.busy":"2024-09-23T11:46:13.445061Z","iopub.execute_input":"2024-09-23T11:46:13.445453Z","iopub.status.idle":"2024-09-23T11:46:13.477299Z","shell.execute_reply.started":"2024-09-23T11:46:13.445405Z","shell.execute_reply":"2024-09-23T11:46:13.476047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plot history loss on each iter\ndef losshistory_plot(train_loss_history, test_loss_history, train_coor_hist,test_coor_hist):\n    epochs = range(1,len(train_loss_history)+1)\n    fig,axe = plt.subplots(1,2,figsize=(10,6))\n    axe[0].plot(epochs, train_loss_history, label='Train Loss')\n    axe[0].plot(epochs, test_loss_history, label = 'Test Loss')\n    axe[0].set_xlabel('Epochs over batch')\n    axe[0].set_ylabel('Loss in total')\n    axe[0].set_title('Train and val Loss on each batch over epochs')\n    axe[0].legend()\n    \n    axe[1].plot(epochs, train_coor_hist, label='Train Loss')\n    axe[1].plot(epochs, test_coor_hist, label = 'Test Loss')\n    axe[1].set_xlabel('Epochs over batch')\n    axe[1].set_ylabel('Loss on Coordinates')\n    axe[1].set_title('Train and val Loss on each batch over epochs')\n    axe[1].legend()\n    \n    plt.tight_layout()\n    #plt.savefig(f'epochs on {model_name}.jpg')\n    plt.show()\n\n################################################################################\n#plot model prediction on images\ndef model_plot(model, test_dataset, list_img):\n    f, ax = plt.subplots(2, 4, figsize=(13, 8))\n    \n    for i, img_id in enumerate(list_img):\n        img, mask,_ = test_dataset[img_id]\n        image = img.permute(1, 2, 0)\n        height, weight = image.shape[:2]\n        \n        # Original image with label scatter plot\n        ax[0, i].imshow(image, cmap='gray')\n        ax[0, i].imshow(mask, cmap='jet', alpha=0.5)  # Overlay mask on the image\n\n        # Model prediction\n        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n        model.to(device)\n        model.eval()\n        img = img.unsqueeze(0).to(device)\n        output = model(img).cpu().detach() \n        _, outputs_mask = torch.max(output, 1)\n        pred_mask = outputs_mask.squeeze(0).numpy() \n\n        ax[1, i].imshow(image, cmap='gray')\n        ax[1, i].imshow(pred_mask, cmap='jet', alpha=0.5)\n    \n    plt.tight_layout()\n    plt.show()\n    \n################################################################################\n","metadata":{"execution":{"iopub.status.busy":"2024-09-23T11:46:13.482012Z","iopub.execute_input":"2024-09-23T11:46:13.482844Z","iopub.status.idle":"2024-09-23T11:46:13.499353Z","shell.execute_reply.started":"2024-09-23T11:46:13.482806Z","shell.execute_reply":"2024-09-23T11:46:13.498114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"################### EVALUATION ########################################\ndef evaluation_mask(model, data_loader):\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    model.to(device)\n    print(device)\n    \n    criterion = nn.CrossEntropyLoss(reduction='mean')  # For left-right prediction\n    \n    total_loss,correct_mask = 0.0, 0.0\n    total_labels = 0.0\n    model.eval()\n    for inputs, mask,_ in data_loader:\n        inputs, labels = inputs.to(device), mask.to(device)\n        \n        outputs = model(inputs)\n        \n        #loss\n        loss = criterion(outputs, labels)\n        total_loss += loss.item()\n        \n        #accuracy\n        _, preds = torch.max(outputs, 1)\n        # Calculate accuracy by comparing predictions to ground truth\n        correct_mask += (preds == labels).sum().item()\n        total_labels += labels.numel() #1k++\n        \n    average_loss = total_loss / len(data_loader)\n    accuracy_mask = correct_mask/total_labels *100\n    \n    print(f'Total Loss: {average_loss}, \\t Total accuracy:{accuracy_mask}')\n    print('-' * 80)\n\ndef evaluation_leftRight(model, data_loader):\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    model.to(device)\n    print(device)\n    \n    criterion = nn.CrossEntropyLoss(reduction='mean')  # For left-right prediction\n    \n    all_test_labels = []\n    all_test_outputs = []\n   \n    total_loss,correct_mask = 0.0, 0.0\n    total_labels = 0.0\n    \n    model.eval()\n    for inputs, _, lr in data_loader:\n        inputs, labels = inputs.to(device), lr.to(device)\n        outputs = model(inputs)\n        \n        #loss\n        loss = criterion(outputs, labels)\n        total_loss += loss.item()\n        \n        #accuracy\n        _, preds = torch.max(outputs, 1)\n        \n        # Calculate accuracy by comparing predictions to ground truth\n        correct_mask += (preds == labels).sum().item()\n        total_labels += labels.numel() \n        #print(labels.numel()) #16\n        \n        # Collect labels and predicted probabilities\n        all_test_labels.extend(labels.cpu().numpy())\n        all_test_outputs.extend(preds.detach().cpu().numpy())\n   \n    average_loss = total_loss / len(data_loader)\n    accuracy_mask = correct_mask/total_labels *100\n    print(f'Total Loss: {average_loss}, \\t Total accuracy:{accuracy_mask}')\n    \n    # Convert to numpy arrays\n    all_labels = np.array(all_test_labels)\n    all_predictions = np.array(all_test_outputs)\n    \n    # Generate and display confusion matrix\n    cm = confusion_matrix(all_labels, all_predictions)\n    disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=np.arange(5))\n    disp.plot(cmap=plt.cm.Blues)\n    plt.title('Confusion Matrix - Left-Right Prediction')\n    plt.show()\n    print('-' * 80)","metadata":{"execution":{"iopub.status.busy":"2024-09-23T11:46:13.500876Z","iopub.execute_input":"2024-09-23T11:46:13.501388Z","iopub.status.idle":"2024-09-23T11:46:13.520321Z","shell.execute_reply.started":"2024-09-23T11:46:13.501341Z","shell.execute_reply":"2024-09-23T11:46:13.519370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Conv2dReLU(nn.Module):\n    def __init__(self, in_channels, out_channels, kernel_size=3, stride=1, padding=1, bias=False):\n        super().__init__()\n        self.conv = nn.Conv2d(in_channels, out_channels, kernel_size, stride, padding, bias=bias)\n        self.bn = nn.BatchNorm2d(out_channels)\n        self.relu = nn.LeakyReLU(0.2, inplace=True)\n\n    def forward(self, x):\n        return self.relu(self.bn(self.conv(x)))\n\nclass UNetVGG16(nn.Module):\n    def __init__(self, in_channels=1, out_channels=6, pretrained=True):\n        super(UNetVGG16, self).__init__()\n\n        # Load VGG16 model\n        vgg16 = models.vgg16(pretrained=pretrained)\n\n        # Modify the first layer for 1 channel input\n        vgg16.features[0] = nn.Conv2d(in_channels, 64, kernel_size=3,stride=1, padding=1)\n\n        # Encoder layers\n        self.enc1 = nn.Sequential(*vgg16.features[:4])  # 64 channels\n        self.enc2 = nn.Sequential(*vgg16.features[4:9])  # 128 channels\n        self.enc3 = nn.Sequential(*vgg16.features[9:16])  # 256 channels\n        self.enc4 = nn.Sequential(*vgg16.features[16:23])  # 512 channels\n        self.enc5 = nn.Sequential(*vgg16.features[23:30])  # 512 channels\n\n        # Bottleneck\n        self.bottleneck = Conv2dReLU(512, 512)\n\n        # Decoder layers\n        self.dec4 = self.upconv(512, 512)  # 512 -> 512\n        self.dec3 = self.upconv(1024, 256)  # 1024 -> 256 (after concatenation)\n        self.dec2 = self.upconv(512, 128)   # 512 -> 128 (after concatenation)\n        self.dec1 = self.upconv(256, 64)    # 256 -> 64 (after concatenation)\n\n        # Final output layer\n        self.final_conv = nn.Conv2d(128, out_channels, kernel_size=1)\n        \n        # Dropout for regularization\n        self.dropout = nn.Dropout(p=0.5)\n\n    def upconv(self, in_channels, out_channels):\n        return nn.Sequential(\n            nn.ConvTranspose2d(in_channels, out_channels, kernel_size=2, stride=2),\n            nn.LeakyReLU(0.2, inplace=True)\n        )\n\n    def forward(self, x):\n        # Encoder\n        enc1 = self.enc1(x)  # [B, 64, H/2, W/2]\n        enc2 = self.enc2(enc1)  # [B, 128, H/4, W/4]\n        enc3 = self.enc3(enc2)  # [B, 256, H/8, W/8]\n        enc4 = self.enc4(enc3)  # [B, 512, H/16, W/16]\n        enc5 = self.enc5(enc4)  # [B, 512, H/32, W/32]\n        \n        # Bottleneck\n        bottleneck = self.dropout(self.bottleneck(enc5))  # [B, 512, H/32, W/32]\n\n\n        # Decoder\n        dec4 = self.dec4(bottleneck)  # [B, 512, H/16, W/16]\n        dec4 = torch.cat((dec4, enc4), dim=1)  # [B, 1024, H/16, W/16\n        \n        dec3 = self.dec3(dec4)  # [B, 256, H/8, W/8]\n        dec3 = torch.cat((dec3, enc3), dim=1)  # [B, 512, H/8, W/8]\n        \n        dec2 = self.dec2(dec3)  # [B, 128, H/4, W/4]\n        dec2 = torch.cat((dec2, enc2), dim=1)  # [B, 256, H/4, W/4]\n        \n        dec1 = self.dec1(dec2)  # [B, 64, H/2, W/2]\n        dec1 = torch.cat((dec1, enc1), dim=1)  # [B, 128, H/2, W/2]\n\n        # Final output\n        out = self.final_conv(dec1)  # [B, out_channels, H/2, W/2]\n        return out","metadata":{"execution":{"iopub.status.busy":"2024-09-23T11:46:13.521735Z","iopub.execute_input":"2024-09-23T11:46:13.522146Z","iopub.status.idle":"2024-09-23T11:46:13.540741Z","shell.execute_reply.started":"2024-09-23T11:46:13.522102Z","shell.execute_reply":"2024-09-23T11:46:13.539726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class subarticular_resnet(nn.Module):\n    def __init__(self, num_classes=2):\n        super(subarticular_resnet, self).__init__()\n        self.resnet = models.resnet50(pretrained=True)\n        \n        # Modify the first convolutional layer to accept 1 channel instead of 3\n        self.resnet.conv1 = nn.Conv2d(1, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n        \n        # Modify the fully connected layer for 2-class output\n        num_ftrs = self.resnet.fc.in_features\n        self.resnet.fc = nn.Linear(num_ftrs, num_classes)\n\n    def forward(self, x):\n        return self.resnet(x)\n    ","metadata":{"execution":{"iopub.status.busy":"2024-09-23T11:46:13.542066Z","iopub.execute_input":"2024-09-23T11:46:13.543078Z","iopub.status.idle":"2024-09-23T11:46:13.555390Z","shell.execute_reply.started":"2024-09-23T11:46:13.543030Z","shell.execute_reply":"2024-09-23T11:46:13.554379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#training process\ndef train_and_evaluate(model_mask, model_rl, train_loader, test_loader, optimizer_mask, optimizer_leris, scheduler_mask, scheduler_leris, num_epochs=10):\n    device_mask = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    device_rl = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n    model_mask.to(device_mask)\n    model_rl.to(device_rl)\n    print(f'model unet in {device_mask}') #', mdoel left right in {device_rl}')\n    \n    criterion = nn.CrossEntropyLoss(reduction='mean')\n    class_weights = torch.tensor([0.4, 1.0, 1.0], dtype=torch.float32).to(device_mask)\n    criterion_mask = nn.CrossEntropyLoss(reduction='mean', weight=class_weights)\n\n    \n    optimizer_mask = optimizer_mask #torch.optim.AdamW(model.parameters(), lr=0.00001)\n    optimizer_leris = optimizer_leris\n    \n    scheduler_mask = scheduler_mask #optim.lr_scheduler.StepLR(optimizer, step_size=20, gamma=0.1) \n    scheduler_leris = scheduler_leris\n    \n    train_loss_mask_set, test_loss_mask_set, train_loss_rl_set, test_loss_rl_set = [], [], [], []\n    #train_coor_set,test_coor_set = [],[]\n    best_test_loss = float('inf')\n    train_start = time.time()\n    \n    for epoch in range(num_epochs):\n        model_mask.train()\n        #model_rl.train()\n        total_loss_mask, total_loss_leris = 0.0, 0.0\n        \n        for inputs, mask,leris in train_loader:\n            inputs_mask,input_leris, mask,leris = inputs.to(device_mask), inputs.to(device_rl), mask.to(device_mask),leris.to(device_rl)\n            \n            optimizer_mask.zero_grad()\n            optimizer_leris.zero_grad()\n            \n            output_mask = model_mask(inputs_mask)\n            #output_leris = model_rl(input_leris)\n            \n            #print(output_leris.shape)  # Should be [batch_size, num_classes]\n            #print(leris.shape)  # Should be [batch_size]\n\n            \n            #loss \n            loss_mask = criterion_mask(output_mask,mask)\n            #loss_leris = criterion(output_leris,leris)\n            \n            #total loss with backward\n            loss_mask.backward()\n            #loss_leris.backward()\n            optimizer_mask.step()\n            #optimizer_leris.step()\n            \n            #accumulated loss\n            total_loss_mask += loss_mask.item()\n            #total_loss_leris += loss_leris.item()\n        \n        avg_train_loss_mask = total_loss_mask / len(train_loader)\n        #avg_train_loss_leris = total_loss_leris / len(train_loader)\n        train_loss_mask_set.append(avg_train_loss_mask)\n        #train_loss_rl_set.append(avg_train_loss_leris)\n        scheduler_mask.step()\n        #scheduler_leris.step()\n        \n        model_mask.eval()\n        #model_rl.eval()\n        total_loss_mask, total_loss_leris, correct_mask, correct_leris, total_mask, total_leris = 0.0, 0.0, 0.0, 0.0, 0.0, 0.0\n        with torch.no_grad(): \n            for inputs,mask,leris in test_loader:\n                inputs_mask,input_leris, mask,leris = inputs.to(device_mask), inputs.to(device_rl), mask.to(device_mask),leris.to(device_rl)\n                output_mask = model_mask(inputs_mask)\n                #output_leris = model_rl(input_leris)\n                \n                #loss \n                loss_mask = criterion_mask(output_mask,mask)\n                #loss_leris = criterion(output_leris,leris)\n                total_loss_mask += loss_mask.item()\n                #total_loss_leris += loss_leris.item()\n                \n                #accuracy\n                _, preds_mask = torch.max(output_mask, 1)\n                #_, preds_leris = torch.max(output_leris, 1)\n\n                # Calculate accuracy by comparing predictions to ground truth\n                correct_mask += (preds_mask == mask).sum().item()\n                #correct_leris += (preds_leris == leris).sum().item()\n                total_mask += mask.numel()\n                #total_leris += leris.numel()\n    \n                \n        avg_test_loss_mask = total_loss_mask / len(test_loader)\n        #avg_test_loss_leris = total_loss_leris / len(test_loader)\n        test_loss_mask_set.append(avg_test_loss_mask)\n        #test_loss_rl_set.append(avg_test_loss_leris)\n        accuracy_mask = correct_mask/total_mask *100\n        #accuracy_leris = correct_leris/total_leris *100\n        \n        current_lr = optimizer_mask.param_groups[0]['lr']\n            \n        if avg_test_loss_mask <best_test_loss:\n            best_test_loss = avg_test_loss_mask\n            torch.save(model_mask.state_dict(),'mask_best_Subarticular.pth')\n            #torch.save(model_rl.state_dict(),'lr_best_Subarticular.pth')\n            print(f'best loss::{best_test_loss}')\n        print(\n            f'Epoch {epoch+1}/{num_epochs}: \\t'\n            f'Lr:{current_lr},\\t'\n            f'train loss mask:{avg_train_loss_mask:.6f} \\t'\n            f'test loss mask:{avg_test_loss_mask:.6f} \\t'\n            f'test acc mask:{accuracy_mask:.2f}% \\t'\n        )\n        print('-'*80)\n            \n    train_end = time.time()\n    time_used = train_end-train_start\n    print(f'Time used for Training:{time_used} sec ')\n    print('-'* 80)\n    \n    train_history_df=pd.DataFrame({\n        'train_loss_mask': train_loss_mask_set,\n        'test_loss_mask': test_loss_mask_set ,\n    })\n    \n    torch.save(model_mask.state_dict(), 'mask_last_subarticular.pth')\n\n    return model_mask,model_rl, train_history_df","metadata":{"execution":{"iopub.status.busy":"2024-09-23T11:46:13.556861Z","iopub.execute_input":"2024-09-23T11:46:13.557495Z","iopub.status.idle":"2024-09-23T11:46:13.577399Z","shell.execute_reply.started":"2024-09-23T11:46:13.557461Z","shell.execute_reply":"2024-09-23T11:46:13.576479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_mask = UNetVGG16(in_channels=1, out_channels=3, pretrained=True)\nmodel_rl = subarticular_resnet(num_classes=5)\n\noptimizer_mask = optim.Adam(model_mask.parameters(), lr=0.001)\n\noptimizer_leris = optim.Adam(model_rl.parameters(), lr=0.001)\n\n\nscheduler_mask = optim.lr_scheduler.StepLR(optimizer_mask, step_size=10, gamma=0.1) \nscheduler_leris = optim.lr_scheduler.StepLR(optimizer_leris, step_size=15, gamma=0.1) \n\nmodel_mask,model_rl, train_history_df = train_and_evaluate(model_mask, model_rl, \n                                                           subarticular_train_loader, subarticular_val_loader, \n                                                           optimizer_mask, optimizer_leris, \n                                                           scheduler_mask, scheduler_leris, \n                                                           num_epochs=30)\n   ","metadata":{"execution":{"iopub.status.busy":"2024-09-23T11:46:13.578728Z","iopub.execute_input":"2024-09-23T11:46:13.579142Z","iopub.status.idle":"2024-09-23T12:21:31.933673Z","shell.execute_reply.started":"2024-09-23T11:46:13.579101Z","shell.execute_reply":"2024-09-23T12:21:31.932415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!export CUDA_LAUNCH_BLOCKING=1","metadata":{"execution":{"iopub.status.busy":"2024-09-23T12:21:31.935442Z","iopub.execute_input":"2024-09-23T12:21:31.935802Z","iopub.status.idle":"2024-09-23T12:21:32.961584Z","shell.execute_reply.started":"2024-09-23T12:21:31.935758Z","shell.execute_reply":"2024-09-23T12:21:32.960494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nEpoch 30/30: \tLr:1.0000000000000002e-06,\ttrain loss mask:0.000414 \ttrain loss lr:0.000909 \ttest loss mask:0.009872 \ttest loss lr:0.412268\ntest acc mask:99.84% \ttest acc lr:90.29% \n'''","metadata":{"execution":{"iopub.status.busy":"2024-09-23T12:22:36.852479Z","iopub.execute_input":"2024-09-23T12:22:36.853119Z","iopub.status.idle":"2024-09-23T12:22:36.859455Z","shell.execute_reply.started":"2024-09-23T12:22:36.853077Z","shell.execute_reply":"2024-09-23T12:22:36.858476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"losshistory_plot(train_history_df['train_loss_mask'],\n                 train_history_df['test_loss_mask'],\n                 train_history_df['train_loss_rl'],\n                 train_history_df['test_loss_rl'])","metadata":{"execution":{"iopub.status.busy":"2024-09-23T12:21:59.919180Z","iopub.execute_input":"2024-09-23T12:21:59.919567Z","iopub.status.idle":"2024-09-23T12:22:00.033840Z","shell.execute_reply.started":"2024-09-23T12:21:59.919530Z","shell.execute_reply":"2024-09-23T12:22:00.032563Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_img=[2,78,157,99]\nmodel_plot(model_mask, subarticular_train_dataset,list_img)","metadata":{"execution":{"iopub.status.busy":"2024-09-23T12:22:11.026481Z","iopub.execute_input":"2024-09-23T12:22:11.026904Z","iopub.status.idle":"2024-09-23T12:22:12.819562Z","shell.execute_reply.started":"2024-09-23T12:22:11.026863Z","shell.execute_reply":"2024-09-23T12:22:12.818380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_img=[2,78,157,99]\nmodel_plot(model_mask, subarticular_val_dataset,list_img)","metadata":{"execution":{"iopub.status.busy":"2024-09-23T12:22:05.169963Z","iopub.execute_input":"2024-09-23T12:22:05.170681Z","iopub.status.idle":"2024-09-23T12:22:07.221274Z","shell.execute_reply.started":"2024-09-23T12:22:05.170637Z","shell.execute_reply":"2024-09-23T12:22:07.220281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_img=[44,90,130,150]\nmodel_plot(model_mask, subarticular_val_dataset,list_img)","metadata":{"execution":{"iopub.status.busy":"2024-09-23T12:22:08.052814Z","iopub.execute_input":"2024-09-23T12:22:08.053724Z","iopub.status.idle":"2024-09-23T12:22:09.964588Z","shell.execute_reply.started":"2024-09-23T12:22:08.053681Z","shell.execute_reply":"2024-09-23T12:22:09.963553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evaluation_leftRight(model_rl, subarticular_val_loader)","metadata":{"execution":{"iopub.status.busy":"2024-09-23T12:21:32.978636Z","iopub.status.idle":"2024-09-23T12:21:32.978988Z","shell.execute_reply.started":"2024-09-23T12:21:32.978814Z","shell.execute_reply":"2024-09-23T12:21:32.978832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evaluation_mask(model_mask, subarticular_val_loader)","metadata":{"execution":{"iopub.status.busy":"2024-09-23T12:21:32.980882Z","iopub.status.idle":"2024-09-23T12:21:32.981241Z","shell.execute_reply.started":"2024-09-23T12:21:32.981067Z","shell.execute_reply":"2024-09-23T12:21:32.981085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#torch.save(model_rl.state_dict(),'lr_Subarticular.pth')\n#torch.save(model_mask.state_dict(),'mask_Subarticular.pth')","metadata":{"execution":{"iopub.status.busy":"2024-09-23T12:21:32.983246Z","iopub.status.idle":"2024-09-23T12:21:32.983741Z","shell.execute_reply.started":"2024-09-23T12:21:32.983493Z","shell.execute_reply":"2024-09-23T12:21:32.983518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model_loaded=vgg16_subarticular(num_classes=4)\n#model_loaded.load_state_dict(torch.load('/kaggle/working/coor_vgg16_subart.pth'))","metadata":{"execution":{"iopub.status.busy":"2024-09-23T12:21:32.984971Z","iopub.status.idle":"2024-09-23T12:21:32.985476Z","shell.execute_reply.started":"2024-09-23T12:21:32.985198Z","shell.execute_reply":"2024-09-23T12:21:32.985224Z"},"trusted":true},"execution_count":null,"outputs":[]}]}