{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":22990,"datasetId":1136396,"databundleVersionId":2048213},{"sourceType":"datasetVersion","sourceId":3521209,"datasetId":2089184,"databundleVersionId":3573956}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -U segmentation-models-pytorch","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:17:11.233404Z","iopub.execute_input":"2025-02-19T14:17:11.233747Z","iopub.status.idle":"2025-02-19T14:17:22.365441Z","shell.execute_reply.started":"2025-02-19T14:17:11.233710Z","shell.execute_reply":"2025-02-19T14:17:22.364211Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport cv2\nfrom glob import glob\nfrom PIL import Image\nimport os\nimport re\nimport matplotlib.pyplot as plt\nfrom matplotlib.patches import Rectangle\nimport seaborn as sns\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:17:22.366625Z","iopub.execute_input":"2025-02-19T14:17:22.367063Z","iopub.status.idle":"2025-02-19T14:17:24.065701Z","shell.execute_reply.started":"2025-02-19T14:17:22.367030Z","shell.execute_reply":"2025-02-19T14:17:24.064611Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/uwmgi-mask-dataset/train.csv\")\ndf_copy = df.copy()\n\ndf_copy= df_copy.drop(['case','day','slice','image_path','height','width','mask_path'],axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:17:24.066757Z","iopub.execute_input":"2025-02-19T14:17:24.067326Z","iopub.status.idle":"2025-02-19T14:17:25.616135Z","shell.execute_reply.started":"2025-02-19T14:17:24.067282Z","shell.execute_reply":"2025-02-19T14:17:25.615178Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_copy.rename(columns= {'class':'classes'} , inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:17:25.618894Z","iopub.execute_input":"2025-02-19T14:17:25.619206Z","iopub.status.idle":"2025-02-19T14:17:25.624261Z","shell.execute_reply.started":"2025-02-19T14:17:25.619181Z","shell.execute_reply":"2025-02-19T14:17:25.623110Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_copy['case'] = df_copy['id'].apply(lambda x : int(x.split('_')[0].replace(\"case\" , \"\")))\ndf_copy['day'] = df_copy['id'].apply(lambda x : int(x.split('_')[1].replace('day',\"\")))\ndf_copy['slice'] = df_copy['id'].apply(lambda x : x.split('_')[3])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:17:25.625796Z","iopub.execute_input":"2025-02-19T14:17:25.626109Z","iopub.status.idle":"2025-02-19T14:17:25.882789Z","shell.execute_reply.started":"2025-02-19T14:17:25.626063Z","shell.execute_reply":"2025-02-19T14:17:25.881743Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train_images = glob(os.path.join(\"/kaggle/input/uwmgi-mask-dataset/png/uw-madison-gi-tract-image-segmentation/train\",\"**\",\"*.png\"),recursive=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:17:25.883940Z","iopub.execute_input":"2025-02-19T14:17:25.884263Z","iopub.status.idle":"2025-02-19T14:17:25.888279Z","shell.execute_reply.started":"2025-02-19T14:17:25.884237Z","shell.execute_reply":"2025-02-19T14:17:25.887255Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"prefix = '/kaggle/input/uwmgi-mask-dataset/png/uw-madison-gi-tract-image-segmentation/train'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:17:25.889393Z","iopub.execute_input":"2025-02-19T14:17:25.889742Z","iopub.status.idle":"2025-02-19T14:17:25.907623Z","shell.execute_reply.started":"2025-02-19T14:17:25.889706Z","shell.execute_reply":"2025-02-19T14:17:25.906443Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"temporary_1 = []\nfor idx, row in df_copy.iterrows():\n\n    temporary_1.append(os.path.join(prefix,\"case\"+str(row['case']),\n                                   \"case\"+str(row['case'])+\"_\"+\"day\"+str(row['day']),\n                                   'scans'))\n\ndf_copy['partial_path'] = temporary_1\ndf_copy","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:17:25.908850Z","iopub.execute_input":"2025-02-19T14:17:25.909265Z","iopub.status.idle":"2025-02-19T14:17:32.441189Z","shell.execute_reply.started":"2025-02-19T14:17:25.909229Z","shell.execute_reply":"2025-02-19T14:17:32.440026Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(df_copy['id'].values[0])\n\nprint(df_copy['partial_path'].values[0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:17:32.442115Z","iopub.execute_input":"2025-02-19T14:17:32.442441Z","iopub.status.idle":"2025-02-19T14:17:32.448242Z","shell.execute_reply.started":"2025-02-19T14:17:32.442407Z","shell.execute_reply":"2025-02-19T14:17:32.447249Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"temporary_1 = []\n\nfor idx,path in enumerate(df_copy['partial_path'].values):\n    for file in glob(os.path.join(path,'slice_'+str(df_copy['slice'].values[idx])+\"*.png\")):\n        temporary_1.append(file)\n\ntemporary_1[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:17:32.449230Z","iopub.execute_input":"2025-02-19T14:17:32.449587Z","iopub.status.idle":"2025-02-19T14:19:02.786186Z","shell.execute_reply.started":"2025-02-19T14:17:32.449552Z","shell.execute_reply":"2025-02-19T14:19:02.785190Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_copy['file_path'] = temporary_1\ndf_copy = df_copy.drop('partial_path',axis=1)\ndf_copy","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:19:02.787042Z","iopub.execute_input":"2025-02-19T14:19:02.787347Z","iopub.status.idle":"2025-02-19T14:19:02.834477Z","shell.execute_reply.started":"2025-02-19T14:19:02.787324Z","shell.execute_reply":"2025-02-19T14:19:02.833417Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_copy['width'] = df_copy['file_path'].apply(lambda x : x.split('_')[3])\ndf_copy['height'] = df_copy['file_path'].apply(lambda x : x.split('_')[4])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:19:02.835489Z","iopub.execute_input":"2025-02-19T14:19:02.835772Z","iopub.status.idle":"2025-02-19T14:19:03.005587Z","shell.execute_reply.started":"2025-02-19T14:19:02.835738Z","shell.execute_reply":"2025-02-19T14:19:03.004620Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### FINAL DATA FRAME","metadata":{"execution":{"iopub.status.busy":"2025-01-23T19:30:03.651808Z","iopub.execute_input":"2025-01-23T19:30:03.652336Z","iopub.status.idle":"2025-01-23T19:30:03.663448Z","shell.execute_reply.started":"2025-01-23T19:30:03.652296Z","shell.execute_reply":"2025-01-23T19:30:03.661358Z"}}},{"cell_type":"code","source":"# Here we take 3 step gap-gap row\n\ntrain_df = pd.DataFrame({'id':df_copy['id'][::3]})\ntrain_df['large_bowel'] = df_copy['segmentation'][::3].values\ntrain_df['small_bowel'] = df_copy['segmentation'][1::3].values\ntrain_df['stomach'] = df_copy['segmentation'][2::3].values\ntrain_df['path'] = df_copy['file_path'][::3]\ntrain_df['case'] = df_copy['case'][::3]\ntrain_df['day'] = df_copy['day'][::3]\ntrain_df['slice'] = df_copy['slice'][::3]\ntrain_df['width'] = df_copy['width'][::3]\ntrain_df['height'] = df_copy['height'][::3]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:19:03.009803Z","iopub.execute_input":"2025-02-19T14:19:03.010159Z","iopub.status.idle":"2025-02-19T14:19:03.027493Z","shell.execute_reply.started":"2025-02-19T14:19:03.010130Z","shell.execute_reply":"2025-02-19T14:19:03.026269Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.reset_index(inplace=True , drop=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:19:03.029668Z","iopub.execute_input":"2025-02-19T14:19:03.030050Z","iopub.status.idle":"2025-02-19T14:19:03.036632Z","shell.execute_reply.started":"2025-02-19T14:19:03.030021Z","shell.execute_reply":"2025-02-19T14:19:03.035573Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df['counts'] = np.sum(train_df.loc[: , ['large_bowel','small_bowel','stomach']] != '',axis=1)\ntrain_df['counts'].values","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:19:03.037342Z","iopub.execute_input":"2025-02-19T14:19:03.037641Z","iopub.status.idle":"2025-02-19T14:19:03.080774Z","shell.execute_reply.started":"2025-02-19T14:19:03.037618Z","shell.execute_reply":"2025-02-19T14:19:03.079901Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df['small_bowel'].unique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:19:03.081763Z","iopub.execute_input":"2025-02-19T14:19:03.082053Z","iopub.status.idle":"2025-02-19T14:19:03.099585Z","shell.execute_reply.started":"2025-02-19T14:19:03.082029Z","shell.execute_reply":"2025-02-19T14:19:03.098552Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"null_cols = train_df.iloc[: , 1:4].isnull().sum().reset_index()\nnull_cols.rename(columns={'index':'column_type',0:'total_null'},inplace=True)\n\nfig , ax = plt.subplots(1 ,1 , figsize=(15,5))\nsns.barplot(null_cols , x='column_type',y='total_null',palette='coolwarm',edgecolor='black',ax=ax)\nax.set_title(\"Number of Null Bar\")\nfor bar in ax.patches:\n    ax.annotate(f\"{bar.get_height():.2f}\",(bar.get_x()+bar.get_width()/2,bar.get_height()),ha='center',\n               va='bottom',fontsize=10,fontweight='bold')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:19:03.100758Z","iopub.execute_input":"2025-02-19T14:19:03.101027Z","iopub.status.idle":"2025-02-19T14:19:03.447497Z","shell.execute_reply.started":"2025-02-19T14:19:03.101005Z","shell.execute_reply":"2025-02-19T14:19:03.446340Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig , ax = plt.subplots(1,1,figsize=(15,5))\nsns.countplot(x='counts',data=train_df,ax=ax)\n\nfor p in ax.patches:\n    ax.annotate(f\"{p.get_height()}\",(p.get_x()+p.get_width()/2,\n               p.get_height()),color='black',fontweight='bold',fontsize=10,\n                 ha='center', va='bottom'\n               )\n\nax.set_xlabel('count')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:19:03.448569Z","iopub.execute_input":"2025-02-19T14:19:03.448947Z","iopub.status.idle":"2025-02-19T14:19:03.603366Z","shell.execute_reply.started":"2025-02-19T14:19:03.448917Z","shell.execute_reply":"2025-02-19T14:19:03.602208Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"height_width_df = train_df.iloc[:,8:].value_counts().reset_index()\n\nfig,ax=plt.subplots(1,1,figsize=(15,5))\nsns.barplot(height_width_df,x=height_width_df.index.values,y='count',palette='inferno',edgecolor='black',ax=ax)\n\nfor bins,(index,row) in zip(ax.patches,height_width_df.iterrows()):\n    ax.annotate(f\"W:{row[0]} x H:{row[1]}\",\n                (bins.get_x()+bins.get_width()/2,bins.get_height()),\n                ha='center',va='bottom',fontweight='bold'\n               )\nax.set_title(\"Width x Height plot\")\nax.set_xticklabels([])\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:19:03.604326Z","iopub.execute_input":"2025-02-19T14:19:03.604634Z","iopub.status.idle":"2025-02-19T14:19:03.816681Z","shell.execute_reply.started":"2025-02-19T14:19:03.604608Z","shell.execute_reply":"2025-02-19T14:19:03.815528Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"width_count = train_df.iloc[: , 8:9].value_counts().reset_index()\nheight_count = train_df.iloc[:,9:10].value_counts().reset_index()\n\nfig , ax = plt.subplots(1 , 2 , figsize=(10 , 6))\n\naxes = ax.flatten()\nsns.barplot(height_count,x='height',y='count',edgecolor='black',palette='coolwarm',ax=axes[0])\nax[0].set_title(\"Height Counts\")\nsns.barplot(width_count,x='width',y='count',edgecolor='black',palette='coolwarm',ax=axes[1])\nax[1].set_title(\"Width Counts\")\n\nfig.suptitle(\"Individual Heigth and Width Counts\",color='Blue',fontsize=10)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:19:03.817779Z","iopub.execute_input":"2025-02-19T14:19:03.818121Z","iopub.status.idle":"2025-02-19T14:19:04.149512Z","shell.execute_reply.started":"2025-02-19T14:19:03.818081Z","shell.execute_reply":"2025-02-19T14:19:04.148316Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df['organ_annotation_count'] = train_df[['large_bowel','small_bowel','stomach']].notnull().sum(axis=1)\ntrain_df['organ_annotation_count'] = train_df['organ_annotation_count'].apply(lambda x : int(x))\nsns.histplot(train_df , x='organ_annotation_count',kde=True)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:19:04.150351Z","iopub.execute_input":"2025-02-19T14:19:04.150716Z","iopub.status.idle":"2025-02-19T14:19:04.637955Z","shell.execute_reply.started":"2025-02-19T14:19:04.150689Z","shell.execute_reply":"2025-02-19T14:19:04.637033Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"case_slices = train_df.groupby(['case']).agg(slice_count = ('slice','count')).reset_index()\nfig , ax = plt.subplots(1 , 1 , figsize=(15,10))\nsns.barplot(case_slices , x='case',y='slice_count',ax=ax)\nax.set_title(\"Number of Slices per case\",fontsize=10, color='blue')\nax.set_xlabel(\"Case\")\nax.set_ylabel(\"Slice Count\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:19:04.638777Z","iopub.execute_input":"2025-02-19T14:19:04.639089Z","iopub.status.idle":"2025-02-19T14:19:05.440583Z","shell.execute_reply.started":"2025-02-19T14:19:04.639044Z","shell.execute_reply":"2025-02-19T14:19:05.439510Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.fillna('', inplace=True) # This is important if we use notna() technique we've to deal with NaN","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:19:05.441833Z","iopub.execute_input":"2025-02-19T14:19:05.442247Z","iopub.status.idle":"2025-02-19T14:19:05.471626Z","shell.execute_reply.started":"2025-02-19T14:19:05.442207Z","shell.execute_reply":"2025-02-19T14:19:05.470371Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df[train_df['large_bowel']!=''].index","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:19:05.472735Z","iopub.execute_input":"2025-02-19T14:19:05.473130Z","iopub.status.idle":"2025-02-19T14:19:05.507309Z","shell.execute_reply.started":"2025-02-19T14:19:05.473064Z","shell.execute_reply":"2025-02-19T14:19:05.506197Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_mask = list(train_df[train_df['large_bowel']!=''].index)\ntrain_mask += list(train_df[train_df['small_bowel']!=''].index)\ntrain_mask += list(train_df[train_df['stomach']!=''].index)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:19:05.508483Z","iopub.execute_input":"2025-02-19T14:19:05.508881Z","iopub.status.idle":"2025-02-19T14:19:05.550195Z","shell.execute_reply.started":"2025-02-19T14:19:05.508843Z","shell.execute_reply":"2025-02-19T14:19:05.549085Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mydf = train_df[train_df.index.isin(train_mask)] # Only train_df.index.isin(train_mask) gives True,False... boolean array\nmydf.reset_index(inplace=True,drop=True)\nmydf.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:19:05.551186Z","iopub.execute_input":"2025-02-19T14:19:05.551481Z","iopub.status.idle":"2025-02-19T14:19:05.571277Z","shell.execute_reply.started":"2025-02-19T14:19:05.551456Z","shell.execute_reply":"2025-02-19T14:19:05.570020Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Validation","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedGroupKFold","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:19:05.572273Z","iopub.execute_input":"2025-02-19T14:19:05.572597Z","iopub.status.idle":"2025-02-19T14:19:05.774135Z","shell.execute_reply.started":"2025-02-19T14:19:05.572571Z","shell.execute_reply":"2025-02-19T14:19:05.773183Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"skf = StratifiedGroupKFold(n_splits=5,shuffle=True,random_state=42)\n\nfor fold, (_, val_idx) in enumerate(skf.split(X=mydf, y=mydf['counts'],groups =mydf['case']), 1):\n    # y: This specifies the target variable used for stratification.\n    mydf.loc[val_idx, 'fold'] = fold","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:19:05.774943Z","iopub.execute_input":"2025-02-19T14:19:05.775238Z","iopub.status.idle":"2025-02-19T14:19:05.852753Z","shell.execute_reply.started":"2025-02-19T14:19:05.775214Z","shell.execute_reply":"2025-02-19T14:19:05.851898Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_ids = mydf[mydf[\"fold\"]!=1].index\nvalidation_ids = mydf[mydf[\"fold\"]==1].index","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:19:05.853772Z","iopub.execute_input":"2025-02-19T14:19:05.854062Z","iopub.status.idle":"2025-02-19T14:19:05.869759Z","shell.execute_reply.started":"2025-02-19T14:19:05.854037Z","shell.execute_reply":"2025-02-19T14:19:05.868549Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"folds_information = mydf.groupby(['fold','counts'])['id'].count().reset_index()\nfolds_information['fold'] = folds_information['fold'].apply(lambda x : int(x))\nfolds_information.rename(columns={'id':'total'},inplace=True)\n\nfig , ax = plt.subplots(1,1,figsize=(15,5))\nsns.lineplot(folds_information,x='fold',y='total',hue='counts',palette='coolwarm',marker='o',ax=ax)\nfor index,row in folds_information.iterrows():\n    ax.annotate(f\"{row['total']}\",(row['fold'],row['total']),fontweight='bold',fontsize=10)\nplt.title(\"Total Count Per Fold\", fontsize=14, fontweight='bold')\nplt.xlabel(\"Fold\", fontsize=12)\nplt.ylabel(\"Total\", fontsize=12)\nplt.legend(title=\"Counts\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:19:05.870735Z","iopub.execute_input":"2025-02-19T14:19:05.871013Z","iopub.status.idle":"2025-02-19T14:19:06.170188Z","shell.execute_reply.started":"2025-02-19T14:19:05.870991Z","shell.execute_reply":"2025-02-19T14:19:06.169197Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.cuda import amp\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torchvision import transforms\nfrom torchvision import models\nimport segmentation_models_pytorch as smp\nimport torchvision.transforms.functional as TF\nfrom tqdm import tqdm\nimport copy","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:19:06.171165Z","iopub.execute_input":"2025-02-19T14:19:06.171445Z","iopub.status.idle":"2025-02-19T14:19:17.317367Z","shell.execute_reply.started":"2025-02-19T14:19:06.171412Z","shell.execute_reply":"2025-02-19T14:19:17.316224Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mydf.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:19:17.318360Z","iopub.execute_input":"2025-02-19T14:19:17.318625Z","iopub.status.idle":"2025-02-19T14:19:17.333643Z","shell.execute_reply.started":"2025-02-19T14:19:17.318603Z","shell.execute_reply":"2025-02-19T14:19:17.332562Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def rle2mask(mask_rle,shape:tuple):\n\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (width,height) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n    Source: https://www.kaggle.com/paulorzp/rle-functions-run-lenght-encode-decode\n    '''\n    s = mask_rle.split()\n    starts,lengths = [np.asarray(x , dtype=int) for x in (s[0:][::2] , s[1:][::2])] # \"10 5 25 10 50 5\"\n    starts -= 1 # zero based indexing\n    ends = starts + lengths\n    img = np.zeros(shape[0]*shape[1] , dtype=np.uint8)\n    for lo,hi in zip(starts , ends):\n        img[lo:hi] = 1\n    return img.reshape(shape).T\n\n# def mask2rle(img):\n#     pixels = img.T.flatten()\n#     pixels = np.pad(pixels,((1,1),))\n#     runs = np.where(pixels[1:] != pixels[:-1])[0]+1\n#     runs[1::2] -= runs[::2]\n#     return ' '.join(str(x) for x in runs)\n\n# rle = \"10 5 25 10 50 5\"\n# mask = rle2mask(rle,(10,10))\n# print(mask2rle(mask))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:19:17.334882Z","iopub.execute_input":"2025-02-19T14:19:17.335316Z","iopub.status.idle":"2025-02-19T14:19:17.368542Z","shell.execute_reply.started":"2025-02-19T14:19:17.335271Z","shell.execute_reply":"2025-02-19T14:19:17.367356Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class MyDataset(Dataset):\n    def __init__(self,df,whatset='train'):\n        self.df = df.reset_index(drop=True)\n        self.whatset = whatset\n        self.img_shape = (224,224)\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self,idx):\n        img_path = self.df['path'][idx]\n        img = self.load_img(img_path)\n        if self.whatset == 'train':\n            mask = self.load_mask(idx)\n            return torch.tensor(img) , torch.tensor(mask)\n\n        return torch.tensor(img)\n        \n    def rle2mask(self,mask_rle):\n        s = mask_rle.split()\n        start,length = [np.asarray(x , dtype=int) for x in (s[:][::2] , s[1:][::2])]\n        start -= 1 # 0 based indexing\n        end = start+length\n        # Careful here i'm taking a linear zero array and later we rehape it\n        img = np.zeros(self.img_shape[0]*self.img_shape[1],dtype=np.uint8) # Img not mask\n        for lo,hi in (zip(start,end)):\n            img[lo:hi] = 1\n        return img.reshape(self.img_shape).T\n        \n    def load_img(self,img):\n        img = cv2.imread(img , cv2.IMREAD_UNCHANGED)\n        img = cv2.normalize(img,None,0,255,norm_type=cv2.NORM_MINMAX)\n        img = cv2.resize(img,self.img_shape)\n        if len(img.shape) == 2:\n           img = np.repeat(img[...,None],3,axis=-1)\n        return img.transpose(2,0,1).astype(np.float32)/255.0\n\n    def load_mask(self,idx):\n        \"\"\"Loads and decodes RLE masks for each class.\"\"\"\n        masks = np.zeros((*self.img_shape,3),dtype=np.float32) # masks pixels always float\n        for channel,label in enumerate(['large_bowel','small_bowel','stomach']):\n            rle = self.df[label].iloc[idx]\n            if not isinstance(rle,str):\n                print(f\"Invalid RLE at index {idx}, label {label}: {rle}\")\n                raise ValueError(\"NOT GOOD\")\n                \n            mask = self.rle2mask(self.df[label].iloc[idx])\n            masks[:,:,channel] = mask.astype(np.float32)\n        return masks.transpose(2,0,1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:19:17.369619Z","iopub.execute_input":"2025-02-19T14:19:17.369928Z","iopub.status.idle":"2025-02-19T14:19:17.393872Z","shell.execute_reply.started":"2025-02-19T14:19:17.369902Z","shell.execute_reply":"2025-02-19T14:19:17.392503Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_transforms = transforms.Compose([\n    transforms.Resize(224,interpolation=transforms.InterpolationMode.NEAREST),\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomVerticalFlip(),\n    transforms.RandomApply([\n        transforms.ColorJitter(contrast=0.2),\n        transforms.ColorJitter(brightness=0.2)\n    ],p=0.2)\n])\nvalidation_transforms = transforms.Compose([\n    transforms.Resize(224 , interpolation=transforms.InterpolationMode.NEAREST)\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:19:17.394931Z","iopub.execute_input":"2025-02-19T14:19:17.395312Z","iopub.status.idle":"2025-02-19T14:19:17.417471Z","shell.execute_reply.started":"2025-02-19T14:19:17.395275Z","shell.execute_reply":"2025-02-19T14:19:17.416395Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cpuCount  = os.cpu_count()\ntorch.set_num_threads(cpuCount)\ntorch.set_num_interop_threads(cpuCount) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:19:17.418434Z","iopub.execute_input":"2025-02-19T14:19:17.418703Z","iopub.status.idle":"2025-02-19T14:19:17.445285Z","shell.execute_reply.started":"2025-02-19T14:19:17.418682Z","shell.execute_reply":"2025-02-19T14:19:17.443205Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_dataset = MyDataset(train_df[train_df.index.isin(train_ids)],whatset=\"train\")\nval_dataset =  MyDataset(train_df[train_df.index.isin(validation_ids)],whatset=\"validation\")\n\ntrain_loader = DataLoader(train_dataset , batch_size=32,num_workers=cpuCount,shuffle=True,pin_memory=True,drop_last=False)\nval_loader = DataLoader(val_dataset , batch_size=64,num_workers=cpuCount,shuffle=False,pin_memory=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:19:17.446325Z","iopub.execute_input":"2025-02-19T14:19:17.446632Z","iopub.status.idle":"2025-02-19T14:19:17.498484Z","shell.execute_reply.started":"2025-02-19T14:19:17.446606Z","shell.execute_reply":"2025-02-19T14:19:17.497484Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.patches as mpatches\nfrom matplotlib.colors import ListedColormap\n\ndef show_img(img,mask):\n    if len(img.shape) == 3 and img.shape[-1] == 3:\n        img = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)\n    clahe = cv2.createCLAHE(clipLimit=2.0,tileGridSize=(8,8))\n    img = clahe.apply(img)\n    plt.imshow(img,cmap='bone')\n    if mask is not None:\n        if mask.shape[-1] == 1:\n            mask = mask.squeeze(-1)\n        cmap = ListedColormap([\"red\",\"green\",\"blue\"])\n        plt.imshow(mask,cmap=cmap,alpha=0.5)\n        handles = [mpatches.Patch(color=c,label=l) for c,l in zip([\"red\",\"green\",\"blue\"],[\"Large Bowel\", \"Small Bowel\", \"Stomach\"])]\n        plt.legend(handles=handles)\n \n    plt.axis('off')\n    \ndef group_plot(imgs,masks,size):\n    plt.figure(figsize=(size * 5, 5))\n    for idx in range(size):\n        plt.subplot(1, 5, idx + 1)\n        # print(imgs[idx].shape) # shaop is 3, 224, 224\n        img = imgs[idx].permute((1,2,0)).cpu().numpy()\n        img = (img * 255).astype(np.uint8)\n        msk = masks[idx].permute((1,2,0)).cpu().numpy()\n        msk = (msk * 255).astype(np.uint8)\n        show_img(img,msk)\n    plt.tight_layout()\n    plt.show()\n\nimgs, masks = next(iter(train_loader))\nprint(\"Image Shape:\", imgs.shape, \"Mask Shape:\", masks.shape)\ngroup_plot(imgs,masks,5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:19:17.499426Z","iopub.execute_input":"2025-02-19T14:19:17.499720Z","iopub.status.idle":"2025-02-19T14:19:19.901298Z","shell.execute_reply.started":"2025-02-19T14:19:17.499695Z","shell.execute_reply":"2025-02-19T14:19:19.900086Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torchsummary import summary\n\nclass DoubleConv(nn.Module):\n    def __init__(self, in_channels, out_channels):\n        super().__init__()\n        self.double_conv = nn.Sequential(\n            nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=3, padding=1, stride=1),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(inplace=True),\n\n            nn.Conv2d(in_channels=out_channels, out_channels=out_channels, kernel_size=3, padding=1, stride=1),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(inplace=True)\n        )\n\n    def forward(self, X):\n        return self.double_conv(X)\n\nclass Down(nn.Module):\n    def __init__(self, in_channels, out_channels):\n        super().__init__()\n        self.down = nn.Sequential(\n            nn.MaxPool2d(2, 2),\n            DoubleConv(in_channels, out_channels)\n        )\n\n    def forward(self, X):\n        return self.down(X)\n\nclass Up(nn.Module):\n\n    \"\"\"\n    Bilinear Upsampling (nn.Upsample):\n    Does not change channels → Needs extra processing (Conv2d) to reduce channels.\n    We manually reduce channels and concatenate with skip connection, so we need +out_channels.\n    \n    Transposed Convolution (nn.ConvTranspose2d):\n    Automatically reduces channels from in_channels to in_channels // 2\t.\n    It directly matches the expected input for DoubleConv, so no need for +out_channels.\n    \"\"\"\n    def __init__(self, in_channels, out_channels, bilinear=True):\n        super().__init__()\n        if bilinear:\n            self.up = nn.Upsample(scale_factor=2, mode=\"bilinear\", align_corners=True)\n            # Reduce the number of channels using a 1x1 convolution\n            self.reduce_channels = nn.Conv2d(in_channels, in_channels // 2, kernel_size=1)\n            self.conv = DoubleConv(in_channels // 2 + out_channels, out_channels)\n        else:\n            self.up = nn.ConvTranspose2d(in_channels, in_channels // 2, kernel_size=2, stride=2)\n            self.conv = DoubleConv(in_channels, out_channels)\n\n    def forward(self, X1, X2):\n        X1 = self.up(X1)\n        X1 = self.reduce_channels(X1) if hasattr(self, 'reduce_channels') else X1  # Apply channel reduction if bilinear\n        diffY = X2.size()[2] - X1.size()[2]\n        diffX = X2.size()[3] - X1.size()[3]\n        X1 = F.pad(X1, (diffX // 2, diffX - diffX // 2, diffY // 2, diffY - diffY // 2))\n        out = torch.cat([X2, X1], dim=1)\n        return self.conv(out)\n\n\nclass Out(nn.Module):\n    def __init__(self, in_channels, n_classes):\n        super().__init__()\n        self.out = nn.Sequential(\n            nn.Conv2d(in_channels=in_channels, out_channels=n_classes, kernel_size=1),\n            nn.Sigmoid()  # Use nn.Softmax(dim=1) for multi-class segmentation\n        )\n\n    def forward(self, X):\n        return self.out(X)\n\nclass UNet(nn.Module):\n    def __init__(self, in_channels, n_channels, n_classes, bilinear=True):\n        super().__init__()\n        self.conv = DoubleConv(in_channels=in_channels, out_channels=n_channels)\n        self.enc1 = Down(in_channels=n_channels, out_channels=2 * n_channels)\n        self.enc2 = Down(in_channels=2 * n_channels, out_channels=4 * n_channels)\n        self.enc3 = Down(in_channels=4 * n_channels, out_channels=8 * n_channels)\n        self.enc4 = Down(in_channels=8 * n_channels, out_channels=16 * n_channels)\n\n        self.dec1 = Up(in_channels=16 * n_channels, out_channels=8 * n_channels, bilinear=bilinear)\n        self.dec2 = Up(in_channels=8 * n_channels, out_channels=4 * n_channels, bilinear=bilinear)\n        self.dec3 = Up(in_channels=4 * n_channels, out_channels=2 * n_channels, bilinear=bilinear)\n        self.dec4 = Up(in_channels=2 * n_channels, out_channels=n_channels, bilinear=bilinear)\n\n        self.out = Out(in_channels=n_channels, n_classes=n_classes)\n\n    def forward(self, X):\n        X1 = self.conv(X)\n        X2 = self.enc1(X1)\n        X3 = self.enc2(X2)\n        X4 = self.enc3(X3)\n        X5 = self.enc4(X4)\n\n        X = self.dec1(X5, X4)\n        X = self.dec2(X, X3)\n        X = self.dec3(X, X2)\n        X = self.dec4(X, X1)\n\n        return self.out(X)\n\n\nx = torch.randn(1, 3, 224, 224)  # Example input\nprint(tuple(x.size()))\nmodel = UNet(in_channels=3, n_classes=3, n_channels=48, bilinear=False)\nsummary(model, (3, 224, 224))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:19:19.902579Z","iopub.execute_input":"2025-02-19T14:19:19.902940Z","iopub.status.idle":"2025-02-19T14:19:21.331383Z","shell.execute_reply.started":"2025-02-19T14:19:19.902912Z","shell.execute_reply":"2025-02-19T14:19:21.330281Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"device= torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\ndef build_model():\n    model = UNet(in_channels=3, n_classes=3, n_channels=48)\n#     model = DeepLabV3Plus(num_classes=3)\n    model.to(device)\n    return model\n\n\ndef load_model(path):\n    model = build_model()\n    model.load_state_dict(torch.load(path))\n    model.eval()\n    return model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:19:21.337722Z","iopub.execute_input":"2025-02-19T14:19:21.338120Z","iopub.status.idle":"2025-02-19T14:19:21.344014Z","shell.execute_reply.started":"2025-02-19T14:19:21.338049Z","shell.execute_reply":"2025-02-19T14:19:21.342800Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"DiceLoss = smp.losses.DiceLoss(mode=\"multilabel\")\nBCELoss = smp.losses.SoftBCEWithLogitsLoss()\n\ndef dice_coef(y_true,y_pred,epsilon=0.001,dim=(2,3),thr=0.5):\n    y_true = y_true.to(torch.float32)\n    y_pred = (y_pred > thr).to(torch.float32) # (y_pred > thr) gives boolean array and .to(torch.float32) converts them to 1. and 0.\n    intersec = (y_true * y_pred).sum(dim=dim)\n    denominator = y_true.sum(dim=dim)+y_pred.sum(dim=dim)\n    dice = ((2*(intersec+epsilon))/(denominator+epsilon)).mean(dim=(1,0))\n    return dice\n\ndef iou_coef(y_true,y_pred,epsilon=0.001,dim=(2,3),thr=0.5):\n    y_true = y_true.to(torch.float32)\n    y_pred = (y_pred > thr).to(torch.float32)\n    intersec = (y_true * y_pred).sum(dim=dim)\n    union = (y_true + y_pred - y_true*y_pred).sum(dim=dim)\n    iou = ((intersec+epsilon)/(union+epsilon)).mean(dim=(1,0))\n    return iou\n\ndef mylossfn(y_pred,y_true):\n    return 0.6 * BCELoss(y_pred, y_true) + 0.4 * DiceLoss(y_pred, y_true)\n\ny_true = torch.randint(0,2,(4, 3, 224, 224)).float()\ny_pred = torch.randn(4, 3, 224, 224)\n\n# Compute Dice and IoU metrics\ndice_score = dice_coef(y_true, torch.sigmoid(y_pred))  # Apply sigmoid for probability output\niou_score = iou_coef(y_true, torch.sigmoid(y_pred))\n\nmyloss_score = mylossfn(y_true,torch.sigmoid(y_pred))\n\nprint(\"My Loss Score :\",myloss_score.item())\nprint(\"Dice Score :\", dice_score.item())\nprint(\"IoU Score :\", iou_score.item())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:19:21.345544Z","iopub.execute_input":"2025-02-19T14:19:21.345815Z","iopub.status.idle":"2025-02-19T14:19:21.429053Z","shell.execute_reply.started":"2025-02-19T14:19:21.345792Z","shell.execute_reply":"2025-02-19T14:19:21.427444Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training and Validation Loop","metadata":{}},{"cell_type":"markdown","source":"## torch.autocast \ntorch.autocast is a powerful tool for mixed precision training in PyTorch. It automatically selects the appropriate precision for each operation, improving training speed and reducing memory usage. Combined with GradScaler, it ensures numerical stability and model accuracy. Use it when training on supported hardware to take advantage of the benefits of mixed precision.\n\n\n1. What is Mixed Precision Training?\n\n   * 16-bit (Half-Precision) Floating Point (FP16):Uses less memory compared to 32-bit floating point (FP32).\n   * Faster computation on hardware that supports FP16 (e.g., NVIDIA GPUs with Tensor Cores).\n\n2. 32-bit (Single-Precision) Floating Point (FP32):\n  \n   * Provides higher numerical precision.\n   * Required for certain operations (e.g., weight updates) to maintain model stability and accuracy.\n     \n3. Mixed precision training combines the benefits of both FP16 and FP32:\n   * FP16 is used for most computations (e.g., forward and backward passes).\n   * FP32 is used for critical operations (e.g., weight updates, loss computation).\n     \n\n### Why Use torch.autocast?\n\n* Faster Training: FP16 computations are faster on supported hardware.\n* Reduced Memory Usage: FP16 tensors use half the memory of FP32 tensors, allowing larger batch sizes or models.\n* Improved Hardware Utilization: NVIDIA GPUs with Tensor Cores can perform FP16 operations much faster than FP32.\n\n\n### How Does torch.autocast Work?\n\ntorch.autocast automatically selects the appropriate precision for each operation:\n* Operations that benefit from FP16 (e.g., matrix multiplications, convolutions) are performed in FP16.\n* Operations that require higher precision (e.g., reductions, loss computations) are performed in FP32.\n\n\n### How to Use torch.autocast\n\n* Enable Autocast: Wrap the forward pass and loss computation in a torch.autocast context manager.\n* Gradient Scaling:Use torch.cuda.amp.GradScaler to scale gradients during backpropagation to prevent underflow in FP16.\n\n### When to Use torch.autocast\n\n* When training on NVIDIA GPUs with Tensor Cores (e.g., Volta, Turing, Ampere architectures).\n* When training large models or using large batch sizes to save memory and speed up training.\n\n","metadata":{}},{"cell_type":"markdown","source":"## Why Use gc.collect()?\n\nPython has automatic garbage collection, so in most cases, you don't need to call it manually. \nHowever, calling gc.collect() can be useful in these cases:\n\n1. Handling Large Objects\n\n\n* When dealing with large NumPy arrays, PyTorch tensors, or Pandas DataFrames, some objects may not be immediately freed.\n* Calling gc.collect() can help reclaim memory.\n\n\n2. Circular References\n\n\n* Python’s garbage collector struggles with circular references (where two or more objects reference each other).\n* gc.collect() forces cleanup of such unreachable objects.\n\n\n3. Memory Management in Long-Running Scripts\n\n\n* If a script runs continuously (e.g., data pipelines, ML training loops), unused objects can accumulate.\n* Calling gc.collect() periodically reduces memory usage.\n","metadata":{}},{"cell_type":"markdown","source":"# The scaler.update() \n\nThe scaler.update() method in PyTorch's GradScaler plays a crucial role in maintaining stable training when using Automatic Mixed Precision (AMP):\n\n## Purpose\n\nThe primary goal of scaler.update() is to adjust the scaling factor used by GradScaler based on the gradients computed during the training step1. It checks if any infinite (inf) or Not-a-Number (NaN) values were present in the gradients.\n\n## How it Works\n\nGradient Check: scaler.update() inspects the gradients to determine if any values are infinite or NaN. These values indicate potential numerical instability during training.\n\n## Scale Adjustment:\n\nIf infs or NaNs are detected, the scaling factor is reduced. This is done to prevent future occurrences of excessively large gradients that can lead to instability.\n `Formula : scaling factor = scaling factor x backoff factor(typically 0.5)`\n\nIf no issues are detected, the scaling factor may be increased. A higher scaling factor can provide better gradient resolution and prevent underflow, especially when using float16 precision.\n  `Formula : S(t+1) = S(t) x g # [g is the growth factor, typically set to 2.0] `\n\nAfter Getting the S update the Gradient :\n`scaled_loss = scaler.scale(loss) # this means scaled_loss = loss * S`\n\n\n\n### Skipping Optimizer Step: \n\n#### Functionality of scaler.step(optimizer)\nGradient Unscaling: When you call scaler.step(optimizer), it first unscales the gradients of the parameters associated with the optimizer. This is necessary because gradients are computed in a scaled format to prevent underflow when using lower precision (like float16).\n\nCheck for Invalid Gradients: After unscaling, the method checks if any of the gradients are either inf or NaN. This is crucial because such values can corrupt the model parameters if an update is attempted.\n\nConditional Update:\n\nIf No Issues: If no inf or NaN values are found in the gradients, the method proceeds to call optimizer.step(), which updates the model parameters based on the unscaled gradients.\n\nIf Issues Detected: If any gradients are found to be inf or NaN, `optimizer.step() is skipped`. This prevents potentially corrupting the model's weights with invalid updates.\n\n## Why is it Important?\n\nPreventing Underflow: When training with mixed precision, gradients with small magnitudes might be flushed to zero due to the limited precision of float16. Gradient scaling helps prevent this by multiplying the loss (and thus the gradients) by a scale factor.\n\nHandling Instability: Large gradients can lead to numerical instability. Reducing the scaling factor when inf or NaN values are detected helps mitigate this issue.\n\nDynamic Adjustment: scaler.update() dynamically adjusts the scaling factor throughout training, allowing the training process to adapt to the specific needs of the model and dataset.\n\n## When to Call\n\nscaler.update() should be called after scaler.step(optimizer) in each training iteration2. It should only be called once after all optimizers used in the iteration have been stepped","metadata":{}},{"cell_type":"code","source":"start = time.time()\nhistory = {}\n\nfor epoch in range(1 , epochs+1):\n    gc.collect()\n\n    model.train()\n    print(f\"EPOCH:{epoch}/{epochs}\")\n    running_loss = 0.0\n    dataset_size = 0\n    scaler = amp.GradScaler()\n    pbar = tqdm(enumerate(train_loader),total=len(train_loader),desc=\"Training:\")\n    for step,(images,masks) in pbar:\n        images = images.to(device,dtype=torch.float)\n        masks = masks.to(device,dtype=torch.float)\n        batch_size=images.size()[0]\n\n        with amp.autocast(enabled=True):\n            y_pred = model(images)\n            loss = mylossfn(y_pred,masks) / n_accumulate\n\n        scaler.scale(loss).backward()\n        if (step+1) % n_accumulate == 0:\n            scaler.step(optimizer)\n            scaler.update()\n            optimizer.zero_grad()\n            if schedular is not None:\n                schedular.step()\n\n        running_loss += loss.item()*batch_size\n        dataset_size += batch_size\n        epoch_loss = runnung_loss / dataset_size\n        mem = torch.cuda.memory_reserved()/1E9 if torch.cuda.is_available() else 0\n        lr = optimizer.param_groups[0][\"lr\"]\n        pbar.set_postfix(Epoch_Loss=f\"{epoch_loss:.4f}\",lr=f\"{lr:.4f}\"Memory=f\"{mem:.4f}\")\n        \n    history[\"Train Loss\"].append(epoch_loss)\n\n    model.eval()\n    running_loss = 0.0\n    dataset_size=0\n    val_scores = []\n    pbar = tqdm(enumerate(val_loader),total=len(val_loader),desc=\"Validating:\")\n    for step , (images,masks) in pbar:\n        images = images.to(device,torch.float)\n        masks = masks.to(device,torch.float)\n        batch_size = images.size()[0]\n\n        with torch.no_grad():\n            y_pred = model(images)\n            loss = mylossfn(y_pred,masks)\n\n        running_loss += loss.item()*batch_size\n        dataset_size += batch_size\n        epoch_loss = running_loss / dataset_size\n\n        y_pred = torch.sigmoid(y_pred)\n        val_dice = dice_coef(masks,y_pred).cpu().numpy()\n        val_jaccard = iou_coef(masks,y_pred).cpu().numpy()\n\n        val_scores.append([val_dice , val_jaccard])\n        lr = optimizer.param_groups[0]['lr']\n        mem = torch.cuda.memory_reserved() / 1E9 if torch.cuda.is_available() else 0\n        pbar.set_postfix(Epoch_Loss=f\"{epoch_loss:.4f}\",lr=f\"{lr:.4f}\"Memory=f\"{mem:.4f}\")\n\n    val_dice,val_jaccard = np.mean(val_scores,axis=0)\n    history[\"Valid Loss\"].append(epoch_loss)\n    history[\"Valid Dice\"].append(val_dice)\n    history[\"val_jaccard\"].append(val_jaccard)\n\n    if val_dice > best_dice:\n         best_dice = val_dice\n         best_jaccard = val_jaccard\n         best_epoch = epoch\n         best_model_wts = copy.deepcopy(model.state_dict())\n         torch.save(best_model_wts,f\"best_epochdeeplab-{fold:02d}.bin\")\n         print(\"Best model saved.\")\n    torch_save(model.state_dict(),f\"last_epochdeeplab-{fold:02d}.bin\")\n\nend = time.time()\nelapsed_time = start-end\nprint(\"Training Complete in : {:.0f}h {:.0f}m {:.0f}s\".format(elapsed_time//3600,(elapsed_time%3600)//60),(elapsed_time%3600)//60)\nprint(\"Best Dice Score: {:.4f} | Best Jaccard Score: {:.4f}\".format(best_dice, best_jaccard))\nmodel.load_state_dict(best_model_wts)\n    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:19:21.429880Z","iopub.execute_input":"2025-02-19T14:19:21.430167Z","iopub.status.idle":"2025-02-19T14:19:21.587269Z","shell.execute_reply.started":"2025-02-19T14:19:21.430142Z","shell.execute_reply":"2025-02-19T14:19:21.585870Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# LEARNING PURPOSE ------->\n\n# from sklearn.model_selection import KFold,StratifiedKFold,StratifiedGroupKFold,GroupKFold,TimeSeriesSplit\n# np.random.seed(42)\n# sample_data = pd.DataFrame({\n#     'id': range(1,21),\n#     'feature1':np.random.rand(20),\n#     'feature2':np.random.rand(20),\n#     'target':np.random.choice([0,1],size=20,p=[0.7,0.3]),\n#     'group': np.random.choice(['A', 'B', 'C', 'D', 'E'], size=20)   \n# })\n\n# X = sample_data.iloc[:,1:3]\n# y = sample_data.loc[:,'target']\n# groups = sample_data['group']\n\n# # Use case: General cross-validation, no need for stratification or grouping.\n# kf = KFold(n_splits=5 , shuffle=True,random_state=42)\n# for fold , (train_idx, test_idx) in enumerate(kf.split(X) , 1):\n#     print(f\"Fold {fold}:\")\n#     print(\"Train indices:\", train_idx)\n#     print(\"Test indices:\", test_idx, \"\\n\")\n\n# print(\"-\"*80)\n\n# #  Use case: Ensures each fold has a similar class distribution.\n# skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n# for fold ,(train_idx , test_idx) in enumerate(skf.split(X , y) , 1):\n#     print(f\"Fold {fold}:\")\n#     print(\"Train target distribution:\",y.iloc[train_idx].value_counts().to_dict())\n#     print(\"Test target distribution:\",y.iloc[test_idx].value_counts().to_dict(),\"\\n\")\n\n# print(\"-\"*80)\n\n# # Use case: Ensures all data from the same group stays together.\n# gkf = GroupKFold(n_splits=3)\n\n# for fold, (train_idx, test_idx) in enumerate(gkf.split(X, y, groups), 1):\n#     print(f\"Fold {fold}:\")\n#     print(\"Train groups:\", groups.iloc[train_idx].unique())\n#     print(\"Test groups:\", groups.iloc[test_idx].unique(), \"\\n\")\n\n# print(\"-\"*80)\n\n# # Use case: Maintains both class distribution and group integrity.\n# sgkf = StratifiedGroupKFold(n_splits=3, shuffle=True, random_state=42)\n\n# for fold, (train_idx, test_idx) in enumerate(sgkf.split(X, y, groups), 1):\n#     print(f\"Fold {fold}:\")\n#     print(\"Train target distribution:\", y.iloc[train_idx].value_counts().to_dict())\n#     print(\"Test target distribution:\", y.iloc[test_idx].value_counts().to_dict())\n#     print(\"Train groups:\", groups.iloc[train_idx].unique())\n#     print(\"Test groups:\", groups.iloc[test_idx].unique(), \"\\n\")\n\n# print(\"-\"*80)\n\n# #Use case: Ensures training data always precedes test data (for time-series forecasting).\n# tscv = TimeSeriesSplit(n_splits=5)\n\n# for fold, (train_idx, test_idx) in enumerate(tscv.split(X), 1):\n#     print(f\"Fold {fold}:\")\n#     print(\"Train indices:\", train_idx)\n#     print(\"Test indices:\", test_idx, \"\\n\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:19:21.587972Z","iopub.status.idle":"2025-02-19T14:19:21.588349Z","shell.execute_reply":"2025-02-19T14:19:21.588215Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.cuda.amp import autocast, GradScaler\n\n# Define a simple model\nmodel = nn.Sequential(\n    nn.Linear(128, 256),\n    nn.ReLU(),\n    nn.Linear(256, 10)\n).cuda()\n\n# Define loss function and optimizer\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), lr=0.001)\n\n# Create a GradScaler for gradient scaling\nscaler = GradScaler()\n\n# Dummy data\ninputs = torch.randn(32, 128).cuda()  # Batch size 32, input size 128\ntargets = torch.randint(0, 10, (32,)).cuda()  # 10 classes\n\n# Training loop\nfor epoch in range(10):\n    optimizer.zero_grad()\n\n    # Forward pass with autocast\n    with autocast(enabled=True):\n        outputs = model(inputs)\n        loss = criterion(outputs, targets)\n\n    # Backward pass with gradient scaling\n    scaler.scale(loss).backward()\n\n    # Update weights\n    scaler.step(optimizer)\n\n    # Update the scale for next iteration\n    scaler.update()\n\n    print(f\"Epoch {epoch + 1}, Loss: {loss.item()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:19:21.589409Z","iopub.status.idle":"2025-02-19T14:19:21.589796Z","shell.execute_reply":"2025-02-19T14:19:21.589660Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\n\ndef auto_n_accumulate(model, batch_size_per_step, desired_batch_size):\n    \"\"\"Automatically find n_accumulate based on GPU memory.\"\"\"\n    total_memory = torch.cuda.get_device_properties(0).total_memory / 1e9  # GB\n    reserved_memory = torch.cuda.memory_reserved(0) / 1e9  # GB\n    available_memory = total_memory - reserved_memory  # GB\n\n    print(f\"GPU Memory: {available_memory:.2f} GB available out of {total_memory:.2f} GB\")\n\n    # If memory is sufficient, reduce accumulation\n    if available_memory > 5:  # Adjust threshold as needed\n        return max(1, desired_batch_size // batch_size_per_step)\n    else:\n        return max(1, (desired_batch_size // batch_size_per_step) // 2)  # Use smaller accumulation if memory is tight\n\n# Example usage\nbatch_size_per_step = 8\ndesired_batch_size = 64\nn_accumulate = auto_n_accumulate(model, batch_size_per_step, desired_batch_size)\nprint(\"Dynamically chosen n_accumulate:\", n_accumulate)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:19:21.590954Z","iopub.status.idle":"2025-02-19T14:19:21.591385Z","shell.execute_reply":"2025-02-19T14:19:21.591239Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(range(1,30000))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:19:21.593114Z","iopub.status.idle":"2025-02-19T14:19:21.593524Z","shell.execute_reply":"2025-02-19T14:19:21.593326Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import KFold\nfrom torch.utils.data import Dataset, DataLoader\n\ndata = {\n    \"feature1\": [1.0, 4.0, 7.0, 10.0, 13.0, 16.0],\n    \"feature2\": [2.0, 5.0, 8.0, 11.0, 14.0, 17.0],\n    \"feature3\": [3.0, 6.0, 9.0, 12.0, 15.0, 18.0],\n    \"label\":    [0, 1, 0, 1, 0, 1]  # Classification labels (0 or 1)\n}\n\n# Convert dictionary to DataFrame\ndf = pd.DataFrame(data)\n\n\n\n# Convert DataFrame to NumPy arrays\nX = df.iloc[:, :-1].values  # Features\ny = df.iloc[:, -1].values   # Labels\n\n# Define a simple Dataset\nclass MyDataset(Dataset):\n    def __init__(self, X, y):\n        self.X = torch.tensor(X, dtype=torch.float32)\n        self.y = torch.tensor(y, dtype=torch.long)\n\n    def __len__(self):\n        return len(self.X)\n\n    def __getitem__(self, idx):\n        return self.X[idx], self.y[idx]\n\n# Define a simple Neural Network\nclass SimpleNN(nn.Module):\n    def __init__(self, input_size):\n        super(SimpleNN, self).__init__()\n        self.fc = nn.Linear(input_size, 2)  # 2 output classes\n        self.relu = nn.ReLU()\n    \n    def forward(self, x):\n        return self.fc(self.relu(x))\n\n# K-Fold Cross-Validation\nkf = KFold(n_splits=3, shuffle=True, random_state=42)\n\nfor fold, (train_idx, val_idx) in enumerate(kf.split(X)):\n    print(f\"Training Fold {fold}\")\n\n    # Split data\n    train_dataset = MyDataset(X[train_idx], y[train_idx])\n    val_dataset = MyDataset(X[val_idx], y[val_idx])\n\n    train_loader = DataLoader(train_dataset, batch_size=2, shuffle=True)\n    val_loader = DataLoader(val_dataset, batch_size=2, shuffle=False)\n\n    # Model, Loss, Optimizer\n    model = SimpleNN(input_size=X.shape[1])\n    criterion = nn.CrossEntropyLoss()\n    optimizer = optim.Adam(model.parameters(), lr=0.01)\n\n    # Training loop\n    for epoch in range(5):  # 5 epochs for demonstration\n        model.train()\n        for inputs, labels in train_loader:\n            optimizer.zero_grad()\n            outputs = model(inputs)\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n\n    # Save the model for this fold\n    torch.save(model.state_dict(), f\"model_fold-{fold}.pth\")\n    print(f\"Saved model for fold {fold}\\n\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T15:07:16.262531Z","iopub.execute_input":"2025-02-19T15:07:16.263004Z","iopub.status.idle":"2025-02-19T15:07:23.839260Z","shell.execute_reply.started":"2025-02-19T15:07:16.262945Z","shell.execute_reply":"2025-02-19T15:07:23.838128Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = torch.tensor([[1,2,3],\n                  [4,5,6]],dtype=torch.float32)\nX[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T14:19:21.596124Z","iopub.status.idle":"2025-02-19T14:19:21.596485Z","shell.execute_reply":"2025-02-19T14:19:21.596351Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.model_selection import KFold\nfrom sklearn.preprocessing import StandardScaler\n\nnp.random.seed(42)\nX = np.random.rand(100, 3) * 10  # 100 samples, 3 features\ny = 3 * X[:, 0] + 2 * X[:, 1] - 5 * X[:, 2] + np.random.randn(100) * 2  # Linear relation with noise\n\nprint(y.shape)\n# Create DataFrame\ndf = pd.DataFrame(X, columns=[\"feature1\", \"feature2\", \"feature3\"])\ndf[\"target\"] = y\n\n# Extract features and target\nX = df.drop(columns=[\"target\"]).values\ny = df[\"target\"].values.reshape(-1, 1)\n\n# Normalize features\nscaler = StandardScaler()\nX = scaler.fit_transform(X)\n \n# Convert to tensors\nX_tensor = torch.tensor(X, dtype=torch.float32)\ny_tensor = torch.tensor(y, dtype=torch.float32)\n\n# Create PyTorch Dataset\nclass RegressionDataset(Dataset):\n    def __init__(self, X, y):\n        self.X = X\n        self.y = y\n\n    def __len__(self):\n        return len(self.X)\n\n    def __getitem__(self, idx):\n        return self.X[idx], self.y[idx]\n\ndataset = RegressionDataset(X_tensor, y_tensor)\n\n# Define PyTorch Model\nclass SimpleNN(nn.Module):\n    def __init__(self, input_dim):\n        super(SimpleNN, self).__init__()\n        self.fc = nn.Linear(input_dim, 1)  # Simple Linear Regression\n\n    def forward(self, x):\n        return self.fc(x)\n\n# Training Setup\nkf = KFold(n_splits=5, shuffle=True, random_state=42)\nepochs = 50\nbatch_size = 16\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n# Perform K-Fold Cross-Validation\nmae_scores = []\nfor fold, (train_idx, test_idx) in enumerate(kf.split(X_tensor), 1):\n    print(f\"Fold {fold}/{kf.get_n_splits()}\")\n\n    # Create DataLoaders\n    train_dataset = torch.utils.data.Subset(dataset, train_idx)\n    test_dataset = torch.utils.data.Subset(dataset, test_idx)\n    \n    train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n    test_loader = DataLoader(test_dataset, batch_size=batch_size)\n\n    # Model, Loss, Optimizer\n    model = SimpleNN(input_dim=3).to(device)\n    criterion = nn.MSELoss()\n    optimizer = optim.Adam(model.parameters(), lr=0.01)\n\n    # Training Loop\n    model.train()\n    for epoch in range(epochs):\n        total_loss = 0\n        for X_batch, y_batch in train_loader:\n            X_batch, y_batch = X_batch.to(device), y_batch.to(device)\n            optimizer.zero_grad()\n            y_pred = model(X_batch)\n            loss = criterion(y_pred, y_batch)\n            loss.backward()\n            optimizer.step()\n            total_loss += loss.item()\n\n        if (epoch + 1) % 10 == 0:\n            print(f\"Epoch {epoch+1}: Loss = {total_loss/len(train_loader):.4f}\")\n\n    # Evaluation\n    model.eval()\n    total_mae = 0\n    with torch.no_grad():\n        for X_batch, y_batch in test_loader:\n            X_batch, y_batch = X_batch.to(device), y_batch.to(device)\n            y_pred = model(X_batch)\n            total_mae += torch.abs(y_pred - y_batch).mean().item()\n\n    mae = total_mae / len(test_loader)\n    mae_scores.append(mae)\n    print(f\"Fold {fold} MAE: {mae:.4f}\")\n\n# Print average MAE\nprint(f\"Average MAE: {sum(mae_scores) / len(mae_scores):.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-19T15:08:30.763204Z","iopub.execute_input":"2025-02-19T15:08:30.763575Z","iopub.status.idle":"2025-02-19T15:08:31.949528Z","shell.execute_reply.started":"2025-02-19T15:08:30.763542Z","shell.execute_reply":"2025-02-19T15:08:31.948476Z"}},"outputs":[],"execution_count":null}]}