{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":20270,"databundleVersionId":1222630,"sourceType":"competition"},{"sourceId":1225697,"sourceType":"datasetVersion","datasetId":701123},{"sourceId":8412844,"sourceType":"datasetVersion","datasetId":4954648}],"dockerImageVersionId":29926,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"\n# Libraries ","metadata":{}},{"cell_type":"code","source":"# Regular Imports\nimport os\nimport pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport matplotlib.image as mpimg\nfrom tabulate import tabulate\nimport missingno as msno \nfrom IPython.display import display_html\nfrom PIL import Image\nimport gc\nimport cv2\n\nimport pydicom # for DICOM images\nfrom skimage.transform import resize\n\n# SKLearn\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.preprocessing import OneHotEncoder\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# Set Color Palettes for the notebook\ncolors_nude = ['#e0798c','#65365a','#da8886','#cfc4c4','#dfd7ca']\nsns.palplot(sns.color_palette(colors_nude))\n\n# Set Style\nsns.set_style(\"whitegrid\")\nsns.despine(left=True, bottom=True)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-05-14T10:52:09.235657Z","iopub.execute_input":"2024-05-14T10:52:09.236092Z","iopub.status.idle":"2024-05-14T10:52:09.355594Z","shell.execute_reply.started":"2024-05-14T10:52:09.236055Z","shell.execute_reply":"2024-05-14T10:52:09.354615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.__version__","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:52:09.358320Z","iopub.execute_input":"2024-05-14T10:52:09.358781Z","iopub.status.idle":"2024-05-14T10:52:09.365309Z","shell.execute_reply.started":"2024-05-14T10:52:09.358732Z","shell.execute_reply":"2024-05-14T10:52:09.364245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list(os.listdir('/kaggle/input/siim-isic-melanoma-classification'))","metadata":{"_kg_hide-input":false,"_kg_hide-output":false,"execution":{"iopub.status.busy":"2024-05-14T10:52:09.367032Z","iopub.execute_input":"2024-05-14T10:52:09.367508Z","iopub.status.idle":"2024-05-14T10:52:09.380036Z","shell.execute_reply.started":"2024-05-14T10:52:09.367460Z","shell.execute_reply":"2024-05-14T10:52:09.378904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. CSV Files - Train📁 + Test📂","metadata":{}},{"cell_type":"code","source":"# Directory\ndirectory = '/kaggle/input/siim-isic-melanoma-classification'\n\n# Import the 2 csv s\ntrain_df = pd.read_csv(directory + '/train.csv')\ntest_df = pd.read_csv(directory + '/test.csv')\n\nprint('Train has {:,} rows and Test has {:,} rows.'.format(len(train_df), len(test_df)))\n\n# Change columns names\nnew_names = ['dcm_name', 'ID', 'sex', 'age', 'anatomy', 'diagnosis', 'benign_malignant', 'target']\ntrain_df.columns = new_names\ntest_df.columns = new_names[:5]","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:52:09.381737Z","iopub.execute_input":"2024-05-14T10:52:09.382240Z","iopub.status.idle":"2024-05-14T10:52:09.490237Z","shell.execute_reply.started":"2024-05-14T10:52:09.382194Z","shell.execute_reply":"2024-05-14T10:52:09.489275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1_styler = train_df.head().style.set_table_attributes(\"style='display:inline'\").set_caption('Head Train Data')\ndf2_styler = test_df.head().style.set_table_attributes(\"style='display:inline'\").set_caption('Head Test Data')\n\ndisplay_html(df1_styler._repr_html_() + df2_styler._repr_html_(), raw=True)\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-05-14T10:52:09.494478Z","iopub.execute_input":"2024-05-14T10:52:09.494831Z","iopub.status.idle":"2024-05-14T10:52:09.510064Z","shell.execute_reply.started":"2024-05-14T10:52:09.494799Z","shell.execute_reply":"2024-05-14T10:52:09.509225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3.1 Missing Values \n\nLet's first visualize the missing values.","metadata":{}},{"cell_type":"code","source":"# This code snippet uses the missingno library to create missing value matrices for the training and test dataframes (train_df and test_df). \n# It then displays these matrices side by side using matplotlib.\n\n\nf, (ax1, ax2) = plt.subplots(1, 2, figsize = (16, 6))\n\nmsno.matrix(train_df, ax = ax1, color=(207/255, 196/255, 171/255), fontsize=10)\nmsno.matrix(test_df, ax = ax2, color=(218/255, 136/255, 130/255), fontsize=10)\n\nax1.set_title('Train Missing Values Map', fontsize = 16)\nax2.set_title('Test Missing Values Map', fontsize = 16);","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2024-05-14T10:52:09.513365Z","iopub.execute_input":"2024-05-14T10:52:09.513742Z","iopub.status.idle":"2024-05-14T10:52:09.917688Z","shell.execute_reply.started":"2024-05-14T10:52:09.513693Z","shell.execute_reply":"2024-05-14T10:52:09.916539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Train: SEX Variable**","metadata":{}},{"cell_type":"code","source":"# Data\n\n#Contains rows where the sex column is missing (NaN).\n\nnan_sex = train_df[train_df['sex'].isna() == True]\n\n#Contains rows where the sex column is not missing.\n\nis_sex = train_df[train_df['sex'].isna() == False]\n\n# Figure\nf, (ax1, ax2) = plt.subplots(1, 2, figsize = (16, 6))\n\na = sns.countplot(nan_sex['anatomy'], ax = ax1, palette=colors_nude)\n\nb = sns.countplot(is_sex['anatomy'], ax = ax2, palette=colors_nude)\n\nax1.set_title('NAN Gender: Anatomy', fontsize=16)\n\nax2.set_title('Rest Gender: Anatomy', fontsize=16)\n\na.set_xticklabels(a.get_xticklabels(), rotation=35, ha=\"right\")\nb.set_xticklabels(b.get_xticklabels(), rotation=35, ha=\"right\")\nsns.despine(left=True, bottom=True);\n\n# Benign/ Malignant check\nprint('Out of 65 NAN values, {} are benign and 0 malignant.'.format(nan_sex['benign_malignant'].value_counts()[0]))","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2024-05-14T10:52:09.919413Z","iopub.execute_input":"2024-05-14T10:52:09.919857Z","iopub.status.idle":"2024-05-14T10:52:10.386970Z","shell.execute_reply.started":"2024-05-14T10:52:09.919810Z","shell.execute_reply":"2024-05-14T10:52:10.385943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# This code snippet checks the distribution of males and females for certain anatomical areas ('lower extremity', 'upper extremity', 'torso') \n# where the target is 0 (indicating benign cases)\n\n# Check how many are males and how many females\n\nanatomy = ['lower extremity', 'upper extremity', 'torso']\n\ntrain_df[(train_df['anatomy'].isin(anatomy)) & (train_df['target'] == 0)]['sex'].value_counts()\n\n","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:52:10.388639Z","iopub.execute_input":"2024-05-14T10:52:10.388984Z","iopub.status.idle":"2024-05-14T10:52:10.418088Z","shell.execute_reply.started":"2024-05-14T10:52:10.388950Z","shell.execute_reply":"2024-05-14T10:52:10.417176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"**Train: AGE Variable**","metadata":{}},{"cell_type":"code","source":"# Data\n\nnan_age = train_df[train_df['age'].isna() == True]\nis_age = train_df[train_df['age'].isna() == False]\n\n# Figure\nf, (ax1, ax2) = plt.subplots(1, 2, figsize = (16, 6))\n\na = sns.countplot(nan_age['anatomy'], ax = ax1, palette=colors_nude)\n\nb = sns.countplot(is_age['anatomy'], ax = ax2, palette=colors_nude)\n\nax1.set_title('NAN age: Anatomy', fontsize=16)\n\nax2.set_title('Rest age: Anatomy', fontsize=16)\n\na.set_xticklabels(a.get_xticklabels(), rotation=35, ha=\"right\")\nb.set_xticklabels(b.get_xticklabels(), rotation=35, ha=\"right\")\nsns.despine(left=True, bottom=True);\n\n# Benign/ Malignant check\nprint('Out of 68 NAN values, {} are benign and 0 malignant.'.format(nan_age['benign_malignant'].value_counts()[0]))","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2024-05-14T10:52:10.419599Z","iopub.execute_input":"2024-05-14T10:52:10.419955Z","iopub.status.idle":"2024-05-14T10:52:10.878643Z","shell.execute_reply.started":"2024-05-14T10:52:10.419921Z","shell.execute_reply":"2024-05-14T10:52:10.877629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the mean age\nanatomy = ['lower extremity', 'upper extremity', 'torso']\nmedian = train_df[(train_df['anatomy'].isin(anatomy)) & (train_df['target'] == 0) & (train_df['sex'] == 'male')]['age'].median()\nprint('Median is:', median)\n\n# Impute the missing values with male\ntrain_df['age'].fillna(median, inplace = True) ","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:52:10.880516Z","iopub.execute_input":"2024-05-14T10:52:10.880965Z","iopub.status.idle":"2024-05-14T10:52:10.905653Z","shell.execute_reply.started":"2024-05-14T10:52:10.880919Z","shell.execute_reply":"2024-05-14T10:52:10.904250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Train: ANATOMY Variable**","metadata":{}},{"cell_type":"code","source":"anatomy = train_df.copy()\n\nanatomy['flag'] = np.where(train_df['anatomy'].isna()==True, 'missing', 'not_missing')\n\n# Figure\nf, (ax1, ax2) = plt.subplots(1, 2, figsize = (16, 6))\n\nsns.countplot(anatomy['flag'], hue=anatomy['sex'], ax=ax1, palette=colors_nude)\n\nsns.distplot(anatomy[anatomy['flag'] == 'missing']['age'], \n hist=False, rug=True, label='Missing', ax=ax2, \n color=colors_nude[2], kde_kws=dict(linewidth=4))\n\nsns.distplot(anatomy[anatomy['flag'] == 'not_missing']['age'], \n hist=False, rug=True, label='Not Missing', ax=ax2, \n color=colors_nude[3], kde_kws=dict(linewidth=4))\n\nax1.set_title('Gender for Anatomy', fontsize=16)\nax2.set_title('Age Distribution for Anatomy', fontsize=16)\nsns.despine(left=True, bottom=True);\n\n# Benign - malignant\nben_mal = anatomy[anatomy['flag'] == 'missing']['benign_malignant'].value_counts()\n\nprint('From all missing values, {} are benign and {} malignant.'.format(ben_mal[0], ben_mal[1]))","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2024-05-14T10:52:10.907094Z","iopub.execute_input":"2024-05-14T10:52:10.907474Z","iopub.status.idle":"2024-05-14T10:52:11.937109Z","shell.execute_reply.started":"2024-05-14T10:52:10.907433Z","shell.execute_reply":"2024-05-14T10:52:11.936212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Impute for anatomy\n\ntrain_df['anatomy'].fillna('torso', inplace = True) ","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:52:11.938538Z","iopub.execute_input":"2024-05-14T10:52:11.938887Z","iopub.status.idle":"2024-05-14T10:52:11.947536Z","shell.execute_reply.started":"2024-05-14T10:52:11.938853Z","shell.execute_reply":"2024-05-14T10:52:11.946529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Test: ANATOMY Variable**","metadata":{}},{"cell_type":"code","source":"anatomy = test_df.copy()\n\nanatomy['flag'] = np.where(test_df['anatomy'].isna()==True, 'missing', 'not_missing')\n\n# Figure\nf, (ax1, ax2) = plt.subplots(1, 2, figsize = (16, 6))\n\nsns.countplot(anatomy['flag'], hue=anatomy['sex'], ax=ax1, palette=colors_nude)\n\nsns.distplot(anatomy[anatomy['flag'] == 'missing']['age'],\n hist=False, rug=True, label='Missing', ax=ax2, \n color=colors_nude[2], kde_kws=dict(linewidth=4, bw=0.1))\n\nsns.distplot(anatomy[anatomy['flag'] == 'not_missing']['age'], \n hist=False, rug=True, label='Not Missing', ax=ax2, \n color=colors_nude[3], kde_kws=dict(linewidth=4, bw=0.1))\n\nax1.set_title('Gender for Anatomy', fontsize=16)\n\nax2.set_title('Age Distribution for Anatomy', fontsize=16)\n\nsns.despine(left=True, bottom=True);","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:52:11.948820Z","iopub.execute_input":"2024-05-14T10:52:11.949161Z","iopub.status.idle":"2024-05-14T10:52:12.896634Z","shell.execute_reply.started":"2024-05-14T10:52:11.949098Z","shell.execute_reply":"2024-05-14T10:52:12.895591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Select most frequent anatomy for age 70 and we will reset index\n\nvalue = test_df[test_df['age'] == 70]['anatomy'].value_counts().reset_index()['index'][0]\n\n# Impute the value\n\ntest_df['anatomy'].fillna(value, inplace = True) ","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:52:12.897982Z","iopub.execute_input":"2024-05-14T10:52:12.898286Z","iopub.status.idle":"2024-05-14T10:52:12.911106Z","shell.execute_reply.started":"2024-05-14T10:52:12.898257Z","shell.execute_reply":"2024-05-14T10:52:12.910091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save the files\n\ntrain_df.to_csv('train_clean.csv', index=False)\ntest_df.to_csv('test_clean.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:52:12.912423Z","iopub.execute_input":"2024-05-14T10:52:12.912723Z","iopub.status.idle":"2024-05-14T10:52:13.191257Z","shell.execute_reply.started":"2024-05-14T10:52:12.912694Z","shell.execute_reply":"2024-05-14T10:52:13.190139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n### 1. Very HIGH class imbalance. We need to take this in consideration when Modeling.\n### 2. Age distribution:\n * Benign: follows a normal distribution\n * Malignant: a little skewed to the left, with the peak oriented towards higher age values.","metadata":{}},{"cell_type":"code","source":"# Figure\nf, (ax1, ax2) = plt.subplots(1, 2, figsize = (16, 6))\n\na = sns.countplot(data = train_df, x = 'benign_malignant', palette=colors_nude[2:4],\n ax=ax1)\nb = sns.distplot(a = train_df[train_df['target']==0]['age'], ax=ax2, color=colors_nude[2], \n hist=False, rug=True, kde_kws=dict(linewidth=4), label='Benign')\nc = sns.distplot(a = train_df[train_df['target']==1]['age'], ax=ax2, color=colors_nude[3], \n hist=False, rug=True, kde_kws=dict(linewidth=4), label='Malignant')\n\nfor p in a.patches:\n a.annotate(format(p.get_height(), ','), \n (p.get_x() + p.get_width() / 2., \n p.get_height()), ha = 'center', va = 'center', \n xytext = (0, 4), textcoords = 'offset points')\n \nax1.set_title('Frequency for Target Variable', fontsize=16)\nax2.set_title('Age Distribution the Target types', fontsize=16)\nsns.despine(left=True, bottom=True);","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2024-05-14T10:52:13.192742Z","iopub.execute_input":"2024-05-14T10:52:13.193080Z","iopub.status.idle":"2024-05-14T10:52:14.139359Z","shell.execute_reply.started":"2024-05-14T10:52:13.193048Z","shell.execute_reply":"2024-05-14T10:52:14.138064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Target and Genders:\n1. There are more males than females in the dataset\n2. However, the percentages are ~ the same","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(16, 6))\n\na = sns.countplot(data=train_df, x='benign_malignant', hue='sex', palette=colors_nude)\n\nfor p in a.patches:\n a.annotate(format(p.get_height(), ','), \n (p.get_x() + p.get_width() / 2., \n p.get_height()), ha = 'center', va = 'center', \n xytext = (0, 4), textcoords = 'offset points')\n\nplt.title('Gender split by Target Variable', fontsize=16)\n\nsns.despine(left=True, bottom=True);","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2024-05-14T10:52:14.141209Z","iopub.execute_input":"2024-05-14T10:52:14.141716Z","iopub.status.idle":"2024-05-14T10:52:14.409594Z","shell.execute_reply.started":"2024-05-14T10:52:14.141662Z","shell.execute_reply":"2024-05-14T10:52:14.408564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Anatomy and Diagnosis:","metadata":{}},{"cell_type":"code","source":"# Figure\nf, (ax1, ax2) = plt.subplots(1, 2, figsize = (16, 6))\n\na = sns.countplot(train_df['anatomy'], ax=ax1, palette = colors_nude)\nb = sns.countplot(train_df['diagnosis'], ax=ax2, palette = colors_nude)\n\na.set_xticklabels(a.get_xticklabels(), rotation=35, ha=\"right\")\nb.set_xticklabels(b.get_xticklabels(), rotation=35, ha=\"right\")\n\nfor p in a.patches:\n a.annotate(format(p.get_height(), ','), \n (p.get_x() + p.get_width() / 2., \n p.get_height()), ha = 'center', va = 'center', \n xytext = (0, 4), textcoords = 'offset points')\n \nfor p in b.patches:\n b.annotate(format(p.get_height(), ','), \n (p.get_x() + p.get_width() / 2., \n p.get_height()), ha = 'center', va = 'center', \n xytext = (0, 4), textcoords = 'offset points')\n \nax1.set_title('Anatomy Frequencies', fontsize=16)\nax2.set_title('Diagnosis Frequencies', fontsize=16)\nsns.despine(left=True, bottom=True);","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2024-05-14T10:52:14.411499Z","iopub.execute_input":"2024-05-14T10:52:14.411952Z","iopub.status.idle":"2024-05-14T10:52:14.950291Z","shell.execute_reply.started":"2024-05-14T10:52:14.411903Z","shell.execute_reply":"2024-05-14T10:52:14.949221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Anatomy and Target\n","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(16, 6))\na = sns.countplot(data=train_df, x='benign_malignant', hue='anatomy', palette=colors_nude)\n\nfor p in a.patches:\n a.annotate(format(p.get_height(), ','), \n (p.get_x() + p.get_width() / 2., \n p.get_height()), ha = 'center', va = 'center', \n xytext = (0, 4), textcoords = 'offset points')\n\nplt.title('Anatomy split by Target Variable', fontsize=16)\nsns.despine(left=True, bottom=True);","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2024-05-14T10:52:14.951746Z","iopub.execute_input":"2024-05-14T10:52:14.952085Z","iopub.status.idle":"2024-05-14T10:52:15.329751Z","shell.execute_reply.started":"2024-05-14T10:52:14.952046Z","shell.execute_reply":"2024-05-14T10:52:15.328748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Diagnosis and Target","metadata":{}},{"cell_type":"code","source":"# Figure\nf, (ax1, ax2) = plt.subplots(1, 2, figsize = (16, 6))\n\na = sns.countplot(train_df[train_df['target']==0]['diagnosis'], ax=ax1, palette = colors_nude)\nb = sns.countplot(train_df[train_df['target']==1]['diagnosis'], ax=ax2, palette = colors_nude)\n\na.set_xticklabels(a.get_xticklabels(), rotation=35, ha=\"right\")\nb.set_xticklabels(b.get_xticklabels(), rotation=35, ha=\"right\")\n\nfor p in a.patches:\n a.annotate(format(p.get_height(), ','), \n (p.get_x() + p.get_width() / 2., \n p.get_height()), ha = 'center', va = 'center', \n xytext = (0, 4), textcoords = 'offset points')\n \nfor p in b.patches:\n b.annotate(format(p.get_height(), ','), \n (p.get_x() + p.get_width() / 2., \n p.get_height()), ha = 'center', va = 'center', \n xytext = (0, 4), textcoords = 'offset points')\n \nax1.set_title('Benign cases: Diagnosis view', fontsize=16)\nax2.set_title('Malignant cases: Diagnosis view', fontsize=16)\nsns.despine(left=True, bottom=True);","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2024-05-14T10:52:15.331390Z","iopub.execute_input":"2024-05-14T10:52:15.331813Z","iopub.status.idle":"2024-05-14T10:52:15.793825Z","shell.execute_reply.started":"2024-05-14T10:52:15.331770Z","shell.execute_reply":"2024-05-14T10:52:15.792764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Test Dataset Overview\n","metadata":{}},{"cell_type":"code","source":"# Figure\nf, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize = (16, 6))\n\na = sns.countplot(test_df['sex'], palette=colors_nude, ax=ax1)\nb = sns.countplot(test_df['anatomy'], ax=ax2, palette = colors_nude)\nc = sns.distplot(a = test_df['age'], ax=ax3, color=colors_nude[3], \n hist=False, rug=True, kde_kws=dict(linewidth=4))\n\nfor p in a.patches:\n a.annotate(format(p.get_height(), ','), \n (p.get_x() + p.get_width() / 2., \n p.get_height()), ha = 'center', va = 'center', \n xytext = (0, 4), textcoords = 'offset points')\n \nfor p in b.patches:\n b.annotate(format(p.get_height(), ','), \n (p.get_x() + p.get_width() / 2., \n p.get_height()), ha = 'center', va = 'center', \n xytext = (0, 4), textcoords = 'offset points')\n \nb.set_xticklabels(b.get_xticklabels(), rotation=35, ha=\"right\")\n\nax1.set_title('Test: Gender Frequencies', fontsize=16)\nax2.set_title('Test: Anatomy Frequencies', fontsize=16)\nax3.set_title('Test: Age Distribution', fontsize=16)\nsns.despine(left=True, bottom=True);","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2024-05-14T10:52:15.795421Z","iopub.execute_input":"2024-05-14T10:52:15.795896Z","iopub.status.idle":"2024-05-14T10:52:16.781983Z","shell.execute_reply.started":"2024-05-14T10:52:15.795850Z","shell.execute_reply":"2024-05-14T10:52:16.781014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Patients\n","metadata":{}},{"cell_type":"code","source":"# Count the number of images per ID\npatients_count_train = train_df.groupby(by='ID')['dcm_name'].count().reset_index()\npatients_count_test = test_df.groupby(by='ID')['dcm_name'].count().reset_index()\n\n# Figure\nf, (ax1, ax2) = plt.subplots(1, 2, figsize = (16, 6))\n\na = sns.distplot(patients_count_train['dcm_name'], kde=False, bins=50, \n ax=ax1, color=colors_nude[0], hist_kws={'alpha': 1})\nb = sns.distplot(patients_count_test['dcm_name'], kde=False, bins=50, \n ax=ax2, color=colors_nude[1], hist_kws={'alpha': 1})\n \nax1.set_title('Train: Images per Patient Distribution', fontsize=16)\nax2.set_title('Test: Images per Patient Distribution', fontsize=16)\nsns.despine(left=True, bottom=True);","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2024-05-14T10:52:16.783327Z","iopub.execute_input":"2024-05-14T10:52:16.783620Z","iopub.status.idle":"2024-05-14T10:52:17.659294Z","shell.execute_reply.started":"2024-05-14T10:52:16.783592Z","shell.execute_reply":"2024-05-14T10:52:17.658221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> Note: Before continuing, let's save the clean files again.","metadata":{}},{"cell_type":"code","source":"# Save the files\n\ntrain_df.to_csv('train_clean.csv', index=False)\ntest_df.to_csv('test_clean.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:52:17.660876Z","iopub.execute_input":"2024-05-14T10:52:17.661249Z","iopub.status.idle":"2024-05-14T10:52:17.938392Z","shell.execute_reply.started":"2024-05-14T10:52:17.661213Z","shell.execute_reply":"2024-05-14T10:52:17.937547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Preprocess .csv files \n\n#### 1. Add Image Path","metadata":{}},{"cell_type":"code","source":"# === DICOM ===\n\n# Create the paths\n\npath_train = directory + '/train/' + train_df['dcm_name'] + '.dcm'\n\npath_test = directory + '/test/' + test_df['dcm_name'] + '.dcm'\n\n# Append to the original dataframes\n\ntrain_df['path_dicom'] = path_train\n\ntest_df['path_dicom'] = path_test\n\n# === JPEG ===\n\n# Create the paths\n\npath_train = directory + '/jpeg/train/' + train_df['dcm_name'] + '.jpg'\n\npath_test = directory + '/jpeg/test/' + test_df['dcm_name'] + '.jpg'\n\n# Append to the original dataframes\n\ntrain_df['path_jpeg'] = path_train\n\ntest_df['path_jpeg'] = path_test","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:52:17.939784Z","iopub.execute_input":"2024-05-14T10:52:17.940107Z","iopub.status.idle":"2024-05-14T10:52:17.979388Z","shell.execute_reply.started":"2024-05-14T10:52:17.940076Z","shell.execute_reply":"2024-05-14T10:52:17.978359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 2. Encoding\nTransforming all categorical features un numerical.\n> Note1: `sex`, `anatomy`, `diagnosis` need to be encoded.\n\n> Note2: `benign_malignant` column will be dropped, as the information is already in the `target` column.","metadata":{}},{"cell_type":"code","source":"# === TRAIN ===\nto_encode = ['sex', 'anatomy', 'diagnosis']\nencoded_all = []\n\nlabel_encoder = LabelEncoder()\n\nfor column in to_encode:\n if train_df[column].dtype == 'float64':\n train_df[column] = train_df[column].astype(str)\n label_encoder = LabelEncoder()\n encoded = label_encoder.fit_transform(train_df[column].astype(str))\n encoded_all.append(encoded)\n \ntrain_df['sex'] = encoded_all[0]\ntrain_df['anatomy'] = encoded_all[1]\ntrain_df['diagnosis'] = encoded_all[2]\n\nif 'benign_malignant' in train_df.columns : train_df.drop(['benign_malignant'], axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:52:17.980986Z","iopub.execute_input":"2024-05-14T10:52:17.981307Z","iopub.status.idle":"2024-05-14T10:52:18.082561Z","shell.execute_reply.started":"2024-05-14T10:52:17.981277Z","shell.execute_reply":"2024-05-14T10:52:18.081698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# === TEST ===\nto_encode = ['sex', 'anatomy']\nencoded_all = []\n\nlabel_encoder = LabelEncoder()\n\nfor column in to_encode:\n encoded = label_encoder.fit_transform(test_df[column])\n encoded_all.append(encoded)\n \ntest_df['sex'] = encoded_all[0]\ntest_df['anatomy'] = encoded_all[1]","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:52:18.084310Z","iopub.execute_input":"2024-05-14T10:52:18.084747Z","iopub.status.idle":"2024-05-14T10:52:18.102073Z","shell.execute_reply.started":"2024-05-14T10:52:18.084699Z","shell.execute_reply":"2024-05-14T10:52:18.101217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> Save the files before continuing.","metadata":{}},{"cell_type":"code","source":"# Save the files\ntrain_df.to_csv('train_clean.csv', index=False)\ntest_df.to_csv('test_clean.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:52:18.103440Z","iopub.execute_input":"2024-05-14T10:52:18.103774Z","iopub.status.idle":"2024-05-14T10:52:18.649719Z","shell.execute_reply.started":"2024-05-14T10:52:18.103742Z","shell.execute_reply":"2024-05-14T10:52:18.648880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# The Images 📸\n\nThere are 2 types of images containing the same information:\n1. `.dcm` files: It's saved in the \"Digital Imaging and Communications in Medicine\" format. It contains an image from a medical scan, such as an ultrasound or MRI + information about the patient.\n2. `.jpeg` files: the DICOM files converted into .jpeg format\n3. `.tfrec` files: [The TFRecord file format is a simple record-oriented binary format for ML training data.]\n## 1. Sanity Check\n> Check if images in `.dcm` and `.jpeg` format have the same number of observations as in `train_df` and `test_df`.","metadata":{}},{"cell_type":"code","source":"print('Train .dcm number of images:', len(list(os.listdir('../input/siim-isic-melanoma-classification/train'))), '\\n' +\n 'Test .dcm number of images:', len(list(os.listdir('../input/siim-isic-melanoma-classification/test'))), '\\n' +\n 'Train .jpeg number of images:', len(list(os.listdir('../input/siim-isic-melanoma-classification/jpeg/train'))), '\\n' +\n 'Test .jpeg number of images:', len(list(os.listdir('../input/siim-isic-melanoma-classification/jpeg/test'))), '\\n' +\n '-----------------------', '\\n' +\n 'There is the same number of images as in train/ test .csv datasets')","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:52:18.651073Z","iopub.execute_input":"2024-05-14T10:52:18.651497Z","iopub.status.idle":"2024-05-14T10:52:18.702046Z","shell.execute_reply.started":"2024-05-14T10:52:18.651443Z","shell.execute_reply":"2024-05-14T10:52:18.700764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Image shapes?\nAlso, let's look at the size of the images (to not overload the memory, we'll check 100 different images). They are pretty different, so we'll need to deal with this in the augmentations part.","metadata":{}},{"cell_type":"code","source":"shapes_train = []\n\nfor k, path in enumerate(train_df['path_jpeg']):\n image = Image.open(path)\n shapes_train.append(image.size)\n \n if k >= 100: break\n \nshapes_train = pd.DataFrame(data = shapes_train, columns = ['H', 'W'], dtype='object')\nshapes_train['Size'] = '[' + shapes_train['H'].astype(str) + ', ' + shapes_train['W'].astype(str) + ']'\nprint(shapes_train)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:52:18.703743Z","iopub.execute_input":"2024-05-14T10:52:18.704176Z","iopub.status.idle":"2024-05-14T10:52:18.896278Z","shell.execute_reply.started":"2024-05-14T10:52:18.704112Z","shell.execute_reply":"2024-05-14T10:52:18.895253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (16, 6))\n\na = sns.countplot(shapes_train['Size'], palette=colors_nude)\n\nfor p in a.patches:\n a.annotate(format(p.get_height(), ','), \n (p.get_x() + p.get_width() / 2., \n p.get_height()), ha = 'center', va = 'center', \n xytext = (0, 4), textcoords = 'offset points')\n \nplt.title('100 Images Shapes', fontsize=16)\nsns.despine(left=True, bottom=True);","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2024-05-14T10:52:18.903698Z","iopub.execute_input":"2024-05-14T10:52:18.904039Z","iopub.status.idle":"2024-05-14T10:52:19.193100Z","shell.execute_reply.started":"2024-05-14T10:52:18.904004Z","shell.execute_reply":"2024-05-14T10:52:19.191882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. DICOM Images\n\n### Malignant vs Benign Images\n\nLet's look at the difference between *malignant* and *benign* melanomas.","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"def show_images(data, n = 5, rows=1, cols=5, title='Default'):\n plt.figure(figsize=(16,4))\n\n for k, path in enumerate(data['path_dicom'][:n]):\n image = pydicom.read_file(path)\n image = image.pixel_array\n \n # image = resize(image, (200, 200), anti_aliasing=True)\n\n plt.suptitle(title, fontsize = 16)\n plt.subplot(rows, cols, k+1)\n plt.imshow(image)\n plt.axis('off')","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:52:19.195166Z","iopub.execute_input":"2024-05-14T10:52:19.195678Z","iopub.status.idle":"2024-05-14T10:52:19.206077Z","shell.execute_reply.started":"2024-05-14T10:52:19.195628Z","shell.execute_reply":"2024-05-14T10:52:19.204905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Show Benign Samples\nshow_images(train_df[train_df['target'] == 0], n=5, rows=1, cols=5, title='Benign Sample')\n","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:52:19.207875Z","iopub.execute_input":"2024-05-14T10:52:19.208390Z","iopub.status.idle":"2024-05-14T10:52:28.114740Z","shell.execute_reply.started":"2024-05-14T10:52:19.208342Z","shell.execute_reply":"2024-05-14T10:52:28.113581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Show Malignant Samples\nshow_images(train_df[train_df['target'] == 1], n=5, rows=1, cols=5, title='Malignant Sample')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-05-14T10:52:28.116340Z","iopub.execute_input":"2024-05-14T10:52:28.116786Z","iopub.status.idle":"2024-05-14T10:52:38.505537Z","shell.execute_reply.started":"2024-05-14T10:52:28.116738Z","shell.execute_reply":"2024-05-14T10:52:38.504661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Class Imbalance \n\nThis is a **very** important topic in this classification problem, as the 2 classes we are dealing with are highly imbalanced, with 98% of the data being *benign* and only 2% of the data being *malignant*.\n\n\nThis is also the kind of problem where you **DON'T** want to have False Negatives. It's waaayyy worse to tell a patient they don't have cancer when they actually do, than to tell em they do have it and they actually don't. So, having balanced classes is *crucial*.\n\n### We can do:\n* **Oversampling**: of the minority class, increasing the number of images through augmentations\n* **Understampling**: of the majority class (we shall see how the process is going)\n\n\n### What is Data Augmentation?\n\nIs *moving, rotating, cropping, flipping, changing color/brightness/hue and whatever else you can come up with* to change the aspect of the original image. It is helpful in Overfitting, as the model learns not only 1 aspect of the image, but multiple (a cat can be standing up straight, or funny upside down, in a b&w image etc.).\n\n\n### Other things to keep in mind:\n
\n

#1: Different skin tones. Might need to find something that levels that.

\n

#2: Different lightings in the image.

\n

#3: Different sizes of the images. We need to resize them.

\n
\n\n## 1. B&W View 🤍🖤","metadata":{}},{"cell_type":"code","source":"fig, axes = plt.subplots(nrows=2, ncols=6, figsize=(16,6))\nplt.suptitle(\"B&W\", fontsize = 16)\n\nfor i in range(0, 2*6):\n data = pydicom.read_file(train_df['path_dicom'][i])\n image = data.pixel_array\n \n # Transform to B&W\n # The function converts an input image from one color space to another.\n image = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)\n image = cv2.resize(image, (200,200))\n \n x = i // 6\n y = i % 6\n axes[x, y].imshow(image, cmap=plt.cm.bone) \n axes[x, y].axis('off')","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2024-05-14T10:52:38.506901Z","iopub.execute_input":"2024-05-14T10:52:38.507216Z","iopub.status.idle":"2024-05-14T10:52:49.060193Z","shell.execute_reply.started":"2024-05-14T10:52:38.507186Z","shell.execute_reply":"2024-05-14T10:52:49.059231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ben Graham: greyscale + Gaussian Blur \n\n`cv2.GaussiaBlur()`: The function convolves the source image with the specified Gaussian kernel.\n\n### #1. Without Gaussian Blur","metadata":{}},{"cell_type":"code","source":"fig, axes = plt.subplots(nrows=2, ncols=6, figsize=(16,6))\n\nplt.suptitle(\"Without Gaussian Blur\", fontsize = 16)\n\nfor i in range(0, 2*6):\n data = pydicom.read_file(train_df['path_dicom'][i])\n image = data.pixel_array\n \n # Transform to B&W\n # The function converts an input image from one color space to another.\n image = cv2.cvtColor(image, cv2.COLOR_RGB2HSV)\n image = cv2.resize(image, (200,200))\n \n x = i // 6\n y = i % 6\n axes[x, y].imshow(image, cmap=plt.cm.bone) \n axes[x, y].axis('off')","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2024-05-14T10:52:49.061578Z","iopub.execute_input":"2024-05-14T10:52:49.061894Z","iopub.status.idle":"2024-05-14T10:52:59.717357Z","shell.execute_reply.started":"2024-05-14T10:52:49.061864Z","shell.execute_reply":"2024-05-14T10:52:59.716162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### #2. With Gaussian Blur","metadata":{}},{"cell_type":"code","source":"fig, axes = plt.subplots(nrows=2, ncols=6, figsize=(16,6))\nplt.suptitle(\"With Gaussian Blur\", fontsize = 16)\n\nfor i in range(0, 2*6):\n data = pydicom.read_file(train_df['path_dicom'][i])\n image = data.pixel_array\n \n # Transform to B&W\n # The function converts an input image from one color space to another.\n image = cv2.cvtColor(image, cv2.COLOR_RGB2HSV)\n image = cv2.resize(image, (200,200))\n image=cv2.addWeighted(image, 4, cv2.GaussianBlur(image, (0,0) ,256/10), -4, 128)\n \n x = i // 6\n y = i % 6\n axes[x, y].imshow(image, cmap=plt.cm.bone) \n axes[x, y].axis('off')","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2024-05-14T10:52:59.718847Z","iopub.execute_input":"2024-05-14T10:52:59.719214Z","iopub.status.idle":"2024-05-14T10:53:10.532137Z","shell.execute_reply.started":"2024-05-14T10:52:59.719179Z","shell.execute_reply":"2024-05-14T10:53:10.531180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3 Hue, Saturation, Brightness","metadata":{}},{"cell_type":"code","source":"fig, axes = plt.subplots(nrows=2, ncols=6, figsize=(16,6))\nplt.suptitle(\"Hue, Saturation, Brightness\", fontsize = 16)\n\nfor i in range(0, 2*6):\n data = pydicom.read_file(train_df['path_dicom'][i])\n image = data.pixel_array\n \n # Transform to B&W\n # The function converts an input image from one color space to another.\n image = cv2.cvtColor(image, cv2.COLOR_RGB2HLS)\n image = cv2.resize(image, (200,200))\n \n x = i // 6\n y = i % 6\n axes[x, y].imshow(image, cmap=plt.cm.bone) \n axes[x, y].axis('off')","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2024-05-14T10:53:10.533395Z","iopub.execute_input":"2024-05-14T10:53:10.533726Z","iopub.status.idle":"2024-05-14T10:53:21.219711Z","shell.execute_reply.started":"2024-05-14T10:53:10.533697Z","shell.execute_reply":"2024-05-14T10:53:21.218666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4 LUV Color Space","metadata":{}},{"cell_type":"code","source":"fig, axes = plt.subplots(nrows=2, ncols=6, figsize=(16,6))\nplt.suptitle(\"LUV Color Space\", fontsize = 16)\n\nfor i in range(0, 2*6):\n data = pydicom.read_file(train_df['path_dicom'][i])\n image = data.pixel_array\n \n # Transform to B&W\n # The function converts an input image from one color space to another.\n image = cv2.cvtColor(image, cv2.COLOR_RGB2LUV)\n image = cv2.resize(image, (200,200))\n \n x = i // 6\n y = i % 6\n axes[x, y].imshow(image, cmap=plt.cm.bone) \n axes[x, y].axis('off')","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2024-05-14T10:53:21.221377Z","iopub.execute_input":"2024-05-14T10:53:21.221801Z","iopub.status.idle":"2024-05-14T10:53:32.307291Z","shell.execute_reply.started":"2024-05-14T10:53:21.221760Z","shell.execute_reply":"2024-05-14T10:53:32.306366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5. Torchvision.transforms 🩹\n\nIt's a library that goes hand in hand with `PyTorch` and it's easily used to augment data. Let's demonstrate.","metadata":{}},{"cell_type":"code","source":"# Necessary Imports\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\nimport torchvision.transforms as transforms\nimport torchvision","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-05-14T10:53:32.308888Z","iopub.execute_input":"2024-05-14T10:53:32.309336Z","iopub.status.idle":"2024-05-14T10:53:32.315279Z","shell.execute_reply.started":"2024-05-14T10:53:32.309297Z","shell.execute_reply":"2024-05-14T10:53:32.314220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Select a small sample of the .jpeg image paths\nimage_list = train_df.sample(12)['path_jpeg']\nimage_list = image_list.reset_index()['path_jpeg']\n\n# Show the sample\nplt.figure(figsize=(16,6))\nplt.suptitle(\"Original View\", fontsize = 16)\n \nfor k, path in enumerate(image_list):\n image = mpimg.imread(path)\n \n plt.subplot(2, 6, k+1)\n plt.imshow(image)\n plt.axis('off')","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:53:32.316993Z","iopub.execute_input":"2024-05-14T10:53:32.317424Z","iopub.status.idle":"2024-05-14T10:53:48.185243Z","shell.execute_reply.started":"2024-05-14T10:53:32.317382Z","shell.execute_reply":"2024-05-14T10:53:48.184207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create PyTorch Dataset Object\nclass DatasetExample(Dataset):\n def __init__(self, image_list, transforms=None):\n self.image_list = image_list\n self.transforms = transforms\n \n # To get item's length\n def __len__(self):\n return (len(self.image_list))\n \n # For indexing\n def __getitem__(self, i):\n # Read in image\n image = plt.imread(self.image_list[i])\n image = Image.fromarray(image).convert('RGB') \n image = np.asarray(image).astype(np.uint8)\n if self.transforms is not None:\n image = self.transforms(image)\n \n return torch.tensor(image, dtype=torch.float)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:53:48.187014Z","iopub.execute_input":"2024-05-14T10:53:48.187491Z","iopub.status.idle":"2024-05-14T10:53:48.200279Z","shell.execute_reply.started":"2024-05-14T10:53:48.187440Z","shell.execute_reply":"2024-05-14T10:53:48.199087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Predefined Show Images Function\ndef show_transform(image, title=\"Default\"):\n plt.figure(figsize=(16,6))\n plt.suptitle(title, fontsize = 16)\n \n # Unnormalize\n image = image / 2 + 0.5 \n npimg = image.numpy()\n npimg = np.clip(npimg, 0., 1.)\n plt.imshow(np.transpose(npimg, (1, 2, 0)))\n plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:53:48.201668Z","iopub.execute_input":"2024-05-14T10:53:48.201992Z","iopub.status.idle":"2024-05-14T10:53:48.215252Z","shell.execute_reply.started":"2024-05-14T10:53:48.201960Z","shell.execute_reply":"2024-05-14T10:53:48.214238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### #1. Crop ✂","metadata":{}},{"cell_type":"code","source":"# Transform\ntransform = transforms.Compose([\n transforms.ToPILImage(),\n transforms.Resize((300, 300)),\n transforms.CenterCrop((100, 100)),\n transforms.ToTensor(),\n transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n ])\n\n# Create the dataset\npytorch_dataset = DatasetExample(image_list=image_list, transforms=transform)\npytorch_dataloader = DataLoader(dataset=pytorch_dataset, batch_size=12, shuffle=True)\n\n# Select the data\nimages = next(iter(pytorch_dataloader))\n \n# show images\nshow_transform(torchvision.utils.make_grid(images, nrow=6), title=\"Crop\")","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:53:48.216529Z","iopub.execute_input":"2024-05-14T10:53:48.216853Z","iopub.status.idle":"2024-05-14T10:53:55.525224Z","shell.execute_reply.started":"2024-05-14T10:53:48.216814Z","shell.execute_reply":"2024-05-14T10:53:55.524046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### #2. ColorJitter \nRandomly change the brightness, contrast and saturation of an image.","metadata":{}},{"cell_type":"code","source":"# Transform\ntransform = transforms.Compose([\n transforms.ToPILImage(),\n transforms.Resize((300, 300)),\n transforms.ColorJitter(brightness=0.7, contrast=0.7, saturation=0.7, hue=0.5),\n transforms.ToTensor(),\n transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n ])\n\n# Create the dataset\npytorch_dataset = DatasetExample(image_list=image_list, transforms=transform)\npytorch_dataloader = DataLoader(dataset=pytorch_dataset, batch_size=12, shuffle=True)\n\n# Select the data\nimages = next(iter(pytorch_dataloader))\n \n# show images\nshow_transform(torchvision.utils.make_grid(images, nrow=6), title=\"Color Jitter\")","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2024-05-14T10:53:55.527016Z","iopub.execute_input":"2024-05-14T10:53:55.527441Z","iopub.status.idle":"2024-05-14T10:54:02.983386Z","shell.execute_reply.started":"2024-05-14T10:53:55.527399Z","shell.execute_reply":"2024-05-14T10:54:02.982374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### #3. RandomGreyscale 🌘\nRandomly convert image to grayscale with a probability of p (default 0.1).","metadata":{}},{"cell_type":"code","source":"# Transform\ntransform = transforms.Compose([\n transforms.ToPILImage(),\n transforms.Resize((300, 300)),\n transforms.RandomGrayscale(p=0.7),\n transforms.ToTensor(),\n transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n ])\n\n# Create the dataset\npytorch_dataset = DatasetExample(image_list=image_list, transforms=transform)\npytorch_dataloader = DataLoader(dataset=pytorch_dataset, batch_size=12, shuffle=True)\n\n# Select the data\nimages = next(iter(pytorch_dataloader))\n \n# show images\nshow_transform(torchvision.utils.make_grid(images, nrow=6), title=\"Random Greyscale\")","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2024-05-14T10:54:02.984961Z","iopub.execute_input":"2024-05-14T10:54:02.985506Z","iopub.status.idle":"2024-05-14T10:54:10.208135Z","shell.execute_reply.started":"2024-05-14T10:54:02.985458Z","shell.execute_reply":"2024-05-14T10:54:10.207142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### #4. RandomVerticalFlip 🌍🌎\nVertically flip the given PIL Image randomly with a given probability.","metadata":{}},{"cell_type":"code","source":"# Transform\ntransform = transforms.Compose([\n transforms.ToPILImage(),\n transforms.Resize((300, 300)),\n transforms.RandomVerticalFlip(p=0.7),\n transforms.ToTensor(),\n transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n ])\n\n# Create the dataset\npytorch_dataset = DatasetExample(image_list=image_list, transforms=transform)\npytorch_dataloader = DataLoader(dataset=pytorch_dataset, batch_size=12, shuffle=True)\n\n# Select the data\nimages = next(iter(pytorch_dataloader))\n \n# show images\nshow_transform(torchvision.utils.make_grid(images, nrow=6), title=\"Random Vertical Flip\")","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:54:10.209872Z","iopub.execute_input":"2024-05-14T10:54:10.210290Z","iopub.status.idle":"2024-05-14T10:54:17.593737Z","shell.execute_reply.started":"2024-05-14T10:54:10.210249Z","shell.execute_reply":"2024-05-14T10:54:17.592625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 7. Hair Removal ","metadata":{}},{"cell_type":"code","source":"# def hair_remove(image):\n# # convert image to grayScale\n# grayScale = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)\n\n# # kernel for morphologyEx\n# kernel = cv2.getStructuringElement(1,(17,17))\n\n# # apply MORPH_BLACKHAT to grayScale image\n# blackhat = cv2.morphologyEx(grayScale, cv2.MORPH_BLACKHAT, kernel)\n\n# # apply thresholding to blackhat\n# _,threshold = cv2.threshold(blackhat,10,255,cv2.THRESH_BINARY)\n\n# # inpaint with original image and threshold image\n# final_image = cv2.inpaint(image,threshold,1,cv2.INPAINT_TELEA)\n\n# return final_image","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:54:17.595152Z","iopub.execute_input":"2024-05-14T10:54:17.595495Z","iopub.status.idle":"2024-05-14T10:54:17.600292Z","shell.execute_reply.started":"2024-05-14T10:54:17.595460Z","shell.execute_reply":"2024-05-14T10:54:17.599129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Select a small sample of the .jpeg image paths\n# # We select some hairy photos on purpose\n# hairy_photos = train_df[train_df[\"sex\"] == 1].reset_index().iloc[[12, 14, 17, 22, 33, 34]]\n# image_list = hairy_photos['path_jpeg']\n# image_list = image_list.reset_index()['path_jpeg']","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:54:17.601886Z","iopub.execute_input":"2024-05-14T10:54:17.602400Z","iopub.status.idle":"2024-05-14T10:54:17.615162Z","shell.execute_reply.started":"2024-05-14T10:54:17.602345Z","shell.execute_reply":"2024-05-14T10:54:17.614084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Show the Augmented Images\n# plt.figure(figsize=(16,3))\n# plt.suptitle(\"Original Hairy Images\", fontsize = 16)\n \n# for k, path in enumerate(image_list):\n# image = mpimg.imread(path)\n# image = cv2.resize(image,(300, 300))\n \n# plt.subplot(1, 6, k+1)\n# plt.imshow(image)\n# plt.axis('off')","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:54:17.616595Z","iopub.execute_input":"2024-05-14T10:54:17.616918Z","iopub.status.idle":"2024-05-14T10:54:17.624335Z","shell.execute_reply.started":"2024-05-14T10:54:17.616888Z","shell.execute_reply":"2024-05-14T10:54:17.623321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Show the sample\n# plt.figure(figsize=(16,3))\n# plt.suptitle(\"Non Hairy Images\", fontsize = 16)\n \n# for k, path in enumerate(image_list):\n# image = mpimg.imread(path)\n# image = cv2.resize(image,(300, 300))\n# image = hair_remove(image)\n \n# plt.subplot(1, 6, k+1)\n# plt.imshow(image)\n# plt.axis('off')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-05-14T10:54:17.625918Z","iopub.execute_input":"2024-05-14T10:54:17.626263Z","iopub.status.idle":"2024-05-14T10:54:17.635648Z","shell.execute_reply.started":"2024-05-14T10:54:17.626231Z","shell.execute_reply":"2024-05-14T10:54:17.634820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"# Models","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"!/opt/conda/bin/python3.7 -m pip install --upgrade pip","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:54:17.636975Z","iopub.execute_input":"2024-05-14T10:54:17.637368Z","iopub.status.idle":"2024-05-14T10:54:25.964295Z","shell.execute_reply.started":"2024-05-14T10:54:17.637335Z","shell.execute_reply":"2024-05-14T10:54:25.963048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -q efficientnet_pytorch","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:54:25.966338Z","iopub.execute_input":"2024-05-14T10:54:25.966876Z","iopub.status.idle":"2024-05-14T10:54:34.176343Z","shell.execute_reply.started":"2024-05-14T10:54:25.966833Z","shell.execute_reply":"2024-05-14T10:54:34.175092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# System\nimport cv2\nimport os, os.path\nfrom PIL import Image # from RBG to YCbCr\nimport gc\nimport time\nimport datetime\n\n# Basics\nimport pandas as pd\nimport numpy as np\nimport random\nimport seaborn as sns\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg # to check images\n# %matplotlib inline\nfrom tqdm.notebook import tqdm # beautiful progression bar\n\n# SKlearn\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold\nfrom sklearn.metrics import accuracy_score, roc_auc_score, confusion_matrix\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn import preprocessing\n\n# PyTorch\nimport torch\nimport torchvision\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch import FloatTensor, LongTensor\nfrom torch.utils.data import Dataset, DataLoader, Subset\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\n\n# Data Augmentation for Image Preprocessing\nfrom albumentations import (ToFloat, Normalize, VerticalFlip, HorizontalFlip, Compose, Resize,\n RandomBrightnessContrast, HueSaturationValue, Blur, GaussNoise,\n Rotate, RandomResizedCrop, Cutout, ShiftScaleRotate)\nfrom albumentations.pytorch import ToTensorV2, ToTensor\n\nfrom torchvision.models import resnet34, resnet50\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:54:34.178828Z","iopub.execute_input":"2024-05-14T10:54:34.179343Z","iopub.status.idle":"2024-05-14T10:54:34.195226Z","shell.execute_reply.started":"2024-05-14T10:54:34.179292Z","shell.execute_reply":"2024-05-14T10:54:34.193960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# System\nimport cv2\nimport os, os.path\nfrom PIL import Image # from RBG to YCbCr\nimport gc\nimport time\nimport datetime\n\n# Basics\nimport pandas as pd\nimport numpy as np\nimport random\nimport seaborn as sns\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg # to check images\n# %matplotlib inline\nfrom tqdm.notebook import tqdm # beautiful progression bar\n\n# SKlearn\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold\nfrom sklearn.metrics import accuracy_score, roc_auc_score, confusion_matrix\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn import preprocessing\n\n# PyTorch\nimport torch\nimport torchvision\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch import FloatTensor, LongTensor\nfrom torch.utils.data import Dataset, DataLoader, Subset\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\n\n# Data Augmentation for Image Preprocessing\nfrom albumentations import (ToFloat, Normalize, VerticalFlip, HorizontalFlip, Compose, Resize,\n RandomBrightnessContrast, HueSaturationValue, Blur, GaussNoise,\n Rotate, RandomResizedCrop, Cutout, ShiftScaleRotate)\nfrom albumentations.pytorch import ToTensorV2, ToTensor\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:54:34.196862Z","iopub.execute_input":"2024-05-14T10:54:34.197318Z","iopub.status.idle":"2024-05-14T10:54:34.212048Z","shell.execute_reply.started":"2024-05-14T10:54:34.197273Z","shell.execute_reply":"2024-05-14T10:54:34.211029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Set the Seeds\n\nSets the seed of the entire notebook so results are the same every time we run.\n This is for REPRODUCIBILITY.","metadata":{}},{"cell_type":"code","source":"\ndef set_seed(seed = 1234):\n \n np.random.seed(seed)\n random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n # When running on the CuDNN backend, two further options must be set\n torch.backends.cudnn.deterministic = True\n # Set a fixed value for the hash seed\n os.environ['PYTHONHASHSEED'] = str(seed)\n \nset_seed()\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint('Device available now:', device)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:54:34.213556Z","iopub.execute_input":"2024-05-14T10:54:34.213943Z","iopub.status.idle":"2024-05-14T10:54:34.227521Z","shell.execute_reply.started":"2024-05-14T10:54:34.213911Z","shell.execute_reply":"2024-05-14T10:54:34.226466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Preparation\nThe data we'll work on has a Train and Test .csv files with coresponding .jpg images. Number of data points also increased with other external sources.\n\n#### 1. Read in the data ","metadata":{}},{"cell_type":"code","source":"# ----- STATICS -----\noutput_size = 1\n# -------------------","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:54:34.229094Z","iopub.execute_input":"2024-05-14T10:54:34.229480Z","iopub.status.idle":"2024-05-14T10:54:34.239442Z","shell.execute_reply.started":"2024-05-14T10:54:34.229449Z","shell.execute_reply":"2024-05-14T10:54:34.238552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# My Train: with imputed missing values + OHE\nmy_train = pd.read_csv('/kaggle/input/melanoma-clean-csv/train_clean.csv')\n\n# Drop path columns and Diagnosis (it won't be available during TEST)\n# We'll rewrite them once the data is concatenated\nto_drop = ['path_dicom','path_jpeg', 'diagnosis']\nfor drop in to_drop:\n if drop in my_train.columns :\n my_train.drop([drop], axis=1, inplace=True)\n\n# Roman's Train: with added data for Malignant category\nroman_train = pd.read_csv('../input/../input/melanoma-external-malignant-256/train_concat.csv')\n\n\n# --- Before concatenatenating both together, let's preprocess roman_train ---\n# Replace NAN with 0 for patient_id\nroman_train['patient_id'] = roman_train['patient_id'].fillna(0)\n\n# OHE\nto_encode = ['sex', 'anatom_site_general_challenge']\nencoded_all = []\n\nroman_train[to_encode[0]] = roman_train[to_encode[0]].astype(str)\nroman_train[to_encode[1]] = roman_train[to_encode[1]].astype(str)\n\nlabel_encoder = LabelEncoder()\n\nfor column in to_encode:\n encoded = label_encoder.fit_transform(roman_train[column])\n encoded_all.append(encoded)\n \nroman_train[to_encode[0]] = encoded_all[0]\nroman_train[to_encode[1]] = encoded_all[1]\n\n# Give all columns the same name\nroman_train.columns = my_train.columns\n\n\n# --- Concatenate info which is not available in my_train ---\ncommon_images = my_train['dcm_name'].unique()\nnew_data = roman_train[~roman_train['dcm_name'].isin(common_images)]\n\n# Merge all together\ntrain_df = pd.concat([my_train, new_data], axis=0)\n\n\n\n# --- Read in Test data (also cleaned, imputed, OHE) ---\ntest_df = pd.read_csv('/kaggle/input/melanoma-clean-csv/test_clean.csv')\n\n# Drop columns\nfor drop in to_drop:\n if drop in test_df.columns :\n test_df.drop([drop], axis=1, inplace=True)\n\n# Create path column to image folder for both Train and Test\npath_train = '../input/melanoma-external-malignant-256/train/train/'\npath_test = '../input/melanoma-external-malignant-256/test/test/'\n\ntrain_df['path_jpg'] = path_train + train_df['dcm_name'] + '.jpg'\ntest_df['path_jpg'] = path_test + test_df['dcm_name'] + '.jpg'\n\n\n# --- Last final thing: NORMALIZE! ---\ntrain_df['age'] = train_df['age'].fillna(-1)\n\nnormalized_train = preprocessing.normalize(train_df[['sex', 'age', 'anatomy']])\nnormalized_test = preprocessing.normalize(test_df[['sex', 'age', 'anatomy']])\n\ntrain_df['sex'] = normalized_train[:, 0]\ntrain_df['age'] = normalized_train[:, 1]\ntrain_df['anatomy'] = normalized_train[:, 2]\n\ntest_df['sex'] = normalized_test[:, 0]\ntest_df['age'] = normalized_test[:, 1]\ntest_df['anatomy'] = normalized_test[:, 2]\n\n\nprint('Len Train: {:,}'.format(len(train_df)), '\\n' +\n 'Len Test: {:,}'.format(len(test_df)))","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:54:34.240707Z","iopub.execute_input":"2024-05-14T10:54:34.241089Z","iopub.status.idle":"2024-05-14T10:54:34.638774Z","shell.execute_reply.started":"2024-05-14T10:54:34.241056Z","shell.execute_reply":"2024-05-14T10:54:34.637615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 2. PyTorch Dataset\nThis class retrieves the data from the train_df or test_df and reads the corresponding information from the image folders.\n\nNote: when reading the images, custom augmentations are applied. Train will have a complex transformation, while valid data will have no augmentations. Test WILL have augmentations like Train because we're doing Test Time Augmentations, meaning that we'll transform the test images, predict and average the result.","metadata":{}},{"cell_type":"code","source":"# ----- STATICS -----\nvertical_flip = 0.5\nhorizontal_flip = 0.5\n\ncsv_columns = ['sex', 'age', 'anatomy']\nno_columns = 3\n# ------------------","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:54:34.640657Z","iopub.execute_input":"2024-05-14T10:54:34.641157Z","iopub.status.idle":"2024-05-14T10:54:34.646403Z","shell.execute_reply.started":"2024-05-14T10:54:34.641101Z","shell.execute_reply":"2024-05-14T10:54:34.645490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Example of csv_data at index=0\nprint(np.array(train_df.iloc[0][csv_columns].values,dtype=np.float32))","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:54:34.647884Z","iopub.execute_input":"2024-05-14T10:54:34.648313Z","iopub.status.idle":"2024-05-14T10:54:34.660849Z","shell.execute_reply.started":"2024-05-14T10:54:34.648279Z","shell.execute_reply":"2024-05-14T10:54:34.659830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# VIT IMPLEMENTATION","metadata":{}},{"cell_type":"code","source":"import os\nimport re\nimport json\nimport glob\nimport time\nimport random\nimport collections\n\nimport numpy as np\nimport pandas as pd\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport cv2\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.nn import functional as torch_functional\nimport torch.optim as optim\nfrom torch.utils import data as torch_data\nfrom torch.utils.data import Dataset\nfrom torch.utils.data import DataLoader\nfrom torch.utils.data.dataloader import default_collate\nimport torchvision.transforms as transform\n\n\n\nfrom PIL import Image\nfrom sklearn import model_selection as sk_model_selection\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn import preprocessing\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.utils import shuffle","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:54:34.662488Z","iopub.execute_input":"2024-05-14T10:54:34.662847Z","iopub.status.idle":"2024-05-14T10:54:34.676186Z","shell.execute_reply.started":"2024-05-14T10:54:34.662811Z","shell.execute_reply":"2024-05-14T10:54:34.675247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"model","metadata":{}},{"cell_type":"code","source":"class MelanomaDataset(Dataset):\n \n def __init__(self, dataframe, vertical_flip, horizontal_flip,\n is_train=True, is_valid=False, is_test=False):\n self.dataframe, self.is_train, self.is_valid = dataframe, is_train, is_valid\n self.vertical_flip, self.horizontal_flip = vertical_flip, horizontal_flip\n \n # Data Augmentation (custom for each dataset type)\n if is_train or is_test:\n self.transform = Compose([RandomResizedCrop(height=224, width=224, scale=(0.4, 1.0)),\n ShiftScaleRotate(rotate_limit=90, scale_limit = [0.8, 1.2]),\n HorizontalFlip(p = self.horizontal_flip),\n VerticalFlip(p = self.vertical_flip),\n HueSaturationValue(sat_shift_limit=[0.7, 1.3], \n hue_shift_limit=[-0.1, 0.1]),\n RandomBrightnessContrast(brightness_limit=[0.7, 1.3],\n contrast_limit= [0.7, 1.3]),\n Normalize(),\n ToTensor()])\n else:\n self.transform = Compose([Normalize(),\n ToTensor()])\n \n def __len__(self):\n return len(self.dataframe)\n \n def __getitem__(self, index):\n # Select path and read image\n image_path = self.dataframe['path_jpg'][index]\n image = cv2.imread(image_path)\n # For this image also import .csv information (sex, age, anatomy)\n csv_data = np.array(self.dataframe.iloc[index][['sex', 'age', 'anatomy']].values, \n dtype=np.float32)\n \n # Apply transforms\n image = self.transform(image=image)\n # Extract image from dictionary\n image = image['image']\n\n # If train/valid: image + class | If test: only image\n if self.is_train or self.is_valid:\n target = self.dataframe['target'][index]\n if isinstance(target, torch.Tensor):\n target = target.item()\n else:\n target = float(target) # Convert to float\n target = torch.tensor([1 - target, target], dtype=torch.float32)\n return (image, csv_data), target\n else:\n return (image, csv_data)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:54:34.678257Z","iopub.execute_input":"2024-05-14T10:54:34.678723Z","iopub.status.idle":"2024-05-14T10:54:34.700960Z","shell.execute_reply.started":"2024-05-14T10:54:34.678677Z","shell.execute_reply":"2024-05-14T10:54:34.699923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\n\n\n\nclass PatchEmbedding(nn.Module):\n def __init__(self, embed_dim=768, patch_size=16):\n super().__init__()\n self.embed_dim = embed_dim\n self.patch_size = patch_size\n self.projection = nn.Conv2d(3, embed_dim, kernel_size=patch_size, stride=patch_size)\n\n def forward(self, x):\n x = self.projection(x) # (n_samples, embed_dim, n_patches ** 0.5, n_patches ** 0.5)\n x = x.flatten(2) # (n_samples, embed_dim, n_patches)\n x = x.transpose(1, 2) # (n_samples, n_patches, embed_dim)\n return x\n\n\n\nclass VisionTransformer(nn.Module):\n def __init__(\n self,\n img_size=224,\n patch_size=16,\n embed_dim=768,\n num_heads=12,\n num_layers=12,\n num_classes=1,\n dropout=0.1,\n no_columns=3,\n ):\n super().__init__()\n self.embed_dim = embed_dim\n self.patch_embedding = PatchEmbedding(embed_dim, patch_size)\n self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))\n n_patches = (img_size // patch_size) ** 2\n self.pos_embedding = None \n self.dropout = nn.Dropout(dropout)\n\n self.transformer = nn.TransformerEncoder(\n nn.TransformerEncoderLayer(d_model=embed_dim, nhead=num_heads),\n num_layers=num_layers,\n )\n\n self.csv_data_projection = nn.Linear(no_columns, embed_dim)\n\n self.classification = nn.Sequential(\n nn.Linear(embed_dim * 2, 500),\n nn.BatchNorm1d(500),\n nn.ReLU(),\n nn.Dropout(p=0.2),\n nn.Linear(500, num_classes),\n )\n\n def forward(self, image, csv_data, prints=False):\n if prints:\n print('Input Image shape:', image.shape, '\\n' +\n 'Input csv_data shape:', csv_data.shape)\n \n# # Image Embedding\n# image_embeddings = self.patch_embedding(image) # (n_samples, n_patches, embed_dim)\n# b = image_embeddings.shape[0]\n# cls_tokens = self.cls_token.expand(b, -1, -1) # (n_samples, 1, embed_dim)\n# image_embeddings = torch.cat((cls_tokens, image_embeddings), dim=1) # (n_samples, 1 + n_patches, embed_dim)\n\n# # Adjust the size of pos_embedding to match the number of patches\n# n_patches = image_embeddings.size(1) - 1\n# self.pos_embedding = nn.Parameter(self.pos_embedding[:, :n_patches + 1, :])\n\n# image_embeddings = image_embeddings + self.pos_embedding # (n_samples, 1 + n_patches, embed_dim)\n# image_embeddings = self.dropout(image_embeddings)\n# image_embeddings = self.transformer(image_embeddings) # (n_samples, 1 + n_patches, embed_dim)\n# image_embeddings = image_embeddings[:, 0] # (n_samples, embed_dim)\n \n # Image Embedding\n image_embeddings = self.patch_embedding(image) # (n_samples, n_patches, embed_dim)\n b = image_embeddings.shape[0]\n cls_tokens = self.cls_token.expand(b, -1, -1) # (n_samples, 1, embed_dim)\n image_embeddings = torch.cat((cls_tokens, image_embeddings), dim=1) # (n_samples, 1 + n_patches, embed_dim)\n\n # Adjust the size of pos_embedding to match the number of patches\n n_patches = image_embeddings.size(1) - 1\n if self.pos_embedding is None or self.pos_embedding.size(1) != n_patches + 1:\n self.pos_embedding = nn.Parameter(torch.randn(1, 1 + n_patches, self.embed_dim))\n\n image_embeddings = image_embeddings + self.pos_embedding.to(image_embeddings.device) # (n_samples, 1 + n_patches, embed_dim)\n image_embeddings = self.dropout(image_embeddings)\n image_embeddings = self.transformer(image_embeddings) # (n_samples, 1 + n_patches, embed_dim)\n image_embeddings = image_embeddings[:, 0] # (n_samples, embed_dim)\n \n \n if prints:\n print('Image Embeddings shape:', image_embeddings.shape)\n\n # CSV Data Embedding\n csv_data_embeddings = self.csv_data_projection(csv_data)\n if prints:\n print('CSV Data Embeddings:', csv_data_embeddings.shape)\n\n # Concatenate Image and CSV Data Embeddings\n combined_embeddings = torch.cat((image_embeddings, csv_data_embeddings), dim=1)\n \n \n\n # Classification\n out = self.classification(combined_embeddings)\n if prints:\n print('Out shape:', out.shape)\n\n return out","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:54:34.717103Z","iopub.execute_input":"2024-05-14T10:54:34.717501Z","iopub.status.idle":"2024-05-14T10:54:34.749302Z","shell.execute_reply.started":"2024-05-14T10:54:34.717464Z","shell.execute_reply":"2024-05-14T10:54:34.748314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_example =VisionTransformer()","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:54:34.750777Z","iopub.execute_input":"2024-05-14T10:54:34.751178Z","iopub.status.idle":"2024-05-14T10:54:34.898714Z","shell.execute_reply.started":"2024-05-14T10:54:34.751102Z","shell.execute_reply":"2024-05-14T10:54:34.897683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training\n\nPrepare OOF and Predictions Matrixes¶\nOOF will be used to assess the overall ROC value of the entire Train data (Train + Valid)","metadata":{}},{"cell_type":"code","source":"\n\n# ----- STATICS -----\ntrain_len = len(train_df)\ntest_len = len(test_df)\n# -------------------\n\n\n# Out of Fold Predictions\noof = np.zeros(shape = (train_len, 1))\n\n# Predictions\npreds_submission = torch.zeros(size = (test_len, 1), dtype=torch.float32, device=device)\n\nprint('oof shape:', oof.shape, '\\n' +\n 'predictions shape:', preds_submission.shape)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:54:34.900765Z","iopub.execute_input":"2024-05-14T10:54:34.901258Z","iopub.status.idle":"2024-05-14T10:54:34.911716Z","shell.execute_reply.started":"2024-05-14T10:54:34.901209Z","shell.execute_reply":"2024-05-14T10:54:34.910447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### GroupKFold() 🦗🦗🦗\nK-fold iterator variant with non-overlapping groups. The same group will not appear in two different folds (the number of distinct groups has to be at least equal to the number of folds).\n\nWe're using patient_id for our grouping column: there are multiple patients with multiple images taken, so we need to be careful with that.","metadata":{}},{"cell_type":"code","source":"# ----- STATICS -----\nk = 6 # number of folds in Group K Fold\n# -------------------","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:54:34.913368Z","iopub.execute_input":"2024-05-14T10:54:34.913811Z","iopub.status.idle":"2024-05-14T10:54:34.922219Z","shell.execute_reply.started":"2024-05-14T10:54:34.913767Z","shell.execute_reply":"2024-05-14T10:54:34.921201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create Object\ngroup_fold = GroupKFold(n_splits = k)\n\n# Generate indices to split data into training and test set.\nfolds = group_fold.split(X = np.zeros(train_len), \n y = train_df['target'], \n groups = train_df['ID'].tolist())","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:54:34.923772Z","iopub.execute_input":"2024-05-14T10:54:34.924237Z","iopub.status.idle":"2024-05-14T10:54:34.935059Z","shell.execute_reply.started":"2024-05-14T10:54:34.924203Z","shell.execute_reply":"2024-05-14T10:54:34.933695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1.Training Loop\n\nReduceLROnPlateau(): Reduce learning rate when a metric has stopped improving. Here patience is set to 1, meaning that if 1 model doesn't improve, then the lr will decrease by a factor of 0.2.\n\npatience: Early Stopping Patience (how many epochs to wait with no improvement until it stops)\n\nTTA: Test Time Augmentation Rounds (creating multiple augmented copies of each image in the test set, having the model make a prediction for each, then returning an ensemble of those predictions)\n","metadata":{}},{"cell_type":"code","source":"# ----- STATICS -----\nepochs = 10\npatience = 3\nTTA = 3\nnum_workers = 8\nlearning_rate = 0.0005\nweight_decay = 0.0\nlr_patience = 1 # 1 model not improving until lr is decreasing\nlr_factor = 0.4 # by how much the lr is decreasing\n\nbatch_size1 = 32\nbatch_size2 = 16\n\nversion = 'v6' # to keep tabs on versions","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:54:34.937216Z","iopub.execute_input":"2024-05-14T10:54:34.937791Z","iopub.status.idle":"2024-05-14T10:54:34.948864Z","shell.execute_reply.started":"2024-05-14T10:54:34.937735Z","shell.execute_reply":"2024-05-14T10:54:34.947514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_folds(preds_submission, model, version = 'v1'):\n # Creates a .txt file that will contain the logs\n f = open(f\"logs_{version}.txt\", \"w+\")\n \n \n for fold, (train_index, valid_index) in enumerate(folds):\n # Append to .txt\n with open(f\"logs_{version}.txt\", 'a+') as f:\n print('-'*10, 'Fold:', fold+1, '-'*10, file=f)\n print('-'*10, 'Fold:', fold+1, '-'*10)\n\n\n # --- Create Instances ---\n # Best ROC score in this fold\n best_roc = None\n # Reset patience before every fold\n patience_f = patience\n \n # Initiate the model\n model = model\n\n optimizer = torch.optim.Adam(model.parameters(), lr = learning_rate, weight_decay=weight_decay)\n scheduler = ReduceLROnPlateau(optimizer=optimizer, mode='max', \n patience=lr_patience, verbose=True, factor=lr_factor)\n criterion = nn.BCEWithLogitsLoss()\n\n\n # --- Read in Data ---\n train_data = train_df.iloc[train_index].reset_index(drop=True)\n valid_data = train_df.iloc[valid_index].reset_index(drop=True)\n\n # Create Data instances\n train = MelanomaDataset(train_data, vertical_flip=vertical_flip, horizontal_flip=horizontal_flip, \n is_train=True, is_valid=False, is_test=False)\n valid = MelanomaDataset(valid_data, vertical_flip=vertical_flip, horizontal_flip=horizontal_flip, \n is_train=False, is_valid=True, is_test=False)\n # Read in test data | Remember! We're using data augmentation like we use for Train data.\n test = MelanomaDataset(test_df, vertical_flip=vertical_flip, horizontal_flip=horizontal_flip,\n is_train=False, is_valid=False, is_test=True)\n\n # Dataloaders\n train_loader = DataLoader(train, batch_size=batch_size1, shuffle=True, num_workers=num_workers)\n # shuffle=False! Otherwise function won't work!!!\n # how do I know? ^^\n valid_loader = DataLoader(valid, batch_size=batch_size2, shuffle=False, num_workers=num_workers)\n test_loader = DataLoader(test, batch_size=batch_size2, shuffle=False, num_workers=num_workers)\n\n\n # === EPOCHS ===\n for epoch in range(epochs):\n start_time = time.time()\n correct = 0\n train_losses = 0\n\n # === TRAIN ===\n # Sets the module in training mode.\n model.train()\n\n for (images, csv_data), labels in train_loader:\n # Save them to device\n images = torch.tensor(images, device=device, dtype=torch.float32)\n csv_data = torch.tensor(csv_data, device=device, dtype=torch.float32)\n labels = torch.tensor(labels, device=device, dtype=torch.float32)\n\n # Clear gradients first; very important, usually done BEFORE prediction\n optimizer.zero_grad()\n\n # Log Probabilities & Backpropagation\n out = model(images, csv_data)\n labels_example = labels.unsqueeze(1).float()\n labels_example = labels_example[:, :, 1]\n \n loss = criterion(out, labels_example)\n loss.backward()\n optimizer.step()\n\n # --- Save information after this batch ---\n # Save loss\n train_losses += loss.item()\n # From log probabilities to actual probabilities\n train_preds = torch.round(torch.sigmoid(out)) # 0 and 1\n # Number of correct predictions\n correct += (train_preds.cpu() == labels.cpu().unsqueeze(1)).sum().item()\n\n # Compute Train Accuracy\n train_acc = correct / len(train_index)\n\n\n # === EVAL ===\n # Sets the model in evaluation mode\n model.eval()\n\n # Create matrix to store evaluation predictions (for accuracy)\n valid_preds = torch.zeros(size = (len(valid_index), 1), device=device, dtype=torch.float32)\n\n\n # Disables gradients (we need to be sure no optimization happens)\n with torch.no_grad():\n for k, ((images, csv_data), labels) in enumerate(valid_loader):\n images = torch.tensor(images, device=device, dtype=torch.float32)\n csv_data = torch.tensor(csv_data, device=device, dtype=torch.float32)\n labels = torch.tensor(labels, device=device, dtype=torch.float32)\n\n out = model(images, csv_data)\n pred = torch.sigmoid(out)\n valid_preds[k*images.shape[0] : k*images.shape[0] + images.shape[0]] = pred\n\n # Compute accuracy\n valid_acc = accuracy_score(valid_data['target'].values, \n torch.round(valid_preds.cpu()))\n # Compute ROC\n valid_roc = roc_auc_score(valid_data['target'].values, \n valid_preds.cpu())\n\n # Compute time on Train + Eval\n duration = str(datetime.timedelta(seconds=time.time() - start_time))[:7]\n\n\n # PRINT INFO\n # Append to .txt file\n with open(f\"logs_{version}.txt\", 'a+') as f:\n print('{} | Epoch: {}/{} | Loss: {:.4} | Train Acc: {:.3} | Valid Acc: {:.3} | ROC: {:.3}'.\\\n format(duration, epoch+1, epochs, train_losses, train_acc, valid_acc, valid_roc), file=f)\n # Print to console\n print('{} | Epoch: {}/{} | Loss: {:.4} | Train Acc: {:.3} | Valid Acc: {:.3} | ROC: {:.3}'.\\\n format(duration, epoch+1, epochs, train_losses, train_acc, valid_acc, valid_roc))\n\n\n # === SAVE MODEL ===\n\n # Update scheduler (for learning_rate)\n scheduler.step(valid_roc)\n\n # Update best_roc\n if not best_roc: # If best_roc = None\n best_roc = valid_roc\n torch.save(model.state_dict(),\n f\"Fold{fold+1}_Epoch{epoch+1}_ValidAcc_{valid_acc:.3f}_ROC_{valid_roc:.3f}.pth\")\n continue\n\n if valid_roc > best_roc:\n best_roc = valid_roc\n # Reset patience (because we have improvement)\n patience_f = patience\n torch.save(model.state_dict(),\n f\"Fold{fold+1}_Epoch{epoch+1}_ValidAcc_{valid_acc:.3f}_ROC_{valid_roc:.3f}.pth\")\n else:\n # Decrease patience (no improvement in ROC)\n patience_f = patience_f - 1\n if patience_f == 0:\n with open(f\"logs_{version}.txt\", 'a+') as f:\n print('Early stopping (no improvement since 3 models) | Best ROC: {}'.\\\n format(best_roc), file=f)\n print('Early stopping (no improvement since 3 models) | Best ROC: {}'.\\\n format(best_roc))\n break\n\n\n # === INFERENCE ===\n # Choose model with best_roc in this fold\n best_model_path = '../working/' + [file for file in os.listdir('../working') if str(round(best_roc, 3)) in file and 'Fold'+str(fold+1) in file][0]\n # Using best model from Epoch Train\n # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n model = VisionTransformer().to(device)\n state_dict = torch.load(best_model_path)\n # Remove pos_embedding key from the state dict\n state_dict.pop('pos_embedding', None)\n model.load_state_dict(state_dict, strict=False)\n \n\n\n # Set the model in evaluation mode\n model.eval()\n\n\n with torch.no_grad():\n # --- EVAL ---\n # Predicting again on Validation data to get preds for OOF\n valid_preds = torch.zeros(size = (len(valid_index), 1), device=device, dtype=torch.float32)\n\n for k, ((images, csv_data), _) in enumerate(valid_loader):\n images = torch.tensor(images, device=device, dtype=torch.float32)\n csv_data = torch.tensor(csv_data, device=device, dtype=torch.float32)\n\n out = model(images, csv_data)\n pred = torch.sigmoid(out)\n valid_preds[k*images.shape[0] : k*images.shape[0] + images.shape[0]] = pred\n\n # Save info to OOF\n oof[valid_index] = valid_preds.cpu().numpy()\n\n\n # --- TEST ---\n # Now (Finally) prediction for our TEST data\n for i in range(TTA):\n for k, (images, csv_data) in enumerate(test_loader):\n images = torch.tensor(images, device=device, dtype=torch.float32)\n csv_data = torch.tensor(csv_data, device=device, dtype=torch.float32)\n\n out = model(images, csv_data)\n # Covert to probablities\n out = torch.sigmoid(out)\n\n # ADDS! the prediction to the matrix we already created\n preds_submission[k*images.shape[0] : k*images.shape[0] + images.shape[0]] += out\n\n\n # Divide Predictions by TTA (to average the results during TTA)\n preds_submission /= TTA\n\n\n # === CLEANING ===\n # Clear memory\n del train, valid, train_loader, valid_loader, images, labels\n # Garbage collector\n gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:54:34.951639Z","iopub.execute_input":"2024-05-14T10:54:34.952445Z","iopub.status.idle":"2024-05-14T10:54:35.041334Z","shell.execute_reply.started":"2024-05-14T10:54:34.952382Z","shell.execute_reply":"2024-05-14T10:54:35.040240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# --- VIT ---\nmodel = VisionTransformer().to(device)\n\n# # ===== Uncomment and Train =====\ntrain_folds(preds_submission = preds_submission, model = model, version = version)\n\n# # Save OOF values\nsave_oof = pd.DataFrame(data = oof, columns=['oof'])\nsave_oof.to_csv(f'oof_{version}.csv', index=False)\n","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:54:35.043050Z","iopub.execute_input":"2024-05-14T10:54:35.043616Z","iopub.status.idle":"2024-05-14T16:10:01.763459Z","shell.execute_reply.started":"2024-05-14T10:54:35.043570Z","shell.execute_reply":"2024-05-14T16:10:01.762434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Print the logs during training\nf = open('/kaggle/working/logs_v6.txt', \"r\")\ncontents = f.read()\nprint(contents)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:10:01.765080Z","iopub.execute_input":"2024-05-14T16:10:01.765462Z","iopub.status.idle":"2024-05-14T16:10:01.771783Z","shell.execute_reply.started":"2024-05-14T16:10:01.765428Z","shell.execute_reply":"2024-05-14T16:10:01.770772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import OOF (pretrained)\noof = pd.read_csv('/kaggle/input/melanoma-clean-csv/oof_v6.csv')\n\n# ROC on full Training data\nprint('OOF ROC: {:.3f}'.format(roc_auc_score(train_df['target'], oof)))","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:22:08.448373Z","iopub.execute_input":"2024-05-14T16:22:08.448767Z","iopub.status.idle":"2024-05-14T16:22:08.495441Z","shell.execute_reply.started":"2024-05-14T16:22:08.448734Z","shell.execute_reply":"2024-05-14T16:22:08.494572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#2. Confusion Matrix¶\n# Make OOF Binary\noof.loc[oof.oof >= 0.5, 'oof'] = 1\noof.loc[oof.oof < 0.5, 'oof'] = 0\n\n# Create Confusion Matrix\ncf_matrix = confusion_matrix(train_df['target'], oof)\n\n# Pretty CM:\ngroup_names = ['True Neg','False Pos','False Neg','True Pos']\n# Format of the absolute numbers\ngroup_counts = ['{:,}'.format(value) for value in cf_matrix.flatten()]\n# Format for relative numbers\ngroup_percentages = ['{0:.1%}'.format(value) for value in cf_matrix.flatten()/np.sum(cf_matrix)]\n\nlabels = [f'{v1}\\n{v2}\\n{v3}' for v1, v2, v3 in zip(group_names,group_counts,group_percentages)]\nlabels = np.asarray(labels).reshape(2,2)\n\n# --- The figure ---\nplt.figure(figsize=(16, 5))\nsns.heatmap(cf_matrix, annot=labels, fmt='', cmap='Oranges',xticklabels=['benign', 'malignant'], \n yticklabels=['benign', 'malignant'], cbar=False)\n\nmatplotlib.rcParams.update({'font.size': 15})\nplt.tick_params(axis='both', labelsize=15)\nplt.title('Confusion Matrix: OOF Data', fontsize=20);","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:22:31.203676Z","iopub.execute_input":"2024-05-14T16:22:31.204063Z","iopub.status.idle":"2024-05-14T16:22:31.567776Z","shell.execute_reply.started":"2024-05-14T16:22:31.204032Z","shell.execute_reply":"2024-05-14T16:22:31.566170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"# Divide predictions by the number of folds\npreds_submission /= k\npreds_submission = preds_submission.cpu().numpy().reshape(-1,)\n\n# Import submission file\nss = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')\n\nss['target'] = preds_submission\nss.to_csv(f'submission_{version}.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:24:40.349561Z","iopub.execute_input":"2024-05-14T16:24:40.349986Z","iopub.status.idle":"2024-05-14T16:24:40.424302Z","shell.execute_reply.started":"2024-05-14T16:24:40.349951Z","shell.execute_reply":"2024-05-14T16:24:40.423460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Bonus: Best Single Model Function ⏩\nFunction that predicts solely on a pretrained model (uses TTA).\n\n","metadata":{}},{"cell_type":"code","source":"# def best_single_model(model, preds_submission, TTA=3):\n \n# test = MelanomaDataset(test_df, vertical_flip=0.5, horizontal_flip=0.5,\n# is_train=False, is_valid=False, is_test=True)\n# test_loader = DataLoader(test, batch_size=16, shuffle=False, num_workers=8)\n \n# model.eval()\n\n# with torch.no_grad():\n# for i in range(TTA):\n# for k, (images, csv_data) in enumerate(test_loader):\n# images = torch.tensor(images, device=device, dtype=torch.float32)\n# csv_data = torch.tensor(csv_data, device=device, dtype=torch.float32)\n\n# out = model(images, csv_data)\n# # Covert to probablities\n# out = torch.sigmoid(out)\n\n# # ADDS! the prediction to the matrix we already created\n# preds_submission[k*images.shape[0] : k*images.shape[0] + images.shape[0]] += out\n\n\n# # Divide Predictions by TTA (to average the results during TTA)\n# preds_submission /= TTA\n \n# return preds_submissio","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:38:39.735902Z","iopub.execute_input":"2024-05-14T16:38:39.736322Z","iopub.status.idle":"2024-05-14T16:38:39.741350Z","shell.execute_reply.started":"2024-05-14T16:38:39.736285Z","shell.execute_reply":"2024-05-14T16:38:39.740262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# # --- Submission ---\n\n","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}