{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Analysis of SIIM-ISIC Melanoma Classification by Images\n\n# Introduction\n\n## The Competition\n\nSkin cancer is common cancer type and despite beign mostly non malignant, due to high case numbers it's pretty serious diasease and can lead serious cases if not detected, treated in time. It's usually diagnosed by eye for primarily and followed by further clinical analysis if needed. Even though the rares outcome is called melanoma it's the most deadly one, so early detection is pretty important. For this task using computer aided diagnosis might be helpful for primarily steps and early detections. Better detection might save thousands of lives.","metadata":{}},{"cell_type":"markdown","source":"# Preparation","metadata":{}},{"cell_type":"code","source":"!pip install -q efficientnet\n!pip install efficientnet tensorflow_addons\n!pip install pydicom","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\n\nimport pydicom as dicom\nimport seaborn as sns\nimport plotly.express as px\n\nimport os\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' \n\nimport random\nimport re\nimport math\nimport time\n\nfrom tqdm import tqdm\nfrom tqdm.keras import TqdmCallback\nfrom pandas_summary import DataFrameSummary\n\nimport warnings\nwarnings.filterwarnings('ignore') # Disabling warnings for clearer outputs\n\nseed_val = 42\nrandom.seed(seed_val)\nnp.random.seed(seed_val)","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.status.busy":"2022-02-14T12:43:59.628676Z","iopub.execute_input":"2022-02-14T12:43:59.629028Z","iopub.status.idle":"2022-02-14T12:44:06.739749Z","shell.execute_reply.started":"2022-02-14T12:43:59.628991Z","shell.execute_reply":"2022-02-14T12:44:06.738985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import\nimport tensorflow as tf, re, math\nimport tensorflow.keras.backend as K\nimport efficientnet.tfkeras as efn\nfrom kaggle_datasets import KaggleDatasets\n\nimport efficientnet.tfkeras as efn\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\n\ntf.random.set_seed(seed_val)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:44:06.740883Z","iopub.execute_input":"2022-02-14T12:44:06.741937Z","iopub.status.idle":"2022-02-14T12:44:07.134467Z","shell.execute_reply.started":"2022-02-14T12:44:06.741892Z","shell.execute_reply":"2022-02-14T12:44:07.133633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set color palette.\ncolor_palette = list(map(lambda x: '#%02x%02x%02x' % (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)), sns.color_palette(\"magma\", n_colors=7)))\n\n# Set plot styling.\nplt.style.use('ggplot')","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:44:07.136264Z","iopub.execute_input":"2022-02-14T12:44:07.136545Z","iopub.status.idle":"2022-02-14T12:44:07.146051Z","shell.execute_reply.started":"2022-02-14T12:44:07.136510Z","shell.execute_reply":"2022-02-14T12:44:07.144813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set file paths for our notebook:\nbase_path = '/kaggle/input/siim-isic-melanoma-classification'\ntrain_img_path = '/kaggle/input/siim-isic-melanoma-classification/jpeg/train/'\ntest_img_path = '/kaggle/input/siim-isic-melanoma-classification/jpeg/test/'\nimg_stats_path = '/kaggle/input/melanoma2020imgtabular'","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:44:10.268679Z","iopub.execute_input":"2022-02-14T12:44:10.269016Z","iopub.status.idle":"2022-02-14T12:44:10.273050Z","shell.execute_reply.started":"2022-02-14T12:44:10.268984Z","shell.execute_reply":"2022-02-14T12:44:10.272434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Try dcm file\nimage_path = '../input/siim-isic-melanoma-classification/train/ISIC_0015719.dcm'\nds = dicom.dcmread(image_path)\nds","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:44:10.720711Z","iopub.execute_input":"2022-02-14T12:44:10.721418Z","iopub.status.idle":"2022-02-14T12:44:10.860181Z","shell.execute_reply.started":"2022-02-14T12:44:10.721380Z","shell.execute_reply":"2022-02-14T12:44:10.859473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading the data\nTrain data has 8 features, 33126 observations and Test data 5 features, 10982 observations.\n\n#### Train dataset consists of:\n\n1. image name -> the filename of specific image for the train set\n2. patient_id -> identifies the unique patient\n3. sex -> gender of the patient\n4. age_approx -> approx age of the patient at time of scanning\n5. anatom_site_general_challenge -> location of the scan site\n6. diagnosis -> information about the diagnosis\n7. benign_malignant - indicates scan result if it's malignant or benign\n8. target -> same as above but better for modelling since it's binary\n\n#### Test dataset consists of:\n\n1. image name -> the filename of specific image for the train set\n2. patient_id -> identifies the unique patient\n3. sex -> gender of the patient\n4. age_approx -> approx age of the patient at time of scanning\n5. anatom_site_general_challenge -> location of the scan site","metadata":{}},{"cell_type":"code","source":"# Load train and test data\ntrain = pd.read_csv(os.path.join(base_path, 'train.csv'))\ntest = pd.read_csv(os.path.join(base_path, 'test.csv'))\nsample = pd.read_csv(os.path.join(base_path, 'sample_submission.csv'))","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:44:12.200957Z","iopub.execute_input":"2022-02-14T12:44:12.201393Z","iopub.status.idle":"2022-02-14T12:44:12.354688Z","shell.execute_reply.started":"2022-02-14T12:44:12.201348Z","shell.execute_reply":"2022-02-14T12:44:12.353242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Observe train and test columns and amount\ntrain.shape, test.shape, train.columns, test.columns","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:44:13.297588Z","iopub.execute_input":"2022-02-14T12:44:13.297852Z","iopub.status.idle":"2022-02-14T12:44:13.304332Z","shell.execute_reply.started":"2022-02-14T12:44:13.297827Z","shell.execute_reply":"2022-02-14T12:44:13.303416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Rename train/test columns\ntrain.columns = ['img_name', 'id', 'sex', 'age', 'location', 'diagnosis', 'benign_malignant', 'target']\ntest.columns = ['img_name', 'id', 'sex', 'age', 'location']","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:44:13.832802Z","iopub.execute_input":"2022-02-14T12:44:13.833765Z","iopub.status.idle":"2022-02-14T12:44:13.839750Z","shell.execute_reply.started":"2022-02-14T12:44:13.833712Z","shell.execute_reply":"2022-02-14T12:44:13.838997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Observe 5 random samples from the train set\ntrain.sample(5)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:44:14.098789Z","iopub.execute_input":"2022-02-14T12:44:14.099411Z","iopub.status.idle":"2022-02-14T12:44:14.126223Z","shell.execute_reply.started":"2022-02-14T12:44:14.099361Z","shell.execute_reply":"2022-02-14T12:44:14.125498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Observe 5 random samples from the test set\ntest.sample(5)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:44:14.441703Z","iopub.execute_input":"2022-02-14T12:44:14.441970Z","iopub.status.idle":"2022-02-14T12:44:14.455883Z","shell.execute_reply.started":"2022-02-14T12:44:14.441943Z","shell.execute_reply":"2022-02-14T12:44:14.454976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['diagnosis'].unique(), train['location'].unique()","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:44:14.816973Z","iopub.execute_input":"2022-02-14T12:44:14.817251Z","iopub.status.idle":"2022-02-14T12:44:14.835003Z","shell.execute_reply.started":"2022-02-14T12:44:14.817225Z","shell.execute_reply":"2022-02-14T12:44:14.834311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nprint('Examples WITH Melanoma')\nimgs = train.loc[train.target == 1].sample(10).img_name.values\nplt.figure(figsize=(20, 8))\nfor i, k in enumerate(imgs):\n    img = cv2.imread('../input/siim-isic-melanoma-classification/jpeg/train/%s.jpg' % k)\n    img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)\n    plt.subplot(2, 5, i+1)\n    plt.axis('off')\n    plt.imshow(img)\nplt.show()\n\nprint('Examples WITHOUT Melanoma')\nimgs = train.loc[train.target == 0].sample(10).img_name.values\nplt.figure(figsize=(20, 8))\nfor i, k in enumerate(imgs):\n    img = cv2.imread('../input/siim-isic-melanoma-classification/jpeg/train/%s.jpg' % k)\n    img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)\n    plt.subplot(2, 5, i+1)\n    plt.axis('off')\n    plt.imshow(img)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:44:15.216625Z","iopub.execute_input":"2022-02-14T12:44:15.217217Z","iopub.status.idle":"2022-02-14T12:44:30.156737Z","shell.execute_reply.started":"2022-02-14T12:44:15.217175Z","shell.execute_reply":"2022-02-14T12:44:30.156143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Observe missing values","metadata":{}},{"cell_type":"code","source":"# Checking missing values:\ndef missing_percentage(df):\n    total = df.isnull().sum().sort_values(ascending=False)[\n        df.isnull().sum().sort_values(ascending=False) != 0]\n    percent = (df.isnull().sum().sort_values(ascending=False) / len(df) * 100)[\n        (df.isnull().sum().sort_values(ascending=False) / len(df) * 100) != 0]\n    return pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\n\nmissing_train = missing_percentage(train)\nmissing_test = missing_percentage(test)\n\nfig, ax = plt.subplots(1, 2, figsize=(16, 6))\nsns.barplot(x=missing_train.index, y='Percent',\n            data=missing_train, palette=color_palette, ax=ax[0])\n\nsns.barplot(x=missing_test.index, y='Percent',\n            data=missing_test, palette=color_palette, ax=ax[1])\n\nax[0].set_title('Train set missing values')\nax[1].set_title('Test set missing values')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-14T12:44:30.158169Z","iopub.execute_input":"2022-02-14T12:44:30.158619Z","iopub.status.idle":"2022-02-14T12:44:30.646426Z","shell.execute_reply.started":"2022-02-14T12:44:30.158575Z","shell.execute_reply":"2022-02-14T12:44:30.644976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Observation of meta features before filling missing values","metadata":{}},{"cell_type":"code","source":"# Create a grid\nfig = plt.figure(constrained_layout=True, figsize=(20, 9))\ngrid = gridspec.GridSpec(ncols=4, nrows=2, figure=fig)\n\n# Gender distribution\nax1 = fig.add_subplot(grid[0, :2])\nax1.set_title('Gender distribution')\nsns.countplot(train.sex.sort_values(ignore_index=True),\n              alpha=0.9, ax=ax1, color=color_palette[0], label='Train')\nsns.countplot(test.sex.sort_values(ignore_index=True),\n              alpha=0.7, ax=ax1, color=color_palette[2], label='Test')\nax1.legend()\n\n# Scanned site distribution\nax2 = fig.add_subplot(grid[0, 2:])\nax2.set_title('Scanned site distribution')\nsns.countplot(train.location,\n              alpha=0.9, ax=ax2, color=color_palette[0],\n              label='Train', order=train['location'].value_counts().index)\nsns.countplot(test.location,\n              alpha=0.7, ax=ax2, color=color_palette[2],\n              label='Test', order=test['location'].value_counts().index)\nax2.legend()\n\n# Age distribution\nax3 = fig.add_subplot(grid[1, :])\nax3.set_title('Age distribution')\nsns.distplot(train.age, ax=ax3, label='Train', color=color_palette[0])\nsns.distplot(test.age, ax=ax3, label='Test', color=color_palette[2])\nax3.legend()\n\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-14T12:44:30.648294Z","iopub.execute_input":"2022-02-14T12:44:30.648626Z","iopub.status.idle":"2022-02-14T12:44:32.400390Z","shell.execute_reply.started":"2022-02-14T12:44:30.648585Z","shell.execute_reply":"2022-02-14T12:44:32.399300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fill missing data","metadata":{}},{"cell_type":"code","source":"# Fill missing scanned site values with 'unknown' tag:\nfor df in [train, test]:\n    df['location'].fillna('unknown', inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:44:32.402505Z","iopub.execute_input":"2022-02-14T12:44:32.402841Z","iopub.status.idle":"2022-02-14T12:44:32.414569Z","shell.execute_reply.started":"2022-02-14T12:44:32.402801Z","shell.execute_reply":"2022-02-14T12:44:32.413639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check\nids_train = train.location.values\nids_test = test.location.values\nids_train_set = set(ids_train)\nids_test_set = set(ids_test)\n\nlocation_not_overlap = list(ids_train_set.symmetric_difference(ids_test_set))\nn_overlap = len(location_not_overlap)\nn_overlap == 0","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:44:32.415936Z","iopub.execute_input":"2022-02-14T12:44:32.416259Z","iopub.status.idle":"2022-02-14T12:44:32.427223Z","shell.execute_reply.started":"2022-02-14T12:44:32.416229Z","shell.execute_reply":"2022-02-14T12:44:32.426321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fill age and sex with appropriate values\ntrain['sex'].fillna(train['sex'].mode()[0], inplace=True)\ntrain['age'].fillna(train['age'].median(), inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:44:33.434050Z","iopub.execute_input":"2022-02-14T12:44:33.434538Z","iopub.status.idle":"2022-02-14T12:44:33.448637Z","shell.execute_reply.started":"2022-02-14T12:44:33.434496Z","shell.execute_reply":"2022-02-14T12:44:33.447802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check missing value counts\ntrain.isnull().sum().sum(), test.isnull().sum().sum()","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:44:33.891375Z","iopub.execute_input":"2022-02-14T12:44:33.892185Z","iopub.status.idle":"2022-02-14T12:44:33.927541Z","shell.execute_reply.started":"2022-02-14T12:44:33.892150Z","shell.execute_reply":"2022-02-14T12:44:33.926977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Observe the data\n\n## Distribution of scanned site between train set and test set","metadata":{}},{"cell_type":"code","source":"# Train set\ncntstr = train.location.value_counts().rename_axis('location').reset_index(name='count')\n\nfig = px.treemap(cntstr,\n                 path=['location'], values='count',\n                 color='count', color_continuous_scale=color_palette,\n                 title='Distribution of scanned site - Train set')\nfig.update_traces(textinfo='label+percent entry')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:44:34.881971Z","iopub.execute_input":"2022-02-14T12:44:34.882399Z","iopub.status.idle":"2022-02-14T12:44:35.867816Z","shell.execute_reply.started":"2022-02-14T12:44:34.882369Z","shell.execute_reply":"2022-02-14T12:44:35.866882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Test set\ncntste = test.location.value_counts().rename_axis('location').reset_index(name='count')\n\nfig = px.treemap(cntste,\n                 path=['location'], values='count',\n                 color='count', color_continuous_scale=color_palette,\n                 title='Distribution of scanned site - Test set')\n\nfig.update_traces(textinfo='label+percent entry')\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-14T12:44:35.869460Z","iopub.execute_input":"2022-02-14T12:44:35.869699Z","iopub.status.idle":"2022-02-14T12:44:35.950867Z","shell.execute_reply.started":"2022-02-14T12:44:35.869670Z","shell.execute_reply":"2022-02-14T12:44:35.950016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Distribution of scanned site between genders and target\n\nSome scanned sites may be more likely to be malignant, head/neck comes first with followed by oral/genital and upper extremity. Scanned sites are similar between males and females with small differences on distribution.","metadata":{}},{"cell_type":"code","source":"# Create a grid\nfig = plt.figure(constrained_layout=True, figsize=(20, 9))\ngrid = gridspec.GridSpec(ncols=4, nrows=2, figure=fig)\n\n# Scanned site - Female\nax1 = fig.add_subplot(grid[1, :2])\nax1.set_title('Scanned site - Female')\nsns.countplot(\n    train[train['sex'] == 'female'].location.sort_values(ignore_index=True),\n    alpha=0.9, ax=ax1, color=color_palette[0], label='Female',\n    order=train['location'].value_counts().index)\nax1.legend()\n\n# Scanned site - Male\nax2 = fig.add_subplot(grid[1, 2:])\nax2.set_title('Scanned site - Male')\nsns.countplot(\n    train[train['sex'] == 'male'].location.sort_values(ignore_index=True),\n    alpha=0.9, ax=ax2, color=color_palette[-1], label='Male', \n    order=train['location'].value_counts().index)\nax2.legend()\n\n# Malignant ratio per scanned site\nax3 = fig.add_subplot(grid[0, :])\nax3.set_title('Malignant ratio per scanned site')\nloc_freq = train.groupby('location')['target'].mean().sort_values(ascending=False)\nsns.barplot(x=loc_freq.index, y=loc_freq, palette=color_palette, ax=ax3)\nax3.legend()\n\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-14T12:44:36.567566Z","iopub.execute_input":"2022-02-14T12:44:36.568273Z","iopub.status.idle":"2022-02-14T12:44:37.473021Z","shell.execute_reply.started":"2022-02-14T12:44:36.568233Z","shell.execute_reply":"2022-02-14T12:44:37.471714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# A general look with sunburst chart\n\n- Only 2% of our targets are malignant\n- On malignant images males are dominant with 62% \n- Gender wise benign images are more balance 52-48% male female ratio\n- Malignant image scan locations differs based on the patients gender:\n    - Meanwhile the torso is most common location in males it's almost half of the scans meanwhile in females it's 39%\n    - Lower extremity is more common with female scans than males 18% males vs 26% females\n    - Again upper extremity malignant scans is common with females than males (23- 17%)\n- Benign image scan locations more similar between male and female patients.","metadata":{}},{"cell_type":"code","source":"# Plot interactive sunburst chart\nfig = px.sunburst(data_frame=train,\n                  path=['benign_malignant', 'sex', 'diagnosis'],\n                  color='sex', color_discrete_sequence=color_palette,\n                  maxdepth=-1, title='Sunburst Chart Benign/Malignant > Sex > Location')\nfig.update_traces(textinfo='label+percent parent')\nfig.update_layout(margin=dict(t=0, l=0, r=0, b=0))\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:44:37.476952Z","iopub.execute_input":"2022-02-14T12:44:37.477485Z","iopub.status.idle":"2022-02-14T12:44:38.317349Z","shell.execute_reply.started":"2022-02-14T12:44:37.477440Z","shell.execute_reply":"2022-02-14T12:44:38.316528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Distribution of target between genders and age\n\nAge is relatively correlated to the target. Getting malignant result is more possible for elder patients than young patients. There is spike for both genders after age of 85, however, there isn't much of 80+ patients which may explain this spike. It's safe to say it's more likely to be malignant scan after age of 60.","metadata":{}},{"cell_type":"code","source":"# Plotting age vs sex vs target:\nfig, ax = plt.subplots(1, 2, figsize=(16, 6))\nsns.lineplot(x='age', y='target', data=train,\n             ax=ax[0], hue='sex', palette=color_palette[2:4], ci=None)\nsns.boxplot(x='benign_malignant', y='age', data=train,\n            ax=ax[1], hue='sex', palette=color_palette[2:4])\nplt.legend(loc='lower right')\nax[0].set_title('Malignant scan ratio by age')\nax[1].set_title('Scan results by age and sex')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:44:38.319184Z","iopub.execute_input":"2022-02-14T12:44:38.319410Z","iopub.status.idle":"2022-02-14T12:44:38.901833Z","shell.execute_reply.started":"2022-02-14T12:44:38.319385Z","shell.execute_reply":"2022-02-14T12:44:38.900888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Age Round Two\n\nWanted to double check age distributions after our previous observations. Age seems evenly distributed on both train and test datasets, we can see small bumps at age 75+ and around 40, these worth investigating.\n\nWe can see again older people are more likely to get malignant scan results. One last thing about age distributions, we see more female patients in younger ages this trend changes with the older patients...","metadata":{}},{"cell_type":"code","source":"# Create a grid\nfig = plt.figure(constrained_layout=True, figsize=(20, 6))\ngrid = gridspec.GridSpec(ncols=4, nrows=1, figure=fig)\n\n# Age distribution by target\nax1 = fig.add_subplot(grid[0, :2])\nax1.set_title('Age distribution by target')\nsns.kdeplot(train[train['target'] == 0]['age'],\n            shade=True, ax=ax1, color=color_palette[2],\n            label='Benign')\nsns.kdeplot(train[train['target'] == 1]['age'],\n            shade=True, ax=ax1, color=color_palette[0],\n            label='Malignant')\nax1.legend()\n\n# Age distribution by gender\nax2 = fig.add_subplot(grid[0, 2:])\nax2.set_title('Age distribution by gender')\nsns.distplot(train[train.sex == 'female'].age,\n             ax=ax2, label='Female', color=color_palette[0])\nsns.distplot(train[train.sex == 'male'].age,\n             ax=ax2, label='Male', color=color_palette[2])\nax2.legend()\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-14T12:44:39.072736Z","iopub.execute_input":"2022-02-14T12:44:39.073053Z","iopub.status.idle":"2022-02-14T12:44:40.537634Z","shell.execute_reply.started":"2022-02-14T12:44:39.073021Z","shell.execute_reply":"2022-02-14T12:44:40.536760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Diagnosis Distribution\n\nThis part we can't use in our model but it's giving us some insights about this disease so we can inspect that too.","metadata":{}},{"cell_type":"code","source":"diag = train.diagnosis.value_counts()\nfig = px.pie(diag,\n             values='diagnosis', names=diag.index, \n             color_discrete_sequence=color_palette, hole=.4)\nfig.update_traces(textinfo='percent+label', pull=0.05)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:44:41.081089Z","iopub.execute_input":"2022-02-14T12:44:41.081379Z","iopub.status.idle":"2022-02-14T12:44:41.170514Z","shell.execute_reply.started":"2022-02-14T12:44:41.081351Z","shell.execute_reply":"2022-02-14T12:44:41.169540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ML algorithm on Meta Feature","metadata":{}},{"cell_type":"code","source":"# Import packages\nimport xgboost as xgb\n\nfrom sklearn.model_selection import StratifiedKFold, train_test_split, cross_val_score, cross_validate\nfrom sklearn.metrics import roc_auc_score, roc_curve","metadata":{"execution":{"iopub.status.busy":"2022-02-12T23:54:58.885242Z","iopub.execute_input":"2022-02-12T23:54:58.885574Z","iopub.status.idle":"2022-02-12T23:54:59.315195Z","shell.execute_reply.started":"2022-02-12T23:54:58.885547Z","shell.execute_reply":"2022-02-12T23:54:59.314038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load lanscape data\ntrain40 = pd.read_csv('../input/melanoma2020imgtabular/train40Features.csv')\ntrainmet = pd.read_csv('../input/melanoma2020imgtabular/trainMetrics.csv')\n\ntest40 = pd.read_csv('../input/melanoma2020imgtabular/test40Features.csv')\ntestmet = pd.read_csv('../input/melanoma2020imgtabular/testMetrics.csv')","metadata":{"execution":{"iopub.status.busy":"2022-02-12T23:55:00.100655Z","iopub.execute_input":"2022-02-12T23:55:00.100938Z","iopub.status.idle":"2022-02-12T23:55:01.90828Z","shell.execute_reply.started":"2022-02-12T23:55:00.100913Z","shell.execute_reply":"2022-02-12T23:55:01.90752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Pre-processing\none_hot = pd.get_dummies(train['sex'], prefix='sex')\ntrain = pd.concat([train, one_hot], axis=1)\none_hot = pd.get_dummies(test['sex'], prefix='sex')\ntest = pd.concat([test, one_hot], axis=1)\n\none_hot = pd.get_dummies(train['location'], prefix='anatom')\ntrain = pd.concat([train, one_hot], axis=1)\none_hot = pd.get_dummies(test['location'], prefix='anatom')\ntest = pd.concat([test, one_hot], axis=1)\n\ntrain.drop(['sex','img_name','id','benign_malignant', 'diagnosis', 'location'], axis=1, inplace=True)\ntest.drop(['sex','img_name','id','benign_malignant', 'diagnosis', 'location'], axis=1, inplace=True)\n\n# Drop duplicate and useless data\ntrain40.drop(['sex', 'age_approx', 'anatom_site_general_challenge'], axis=1, inplace=True)\ntest40.drop(['sex', 'age_approx', 'anatom_site_general_challenge'], axis=1, inplace=True)\n\n# Merge data\ntrain = pd.concat([train, train40, trainmet], axis=1)\ntest = pd.concat([test, test40, testmet], axis=1)\n\n# Devide train set and label\nX = train.drop('target', axis=1)\ny = train.target","metadata":{"execution":{"iopub.status.busy":"2022-02-12T23:55:01.909692Z","iopub.execute_input":"2022-02-12T23:55:01.910117Z","iopub.status.idle":"2022-02-12T23:55:02.008019Z","shell.execute_reply.started":"2022-02-12T23:55:01.910084Z","shell.execute_reply":"2022-02-12T23:55:02.006704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Set up cross-validation","metadata":{}},{"cell_type":"code","source":"# taking holdout set for validating with stratified y\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, stratify=y, random_state=42)\n\n# 5 fold stratify for cv\ncv = StratifiedKFold(5, shuffle=True, random_state=42)\n\n# Initialization\nxg = xgb.XGBClassifier(\n    n_estimators=750,\n    min_child_weight=0.81,\n    learning_rate=0.025,\n    max_depth=2,\n    subsample=0.80,\n    colsample_bytree=0.42,\n    gamma=0.10,\n    random_state=42,\n    n_jobs=-1)","metadata":{"execution":{"iopub.status.busy":"2022-02-12T23:55:04.557638Z","iopub.execute_input":"2022-02-12T23:55:04.558026Z","iopub.status.idle":"2022-02-12T23:55:04.632009Z","shell.execute_reply.started":"2022-02-12T23:55:04.55799Z","shell.execute_reply":"2022-02-12T23:55:04.631377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Run cross-validation\ncv_results = cross_validate(xg, X_train, y_train, \n                            cv=cv, scoring='roc_auc', \n                            return_train_score=True, n_jobs=-1)\n\ncv_results","metadata":{"execution":{"iopub.status.busy":"2022-02-12T23:55:05.368624Z","iopub.execute_input":"2022-02-12T23:55:05.369082Z","iopub.status.idle":"2022-02-12T23:57:03.939841Z","shell.execute_reply.started":"2022-02-12T23:55:05.36905Z","shell.execute_reply":"2022-02-12T23:57:03.938207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluation on test set\nxg.fit(X_train, y_train)\nvalidation = xg.predict_proba(X_test)[:, 1]\nroc_auc_score(y_test, validation)","metadata":{"execution":{"iopub.status.busy":"2022-02-12T23:57:03.942599Z","iopub.execute_input":"2022-02-12T23:57:03.943046Z","iopub.status.idle":"2022-02-12T23:58:08.470721Z","shell.execute_reply.started":"2022-02-12T23:57:03.942994Z","shell.execute_reply":"2022-02-12T23:58:08.469651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_roc_feat(y_trues, y_preds, est, x_max=1.0):\n    color_palette = list(map(lambda x: '#%02x%02x%02x' % (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)), sns.color_palette(\"magma\", n_colors=20)))\n    \n    fig, ax = plt.subplots(1,2, figsize=(16,6))\n    for i, y_pred in enumerate(y_preds):\n        y_true = y_trues[i]\n        fpr, tpr, thresholds = roc_curve(y_true, y_pred)\n        auc = roc_auc_score(y_true, y_pred)\n        ax[0].plot(fpr, tpr, label='AUC=%.3f' % auc, marker='o', markersize=1, c=color_palette[0])\n\n    ax[0].legend()\n    ax[0].grid()\n    ax[0].plot(np.linspace(0, 1, 20), np.linspace(0, 1, 20), linestyle='--', c=color_palette[-10])\n    ax[0].set_title('ROC curve')\n    ax[0].set_xlabel('False Positive Rate')\n    ax[0].set_xlim([-0.01, x_max])\n    _ = ax[0].set_ylabel('True Positive Rate')\n    \n    \n    feature_importance = est.get_booster().get_score(importance_type='weight')\n\n    keys = list(feature_importance.keys())\n    values = list(feature_importance.values())\n\n    importance = pd.DataFrame(data=values, index=keys, \n                              columns=['score']).sort_values(by='score', ascending=False)\n    \n    sns.barplot(x=importance.score.iloc[:20], y=importance.index[:20],\n            orient='h', palette=color_palette, ax=ax[1])\n    ax[1].set_title('Feature Importances')","metadata":{"execution":{"iopub.status.busy":"2022-02-13T00:05:01.951848Z","iopub.execute_input":"2022-02-13T00:05:01.952147Z","iopub.status.idle":"2022-02-13T00:05:01.965341Z","shell.execute_reply.started":"2022-02-13T00:05:01.952121Z","shell.execute_reply":"2022-02-13T00:05:01.964282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_roc_feat([y_test], [validation], xg)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T00:05:08.946868Z","iopub.execute_input":"2022-02-13T00:05:08.947173Z","iopub.status.idle":"2022-02-13T00:05:09.37507Z","shell.execute_reply.started":"2022-02-13T00:05:08.947144Z","shell.execute_reply":"2022-02-13T00:05:09.374114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Neural Networks\n\nFor this part I was inspired by AgentAuers's 'Incredible TPUs' [here](https://www.kaggle.com/agentauers/incredible-tpus-finetune-effnetb0-b6-at-once).\nWe start by importing neccesary packages and setting random seed.","metadata":{}},{"cell_type":"code","source":"# Set TPU as main device for training, if you get warnings while working with tpu's ignore them.\nDEVICE = 'TPU'\nif DEVICE == 'TPU':\n    print('connecting to TPU...')\n    try:        \n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        print('Could not connect to TPU')\n        tpu = None\n\n    if tpu:\n        try:\n            print('Initializing  TPU...')\n            tf.config.experimental_connect_to_cluster(tpu)\n            tf.tpu.experimental.initialize_tpu_system(tpu)\n            strategy = tf.distribute.experimental.TPUStrategy(tpu)\n            print('TPU initialized')\n        except _:\n            print('Failed to initialize TPU!')\n    else:\n        DEVICE = 'GPU'\n\nif DEVICE != 'TPU':\n    print('Using default strategy for CPU and single GPU')\n    strategy = tf.distribute.get_strategy()\n\nif DEVICE == 'GPU':\n    print('Num GPUs Available: ',\n          len(tf.config.experimental.list_physical_devices('GPU')))\n\nprint('REPLICAS: ', strategy.num_replicas_in_sync)\nAUTO = tf.data.experimental.AUTOTUNE","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:44:46.433774Z","iopub.execute_input":"2022-02-14T12:44:46.435285Z","iopub.status.idle":"2022-02-14T12:44:52.215606Z","shell.execute_reply.started":"2022-02-14T12:44:46.435211Z","shell.execute_reply":"2022-02-14T12:44:52.214659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Configuration\ncfg = dict(batch_size=32,\n           img_size=384,\n    \n           lr_start=0.000005,\n           lr_max=0.00000125,\n           lr_min=0.000001,\n           lr_rampup=5,\n           lr_sustain=0,\n           lr_decay=0.8,\n           epochs=10,\n    \n           transform_prob=1.0,\n           rot=180.0,\n           shr=2.0,\n           hzoom=8.0,\n           wzoom=8.0,\n           hshift=8.0,\n           wshift=8.0,\n    \n           optimizer='adam',\n           label_smooth_fac=0.05,\n           tta_steps=20)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:44:52.218156Z","iopub.execute_input":"2022-02-14T12:44:52.218488Z","iopub.status.idle":"2022-02-14T12:44:52.225692Z","shell.execute_reply.started":"2022-02-14T12:44:52.218444Z","shell.execute_reply":"2022-02-14T12:44:52.224573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_mat(rotation, shear, height_zoom, width_zoom, height_shift,\n            width_shift):\n    \n    ''' Settings for image preparations '''\n\n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear = math.pi * shear / 180.\n\n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1], dtype='float32')\n    zero = tf.constant([0], dtype='float32')\n    rotation_matrix = tf.reshape(tf.concat([c1, s1, zero, -s1, c1, zero, zero, zero, one], axis=0),\n        [3, 3])\n\n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape(tf.concat([one, s2, zero, zero, c2, zero, zero, zero, one], axis=0), [3, 3])\n\n    # ZOOM MATRIX\n    zoom_matrix = tf.reshape(\n        tf.concat([one / height_zoom, zero, zero, zero, one / width_zoom, zero, zero, zero, one],\n                  axis=0), [3, 3])\n\n    # SHIFT MATRIX\n    shift_matrix = tf.reshape(\n        tf.concat([one, zero, height_shift, zero, one, width_shift, zero, zero, one], axis=0), [3, 3])\n\n    return K.dot(K.dot(rotation_matrix, shear_matrix),\n                 K.dot(zoom_matrix, shift_matrix))\n\n\ndef transform(image, cfg):\n    \n    ''' This function takes input images of [: , :, 3] sizes and returns them as randomly rotated, sheared, shifted and zoomed. '''\n\n    DIM = cfg['img_size']\n    XDIM = DIM % 2  # fix for size 331\n\n    rot = cfg['rot'] * tf.random.normal([1], dtype='float32')\n    shr = cfg['shr'] * tf.random.normal([1], dtype='float32')\n    h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / cfg['hzoom']\n    w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / cfg['wzoom']\n    h_shift = cfg['hshift'] * tf.random.normal([1], dtype='float32')\n    w_shift = cfg['wshift'] * tf.random.normal([1], dtype='float32')\n\n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot, shr, h_zoom, w_zoom, h_shift, w_shift)\n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat(tf.range(DIM // 2, -DIM // 2, -1), DIM)\n    y = tf.tile(tf.range(-DIM // 2, DIM // 2), [DIM])\n    z = tf.ones([DIM * DIM], dtype='int32')\n    idx = tf.stack([x, y, z])\n\n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m, tf.cast(idx, dtype='float32'))\n    idx2 = K.cast(idx2, dtype='int32')\n    idx2 = K.clip(idx2, -DIM // 2 + XDIM + 1, DIM // 2)\n\n    # FIND ORIGIN PIXEL VALUES\n    idx3 = tf.stack([DIM // 2 - idx2[0, ], DIM // 2 - 1 + idx2[1, ]])\n    d = tf.gather_nd(image, tf.transpose(idx3))\n\n    return tf.reshape(d, [DIM, DIM, 3])\n\ndef prepare_image(img, cfg=None, augment=True):\n    \n    ''' This function loads the image, resizes it, casts a tensor to a new type float32 in our case, transforms it using the function just above, then applies the augmentations.'''\n    \n    img = tf.image.decode_jpeg(img, channels=3)\n    img = tf.image.resize(img, [cfg['img_size'], cfg['img_size']], antialias=True)\n    img = tf.cast(img, tf.float32) / 255.0\n\n    if augment:\n        if cfg['transform_prob'] > tf.random.uniform([1], minval=0, maxval=1):\n            img = transform(img, cfg)\n\n        img = tf.image.random_flip_left_right(img)\n        img = tf.image.random_saturation(img, 0.7, 1.3)\n        img = tf.image.random_contrast(img, 0.8, 1.2)\n        img = tf.image.random_brightness(img, 0.1)\n\n    return img","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:44:52.227460Z","iopub.execute_input":"2022-02-14T12:44:52.227801Z","iopub.status.idle":"2022-02-14T12:44:52.256257Z","shell.execute_reply.started":"2022-02-14T12:44:52.227762Z","shell.execute_reply":"2022-02-14T12:44:52.255400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'image_name': tf.io.FixedLenFeature([], tf.string),\n        'patient_id': tf.io.FixedLenFeature([], tf.int64),\n        'sex': tf.io.FixedLenFeature([], tf.int64),\n        'age_approx': tf.io.FixedLenFeature([], tf.int64),\n        'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),\n        'diagnosis': tf.io.FixedLenFeature([], tf.int64),\n        'target': tf.io.FixedLenFeature([], tf.int64)}\n\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    return example['image'], example['target']\n\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'image_name': tf.io.FixedLenFeature([], tf.string),\n        'patient_id': tf.io.FixedLenFeature([], tf.int64),\n        'sex': tf.io.FixedLenFeature([], tf.int64),\n        'age_approx': tf.io.FixedLenFeature([], tf.int64),\n        'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64)}\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    return example['image'], example['image_name']\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r'-([0-9]*)\\.').search(filename).group(1))\n         for filename in filenames]\n    return np.sum(n)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:44:52.258494Z","iopub.execute_input":"2022-02-14T12:44:52.259253Z","iopub.status.idle":"2022-02-14T12:44:52.273168Z","shell.execute_reply.started":"2022-02-14T12:44:52.259196Z","shell.execute_reply":"2022-02-14T12:44:52.272340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def getTrainDataset(files, cfg, augment=True, shuffle=True, repeat=True):\n    ''' This function reads the tfrecord train images, shuffles them, apply augmentations to them and prepares the data for training. '''\n    \n    ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)\n    ds = ds.cache()\n\n    if shuffle:\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n\n    ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)\n    if repeat:\n        ds = ds.repeat()\n        \n    if shuffle:\n        ds = ds.shuffle(2048)\n        \n    ds = ds.map(lambda img, label:\n                (prepare_image(img, augment=augment, cfg=cfg), label),\n                num_parallel_calls=AUTO)\n    ds = ds.batch(cfg['batch_size'] * strategy.num_replicas_in_sync)\n    ds = ds.prefetch(AUTO)\n    return ds\n\ndef getTestDataset(files, cfg, augment=False, repeat=False):\n    ''' This function reads the tfrecord test images and prepares the data for predicting. '''\n    \n    ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)\n    ds = ds.cache()\n    if repeat:\n        ds = ds.repeat()\n    ds = ds.map(read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    ds = ds.map(lambda img, idnum:\n                (prepare_image(img, augment=augment, cfg=cfg), idnum),\n                num_parallel_calls=AUTO)\n    ds = ds.batch(cfg['batch_size'] * strategy.num_replicas_in_sync)\n    ds = ds.prefetch(AUTO)\n    return ds","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:44:52.274621Z","iopub.execute_input":"2022-02-14T12:44:52.274832Z","iopub.status.idle":"2022-02-14T12:44:52.287837Z","shell.execute_reply.started":"2022-02-14T12:44:52.274807Z","shell.execute_reply":"2022-02-14T12:44:52.286856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model(weights):\n    ''' This function gets the layers inclunding efficientnet ones. '''\n    \n    model_input = tf.keras.Input(shape=(cfg['img_size'], cfg['img_size'], 3), name='img_input')\n\n    x = efn.EfficientNetB5(include_top=False, weights=weights,\n                           input_shape=(cfg['img_size'], cfg['img_size'], 3), pooling='avg')(model_input)\n    x = tf.keras.layers.Dense(1, activation='sigmoid')(x)\n    \n    model = tf.keras.Model(model_input, x, name='EffNet')\n    return model\n\ndef compileModel(cfg, weights='noisy-student'):\n    ''' Configuring the model with losses and metrics. '''    \n    \n    with strategy.scope():\n        model = get_model(weights=weights)\n\n    with strategy.scope():\n        model.compile(optimizer=cfg['optimizer'],\n                      loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=cfg['label_smooth_fac']),\n                      metrics=[tf.keras.metrics.AUC(name='auc')])\n    return model\n\n\ndef getLearnRateCallback(cfg):\n    ''' Using callbacks for learning rate adjustments. '''\n    \n    lr_start = cfg['lr_start']\n    lr_max = cfg['lr_max'] * strategy.num_replicas_in_sync * cfg['batch_size']\n    lr_min = cfg['lr_min']\n    lr_rampup = cfg['lr_rampup']\n    lr_sustain = cfg['lr_sustain']\n    lr_decay = cfg['lr_decay']\n\n    def lrfn(epoch):\n        if epoch < lr_rampup:\n            lr = (lr_max - lr_start) / lr_rampup * epoch + lr_start\n        elif epoch < lr_rampup + lr_sustain:\n            lr = lr_max\n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_rampup -\n                                                lr_sustain) + lr_min\n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:44:52.288766Z","iopub.execute_input":"2022-02-14T12:44:52.289091Z","iopub.status.idle":"2022-02-14T12:44:52.303275Z","shell.execute_reply.started":"2022-02-14T12:44:52.289059Z","shell.execute_reply":"2022-02-14T12:44:52.302314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_history(cfg, history, network='noisy-student'):\n\n    plt.figure(figsize=(15,5))\n\n    plt.plot(np.arange(cfg['epochs']), history.history['auc'],'-o', \n             label='Train AUC', color=color_palette[0])\n    plt.plot(np.arange(cfg['epochs']), history.history['val_auc'],'-o', \n             label='Val AUC', color=color_palette[-2])\n\n    x = np.argmax(history.history['val_auc'])\n    y = np.max(history.history['val_auc'])\n    xdist = plt.xlim()[1] - plt.xlim()[0]\n    ydist = plt.ylim()[1] - plt.ylim()[0]\n    plt.scatter(x, y, s=200, color=color_palette[-2])\n    plt.text(x-0.03*xdist, y-0.13*ydist, 'max auc\\n%.2f'%y, size=14)\n    plt.ylabel('AUC', size=14)\n    plt.xlabel('Epoch', size=14)\n    plt.legend(loc=2)\n\n    plt2 = plt.gca().twinx()\n    plt2.plot(np.arange(cfg['epochs']), history.history['loss'],'-o', \n              label='Train Loss', color=color_palette[1])\n    plt2.plot(np.arange(cfg['epochs']), history.history['val_loss'],'-o', \n              label='Val Loss', color=color_palette[-3])\n    x = np.argmin(history.history['val_loss'])\n    y = np.min(history.history['val_loss'])\n    ydist = plt.ylim()[1] - plt.ylim()[0]\n    plt.scatter(x, y, s=200, color=color_palette[-3])\n    plt.text(x-0.03*xdist, y+0.05*ydist, 'min loss', size=14)\n    plt.ylabel('Loss', size=14)\n\n    plt.title('Efficientnet-B5 ' + network + ' on melanoma 384 x 384')\n    plt.legend(loc=3)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:44:52.304579Z","iopub.execute_input":"2022-02-14T12:44:52.304867Z","iopub.status.idle":"2022-02-14T12:44:52.320207Z","shell.execute_reply.started":"2022-02-14T12:44:52.304834Z","shell.execute_reply":"2022-02-14T12:44:52.319339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_roc(y_trues, y_preds, x_max=1.0):\n    color_palette = list(map(lambda x: '#%02x%02x%02x' % (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)), sns.color_palette(\"magma\", n_colors=20)))\n    \n    fig, ax = plt.subplots(1,2, figsize=(16,6))\n    for i, y_pred in enumerate(y_preds):\n        y_true = y_trues[i]\n        fpr, tpr, thresholds = roc_curve(y_true, y_pred)\n        auc = roc_auc_score(y_true, y_pred)\n        ax[0].plot(fpr, tpr, label='AUC=%.3f' % auc, marker='o', markersize=1, c=color_palette[0])\n\n    ax[0].legend()\n    ax[0].grid()\n    ax[0].plot(np.linspace(0, 1, 20), np.linspace(0, 1, 20), linestyle='--', c=color_palette[-10])\n    ax[0].set_title('ROC curve')\n    ax[0].set_xlabel('False Positive Rate')\n    ax[0].set_xlim([-0.01, x_max])\n    _ = ax[0].set_ylabel('True Positive Rate')","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:44:52.321725Z","iopub.execute_input":"2022-02-14T12:44:52.322350Z","iopub.status.idle":"2022-02-14T12:44:52.334990Z","shell.execute_reply.started":"2022-02-14T12:44:52.322315Z","shell.execute_reply":"2022-02-14T12:44:52.334013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Run Models","metadata":{}},{"cell_type":"markdown","source":"## Compare epoch","metadata":{}},{"cell_type":"code","source":"GCS_PATH = KaggleDatasets().get_gcs_path('melanoma-384x384')\ncfg['epochs'] = 20\n\nidx_train, idx_valid = [0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 12, 13, 14], [8]\n\nfiles_train = np.array(tf.io.gfile.glob([GCS_PATH + '/train%.2i*.tfrec' % x for x in idx_train]))\nfiles_valid = np.array(tf.io.gfile.glob([GCS_PATH + '/train%.2i*.tfrec'% x for x in idx_valid]))\n\n# print(files_train, files_valid, sep='\\n')\n\nmodel = compileModel(cfg)\nmodel.summary()\n\nds_train = getTrainDataset(files_train, cfg)\nds_valid = getTrainDataset(files_valid, cfg, augment=False, shuffle=False, repeat=False)\nstepsTrain = count_data_items(files_train) / (cfg['batch_size'] * strategy.num_replicas_in_sync)\n\nhistory = model.fit(ds_train,\n                    validation_data=ds_valid,\n                    steps_per_epoch=stepsTrain,\n                    epochs=cfg['epochs'], verbose=1,\n                    callbacks=[getLearnRateCallback(cfg)])\n\nplot_history(cfg, history)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T12:45:56.654251Z","iopub.execute_input":"2022-02-14T12:45:56.654595Z","iopub.status.idle":"2022-02-14T13:16:20.137865Z","shell.execute_reply.started":"2022-02-14T12:45:56.654559Z","shell.execute_reply":"2022-02-14T13:16:20.136974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Compare Pre-trained type","metadata":{}},{"cell_type":"code","source":"# imagenet EfficientNet test run\nGCS_PATH = KaggleDatasets().get_gcs_path('melanoma-384x384')\n\nidx_train, idx_valid = [0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 12, 13, 14], [8]\n\nfiles_train = np.array(tf.io.gfile.glob([GCS_PATH + '/train%.2i*.tfrec' % x for x in idx_train]))\nfiles_valid = np.array(tf.io.gfile.glob([GCS_PATH + '/train%.2i*.tfrec'% x for x in idx_valid]))\n\n# print(files_train, files_valid, sep='\\n')\n\nmodel = compileModel(cfg, weights='imagenet')\nmodel.summary()\n\nds_train = getTrainDataset(files_train, cfg)\nds_valid = getTrainDataset(files_valid, cfg, augment=False, shuffle=False, repeat=False)\nstepsTrain = count_data_items(files_train) / (cfg['batch_size'] * strategy.num_replicas_in_sync)\n\nhistory = model.fit(ds_train,\n                    validation_data=ds_valid,\n                    steps_per_epoch=stepsTrain,\n                    epochs=cfg['epochs'], verbose=1,\n                    callbacks=[getLearnRateCallback(cfg)])\n\nplot_history(cfg, history, network='pre_trained imagenet')","metadata":{"execution":{"iopub.status.busy":"2022-02-14T13:23:49.048422Z","iopub.execute_input":"2022-02-14T13:23:49.048848Z","iopub.status.idle":"2022-02-14T13:40:30.077429Z","shell.execute_reply.started":"2022-02-14T13:23:49.048785Z","shell.execute_reply":"2022-02-14T13:40:30.076467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# noisy-student EfficientNet test run\nGCS_PATH = KaggleDatasets().get_gcs_path('melanoma-384x384')\n\nidx_train, idx_valid = [0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 12, 13, 14], [8]\n\nfiles_train = np.array(tf.io.gfile.glob([GCS_PATH + '/train%.2i*.tfrec' % x for x in idx_train]))\nfiles_valid = np.array(tf.io.gfile.glob([GCS_PATH + '/train%.2i*.tfrec'% x for x in idx_valid]))\n\n# print(files_train, files_valid, sep='\\n')\n\nmodel = compileModel(cfg)\nmodel.summary()\n\nds_train = getTrainDataset(files_train, cfg)\nds_valid = getTrainDataset(files_valid, cfg, augment=False, shuffle=False, repeat=False)\nstepsTrain = count_data_items(files_train) / (cfg['batch_size'] * strategy.num_replicas_in_sync)\n\nhistory = model.fit(ds_train,\n                    validation_data=ds_valid,\n                    steps_per_epoch=stepsTrain,\n                    epochs=cfg['epochs'], verbose=1,\n                    callbacks=[getLearnRateCallback(cfg)])\n\nplot_history(cfg, history)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T13:40:30.079282Z","iopub.execute_input":"2022-02-14T13:40:30.079522Z","iopub.status.idle":"2022-02-14T13:57:15.602325Z","shell.execute_reply.started":"2022-02-14T13:40:30.079495Z","shell.execute_reply":"2022-02-14T13:57:15.601392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## KFold","metadata":{}},{"cell_type":"code","source":"# KFold run\nhistories = []\nGCS_PATH = KaggleDatasets().get_gcs_path('melanoma-384x384')\nfiles_test  = np.sort(tf.io.gfile.glob(GCS_PATH + '/test*.tfrec'))\n\nskf = KFold(n_splits=5, shuffle=True, random_state=seed_val)\nfor fold, (idx_train, idx_valid) in enumerate(skf.split(np.arange(15))):\n    \n    files_train = np.array(tf.io.gfile.glob([GCS_PATH + '/train%.2i*.tfrec' % x for x in idx_train]))\n    files_valid = np.array(tf.io.gfile.glob([GCS_PATH + '/train%.2i*.tfrec'% x for x in idx_valid]))\n    \n    model = compileModel(cfg)\n    if fold == 0:\n        model.summary()\n    \n    ds_train = getTrainDataset(files_train, cfg)\n    ds_valid = getTrainDataset(files_valid, cfg, augment=False, shuffle=False, repeat=False)\n    stepsTrain = count_data_items(files_train) / (cfg['batch_size'] * strategy.num_replicas_in_sync)\n    \n    print(\"Fold\", fold + 1, '-' * 15, '\\n')\n    history = model.fit(ds_train,\n                        validation_data=ds_valid,\n                        steps_per_epoch=stepsTrain,\n                        epochs=cfg['epochs'], verbose=1,\n                        callbacks=[getLearnRateCallback(cfg)])\n    \n    plot_history(cfg, history)\n    histories.append(history)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T00:47:20.226265Z","iopub.execute_input":"2022-02-13T00:47:20.226498Z","iopub.status.idle":"2022-02-13T02:03:32.219766Z","shell.execute_reply.started":"2022-02-13T00:47:20.226473Z","shell.execute_reply":"2022-02-13T02:03:32.218842Z"},"trusted":true},"execution_count":null,"outputs":[]}]}