{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport os\n\nsns.set_style('darkgrid')\nplt.style.use('seaborn-notebook')","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:32.852019Z","iopub.execute_input":"2023-02-02T10:01:32.852525Z","iopub.status.idle":"2023-02-02T10:01:34.209429Z","shell.execute_reply.started":"2023-02-02T10:01:32.852425Z","shell.execute_reply":"2023-02-02T10:01:34.208190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#read in train and test csv files to calculate descriptive stats and characteristics\n\ntrain = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv')\n\ntest = pd.read_csv('../input/siim-isic-melanoma-classification/test.csv')","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:34.211275Z","iopub.execute_input":"2023-02-02T10:01:34.211792Z","iopub.status.idle":"2023-02-02T10:01:34.408039Z","shell.execute_reply.started":"2023-02-02T10:01:34.211756Z","shell.execute_reply":"2023-02-02T10:01:34.407157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:34.409316Z","iopub.execute_input":"2023-02-02T10:01:34.410239Z","iopub.status.idle":"2023-02-02T10:01:34.441080Z","shell.execute_reply.started":"2023-02-02T10:01:34.410206Z","shell.execute_reply":"2023-02-02T10:01:34.439039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:34.444208Z","iopub.execute_input":"2023-02-02T10:01:34.444684Z","iopub.status.idle":"2023-02-02T10:01:34.452211Z","shell.execute_reply.started":"2023-02-02T10:01:34.444642Z","shell.execute_reply":"2023-02-02T10:01:34.451172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:34.453809Z","iopub.execute_input":"2023-02-02T10:01:34.455610Z","iopub.status.idle":"2023-02-02T10:01:34.507688Z","shell.execute_reply.started":"2023-02-02T10:01:34.455537Z","shell.execute_reply":"2023-02-02T10:01:34.505966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe()","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:34.509980Z","iopub.execute_input":"2023-02-02T10:01:34.510577Z","iopub.status.idle":"2023-02-02T10:01:34.559869Z","shell.execute_reply.started":"2023-02-02T10:01:34.510537Z","shell.execute_reply":"2023-02-02T10:01:34.557598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ax1 = sns.countplot(data=train, x='anatom_site_general_challenge')\nax1.set_title('Distribution of Anatomical Sites Training Set')\nax1.tick_params(axis='x', labelrotation = 45, labelsize = 12)\n\n#note: largest category by far is the torso, oral/genital is the least common category","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:34.561650Z","iopub.execute_input":"2023-02-02T10:01:34.562143Z","iopub.status.idle":"2023-02-02T10:01:34.826970Z","shell.execute_reply.started":"2023-02-02T10:01:34.562097Z","shell.execute_reply":"2023-02-02T10:01:34.825157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ax2 = sns.countplot(data=train, x='sex')\nax2.set_title('Distribution of Gender')\n\n#note: gender is pretty evenly distributed amongst the image dataset","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:34.830393Z","iopub.execute_input":"2023-02-02T10:01:34.831326Z","iopub.status.idle":"2023-02-02T10:01:35.010380Z","shell.execute_reply.started":"2023-02-02T10:01:34.831244Z","shell.execute_reply":"2023-02-02T10:01:35.009015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ax3 = sns.countplot(data=train, x='diagnosis')\nax3.tick_params(axis='x', labelrotation = 45, labelsize = 12)\nax3.set_title('Distribution of Diagnosis')\n\n#note: the vast majority of the images have no associated diagnosis, some have the nevus diagnosis, and very few have the melanoma diagnosis. \n#Melanoma is the target we're actually looking for.","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:35.014856Z","iopub.execute_input":"2023-02-02T10:01:35.015475Z","iopub.status.idle":"2023-02-02T10:01:35.287001Z","shell.execute_reply.started":"2023-02-02T10:01:35.015436Z","shell.execute_reply":"2023-02-02T10:01:35.285952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ax4 = sns.countplot(data=train, x='benign_malignant')\nax4.tick_params(axis='x', labelrotation = 45, labelsize = 12)\nax4.set_title('Distribution of Diagnosis')\n\n#note: malignant here is defined as a have a \"melanoma\" disease label","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:35.292255Z","iopub.execute_input":"2023-02-02T10:01:35.292743Z","iopub.status.idle":"2023-02-02T10:01:35.491342Z","shell.execute_reply.started":"2023-02-02T10:01:35.292707Z","shell.execute_reply":"2023-02-02T10:01:35.490169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ax5 = sns.histplot(data=train, x='age_approx', bins=18)\nax5.tick_params(axis='x', labelrotation = 45, labelsize = 12)\nax5.set_title('Distribution of Age')\n\n#age follows a normal distribution","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:35.493432Z","iopub.execute_input":"2023-02-02T10:01:35.494008Z","iopub.status.idle":"2023-02-02T10:01:35.740814Z","shell.execute_reply.started":"2023-02-02T10:01:35.493956Z","shell.execute_reply":"2023-02-02T10:01:35.739339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('n rows where target != benign_malignant: {}'.format(len(train.loc[(train['target'] == 1) & (train['benign_malignant'] != 'malignant')])))\nprint('n rows where benign_malignant != melanoma diagnosis: {}'.format(len(train.loc[(train['diagnosis'] != 'melanoma') & (train['benign_malignant'] == 'malignant')])))\nprint('n rows where target != benign_malignant: {}'.format(len(train.loc[(train['target'] == 1) & (train['benign_malignant'] != 'malignant')])))\n\n#my assumptions hold--this dataset is labeled based off of the 'melanoma' diagnosis, \n#and a positive value in the target column indicates a malignant melanoma tumor","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:35.742651Z","iopub.execute_input":"2023-02-02T10:01:35.743305Z","iopub.status.idle":"2023-02-02T10:01:35.771917Z","shell.execute_reply.started":"2023-02-02T10:01:35.743260Z","shell.execute_reply":"2023-02-02T10:01:35.768958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#print the number of missing values from each col in train\nfor col in train.columns:\n    print(col + ' missing values: ' + str(train[col].isna().sum()))\n    ","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:35.776780Z","iopub.execute_input":"2023-02-02T10:01:35.778848Z","iopub.status.idle":"2023-02-02T10:01:35.809426Z","shell.execute_reply.started":"2023-02-02T10:01:35.778790Z","shell.execute_reply":"2023-02-02T10:01:35.806869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Pull out just the observations that are positive for the target condition\nmalignant = train[train['target'] == 1]\nprint(malignant.head())\nprint(malignant.shape)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:35.813174Z","iopub.execute_input":"2023-02-02T10:01:35.814834Z","iopub.status.idle":"2023-02-02T10:01:35.840026Z","shell.execute_reply.started":"2023-02-02T10:01:35.814726Z","shell.execute_reply":"2023-02-02T10:01:35.838016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in malignant.columns:\n    print(col + ' missing values: ' + str(malignant[col].isna().sum()))\n","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:35.842337Z","iopub.execute_input":"2023-02-02T10:01:35.842944Z","iopub.status.idle":"2023-02-02T10:01:35.858413Z","shell.execute_reply.started":"2023-02-02T10:01:35.842866Z","shell.execute_reply":"2023-02-02T10:01:35.855605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#fill missing anatom site values with \"unknown or other\"\ntrain.anatom_site_general_challenge.fillna('other or unknown', inplace=True)\n\n#verify there are no more missing values in that column\nprint('anatom_site_general_challenge' + ' missing values: ' + str(train['anatom_site_general_challenge'].isna().sum()))\n\n#plot the new distribution\nax = sns.countplot(data=train, x='anatom_site_general_challenge')\n#train.anatom_site_general_challenge.hist()\nax.set_title('Distribution of Anatomical Sites Training Set')\nax.tick_params(axis='x', labelrotation = 45, labelsize = 12)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:35.860542Z","iopub.execute_input":"2023-02-02T10:01:35.861096Z","iopub.status.idle":"2023-02-02T10:01:36.107115Z","shell.execute_reply.started":"2023-02-02T10:01:35.861047Z","shell.execute_reply":"2023-02-02T10:01:36.105680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Test Set exploration**","metadata":{}},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:36.108485Z","iopub.execute_input":"2023-02-02T10:01:36.109753Z","iopub.status.idle":"2023-02-02T10:01:36.125457Z","shell.execute_reply.started":"2023-02-02T10:01:36.109698Z","shell.execute_reply":"2023-02-02T10:01:36.123423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.shape","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:36.127008Z","iopub.execute_input":"2023-02-02T10:01:36.127405Z","iopub.status.idle":"2023-02-02T10:01:36.142845Z","shell.execute_reply.started":"2023-02-02T10:01:36.127373Z","shell.execute_reply":"2023-02-02T10:01:36.140849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#print the number of missing values from each col in train\nfor col in test.columns:\n    print(col + ' missing values: ' + str(test[col].isna().sum()))\n","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:36.144439Z","iopub.execute_input":"2023-02-02T10:01:36.144834Z","iopub.status.idle":"2023-02-02T10:01:36.157702Z","shell.execute_reply.started":"2023-02-02T10:01:36.144801Z","shell.execute_reply":"2023-02-02T10:01:36.156947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#how many images are associated with each patient?\nindividuals_count = train.groupby('patient_id').count()\n\nindividuals_count.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:36.159013Z","iopub.execute_input":"2023-02-02T10:01:36.159476Z","iopub.status.idle":"2023-02-02T10:01:36.195195Z","shell.execute_reply.started":"2023-02-02T10:01:36.159447Z","shell.execute_reply":"2023-02-02T10:01:36.193381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#show the disribution of image counts\nfig, ax = plt.subplots(figsize=(25,10))\nax = sns.histplot(data=individuals_count, x='image_name')\nax.set_title('Images per Patient')\nax.set_xlabel('Unique Images per Patient')\nax.set_ylabel('Number of Patients')\nax.set_xticks(ticks=range(0,120,10))\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:36.197060Z","iopub.execute_input":"2023-02-02T10:01:36.197478Z","iopub.status.idle":"2023-02-02T10:01:36.578860Z","shell.execute_reply.started":"2023-02-02T10:01:36.197430Z","shell.execute_reply":"2023-02-02T10:01:36.577901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('the mean number of images per patient is: {}'.format(round(individuals_count.image_name.mean(),2)))\nprint('the median number of images per patient is: {}'.format(round(individuals_count.image_name.median(),2)))\nprint('the standard deviation of the number of images per patient is: {}'.format(round(individuals_count.image_name.std(),2)))","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:36.580565Z","iopub.execute_input":"2023-02-02T10:01:36.582417Z","iopub.status.idle":"2023-02-02T10:01:36.591908Z","shell.execute_reply.started":"2023-02-02T10:01:36.582367Z","shell.execute_reply":"2023-02-02T10:01:36.590427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check out the distribution of positive cases grouped by patient\nindividuals_sum = train.groupby('patient_id').sum()\n\nprint(individuals_sum.target.describe())\n\n#fig, ax = plt.subplots(figsize=(18,10))\nax = sns.histplot(data=individuals_sum, x='target', bins=10)\nax.set_title('Quantity of Malignant Tumors per Patient')\nax.set_xlabel('Sum of Positives')\nax.set_ylabel('Number of Patients')\n#ax.set_xticks(ticks=range(0,20))\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:36.593943Z","iopub.execute_input":"2023-02-02T10:01:36.594620Z","iopub.status.idle":"2023-02-02T10:01:36.901546Z","shell.execute_reply.started":"2023-02-02T10:01:36.594583Z","shell.execute_reply":"2023-02-02T10:01:36.900688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Another pass at missing sex and age values**","metadata":{}},{"cell_type":"code","source":"#let's pull out individuals from individuals_count who have fewer counts of sex or age_approx than image_name\nindividuals_count.loc[(individuals_count['sex'] < individuals_count['image_name']) | (individuals_count['age_approx'] < individuals_count['image_name'])]","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:36.903264Z","iopub.execute_input":"2023-02-02T10:01:36.903875Z","iopub.status.idle":"2023-02-02T10:01:36.933226Z","shell.execute_reply.started":"2023-02-02T10:01:36.903835Z","shell.execute_reply":"2023-02-02T10:01:36.931198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pats = ['IP_0550106','IP_5205991', 'IP_9835712']\nindividuals_sum.reset_index(inplace=True)\n\n\nindividuals_sum.query('patient_id in @pats')","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:36.935828Z","iopub.execute_input":"2023-02-02T10:01:36.936277Z","iopub.status.idle":"2023-02-02T10:01:36.971241Z","shell.execute_reply.started":"2023-02-02T10:01:36.936237Z","shell.execute_reply":"2023-02-02T10:01:36.969057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('percentage of missing patient data= {}%'.format(round((3+48+17)/len(train)*100,2)))","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:36.973160Z","iopub.execute_input":"2023-02-02T10:01:36.973613Z","iopub.status.idle":"2023-02-02T10:01:36.981410Z","shell.execute_reply.started":"2023-02-02T10:01:36.973569Z","shell.execute_reply":"2023-02-02T10:01:36.979929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#drop all rows from the train dataframe that are associated with the 3 patients identified above.\ntrain.drop(train[(train['patient_id'] == 'IP_0550106') | (train['patient_id'] == 'IP_5205991') |(train['patient_id'] == 'IP_9835712')].index, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:36.982652Z","iopub.execute_input":"2023-02-02T10:01:36.982997Z","iopub.status.idle":"2023-02-02T10:01:37.006598Z","shell.execute_reply.started":"2023-02-02T10:01:36.982968Z","shell.execute_reply":"2023-02-02T10:01:37.004394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check for any more missing values:\nfor col in train.columns:\n    print(col + ' missing values: ' + str(train[col].isna().sum()))","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:37.015753Z","iopub.execute_input":"2023-02-02T10:01:37.016791Z","iopub.status.idle":"2023-02-02T10:01:37.037036Z","shell.execute_reply.started":"2023-02-02T10:01:37.016747Z","shell.execute_reply":"2023-02-02T10:01:37.035951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Relationships between positive cases and other variables.**","metadata":{}},{"cell_type":"code","source":"#make sure index is the patient id so we can join these aggregate dfs\n\nindividuals_sum.set_index('patient_id', inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:37.038786Z","iopub.execute_input":"2023-02-02T10:01:37.039174Z","iopub.status.idle":"2023-02-02T10:01:37.046226Z","shell.execute_reply.started":"2023-02-02T10:01:37.039139Z","shell.execute_reply":"2023-02-02T10:01:37.043948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"patients = individuals_count.join(individuals_sum, lsuffix='_count', rsuffix='_sum')\npatients.drop(columns=['sex','age_approx_count','anatom_site_general_challenge','diagnosis', 'benign_malignant','target_count', 'age_approx_sum'], inplace=True)\npatients.rename(columns={'image_name':'n_images'}, inplace=True)\npatients.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:37.047736Z","iopub.execute_input":"2023-02-02T10:01:37.048225Z","iopub.status.idle":"2023-02-02T10:01:37.080783Z","shell.execute_reply.started":"2023-02-02T10:01:37.048177Z","shell.execute_reply":"2023-02-02T10:01:37.078929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#join with train df to include sex and age data for each patient\npatients = patients.join(train.set_index('patient_id'), on='patient_id')","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:37.083566Z","iopub.execute_input":"2023-02-02T10:01:37.084046Z","iopub.status.idle":"2023-02-02T10:01:37.130248Z","shell.execute_reply.started":"2023-02-02T10:01:37.084000Z","shell.execute_reply":"2023-02-02T10:01:37.128148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#clean it up\npatients.reset_index(inplace=True)\npatients.drop_duplicates(subset='patient_id', inplace=True)\npatients.drop(columns=['image_name','anatom_site_general_challenge', 'diagnosis',   'benign_malignant', 'target'], inplace=True)\npatients.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:37.133387Z","iopub.execute_input":"2023-02-02T10:01:37.134697Z","iopub.status.idle":"2023-02-02T10:01:37.178148Z","shell.execute_reply.started":"2023-02-02T10:01:37.134633Z","shell.execute_reply":"2023-02-02T10:01:37.177171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plot n_images v. target_sum\nax1 = sns.scatterplot(data=patients, x='n_images', y ='target_sum', hue='sex')\nax1.set_title('n_images per patient v. target_sum')\n#ax1.tick_params(axis='x', labelrotation = 45, labelsize = 12)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:37.179562Z","iopub.execute_input":"2023-02-02T10:01:37.180017Z","iopub.status.idle":"2023-02-02T10:01:37.604022Z","shell.execute_reply.started":"2023-02-02T10:01:37.179990Z","shell.execute_reply":"2023-02-02T10:01:37.602857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ax1 = sns.violinplot(data=patients, y='n_images', x ='target_sum')\nax1.set_title('n_images per patient v. target_sum')\n","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:37.605401Z","iopub.execute_input":"2023-02-02T10:01:37.605961Z","iopub.status.idle":"2023-02-02T10:01:37.920629Z","shell.execute_reply.started":"2023-02-02T10:01:37.605925Z","shell.execute_reply":"2023-02-02T10:01:37.919061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ax1 = sns.stripplot(data=patients, y='target_sum', x ='sex')\nax1.set_title('target_sum distribution by gender')","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:37.922391Z","iopub.execute_input":"2023-02-02T10:01:37.922922Z","iopub.status.idle":"2023-02-02T10:01:38.123150Z","shell.execute_reply.started":"2023-02-02T10:01:37.922847Z","shell.execute_reply":"2023-02-02T10:01:38.122326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plot age_approx v target_sum\nax1 = sns.stripplot(data=patients, x='age_approx', y ='target_sum')\nax1.set_title('age_approx v. target_sum')\n","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:38.124377Z","iopub.execute_input":"2023-02-02T10:01:38.125135Z","iopub.status.idle":"2023-02-02T10:01:38.489513Z","shell.execute_reply.started":"2023-02-02T10:01:38.125100Z","shell.execute_reply":"2023-02-02T10:01:38.487430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plot n_images v. target_sum\nax1 = sns.histplot(data=patients, x='age_approx', bins=16)\nax1.set_title('Ditribution of Patient Ages in Dataset')","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:38.491046Z","iopub.execute_input":"2023-02-02T10:01:38.491941Z","iopub.status.idle":"2023-02-02T10:01:38.801489Z","shell.execute_reply.started":"2023-02-02T10:01:38.491857Z","shell.execute_reply":"2023-02-02T10:01:38.800130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Visualizing Images**\n","metadata":{}},{"cell_type":"code","source":"#open one image\n\n#import ImageIO\nimport imageio.v2 as imageio\n\n#load an image\nim = imageio.imread('../input/siim-isic-melanoma-classification/jpeg/train/ISIC_0015719.jpg')\n\n#plot image\nplt.imshow(im)\nplt.axis('off')\n#plt.show()\n\n#print metadata\nprint(im.meta.keys())","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:38.803928Z","iopub.execute_input":"2023-02-02T10:01:38.804273Z","iopub.status.idle":"2023-02-02T10:01:42.527310Z","shell.execute_reply.started":"2023-02-02T10:01:38.804242Z","shell.execute_reply":"2023-02-02T10:01:42.525406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Plot malignant lesions**","metadata":{}},{"cell_type":"code","source":"#Now, let's see if we can show all the malignant images\n\n#make a list of image names that have the malignant target value = 1\nmal_ims = train[train['target']==1].image_name\n\n#convert those image names into path names to access each photo\n#mal_im_paths = ['../input/siim-isic-melanoma-classification/jpeg/train/' + str(im) + '.jpg' for im in mal_ims]","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:42.529058Z","iopub.execute_input":"2023-02-02T10:01:42.529479Z","iopub.status.idle":"2023-02-02T10:01:42.537474Z","shell.execute_reply.started":"2023-02-02T10:01:42.529442Z","shell.execute_reply":"2023-02-02T10:01:42.536371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plot all malignant images\n\nfig, axs = plt.subplots(nrows=146, ncols=4, tight_layout=True, figsize=(15,365))\nfig.suptitle('Malignant Lesions', fontsize=14)\n\nfor ax, image in zip(axs.ravel(), mal_ims):\n    path = '../input/siim-isic-melanoma-classification/jpeg/train/' + str(image) + '.jpg'\n    im = imageio.imread(path)\n    ax.imshow(im)\n    ax.axis('off')\n    ax.set_title(image)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:01:42.538825Z","iopub.execute_input":"2023-02-02T10:01:42.539198Z","iopub.status.idle":"2023-02-02T10:10:35.443365Z","shell.execute_reply.started":"2023-02-02T10:01:42.539140Z","shell.execute_reply":"2023-02-02T10:10:35.440468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\nrandom.seed(42)\n\n#make a list of image names that have the malignant target value = 0\nben_ims = train[train['target']==0].image_name\n\n#take a random sample of 100 of these images\nrand_ben_ims = random.sample(list(ben_ims),100)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:10:35.445171Z","iopub.execute_input":"2023-02-02T10:10:35.445579Z","iopub.status.idle":"2023-02-02T10:10:35.461915Z","shell.execute_reply.started":"2023-02-02T10:10:35.445543Z","shell.execute_reply":"2023-02-02T10:10:35.460821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plot the subset of 100 benign images\n\nfig, axs = plt.subplots(nrows=25, ncols=4, tight_layout=True, figsize=(15,62.5))\nfig.suptitle('Subset of Benign Lesions', fontsize=14, y=0.98)\n\nfor ax, image in zip(axs.ravel(), rand_ben_ims):\n    path = '../input/siim-isic-melanoma-classification/jpeg/train/' + str(image) + '.jpg'\n    im = imageio.imread(path)\n    ax.imshow(im)\n    ax.axis('off')\n    ax.set_title(image)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:10:35.463103Z","iopub.execute_input":"2023-02-02T10:10:35.463628Z","iopub.status.idle":"2023-02-02T10:13:02.653298Z","shell.execute_reply.started":"2023-02-02T10:10:35.463591Z","shell.execute_reply":"2023-02-02T10:13:02.651309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# System\nimport cv2\nimport os, os.path\nfrom PIL import Image             \nimport gc\nimport time\nimport datetime\n\n# Basics\nimport pandas as pd\nimport numpy as np\nimport random\nimport seaborn as sns\nimport matplotlib\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nfrom tqdm.notebook import tqdm     \n\n# SKlearn\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold\nfrom sklearn.metrics import accuracy_score, roc_auc_score, confusion_matrix\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn import preprocessing\n\nfrom skimage import io\n\n# PyTorch\nimport torch\nimport torchvision\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader, Subset\n\nfrom torchvision.models import resnet34\nfrom torchvision import transforms\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nprint(\"PyTorch Version: \",torch.__version__)\nprint(\"Torchvision Version: \",torchvision.__version__)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:13:02.655137Z","iopub.execute_input":"2023-02-02T10:13:02.655562Z","iopub.status.idle":"2023-02-02T10:13:07.210981Z","shell.execute_reply.started":"2023-02-02T10:13:02.655521Z","shell.execute_reply":"2023-02-02T10:13:07.209458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For reproducibility\nseed = 1234\n\nnp.random.seed(seed)\nrandom.seed(seed)\ntorch.manual_seed(seed)\ntorch.cuda.manual_seed(seed)\n","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:13:07.212333Z","iopub.execute_input":"2023-02-02T10:13:07.212978Z","iopub.status.idle":"2023-02-02T10:13:07.226331Z","shell.execute_reply.started":"2023-02-02T10:13:07.212942Z","shell.execute_reply":"2023-02-02T10:13:07.225042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint('Device available now:', device)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:13:07.228031Z","iopub.execute_input":"2023-02-02T10:13:07.228619Z","iopub.status.idle":"2023-02-02T10:13:07.238411Z","shell.execute_reply.started":"2023-02-02T10:13:07.228584Z","shell.execute_reply":"2023-02-02T10:13:07.237000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv')[1:1000]\n# train_df = train_df.head(1000)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:13:07.240119Z","iopub.execute_input":"2023-02-02T10:13:07.240839Z","iopub.status.idle":"2023-02-02T10:13:07.288409Z","shell.execute_reply.started":"2023-02-02T10:13:07.240799Z","shell.execute_reply":"2023-02-02T10:13:07.287616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.target.value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:13:07.289485Z","iopub.execute_input":"2023-02-02T10:13:07.289876Z","iopub.status.idle":"2023-02-02T10:13:07.297751Z","shell.execute_reply.started":"2023-02-02T10:13:07.289851Z","shell.execute_reply":"2023-02-02T10:13:07.296960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_df)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:13:07.298768Z","iopub.execute_input":"2023-02-02T10:13:07.299628Z","iopub.status.idle":"2023-02-02T10:13:07.310197Z","shell.execute_reply.started":"2023-02-02T10:13:07.299590Z","shell.execute_reply":"2023-02-02T10:13:07.308827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MelanomaDataset(Dataset):\n    \n    def __init__(self, dataframe,is_train=True, is_valid=False, is_test=False):\n        self.dataframe, self.is_train, self.is_valid = dataframe, is_train, is_valid\n        \n        # Data Augmentation\n        if is_train or is_test:\n            self.transform = transforms.Compose([transforms.ToPILImage(),\n                                                 transforms.RandomResizedCrop((224,224), scale=(0.4, 1.0)),\n                                                 transforms.RandomHorizontalFlip(p = 0.3),\n                                                 transforms.RandomVerticalFlip(p = 0.3),\n                                                 transforms.ToTensor(),\n                                                 transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))])\n        else:\n            self.transform = transforms.Compose([transforms.ToPILImage(),\n                                                 transforms.Resize((224,224)),\n                                                 transforms.ToTensor(),\n                                                 transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))])\n            \n    def __len__(self):\n        return len(self.dataframe)\n    \n    def __getitem__(self, index):\n        # Select path and read image\n        img_name = self.dataframe['image_name'][index]\n        image_path = f'../input/siim-isic-melanoma-classification/jpeg/train/{img_name}.jpg'\n        image = io.imread(image_path)\n        \n        # Apply transforms\n        image = self.transform(image)\n\n        \n        # If train/valid: image + class | If test: only image\n        if self.is_train or self.is_valid:\n            return (image, self.dataframe['target'][index])\n        else:\n            return (image)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:13:07.311998Z","iopub.execute_input":"2023-02-02T10:13:07.312605Z","iopub.status.idle":"2023-02-02T10:13:07.324541Z","shell.execute_reply.started":"2023-02-02T10:13:07.312568Z","shell.execute_reply":"2023-02-02T10:13:07.322936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(resnet34(pretrained=True))","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:13:07.325981Z","iopub.execute_input":"2023-02-02T10:13:07.326361Z","iopub.status.idle":"2023-02-02T10:13:08.229660Z","shell.execute_reply.started":"2023-02-02T10:13:07.326327Z","shell.execute_reply":"2023-02-02T10:13:08.228734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ResNet34Network(nn.Module):\n    def __init__(self):\n        super().__init__()\n        \n        # Define Feature part (IMAGE)\n        self.features = resnet34(pretrained=True) # 1000 neurons out\n        \n        for param in self.features.parameters():\n            param.requires_grad = False\n  \n        # Define Classification part\n        self.classification = nn.Linear(1000, 1)\n        \n        \n    def forward(self, image):\n        # Image CNN\n        image = self.features(image)\n        \n        # Classifier\n        out = self.classification(image)\n        \n        return out","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:13:08.231119Z","iopub.execute_input":"2023-02-02T10:13:08.231726Z","iopub.status.idle":"2023-02-02T10:13:08.240643Z","shell.execute_reply.started":"2023-02-02T10:13:08.231687Z","shell.execute_reply":"2023-02-02T10:13:08.238063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = ResNet34Network()\nmodel = model.to(device)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:13:08.242664Z","iopub.execute_input":"2023-02-02T10:13:08.243857Z","iopub.status.idle":"2023-02-02T10:13:08.604928Z","shell.execute_reply.started":"2023-02-02T10:13:08.243808Z","shell.execute_reply":"2023-02-02T10:13:08.604103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(model)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:13:08.606875Z","iopub.execute_input":"2023-02-02T10:13:08.607327Z","iopub.status.idle":"2023-02-02T10:13:08.614589Z","shell.execute_reply.started":"2023-02-02T10:13:08.607286Z","shell.execute_reply":"2023-02-02T10:13:08.612869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Data object and Loader\ndataset = MelanomaDataset(train_df, is_train=True, is_valid=False, is_test=False)\nloader = DataLoader(dataset,batch_size=3, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:13:08.616143Z","iopub.execute_input":"2023-02-02T10:13:08.616546Z","iopub.status.idle":"2023-02-02T10:13:08.627530Z","shell.execute_reply.started":"2023-02-02T10:13:08.616509Z","shell.execute_reply":"2023-02-02T10:13:08.626309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get a sample\nfor image, labels in loader:\n    image_example = image\n    labels_example = torch.tensor(labels, dtype=torch.float32)\n    break\n    \nprint('Data shape:', image_example.shape)\nprint('Label:', labels_example)\n","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:13:08.630558Z","iopub.execute_input":"2023-02-02T10:13:08.631097Z","iopub.status.idle":"2023-02-02T10:13:09.025295Z","shell.execute_reply.started":"2023-02-02T10:13:08.631050Z","shell.execute_reply":"2023-02-02T10:13:09.024017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learning_rate = 0.0005\nepochs = 5\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nprint(device)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:13:09.026791Z","iopub.execute_input":"2023-02-02T10:13:09.027284Z","iopub.status.idle":"2023-02-02T10:13:09.032926Z","shell.execute_reply.started":"2023-02-02T10:13:09.027252Z","shell.execute_reply":"2023-02-02T10:13:09.031929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initiate the model\nmodel = model\nmodel = model.to(device)\noptimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr = learning_rate)\ncriterion = nn.BCEWithLogitsLoss()  ","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:13:09.034010Z","iopub.execute_input":"2023-02-02T10:13:09.034409Z","iopub.status.idle":"2023-02-02T10:13:09.049365Z","shell.execute_reply.started":"2023-02-02T10:13:09.034382Z","shell.execute_reply":"2023-02-02T10:13:09.048343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_size = int(len(train_df) * 0.6)\nval_size = int(len(train_df) * 0.2)\ntest_size = int(len(train_df) * 0.2)\n\nprint(f'train size : {train_size}, val size : {val_size}, test size : {test_size}')","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:13:09.050483Z","iopub.execute_input":"2023-02-02T10:13:09.050804Z","iopub.status.idle":"2023-02-02T10:13:09.062772Z","shell.execute_reply.started":"2023-02-02T10:13:09.050772Z","shell.execute_reply":"2023-02-02T10:13:09.061602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# --- Read in Data ---\ntrain_data = train_df.iloc[:train_size].reset_index(drop=True)\nvalid_data = train_df.iloc[train_size:train_size + val_size].reset_index(drop=True)\ntest_data = train_df.iloc[train_size + val_size:].reset_index(drop=True)\n# Create Data instances\ntrain = MelanomaDataset(train_data, is_train=True, is_valid=False, is_test=False)\nvalid = MelanomaDataset(valid_data, is_train=False, is_valid=True, is_test=False)\ntest = MelanomaDataset(valid_data, is_train=False, is_valid=True, is_test=False)\n\n# Dataloaders\ntrain_loader = DataLoader(train, batch_size=16, shuffle=True)\nvalid_loader = DataLoader(valid, batch_size=8, shuffle=True)\ntest_loader = DataLoader(valid, batch_size=1, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:13:09.063937Z","iopub.execute_input":"2023-02-02T10:13:09.064747Z","iopub.status.idle":"2023-02-02T10:13:09.076478Z","shell.execute_reply.started":"2023-02-02T10:13:09.064720Z","shell.execute_reply":"2023-02-02T10:13:09.075680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# === EPOCHS ===\nfor epoch in range(epochs):\n    print(f'epoch : {epoch} start!')\n    start_time = time.time()\n    correct = 0\n    train_losses = 0\n\n    # === TRAIN ===\n    # Sets the module in training mode.\n    model.train()\n\n    for images, labels in train_loader:\n        # Save them to device\n        images = torch.tensor(images, device=device, dtype=torch.float32)\n        labels = torch.tensor(labels, device=device, dtype=torch.float32)\n\n        # Clear gradients first; very important, usually done BEFORE prediction\n        optimizer.zero_grad()\n\n        # Log Probabilities & Backpropagation\n        out = model(images)\n        loss = criterion(out, labels.unsqueeze(1))\n        loss.backward()\n        optimizer.step()\n\n        train_losses += loss.item()\n        # From log probabilities to actual probabilities\n        train_preds = torch.round(torch.sigmoid(out)) # 0 and 1\n        # Number of correct predictions\n        correct += (train_preds.cpu() == labels.cpu().unsqueeze(1)).sum().item()\n\n    # Compute Train Accuracy\n    train_acc = correct*100 / train_size\n    print(f'Epoch :{epoch + 1} - train accuracy: {train_acc}')\n    \n    # === EVAL ===\n    model.eval()\n\n    # Create matrix to store evaluation predictions (for accuracy)\n    valid_preds = torch.zeros(size = (len(valid_data), 1), device=device, dtype=torch.float32)\n\n\n    # Disables gradients (we need to be sure no optimization happens)\n    with torch.no_grad():\n        for k, (images, labels) in enumerate(valid_loader):\n            images = torch.tensor(images, device=device, dtype=torch.float32)\n            labels = torch.tensor(labels, device=device, dtype=torch.float32)\n\n            out = model(images)\n            pred = torch.sigmoid(out)\n            valid_preds[k*images.shape[0] : k*images.shape[0] + images.shape[0]] = pred\n\n        # Compute accuracy\n        valid_acc = accuracy_score(valid_data['target'].values, \n                                           torch.round(valid_preds.cpu()))*100\n        # Compute ROC\n        valid_roc = roc_auc_score(valid_data['target'].values, \n                                          valid_preds.cpu())\n\n        # Compute time on Train + Eval\n        duration = str(datetime.timedelta(seconds=time.time() - start_time))[:7]\n\n\n        # PRINT INFO\n        print('{} | Epoch: {}/{} | Loss: {:.4} | Train Acc: {:.3} | Valid Acc: {:.3} | ROC: {:.3}'.\\\n                    format(duration, epoch+1, epochs, train_losses, train_acc, valid_acc, valid_roc))\n        \n        \ntorch.save(model.state_dict(), './model.pt')\n\nfrom IPython.display import FileLink\nFileLink(r'model.pt')","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:13:09.077549Z","iopub.execute_input":"2023-02-02T10:13:09.078247Z","iopub.status.idle":"2023-02-02T10:32:07.617861Z","shell.execute_reply.started":"2023-02-02T10:13:09.078210Z","shell.execute_reply":"2023-02-02T10:32:07.615653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.save(model.state_dict(), './model.pt')","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:32:07.621418Z","iopub.execute_input":"2023-02-02T10:32:07.623266Z","iopub.status.idle":"2023-02-02T10:32:07.854447Z","shell.execute_reply.started":"2023-02-02T10:32:07.623228Z","shell.execute_reply":"2023-02-02T10:32:07.852447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Data object and Loader\ndataset = MelanomaDataset(train_df, is_train=True, is_valid=False, is_test=False)\nloader = DataLoader(dataset,batch_size=1, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:32:07.856762Z","iopub.execute_input":"2023-02-02T10:32:07.857307Z","iopub.status.idle":"2023-02-02T10:32:07.865522Z","shell.execute_reply.started":"2023-02-02T10:32:07.857260Z","shell.execute_reply":"2023-02-02T10:32:07.864334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get a sample\nfor image, label in test_loader:\n    image = torch.tensor(image, device=device, dtype=torch.float32)\n    label = torch.tensor(label, device=device, dtype=torch.float32)\n    out = model(image)\n    loss = criterion(out, label.unsqueeze(1))\n    pred = torch.sigmoid(out)\n    print('loss: ', loss)\n    print('Label:', label)\n    print('Pred:', pred)\n    break","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:32:07.867332Z","iopub.execute_input":"2023-02-02T10:32:07.868006Z","iopub.status.idle":"2023-02-02T10:32:07.999773Z","shell.execute_reply.started":"2023-02-02T10:32:07.867966Z","shell.execute_reply":"2023-02-02T10:32:07.998509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import FileLink\nFileLink(r'model.pt')","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:32:08.001265Z","iopub.execute_input":"2023-02-02T10:32:08.001589Z","iopub.status.idle":"2023-02-02T10:32:08.008604Z","shell.execute_reply.started":"2023-02-02T10:32:08.001557Z","shell.execute_reply":"2023-02-02T10:32:08.007931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_lo = ResNet34Network()\nmodel_lo = model_lo.to(device)\nmodel_lo.load_state_dict(torch.load('model.pt'))\nmodel_lo.eval()","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:32:08.009962Z","iopub.execute_input":"2023-02-02T10:32:08.010246Z","iopub.status.idle":"2023-02-02T10:32:08.434924Z","shell.execute_reply.started":"2023-02-02T10:32:08.010220Z","shell.execute_reply":"2023-02-02T10:32:08.433125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_df.loc[train_df['target']==1].head().values)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:32:08.436671Z","iopub.execute_input":"2023-02-02T10:32:08.437108Z","iopub.status.idle":"2023-02-02T10:32:08.451433Z","shell.execute_reply.started":"2023-02-02T10:32:08.437062Z","shell.execute_reply":"2023-02-02T10:32:08.449166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.columns","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:32:08.453116Z","iopub.execute_input":"2023-02-02T10:32:08.453582Z","iopub.status.idle":"2023-02-02T10:32:08.466256Z","shell.execute_reply.started":"2023-02-02T10:32:08.453532Z","shell.execute_reply":"2023-02-02T10:32:08.464972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Data object and Loader\ndf = pd.DataFrame({'image_name':['ISIC_2637011', 'ISIC_0149568'], 'patient_id':['IP_7279968', 'IP_0962375'], 'sex':['male', 'female'], 'age_approx':[45.0, 55.0],\n                  'anatom_site_general_challenge':['head/neck', 'upper extremity'], 'diagnosis':['unknown', 'melanoma'],\n                   'benign_malignant':['benign', 'malignant'], 'target':[0,1]})\n\n\ndataset = MelanomaDataset(df, is_train=True, is_valid=False, is_test=False)\nloader = DataLoader(dataset,batch_size=1, shuffle=False)\n# Get a sample\nfor image, label in loader:\n    image = torch.tensor(image, device=device, dtype=torch.float32)\n    label = torch.tensor(label, device=device, dtype=torch.float32)\n    out = model(image)\n    loss = criterion(out, label.unsqueeze(1))\n    pred = torch.sigmoid(out)\n    \n#     print('loss: ', loss)\n#     print('Label:', label)\n    print('Pred:', pred)\n    print('out:', torch.round(pred))\n    print('__________________________________________________________________')","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:32:08.468776Z","iopub.execute_input":"2023-02-02T10:32:08.469338Z","iopub.status.idle":"2023-02-02T10:32:09.396987Z","shell.execute_reply.started":"2023-02-02T10:32:08.469300Z","shell.execute_reply":"2023-02-02T10:32:09.395975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.\nimport cv2\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset,DataLoader\nfrom torchvision import transforms,models\nfrom tqdm import tqdm_notebook as tqdm","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:32:09.398657Z","iopub.execute_input":"2023-02-02T10:32:09.399269Z","iopub.status.idle":"2023-02-02T10:33:47.255458Z","shell.execute_reply.started":"2023-02-02T10:32:09.399232Z","shell.execute_reply":"2023-02-02T10:33:47.253719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/test.csv')","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:44:00.622166Z","iopub.execute_input":"2023-02-02T10:44:00.622641Z","iopub.status.idle":"2023-02-02T10:44:00.641149Z","shell.execute_reply.started":"2023-02-02T10:44:00.622601Z","shell.execute_reply":"2023-02-02T10:44:00.639738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class GraphemeDataset(Dataset):\n    def __init__(self,df,_type='train'):\n        self.df = df\n    def __len__(self):\n        return len(self.df)\n    def __getitem__(self,idx):\n        image = self.df.iloc[idx][1:].values.reshape(64,64).astype(float)\n        return image","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:44:08.479314Z","iopub.execute_input":"2023-02-02T10:44:08.479829Z","iopub.status.idle":"2023-02-02T10:44:08.492045Z","shell.execute_reply.started":"2023-02-02T10:44:08.479788Z","shell.execute_reply":"2023-02-02T10:44:08.490122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model_resnet34 = ResNet34().to(device)\n#summary(model_resnet34, (1, 64, 64))","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:45:50.757733Z","iopub.execute_input":"2023-02-02T10:45:50.758262Z","iopub.status.idle":"2023-02-02T10:45:50.765481Z","shell.execute_reply.started":"2023-02-02T10:45:50.758221Z","shell.execute_reply":"2023-02-02T10:45:50.763730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nmodel = model\nmodel = model.to(device)\noptimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr = learning_rate)\ncriterion = nn.BCEWithLogitsLoss()  ","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:33:47.491821Z","iopub.status.idle":"2023-02-02T10:33:47.493446Z","shell.execute_reply.started":"2023-02-02T10:33:47.492762Z","shell.execute_reply":"2023-02-02T10:33:47.492816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def Resize(df,size=64):\n    resized = {} \n    df = df.set_index('image_id')\n    for i in tqdm(range(df.shape[0])):\n        image = cv2.resize(df.loc[df.index[i]].values.reshape(137,236),(size,size))\n        resized[df.index[i]] = image.reshape(-1)\n    resized = pd.DataFrame(resized).T.reset_index()\n    resized.columns = resized.columns.astype(str)\n    resized.rename(columns={'index':'image_id'},inplace=True)\n    return resized","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:36:29.741715Z","iopub.execute_input":"2023-02-02T10:36:29.742191Z","iopub.status.idle":"2023-02-02T10:36:29.750827Z","shell.execute_reply.started":"2023-02-02T10:36:29.742147Z","shell.execute_reply":"2023-02-02T10:36:29.749554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.eval()\ntest_data = ['test_image_data_0.parquet','test_image_data_1.parquet','test_image_data_2.parquet','test_image_data_3.parquet']\npredictions = []\nbatch_size=1\nfor fname in test_data:\n    data = pd.read_parquet(f'/kaggle/input/siim-isic-melanoma-classification/')\n    data = Resize(data)\n    test_image = GraphemeDataset(data)\n    test_loader = torch.utils.data.DataLoader(test_image,batch_size=1,shuffle=False)\n    with torch.no_grad():\n        for idx, (inputs) in tqdm(enumerate(test_loader),total=len(test_loader)):\n            inputs.to(device)\n            \n            outputs1,outputs2,outputs3 = model(inputs.unsqueeze(1).float().cuda())\n            predictions.append(outputs3.argmax(1).cpu().detach().numpy())\n            predictions.append(outputs2.argmax(1).cpu().detach().numpy())\n            predictions.append(outputs1.argmax(1).cpu().detach().numpy())","metadata":{"execution":{"iopub.status.busy":"2023-02-02T10:52:28.829159Z","iopub.execute_input":"2023-02-02T10:52:28.829638Z","iopub.status.idle":"2023-02-02T10:53:18.565911Z","shell.execute_reply.started":"2023-02-02T10:52:28.829601Z","shell.execute_reply":"2023-02-02T10:53:18.563697Z"},"trusted":true},"execution_count":null,"outputs":[]}]}