{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":20270,"databundleVersionId":1222630,"sourceType":"competition"}],"dockerImageVersionId":29963,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# ## SIIM-ISIC Melanoma Classification dataset\n\n\nThis notebook shows a few methods to load and display images from the SIIM-ISIC Melanoma Classification challenge dataset. \n","metadata":{}},{"cell_type":"code","source":"#plotly\n!pip install --upgrade pip --quiet\n!pip install chart_studio --quiet\n\nimport os\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport numpy as np\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport plotly.express as px\nimport chart_studio.plotly as py\nimport plotly.graph_objs as go\nfrom plotly.offline import iplot\nimport cufflinks\ncufflinks.go_offline()\ncufflinks.set_config_file(world_readable=True, theme='pearl')\n\nimport seaborn as sns\nsns.set(style=\"whitegrid\")\n\n#pydicom\nimport pydicom\n# Suppress warnings \nimport warnings\nwarnings.filterwarnings('ignore')\n\n\n# Settings for pretty nice plots\nplt.style.use('fivethirtyeight')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-21T21:07:12.350124Z","iopub.execute_input":"2023-09-21T21:07:12.35047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir('../input/siim-isic-melanoma-classification/')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#os.listdir('../input/siim-isic-melanoma-classification/jpeg/train/')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.1 Loading Dataset","metadata":{}},{"cell_type":"code","source":"BASE_PATH = '../input/siim-isic-melanoma-classification'\n\n\n\nprint('Reading data...')\ntrain = pd.read_csv(f'{BASE_PATH}/train.csv')\ntest = pd.read_csv(f'{BASE_PATH}/test.csv')\nsubmission = pd.read_csv(f'{BASE_PATH}/sample_submission.csv')\nprint('Reading data completed')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train.head())\nprint(\"Shape of train :\", train.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(test.head())\nprint(\"Shape of test :\", test.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.2 Checking for Null values\n\n**train**","metadata":{}},{"cell_type":"code","source":"# checking missing data\ntotal = train.isnull().sum().sort_values(ascending = False)\npercent = (train.isnull().sum()/train.isnull().count()*100).sort_values(ascending = False)\nmissing_train_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_train_data.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**test**","metadata":{}},{"cell_type":"code","source":"# checking missing data\ntotal = test.isnull().sum().sort_values(ascending = False)\npercent = (test.isnull().sum()/test.isnull().count()*100).sort_values(ascending = False)\nmissing_test_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_test_data.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. Basic EDA\n\n### Let's Start with distribution of variables in train.csv\n","metadata":{}},{"cell_type":"markdown","source":"### 3.1checking for `benign_malignant` distribution","metadata":{}},{"cell_type":"code","source":"def plot_count(df, feature, title='', size=2.5):\n    f, ax = plt.subplots(1,1, figsize=(4*size,3*size))\n    total = float(len(df))\n    sns.countplot(df[feature],order = df[feature].value_counts().index, palette='Set2')\n    plt.title(title)\n    for p in ax.patches:\n        height = p.get_height()\n        ax.text(p.get_x()+p.get_width()/2.,\n                height + 3,\n                '{:1.2f}%'.format(100*height/total),\n                ha=\"center\") \n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_count(train, 'benign_malignant')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.2 checking for `gender` distribution","metadata":{}},{"cell_type":"code","source":"plot_count(train, 'sex')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.3 checking for `anatom_site_general_challenge` distribution","metadata":{}},{"cell_type":"code","source":"\nplot_count(train, 'anatom_site_general_challenge')\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.4 checking for `diagnosis` distribution","metadata":{}},{"cell_type":"code","source":"\ntrain['diagnosis'].value_counts(normalize=True).sort_values().iplot(kind='barh',\n                                                      xTitle='Percentage', \n                                                      linecolor='black', \n                                                      opacity=0.7,\n                                                      color='blue',\n                                                      theme='pearl',\n                                                      bargap=0.2,\n                                                      gridcolor='white',\n                                                      title='Distribution in the training set'\n                                                    )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.4 Let's check for relative distribution of `sex` and `target`","metadata":{}},{"cell_type":"code","source":"def plot_relative_distribution(df, feature, hue, title='', size=2):\n    f, ax = plt.subplots(1,1, figsize=(4*size,3*size))\n    total = float(len(df))\n    sns.countplot(x=feature, hue=hue, data=df, palette='Set2')\n    plt.title(title)\n    for p in ax.patches:\n        height = p.get_height()\n        ax.text(p.get_x()+p.get_width()/2.,\n                height + 3,\n                '{:1.2f}%'.format(100*height/total),\n                ha=\"center\") \n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_relative_distribution(\n    df=train,\n    feature='sex',\n    hue='benign_malignant',\n    title = 'relative count plot of sex with benign_malignant',\n    size=2.8\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.4 Let's check for relative distribution of `anatom_site_general_challenge` and `benign_malignant`","metadata":{}},{"cell_type":"code","source":"plot_relative_distribution(\n    df=train,\n    feature='anatom_site_general_challenge',\n    hue='benign_malignant',\n    title = 'relative count plot of anatom_site_general_challenge with benign_malignant',\n    size=3\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.5 Age distribution","metadata":{}},{"cell_type":"code","source":"train['age_approx'].iplot(\n    kind='hist',\n    bins=30,\n    color='blue',\n    xTitle='Age',\n    yTitle='Count',\n    title='Age Distribution'\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4. Randomly displaying few images\n","metadata":{}},{"cell_type":"code","source":"import PIL\nfrom PIL import Image, ImageDraw\n\n\ndef display_images(images, title=None): \n    f, ax = plt.subplots(5,3, figsize=(18,22))\n    if title:\n        f.suptitle(title, fontsize = 30)\n\n    for i, image_id in enumerate(images):\n        image_path = os.path.join(BASE_PATH, f'jpeg/train/{image_id}.jpg')\n        image = Image.open(image_path)\n        \n        ax[i//3, i%3].imshow(image) \n        image.close()       \n        ax[i//3, i%3].axis('off')\n\n        benign_malignant = train[train['image_name'] == image_id]['benign_malignant'].values[0]\n        ax[i//3, i%3].set_title(f\"image_name: {image_id}\\nSource: {benign_malignant}\", fontsize=\"15\")\n\n    plt.show() ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4.1 Displaying benign images","metadata":{}},{"cell_type":"code","source":"benign = train[train.benign_malignant == 'benign'].sample(n=15, random_state=42)\ndisplay_images(benign.image_name.values, title = 'benign images')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4.2 Displaying malignant images","metadata":{}},{"cell_type":"code","source":"malignant = train[train.benign_malignant == 'malignant'].sample(n=15, random_state=42)\ndisplay_images(malignant.image_name.values, title='malignant images')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4.3 Displaying benign images for sex == 'female'","metadata":{}},{"cell_type":"code","source":"female_patients = train[train.sex == 'female']\nbenign = female_patients[female_patients.benign_malignant == 'benign'].sample(n=15, random_state=42)\ndisplay_images(benign.image_name.values, title='benign images for female patients')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4.4 Displaying malignant images for sex == 'female'","metadata":{}},{"cell_type":"code","source":"female_patients = train[train.sex == 'female']\nmalignant = female_patients[female_patients.benign_malignant == 'malignant'].sample(n=15, random_state=42)\ndisplay_images(malignant.image_name.values, title='malignant images for female patients')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4.5 Displaying benign images for sex == 'male'","metadata":{}},{"cell_type":"code","source":"male_patients = train[train.sex == 'male']\nbenign = male_patients[male_patients.benign_malignant == 'benign'].sample(n=15, random_state=42)\ndisplay_images(benign.image_name.values, title='benign images for male patients')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4.6 Displaying malignant images for sex == 'male'","metadata":{}},{"cell_type":"code","source":"malignant = male_patients[male_patients.benign_malignant == 'malignant'].sample(n=15, random_state=42)\ndisplay_images(malignant.image_name.values, title='malignant images for male patients')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4.7 Displaying images based on `anatom_site_general_challenge`","metadata":{}},{"cell_type":"code","source":"anatom_sites = [ site for site in list(train.anatom_site_general_challenge.unique()) if type(site) != float ]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for site in anatom_sites[:2]:\n    site_df = train[train.anatom_site_general_challenge == site].sample(n=15, random_state=42)\n    display_images(site_df.image_name.values, title = f'patient images for anatom_site == {site}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Preparation","metadata":{}},{"cell_type":"code","source":"#os.listdir('../input/siim-isic-melanoma-classification/jpeg/train/')\ndef imtocsv(path,resize):\n    from PIL import Image\n    r=resize\n    A=list(os.listdir(path))\n    D=np.zeros((len(A),r*r*3))\n    for i in range(len(A)):\n        image = Image.open(path+A[i])\n        out=image.resize((r,r))\n        out=np.array(out)\n        out=out.flatten()\n        D[i,]=out\n    col_list = ['x' + str(x) for x in range(0,r*r*3)]\n    df= pd.DataFrame(D,columns=col_list)\n    image_name=[]\n    for j in range(len(A)):\n        image_name.append(A[j][:-4])\n    df.insert(0,'image_name',image_name,True)\n    return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tr='../input/siim-isic-melanoma-classification/jpeg/train/'\nts='../input/siim-isic-melanoma-classification/jpeg/test/'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df=imtocsv(tr,28)\ntest_df=imtocsv(ts,28)\nprint(train_df.head())\nprint(test_df.head())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.drop(['patient_id','sex','age_approx','anatom_site_general_challenge','diagnosis','benign_malignant'],axis=1,inplace=True)\ntest.drop(['patient_id','sex','age_approx','anatom_site_general_challenge'],axis=1,inplace=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tr=pd.merge(train,train_df,how='left',on='image_name')\nts=pd.merge(test,test_df,how='left',on='image_name')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tr.to_csv(\"train_28.csv\", index=False)\nts.to_csv(\"test_28.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tr.drop('image_name',axis=1,inplace=True)\nts.drop('image_name',axis=1,inplace=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#tr[tr.image_name=='ISIC_7685852']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_cols=tr.columns[tr.columns!='target']\nx_cols","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = pd.DataFrame(tr[x_cols])\ny = pd.Series(tr.target.values)\nX_train, X_test, y_train, y_test = train_test_split(X,\n                                                    y,\n                                                    test_size=.3,\n                                                    random_state=1234123)\n\n# look at the distrubution of labels in the train set\npd.Series(y_train).value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.svm import SVC\n# define support vector classifier\nsvm = SVC(kernel='rbf', probability=True, random_state=42)\n\n# fit model\nsvm.fit(X_train, y_train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score, auc, roc_curve\n\n# generate predictions\ny_pred = svm.predict(X_test)\n\n# calculate accuracy\naccuracy = accuracy_score(y_test, y_pred)\nprint('Model accuracy is: ', accuracy)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predict probabilities for X_test using predict_proba\nprobabilities = svm.predict_proba(X_test)\n\n# select the probabilities for label 1.0\ny_proba = probabilities[:, 1]\n\n# calculate false positive rate and true positive rate at different thresholds\nfalse_positive_rate, true_positive_rate, thresholds = roc_curve(y_test, y_proba, pos_label=1)\n\n# calculate AUC\nroc_auc = auc(false_positive_rate, true_positive_rate)\n\nplt.title('Receiver Operating Characteristic')\n# plot the false positive rate on the x axis and the true positive rate on the y axis\nroc_plot = plt.plot(false_positive_rate,\n                    true_positive_rate,\n                    label='AUC = {:0.2f}'.format(roc_auc))\n\nplt.legend(loc=0)\nplt.plot([0,1], [0,1], ls='--')\nplt.ylabel('True Positive Rate')\nplt.xlabel('False Positive Rate');","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# generate predictions\ny_pred = svm.predict(ts)\nprobabilities = svm.predict_proba(X_test)\nprint(probabilities.head())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = pd.DataFrame(svm.predict_proba(ts))\nsample = pd.read_csv(f\"{BASE_PATH}/sample_submission.csv\")\nsample.loc[:, \"target\"] = predictions[1]\n\nsample.to_csv(\"submission_svm.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}