{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport pydicom as dicom\nfrom sklearn.metrics import accuracy_score, confusion_matrix\nfrom sklearn.metrics import mean_squared_error, r2_score","metadata":{"execution":{"iopub.status.busy":"2022-07-26T12:33:48.91936Z","iopub.execute_input":"2022-07-26T12:33:48.919986Z","iopub.status.idle":"2022-07-26T12:33:49.987599Z","shell.execute_reply.started":"2022-07-26T12:33:48.919877Z","shell.execute_reply":"2022-07-26T12:33:49.986822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/train.csv')\ndf.info()\ndf = df.rename(columns = {'anatom_site_general_challenge':'site'})\ndf.drop([\"target\"],axis = 1,inplace = True)\nprint(df)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-26T12:33:49.989246Z","iopub.execute_input":"2022-07-26T12:33:49.989538Z","iopub.status.idle":"2022-07-26T12:33:50.116563Z","shell.execute_reply.started":"2022-07-26T12:33:49.98948Z","shell.execute_reply":"2022-07-26T12:33:50.11581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.isnull().sum()\n","metadata":{"execution":{"iopub.status.busy":"2022-07-26T12:33:50.117856Z","iopub.execute_input":"2022-07-26T12:33:50.118246Z","iopub.status.idle":"2022-07-26T12:33:50.154862Z","shell.execute_reply.started":"2022-07-26T12:33:50.118207Z","shell.execute_reply":"2022-07-26T12:33:50.152086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndf[\"age_approx\"].fillna(value = df[\"age_approx\"].mean(),inplace = True)\ndf[\"sex\"].fillna(value = df[\"sex\"].mode()[0],inplace = True)\ndf[\"site\"].fillna(value = df[\"site\"].mode()[0],inplace = True)\ndf.isnull().sum()\n","metadata":{"execution":{"iopub.status.busy":"2022-07-26T12:33:50.157104Z","iopub.execute_input":"2022-07-26T12:33:50.157441Z","iopub.status.idle":"2022-07-26T12:33:50.205564Z","shell.execute_reply.started":"2022-07-26T12:33:50.157403Z","shell.execute_reply":"2022-07-26T12:33:50.204702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"****Data Visualization and Analysis\n\n","metadata":{}},{"cell_type":"code","source":"\n\nsex_values = []\nfor k in range(len(df[\"benign_malignant\"])):\n    if df[\"benign_malignant\"][k]==\"malignant\":\n        sex_values.append(df[\"sex\"][k])\nse = [\"male\",'female']\nval = [sex_values.count(\"male\"),sex_values.count(\"female\")]\nplt.bar(se,val)\nplt.xlabel(\"sex\")\nplt.ylabel(\"Count\")\nplt.show()\n\n\n#Male suffers more than female","metadata":{"execution":{"iopub.status.busy":"2022-07-26T12:33:50.207139Z","iopub.execute_input":"2022-07-26T12:33:50.207402Z","iopub.status.idle":"2022-07-26T12:33:50.617115Z","shell.execute_reply.started":"2022-07-26T12:33:50.207365Z","shell.execute_reply":"2022-07-26T12:33:50.616386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ages = []\nfor k in range(len(df[\"benign_malignant\"])):\n    if df[\"benign_malignant\"][k]==\"malignant\":\n        ages.append(df[\"age_approx\"][k])\nsns.displot(ages)\n\n#People age between 50 to 80 suffers more","metadata":{"execution":{"iopub.status.busy":"2022-07-26T12:33:50.618516Z","iopub.execute_input":"2022-07-26T12:33:50.619348Z","iopub.status.idle":"2022-07-26T12:33:51.151211Z","shell.execute_reply.started":"2022-07-26T12:33:50.619307Z","shell.execute_reply":"2022-07-26T12:33:51.150354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sites = []\nfor k in range(len(df[\"benign_malignant\"])):\n    if df[\"benign_malignant\"][k]==\"malignant\":\n        sites.append(df[\"site\"][k])\nsns.countplot(sites)\n\n# torso is the major site","metadata":{"execution":{"iopub.status.busy":"2022-07-26T12:33:51.15278Z","iopub.execute_input":"2022-07-26T12:33:51.153193Z","iopub.status.idle":"2022-07-26T12:33:51.582126Z","shell.execute_reply.started":"2022-07-26T12:33:51.153154Z","shell.execute_reply":"2022-07-26T12:33:51.581381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# Patient id is not useful\ndf.drop([\"patient_id\"],axis =1,inplace = True)","metadata":{"execution":{"iopub.status.busy":"2022-07-26T12:33:51.583731Z","iopub.execute_input":"2022-07-26T12:33:51.584325Z","iopub.status.idle":"2022-07-26T12:33:51.593097Z","shell.execute_reply.started":"2022-07-26T12:33:51.584283Z","shell.execute_reply":"2022-07-26T12:33:51.592292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"ENCODINGS OF  CATEGORICAL FIELDS\n","metadata":{}},{"cell_type":"code","source":"\ndf[\"sex\"].replace([\"male\",\"female\"],[0,1],inplace = True)\ndf[\"benign_malignant\"].replace([\"benign\",\"malignant\"],[0,1],inplace = True)\ndf = pd.get_dummies(df,columns  = [\"site\",\"diagnosis\"],drop_first = True)\nprint(df)","metadata":{"execution":{"iopub.status.busy":"2022-07-26T12:33:51.595236Z","iopub.execute_input":"2022-07-26T12:33:51.595927Z","iopub.status.idle":"2022-07-26T12:33:51.66594Z","shell.execute_reply.started":"2022-07-26T12:33:51.59589Z","shell.execute_reply":"2022-07-26T12:33:51.665006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"CORELATIONS BETWEEN COLUMNS AND BENIGN_MALIGNANT COLUMN","metadata":{}},{"cell_type":"code","source":"sns.heatmap(df.corr())\nprint(df.corr())","metadata":{"execution":{"iopub.status.busy":"2022-07-26T12:33:51.668831Z","iopub.execute_input":"2022-07-26T12:33:51.669748Z","iopub.status.idle":"2022-07-26T12:33:52.20696Z","shell.execute_reply.started":"2022-07-26T12:33:51.669706Z","shell.execute_reply":"2022-07-26T12:33:52.206131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.heatmap(df.corr()[[\"benign_malignant\"]].sort_values('benign_malignant').tail(16),annot = True)\n\n# The benign_malignant column depends on sex,site_upper extremity columns,age_approx","metadata":{"execution":{"iopub.status.busy":"2022-07-26T12:33:52.20854Z","iopub.execute_input":"2022-07-26T12:33:52.208816Z","iopub.status.idle":"2022-07-26T12:33:52.604264Z","shell.execute_reply.started":"2022-07-26T12:33:52.208775Z","shell.execute_reply":"2022-07-26T12:33:52.603432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image = '/kaggle/input/siim-isic-melanoma-classification/train/' + df['image_name'][91] +'.dcm'\nds = dicom.dcmread(image)\n\nplt.imshow(ds.pixel_array)","metadata":{"execution":{"iopub.status.busy":"2022-07-26T12:33:52.605653Z","iopub.execute_input":"2022-07-26T12:33:52.606156Z","iopub.status.idle":"2022-07-26T12:33:55.748223Z","shell.execute_reply.started":"2022-07-26T12:33:52.606117Z","shell.execute_reply":"2022-07-26T12:33:55.747395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**TRAINING BASED ON IMAGES**","metadata":{}},{"cell_type":"code","source":"images = []\nfor x in range(len(df[\"image_name\"][:270])):\n    image = '/kaggle/input/siim-isic-melanoma-classification/train/' + df[\"image_name\"][x] +'.dcm'\n    ds = dicom.dcmread(image)\n    pixels = ds.pixel_array\n\n    arr = pixels.flatten()\n\n    images.append(arr)\nimport tensorflow as tf\nimages = tf.keras.preprocessing.sequence.pad_sequences(\n  images,\n  maxlen = 256,\n  dtype = \"int32\",\n  padding = \"pre\",\n  truncating = \"pre\",\n  value = 0\n)\n\n\ntest = df.tail(100)\ntest_images = []\nfor x in test['image_name']:\n    image = '/kaggle/input/siim-isic-melanoma-classification/train/' + x +'.dcm'\n    ds = dicom.dcmread(image)\n    pixels = ds.pixel_array\n    \n    test_images.append(pixels.flatten())\ntest_images = tf.keras.preprocessing.sequence.pad_sequences(\n  test_images,\n  maxlen = 256,\n  dtype = \"int32\",\n  padding = \"pre\",\n  truncating = \"pre\",\n  value = 0\n)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-07-26T12:33:55.749286Z","iopub.execute_input":"2022-07-26T12:33:55.749593Z","iopub.status.idle":"2022-07-26T12:36:01.036269Z","shell.execute_reply.started":"2022-07-26T12:33:55.749556Z","shell.execute_reply":"2022-07-26T12:36:01.035364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\nX = images\ny = df[\"benign_malignant\"][:270]\nclassifier_lr = LogisticRegression(solver = 'liblinear')\nclassifier_lr.fit(X,y)\nX_test = test_images\ny_test = df[\"benign_malignant\"].tail(100)\ny_pred_lr = classifier_lr.predict(X_test)\nprint(accuracy_score(y_test,y_pred_lr))\nprint(confusion_matrix(y_test,y_pred_lr))","metadata":{"execution":{"iopub.status.busy":"2022-07-26T12:36:01.037592Z","iopub.execute_input":"2022-07-26T12:36:01.037875Z","iopub.status.idle":"2022-07-26T12:36:01.149302Z","shell.execute_reply.started":"2022-07-26T12:36:01.037837Z","shell.execute_reply":"2022-07-26T12:36:01.148402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn import svm\nX = images\ny = df[\"benign_malignant\"][:270]\nclassifier_svm = svm.SVC()\nclassifier_svm.fit(X,y)\nX_test = test_images\ny_test = df[\"benign_malignant\"].tail(100)\ny_pred_svm = classifier_svm.predict(X_test)\nprint(accuracy_score(y_test,y_pred_svm))\nprint(confusion_matrix(y_test,y_pred_svm))","metadata":{"execution":{"iopub.status.busy":"2022-07-26T12:36:01.150781Z","iopub.execute_input":"2022-07-26T12:36:01.151679Z","iopub.status.idle":"2022-07-26T12:36:01.178691Z","shell.execute_reply.started":"2022-07-26T12:36:01.151624Z","shell.execute_reply":"2022-07-26T12:36:01.177854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.tree import DecisionTreeClassifier\nX = images\ny = df[\"benign_malignant\"][:270]\nclassifier_dt = DecisionTreeClassifier()\nclassifier_dt.fit(X,y)\nX_test = test_images\ny_test = df[\"benign_malignant\"].tail(100)\n\ny_pred_dt = classifier_dt.predict(X_test)\nprint(accuracy_score(y_test,y_pred_dt))\nprint(confusion_matrix(y_test,y_pred_dt))","metadata":{"execution":{"iopub.status.busy":"2022-07-26T12:36:01.180161Z","iopub.execute_input":"2022-07-26T12:36:01.180419Z","iopub.status.idle":"2022-07-26T12:36:01.286909Z","shell.execute_reply.started":"2022-07-26T12:36:01.180381Z","shell.execute_reply":"2022-07-26T12:36:01.286186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nX = images\ny = df[\"benign_malignant\"][:270]\nclassifier_rf = RandomForestClassifier()\nclassifier_rf.fit(X,y)\nX_test = test_images\ny_test = df[\"benign_malignant\"].tail(100)\ny_pred_rf = classifier_rf.predict(X_test)\nprint(accuracy_score(y_test,y_pred_rf))\nprint(confusion_matrix(y_test,y_pred_rf))\n","metadata":{"execution":{"iopub.status.busy":"2022-07-26T12:36:01.288236Z","iopub.execute_input":"2022-07-26T12:36:01.288666Z","iopub.status.idle":"2022-07-26T12:36:01.503543Z","shell.execute_reply.started":"2022-07-26T12:36:01.288622Z","shell.execute_reply":"2022-07-26T12:36:01.502831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**TRAINING BASED ON RELATED FEATURES**","metadata":{}},{"cell_type":"code","source":"sns.heatmap(df.corr()[[\"benign_malignant\"]].sort_values('benign_malignant').tail(16),annot = True)","metadata":{"execution":{"iopub.status.busy":"2022-07-26T12:36:01.504702Z","iopub.execute_input":"2022-07-26T12:36:01.505225Z","iopub.status.idle":"2022-07-26T12:36:02.084803Z","shell.execute_reply.started":"2022-07-26T12:36:01.505183Z","shell.execute_reply":"2022-07-26T12:36:02.083886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = []\nma =df[\"age_approx\"][:270].max()\nfor k in range(270):\n    train.append([df[\"diagnosis_melanoma\"][k],df[\"site_oral/genital\"][k],df[\"site_upper extremity\"][k],(df[\"age_approx\"][k])/ma])\n","metadata":{"execution":{"iopub.status.busy":"2022-07-26T12:36:02.086462Z","iopub.execute_input":"2022-07-26T12:36:02.086749Z","iopub.status.idle":"2022-07-26T12:36:02.104863Z","shell.execute_reply.started":"2022-07-26T12:36:02.086711Z","shell.execute_reply":"2022-07-26T12:36:02.103888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = []\nind = 0\nma =  df[\"age_approx\"][33026:].max()\n\nfor k in range(33026,33126):\n\n    test.append([df[\"diagnosis_melanoma\"][k],df[\"site_oral/genital\"][k],df[\"site_upper extremity\"][k],(df[\"age_approx\"][k])/ma])\n    ","metadata":{"execution":{"iopub.status.busy":"2022-07-26T12:36:02.10685Z","iopub.execute_input":"2022-07-26T12:36:02.107136Z","iopub.status.idle":"2022-07-26T12:36:02.122914Z","shell.execute_reply.started":"2022-07-26T12:36:02.107098Z","shell.execute_reply":"2022-07-26T12:36:02.122032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\nX = train\ny = df[\"benign_malignant\"][:270]\nclassifier_lr = LogisticRegression(solver = 'liblinear')\nclassifier_lr.fit(X,y)\nX_test = test\ny_test = df[\"benign_malignant\"].tail(100)\ny_pred_lr = classifier_lr.predict(X_test)\nprint(accuracy_score(y_test,y_pred_lr))\nprint(confusion_matrix(y_test,y_pred_lr))","metadata":{"execution":{"iopub.status.busy":"2022-07-26T12:36:02.124828Z","iopub.execute_input":"2022-07-26T12:36:02.125166Z","iopub.status.idle":"2022-07-26T12:36:02.141222Z","shell.execute_reply.started":"2022-07-26T12:36:02.125126Z","shell.execute_reply":"2022-07-26T12:36:02.140398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn import svm\nX = train\ny = df[\"benign_malignant\"][:250]\nclassifier_svm = svm.SVC()\nclassifier_svm.fit(X,y)\nX_test = test\ny_test = df[\"benign_malignant\"].tail(100)\ny_pred_svm = classifier_svm.predict(X_test)\nprint(accuracy_score(y_test,y_pred_svm))\nprint(confusion_matrix(y_test,y_pred_svm))","metadata":{"execution":{"iopub.status.busy":"2022-07-26T12:36:02.145405Z","iopub.execute_input":"2022-07-26T12:36:02.146063Z","iopub.status.idle":"2022-07-26T12:36:02.476419Z","shell.execute_reply.started":"2022-07-26T12:36:02.146024Z","shell.execute_reply":"2022-07-26T12:36:02.475392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.tree import DecisionTreeClassifier\nX = train\ny = df[\"benign_malignant\"][:270]\nclassifier_dt = DecisionTreeClassifier()\nclassifier_dt.fit(X,y)\nX_test = test\ny_test = df[\"benign_malignant\"].tail(100)\n\ny_pred_dt = classifier_dt.predict(X_test)\nprint(accuracy_score(y_test,y_pred_dt))\nprint(confusion_matrix(y_test,y_pred_dt))","metadata":{"execution":{"iopub.status.busy":"2022-07-26T12:36:02.477622Z","iopub.status.idle":"2022-07-26T12:36:02.478184Z","shell.execute_reply.started":"2022-07-26T12:36:02.477948Z","shell.execute_reply":"2022-07-26T12:36:02.477973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nX = train\ny = df[\"benign_malignant\"][:270]\nclassifier_rf = RandomForestClassifier()\nclassifier_rf.fit(X,y)\nX_test = test\ny_test = df[\"benign_malignant\"].tail(100)\ny_pred_dt = classifier_rf.predict(X_test)\nprint(accuracy_score(y_test,y_pred_dt))\nprint(confusion_matrix(y_test,y_pred_dt))\n","metadata":{"execution":{"iopub.status.busy":"2022-07-26T12:36:02.479319Z","iopub.status.idle":"2022-07-26T12:36:02.479888Z","shell.execute_reply.started":"2022-07-26T12:36:02.479637Z","shell.execute_reply":"2022-07-26T12:36:02.479662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**PEDICTION OF AN IMAGE**","metadata":{}},{"cell_type":"code","source":"image_path = '/kaggle/input/siim-isic-melanoma-classification/train/' + df['image_name'][91] +'.dcm'\n\nimage_to_test = []\nds = dicom.dcmread(image_path)\npixels = ds.pixel_array\nimage_to_test.append(pixels.flatten())\nimage_to_test = tf.keras.preprocessing.sequence.pad_sequences(\n  image_to_test,\n  maxlen = 256,\n  dtype = \"int32\",\n  padding = \"pre\",\n  truncating = \"pre\",\n  value = 0\n)\n\nnp.append(image_to_test,[df[\"sex\"][91],df[\"site_upper extremity\"][91],df[\"age_approx\"][91]],axis = None)\n\nif classifier_lr.predict(image_to_test) == [1]:\n    plt.imshow(pixels)\n    print('Malignant:Need immediate Treatment')\nelse:\n    plt.imshow(pixels)\n    print('Benign')\n","metadata":{"execution":{"iopub.status.busy":"2022-07-26T12:36:02.480948Z","iopub.status.idle":"2022-07-26T12:36:02.481535Z","shell.execute_reply.started":"2022-07-26T12:36:02.481278Z","shell.execute_reply":"2022-07-26T12:36:02.481302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}