{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gc\nimport os\nimport cv2\nimport time\nimport datetime\nimport warnings\nimport random\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Load Results -Original Data + prediction values column given by the model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/test.csv')\nben_pred = pd.read_csv(\"../input/predictions-2/benign_with_preds (1).csv\")\nmal_pred = pd.read_csv(\"../input/predictions-2/malignant_with_preds (1).csv\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Slicing the dataset into SAMPLE_NUM number of samples","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"SAMPLE_NUM = 600\n\nnew_benign = ben_pred[0:SAMPLE_NUM]\nbenign_predictios = new_benign['predictions']\n\nnew_mal = mal_pred[0:SAMPLE_NUM]\nmalignant_predictions = new_mal['predictions']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Drop non-relevant columns, also notice that the test_df needs one-hot encoding","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.concat([test_df,pd.get_dummies(test_df['anatom_site_general_challenge'], prefix='site')],axis=1)\ntest_df.drop(['image_name'],axis=1, inplace=True)\ntest_df.drop(['patient_id'],axis=1, inplace=True)\ntest_df.drop(['anatom_site_general_challenge'],axis=1, inplace=True)\n\nnew_benign.drop(['anatom_site_general_challenge'],axis=1, inplace=True)\nnew_benign.drop(['patient_id'],axis=1, inplace=True)\nnew_benign.drop(['image_name'],axis=1, inplace=True)\nnew_benign.drop(['Unnamed: 0'],axis=1, inplace=True)\nnew_benign.drop(['target', 'site_anterior torso', 'site_lateral torso', 'site_posterior torso', 'site_nan'], inplace=True, axis=1)\n\n\n\nnew_mal.drop(['anatom_site_general_challenge'],axis=1, inplace=True)\nnew_mal.drop(['patient_id'],axis=1, inplace=True)\nnew_mal.drop(['image_name'],axis=1, inplace=True)\nnew_mal.drop(['Unnamed: 0'],axis=1, inplace=True)\nnew_mal.drop(['target', 'site_anterior torso', 'site_lateral torso', 'site_posterior torso', 'site_nan'], inplace=True, axis=1)\n\n\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Display histogram of prediction values for training predictions","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import random\nfrom matplotlib import pyplot\n\nfrom matplotlib.pyplot import figure\nfigure(num=None, figsize=(10, 10), dpi=80, facecolor='w', edgecolor='k')\n\n\n#plot both histograms(range from -10 to 10), bins set to 100\npyplot.hist([benign_predictios, malignant_predictions], bins= 100, range=[0,1.1], label=['ben', 'mal'], density = True)\n#plot legend\npyplot.legend(loc='upper right')\n#show it\npyplot.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Filter the groups by the prediction values - Take out only the sample with prediction value between 0.15 and 0.85. Group A is benign samples and group M is malignant samples - all in this range of prediction values.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"benign_first = new_benign['predictions'] > 0.15\nnew_benign = new_benign[benign_first]\n\nm_1 =  new_mal['predictions'] < 0.85\nnew_mal = new_mal[m_1]\n\nnew_benign['group'] = 'A'\nnew_mal['group'] = 'M'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Concat the training samples, normalize the age column and drop non relevant columns. If you wish to use the test_df instead of the training samples - uncomment the code in the bottom","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"all_preds = pd.concat([new_benign,new_mal], axis = 0)\n\na = (all_preds['age_approx'] -all_preds['age_approx'].mean())/all_preds['age_approx'].std()\nall_preds['age_approx'] = a\n\nall_preds_meta = all_preds.loc[:, all_preds.columns.str.contains('^g')]\n\nall_preds.drop(['predictions', 'group'], inplace=True, axis=1)\nall_preds\n\n# all_preds = test_df\n# a = (all_preds['age_approx'] -all_preds['age_approx'].mean())/all_preds['age_approx'].std()\n# all_preds['age_approx'] = a\n# all_preds.fillna(0, inplace = True)\n# all_preds['sex'] = all_preds['sex'].map({'male': 1, 'female': 0})\n# all_preds['sex'] = all_preds['sex'].fillna(-1)\n# all_preds","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Apply SVD algorithm on the full Data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nu, s, v = np.linalg.svd(all_preds, full_matrices=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This is Code for loading pre existing matrix of s and v","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# S = np.loadtxt('/kaggle/input/matricesnew/S.txt', dtype=float)\n# S = np.diag(S)\n# V = np.loadtxt('/kaggle/input/matricesnew/V.txt', dtype=float)\n# M = all_preds.values\n# v = V.transpose() \n# s = np.linalg.inv(s)\n# U = np.matmul(np.matmul(M, v), s)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"SVD Metrics","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"var_explained = np.round(s**2/np.sum(s**2), decimals=3)\nvar_explained\n \nsns.barplot(x=list(range(1,len(var_explained)+1)),\n            y=var_explained, color=\"limegreen\")\nplt.xlabel('SVs', fontsize=16)\nplt.ylabel('Percent Variance Explained', fontsize=16)\nplt.savefig('svd_scree_plot.png',dpi=100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels= ['SV'+str(i) for i in range(1,10)]\nsvd_df = pd.DataFrame(u[:,0:9], index=all_preds_meta[\"group\"].tolist(), columns=labels)\nsvd_df=svd_df.reset_index()\nsvd_df.rename(columns={'index':'Group'}, inplace=True)\nsvd_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Group division - not relevant for test set","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"A = svd_df[svd_df['Group'] == 'A']  \nA_M = pd.concat([A,M],axis=0)\nM = svd_df[svd_df['Group'] == 'M']  ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"1D histogram for SV values","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from matplotlib.pyplot import figure\nfigure(num=None, figsize=(20, 20), dpi=80, facecolor='w', edgecolor='k')\n\n\n#plot both histograms(range from -10 to 10), bins set to 100\nn, bins, _ =pyplot.hist([A['SV1'], M['SV1']], bins= 100, range=[-5,5], label=['A', 'M'], density = True)\n#plot legend\npyplot.legend(loc='upper right')\n#show it\npyplot.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 3D comparison","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import itertools\nl = [1,2,3,4,5,6,7,8,9,10]\n\nperms = list(itertools.combinations(l, 3)) \nprint(len(perms))\nprint(perms)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from mpl_toolkits.mplot3d import Axes3D\nimport matplotlib.pyplot as plt\nfrom matplotlib.pyplot import figure\nimport plotly.express as px\n\ni = -1\n\n\nX = 'SV' + str(perms[i][0])\nY = 'SV' + str(perms[i][1])\nZ = 'SV' + str(perms[i][2])\n\nfig = px.scatter_3d(A_M, x=X, y=Y, z=Z, color='Group',\n                    title=\"3D Scatter Plot\")\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2D comparison","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import itertools\nl = [1,2,3,4,5,6,7,8,9,10]\n\nperms = list(itertools.combinations(l, 2))\nprint(len(perms))\nprint(perms)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from mpl_toolkits.mplot3d import Axes3D\nimport matplotlib.pyplot as plt\nfrom matplotlib.pyplot import figure\nimport plotly.express as px\n\ni=perms.index((8, 9))\nX = 'SV' + str(perms[i][0])\nY = 'SV' + str(perms[i][1])\n\nfig = px.scatter(A_M, x=X, y=Y, color='Group',\n             width=700, height=500,\n             title=\"2D Scatter Plot\")\nfig.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}