{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.cluster import KMeans\nimport itertools\n#from sklearn.mixture import GaussianMixture\nfrom sklearn.mixture import BayesianGaussianMixture\nfrom sklearn.model_selection import train_test_split\nimport eli5\nfrom eli5.sklearn import PermutationImportance\n\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-31T14:37:46.278704Z","iopub.execute_input":"2022-07-31T14:37:46.279117Z","iopub.status.idle":"2022-07-31T14:37:57.088143Z","shell.execute_reply.started":"2022-07-31T14:37:46.279022Z","shell.execute_reply":"2022-07-31T14:37:57.087062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# read data","metadata":{}},{"cell_type":"code","source":"train=pd.read_csv(\"../input/tabular-playground-series-jul-2022/data.csv\")\nsubmission=pd.read_csv(\"../input/tabular-playground-series-jul-2022/sample_submission.csv\")\nprint('train data shape is :',train.shape)\nprint('submission data shape is :',submission.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-31T14:37:57.089835Z","iopub.execute_input":"2022-07-31T14:37:57.090463Z","iopub.status.idle":"2022-07-31T14:37:57.989444Z","shell.execute_reply.started":"2022-07-31T14:37:57.090433Z","shell.execute_reply":"2022-07-31T14:37:57.988737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# check data simply","metadata":{}},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-31T14:37:57.990665Z","iopub.execute_input":"2022-07-31T14:37:57.990957Z","iopub.status.idle":"2022-07-31T14:37:58.021756Z","shell.execute_reply.started":"2022-07-31T14:37:57.990931Z","shell.execute_reply":"2022-07-31T14:37:58.020804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-07-31T14:37:58.023687Z","iopub.execute_input":"2022-07-31T14:37:58.023958Z","iopub.status.idle":"2022-07-31T14:37:58.030770Z","shell.execute_reply.started":"2022-07-31T14:37:58.023932Z","shell.execute_reply":"2022-07-31T14:37:58.030071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe(include='all').T","metadata":{"execution":{"iopub.status.busy":"2022-07-31T14:37:58.031718Z","iopub.execute_input":"2022-07-31T14:37:58.032398Z","iopub.status.idle":"2022-07-31T14:37:58.217497Z","shell.execute_reply.started":"2022-07-31T14:37:58.032370Z","shell.execute_reply":"2022-07-31T14:37:58.216437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a=[col for col in train.columns if train[col].isnull().any()]\nprint(a)","metadata":{"execution":{"iopub.status.busy":"2022-07-31T14:37:58.218569Z","iopub.execute_input":"2022-07-31T14:37:58.219121Z","iopub.status.idle":"2022-07-31T14:37:58.230460Z","shell.execute_reply.started":"2022-07-31T14:37:58.219093Z","shell.execute_reply":"2022-07-31T14:37:58.229418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"there is no missing value","metadata":{}},{"cell_type":"code","source":"train[train.iloc[:,1:].duplicated()]","metadata":{"execution":{"iopub.status.busy":"2022-07-31T14:37:58.231492Z","iopub.execute_input":"2022-07-31T14:37:58.231865Z","iopub.status.idle":"2022-07-31T14:37:58.466918Z","shell.execute_reply.started":"2022-07-31T14:37:58.231838Z","shell.execute_reply":"2022-07-31T14:37:58.465970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"there is no dumplicate rows","metadata":{}},{"cell_type":"markdown","source":"# simple EDA","metadata":{}},{"cell_type":"markdown","source":"**check data distribution**","metadata":{}},{"cell_type":"code","source":"col_list=train.columns\nncols = 5\nnrows = 6\n\nfig, axes = plt.subplots(nrows, ncols, figsize=(30,30), facecolor='#EAEAF2')\n\nfor r in range(nrows):\n    for c in range(ncols):\n        col = col_list[r*ncols+c]\n        sns.histplot(x=train[col], ax=axes[r, c])\n        axes[r, c].set_ylabel('')\n        axes[r, c].set_xlabel(col, fontsize=15, fontweight='bold')\n        axes[r, c].tick_params(labelsize=10, width=0.5)\n        axes[r, c].xaxis.offsetText.set_fontsize(20)\n        axes[r, c].yaxis.offsetText.set_fontsize(20)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-31T14:37:58.468085Z","iopub.execute_input":"2022-07-31T14:37:58.468372Z","iopub.status.idle":"2022-07-31T14:38:09.757306Z","shell.execute_reply.started":"2022-07-31T14:37:58.468345Z","shell.execute_reply":"2022-07-31T14:38:09.756292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**check single correlation heatmap**","metadata":{}},{"cell_type":"code","source":"sns.set(rc = {'figure.figsize':(13,8)})\ndf_corr=train.iloc[:,1:].corr()\nsns.heatmap(df_corr, vmax=1, vmin=-1, center=0)","metadata":{"execution":{"iopub.status.busy":"2022-07-31T14:38:09.758683Z","iopub.execute_input":"2022-07-31T14:38:09.759077Z","iopub.status.idle":"2022-07-31T14:38:10.602148Z","shell.execute_reply.started":"2022-07-31T14:38:09.759031Z","shell.execute_reply":"2022-07-31T14:38:10.601279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# prepare training data","metadata":{}},{"cell_type":"markdown","source":"**standardization**","metadata":{}},{"cell_type":"code","source":"train_data=train.iloc[:,1:]\n\nscaler = StandardScaler()\nscaler.fit(train_data)\nX=pd.DataFrame(scaler.transform(train_data),columns=train_data.columns)\nX.describe().T","metadata":{"execution":{"iopub.status.busy":"2022-07-31T14:38:10.605213Z","iopub.execute_input":"2022-07-31T14:38:10.605520Z","iopub.status.idle":"2022-07-31T14:38:10.846955Z","shell.execute_reply.started":"2022-07-31T14:38:10.605493Z","shell.execute_reply":"2022-07-31T14:38:10.845911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**how may colusters is better ?**  ","metadata":{}},{"cell_type":"code","source":"score=[]\ndistances = []\n\nfor i in range(2, 20):\n    model = KMeans(n_clusters=i, random_state=0, init='random')\n    model.fit(train_data.iloc[:,1:])\n    result_list=model.cluster_centers_\n    a=list(itertools.combinations(result_list, 2))\n    diff_sum=0\n    for list_ in list(itertools.combinations(result_list, 2)):\n        diff_sum=diff_sum+np.abs(np.linalg.norm(list_[0]-list_[1]))\n    print('number of columns is : ',i,' , score is : ',diff_sum/(i*(i-1)/2))\n    distances.append(diff_sum/(i*(i-1)/2))\n\nsns.set(rc = {'figure.figsize':(6,4)})\nplt.plot(range(2, 20), distances, marker='o')\nplt.xlabel('number of clusters')\nplt.ylabel('distances')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-31T14:38:10.848191Z","iopub.execute_input":"2022-07-31T14:38:10.848577Z","iopub.status.idle":"2022-07-31T14:40:50.817139Z","shell.execute_reply.started":"2022-07-31T14:38:10.848550Z","shell.execute_reply":"2022-07-31T14:40:50.815851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"from above chart, I thought the number of clustter is 7 or 11 are good one","metadata":{}},{"cell_type":"markdown","source":"**sort columns in order of importance  with PermutationImportance**","metadata":{}},{"cell_type":"code","source":"X_train, X_valid = train_test_split(X,random_state=0)\ncls_gaussian= BayesianGaussianMixture(n_components=7,tol=0.001,covariance_type='full',\n                         max_iter=7000,init_params='kmeans',n_init=3)\ncls_gaussian.fit(X_train)\ny=cls_gaussian.predict(X_valid)\nprint('score is :', np.abs(cls_gaussian.score(X_valid)))\nperm = PermutationImportance(cls_gaussian, random_state=1).fit(X_valid, y)\neli5.show_weights(perm, feature_names = X.columns.tolist(),top=50)","metadata":{"execution":{"iopub.status.busy":"2022-07-31T14:40:50.819295Z","iopub.execute_input":"2022-07-31T14:40:50.819747Z","iopub.status.idle":"2022-07-31T14:44:51.977991Z","shell.execute_reply.started":"2022-07-31T14:40:50.819715Z","shell.execute_reply":"2022-07-31T14:44:51.976893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**make some feature**","metadata":{}},{"cell_type":"code","source":"from sklearn.decomposition import FastICA\n\nica = FastICA(n_components=5)\nS_1 = ica.fit_transform(train_data)\n\ncomp_df_1=pd.DataFrame(S_1,columns=['01','02','03','04','05'])\ncomp_df_1.head()\n\nX=pd.concat([train_data,comp_df_1],axis=1)\nX.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-31T14:44:51.979414Z","iopub.execute_input":"2022-07-31T14:44:51.980121Z","iopub.status.idle":"2022-07-31T14:44:59.185054Z","shell.execute_reply.started":"2022-07-31T14:44:51.980076Z","shell.execute_reply":"2022-07-31T14:44:59.183953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_valid = train_test_split(X,random_state=0)\ncls_gaussian= BayesianGaussianMixture(n_components=7,tol=0.001,covariance_type='full',\n                         max_iter=7000,init_params='kmeans',n_init=3)\ncls_gaussian.fit(X_train)\ny=cls_gaussian.predict(X_valid)\nprint('score is :', np.abs(cls_gaussian.score(X_valid)))\nperm = PermutationImportance(cls_gaussian, random_state=1).fit(X_valid, y)\neli5.show_weights(perm, feature_names = X.columns.tolist(),top=50)","metadata":{"execution":{"iopub.status.busy":"2022-07-31T14:44:59.186604Z","iopub.execute_input":"2022-07-31T14:44:59.187315Z","iopub.status.idle":"2022-07-31T14:48:06.315161Z","shell.execute_reply.started":"2022-07-31T14:44:59.187273Z","shell.execute_reply":"2022-07-31T14:48:06.313948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_list=['01','02','03','04','05','f_08','f_09','f_10','f_11','f_12','f_13','f_07','f_26','f_22','f_23','f_24','f_28','f_27','f_25']","metadata":{"execution":{"iopub.status.busy":"2022-07-31T14:48:06.316843Z","iopub.execute_input":"2022-07-31T14:48:06.323199Z","iopub.status.idle":"2022-07-31T14:48:06.332894Z","shell.execute_reply.started":"2022-07-31T14:48:06.323135Z","shell.execute_reply":"2022-07-31T14:48:06.331568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# create submission file","metadata":{}},{"cell_type":"code","source":"X=X[test_list]\ncls_gaussian= BayesianGaussianMixture(n_components=7,tol=0.0001,\n                             max_iter=10000,init_params='kmeans',n_init=5)\nsubmission['Predicted']=cls_gaussian.fit_predict(X)\nsubmission.to_csv('submission.csv',index=False)\nprint(abs(cls_gaussian.score(X)))","metadata":{"execution":{"iopub.status.busy":"2022-07-31T14:48:06.335162Z","iopub.execute_input":"2022-07-31T14:48:06.336159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.pie(submission['Predicted'].value_counts(),counterclock=False, startangle=90,\n       autopct='%1.1f%%', pctdistance=0.7)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}