{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom matplotlib import pyplot as plt\nplt.rcParams['figure.figsize'] = (12, 8)\nplt.style.use('seaborn-darkgrid')\nimport seaborn as sns\nsns.set_style('darkgrid')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-08T05:53:07.755526Z","iopub.execute_input":"2022-08-08T05:53:07.756136Z","iopub.status.idle":"2022-08-08T05:53:08.447855Z","shell.execute_reply.started":"2022-08-08T05:53:07.756085Z","shell.execute_reply":"2022-08-08T05:53:08.446813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/tabular-playground-series-aug-2022/train.csv')\ntest = pd.read_csv('../input/tabular-playground-series-aug-2022/test.csv')\nsub = pd.read_csv('../input/tabular-playground-series-aug-2022/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:53:08.452622Z","iopub.execute_input":"2022-08-08T05:53:08.453713Z","iopub.status.idle":"2022-08-08T05:53:08.774664Z","shell.execute_reply.started":"2022-08-08T05:53:08.453673Z","shell.execute_reply":"2022-08-08T05:53:08.773405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:53:08.778009Z","iopub.execute_input":"2022-08-08T05:53:08.778441Z","iopub.status.idle":"2022-08-08T05:53:08.834502Z","shell.execute_reply.started":"2022-08-08T05:53:08.778399Z","shell.execute_reply":"2022-08-08T05:53:08.833339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.drop('id', axis = 1, inplace = True)\ntest.drop('id', axis = 1, inplace = True)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:53:08.837593Z","iopub.execute_input":"2022-08-08T05:53:08.838382Z","iopub.status.idle":"2022-08-08T05:53:08.855065Z","shell.execute_reply.started":"2022-08-08T05:53:08.838333Z","shell.execute_reply":"2022-08-08T05:53:08.853814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:53:08.856819Z","iopub.execute_input":"2022-08-08T05:53:08.857299Z","iopub.status.idle":"2022-08-08T05:53:08.928311Z","shell.execute_reply.started":"2022-08-08T05:53:08.857254Z","shell.execute_reply":"2022-08-08T05:53:08.927220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['product_code']","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:53:08.929998Z","iopub.execute_input":"2022-08-08T05:53:08.931202Z","iopub.status.idle":"2022-08-08T05:53:08.942901Z","shell.execute_reply.started":"2022-08-08T05:53:08.931157Z","shell.execute_reply":"2022-08-08T05:53:08.941649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['product_code'].unique()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:53:08.944923Z","iopub.execute_input":"2022-08-08T05:53:08.945630Z","iopub.status.idle":"2022-08-08T05:53:08.957955Z","shell.execute_reply.started":"2022-08-08T05:53:08.945586Z","shell.execute_reply":"2022-08-08T05:53:08.957137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:53:08.959412Z","iopub.execute_input":"2022-08-08T05:53:08.960537Z","iopub.status.idle":"2022-08-08T05:53:08.971429Z","shell.execute_reply.started":"2022-08-08T05:53:08.960493Z","shell.execute_reply":"2022-08-08T05:53:08.970209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = train.pop('failure')\ny","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:53:08.974396Z","iopub.execute_input":"2022-08-08T05:53:08.975119Z","iopub.status.idle":"2022-08-08T05:53:08.986120Z","shell.execute_reply.started":"2022-08-08T05:53:08.975069Z","shell.execute_reply":"2022-08-08T05:53:08.985000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y.value_counts().plot.bar()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:53:08.990637Z","iopub.execute_input":"2022-08-08T05:53:08.991622Z","iopub.status.idle":"2022-08-08T05:53:09.249591Z","shell.execute_reply.started":"2022-08-08T05:53:08.991573Z","shell.execute_reply":"2022-08-08T05:53:09.248293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:53:09.251165Z","iopub.execute_input":"2022-08-08T05:53:09.251523Z","iopub.status.idle":"2022-08-08T05:53:09.267990Z","shell.execute_reply.started":"2022-08-08T05:53:09.251491Z","shell.execute_reply":"2022-08-08T05:53:09.266772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:53:09.269497Z","iopub.execute_input":"2022-08-08T05:53:09.270130Z","iopub.status.idle":"2022-08-08T05:53:09.284215Z","shell.execute_reply.started":"2022-08-08T05:53:09.270096Z","shell.execute_reply":"2022-08-08T05:53:09.283031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"isna = ['loading'] + [f'measurement_{i}' for i in range(3, 18)]\ntrain[[f'{i}_isna' for i in isna]] = train[isna].isna().astype(int)\ntest[[f'{i}_isna' for i in isna]] = test[isna].isna().astype(int)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:53:09.285435Z","iopub.execute_input":"2022-08-08T05:53:09.286260Z","iopub.status.idle":"2022-08-08T05:53:09.317198Z","shell.execute_reply.started":"2022-08-08T05:53:09.286227Z","shell.execute_reply":"2022-08-08T05:53:09.316382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tofill = train.columns.tolist()\ntofill.remove('product_code')\ntofill","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:53:09.318643Z","iopub.execute_input":"2022-08-08T05:53:09.319219Z","iopub.status.idle":"2022-08-08T05:53:09.326409Z","shell.execute_reply.started":"2022-08-08T05:53:09.319186Z","shell.execute_reply":"2022-08-08T05:53:09.325455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm.auto import tqdm\ntqdm.pandas()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:53:09.327777Z","iopub.execute_input":"2022-08-08T05:53:09.328584Z","iopub.status.idle":"2022-08-08T05:53:09.342797Z","shell.execute_reply.started":"2022-08-08T05:53:09.328550Z","shell.execute_reply":"2022-08-08T05:53:09.341632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in tqdm(train.columns) :\n    train[col] = train.groupby('product_code')[col].fillna(train[col].mode()[0]) if train[col].dtypes == 'object' else train.groupby('product_code')[col].fillna(train[col].mean())\n    test[col] = test.groupby('product_code')[col].fillna(test[col].mode()[0]) if test[col].dtypes == 'object' else test.groupby('product_code')[col].fillna(test[col].mean())","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:53:09.344752Z","iopub.execute_input":"2022-08-08T05:53:09.345363Z","iopub.status.idle":"2022-08-08T05:53:10.097009Z","shell.execute_reply.started":"2022-08-08T05:53:09.345329Z","shell.execute_reply":"2022-08-08T05:53:10.095741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:53:10.098672Z","iopub.execute_input":"2022-08-08T05:53:10.099152Z","iopub.status.idle":"2022-08-08T05:53:10.147008Z","shell.execute_reply.started":"2022-08-08T05:53:10.099105Z","shell.execute_reply":"2022-08-08T05:53:10.145762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:53:10.149089Z","iopub.execute_input":"2022-08-08T05:53:10.149547Z","iopub.status.idle":"2022-08-08T05:53:10.192032Z","shell.execute_reply.started":"2022-08-08T05:53:10.149501Z","shell.execute_reply":"2022-08-08T05:53:10.191019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.drop('product_code', axis = 1, inplace = True)\ntest.drop('product_code', axis = 1, inplace = True)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:53:10.193379Z","iopub.execute_input":"2022-08-08T05:53:10.194106Z","iopub.status.idle":"2022-08-08T05:53:10.217329Z","shell.execute_reply.started":"2022-08-08T05:53:10.194058Z","shell.execute_reply":"2022-08-08T05:53:10.215958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f, axes = plt.subplots(8, 5, figsize = (30, 20))\nfor col, ax in zip(train.columns, axes.ravel()) :\n    _ = sns.kdeplot(train[col], ax = ax, hue = y) if not train[col].dtypes == 'object' else sns.countplot(train[col], ax = ax, hue = y)\n    ax.set_xlabel('')\n    ax.set_title(col)\nf.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:53:10.220208Z","iopub.execute_input":"2022-08-08T05:53:10.221402Z","iopub.status.idle":"2022-08-08T05:53:23.846986Z","shell.execute_reply.started":"2022-08-08T05:53:10.221354Z","shell.execute_reply":"2022-08-08T05:53:23.846067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:53:23.848356Z","iopub.execute_input":"2022-08-08T05:53:23.848926Z","iopub.status.idle":"2022-08-08T05:53:23.886609Z","shell.execute_reply.started":"2022-08-08T05:53:23.848894Z","shell.execute_reply":"2022-08-08T05:53:23.885656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = sns.heatmap(train.corr())","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:53:23.887813Z","iopub.execute_input":"2022-08-08T05:53:23.888636Z","iopub.status.idle":"2022-08-08T05:53:25.074606Z","shell.execute_reply.started":"2022-08-08T05:53:23.888597Z","shell.execute_reply":"2022-08-08T05:53:25.073526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat = ['attribute_0', 'attribute_1']\ntrain_ohe = pd.get_dummies(train, columns = cat)\ntest_ohe = pd.get_dummies(test, columns = cat)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:53:25.075894Z","iopub.execute_input":"2022-08-08T05:53:25.076332Z","iopub.status.idle":"2022-08-08T05:53:25.119501Z","shell.execute_reply.started":"2022-08-08T05:53:25.076300Z","shell.execute_reply":"2022-08-08T05:53:25.118476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.heatmap(train_ohe.corr())","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:53:27.421557Z","iopub.execute_input":"2022-08-08T05:53:27.422033Z","iopub.status.idle":"2022-08-08T05:53:28.966107Z","shell.execute_reply.started":"2022-08-08T05:53:27.421997Z","shell.execute_reply":"2022-08-08T05:53:28.964879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.feature_selection import mutual_info_classif","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:53:31.978402Z","iopub.execute_input":"2022-08-08T05:53:31.979895Z","iopub.status.idle":"2022-08-08T05:53:32.243161Z","shell.execute_reply.started":"2022-08-08T05:53:31.979853Z","shell.execute_reply":"2022-08-08T05:53:32.242013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ohe","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:53:33.998930Z","iopub.execute_input":"2022-08-08T05:53:33.999409Z","iopub.status.idle":"2022-08-08T05:53:34.036113Z","shell.execute_reply.started":"2022-08-08T05:53:33.999367Z","shell.execute_reply":"2022-08-08T05:53:34.034782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mi_scores = mutual_info_classif(train_ohe, y, random_state = 0, discrete_features = [1, 2])\nmi_scores = pd.Series(\n    mi_scores,\n    index = train_ohe.columns\n).sort_values(ascending = False)\nmi_scores","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:54:12.790666Z","iopub.execute_input":"2022-08-08T05:54:12.791085Z","iopub.status.idle":"2022-08-08T05:54:17.856583Z","shell.execute_reply.started":"2022-08-08T05:54:12.791051Z","shell.execute_reply":"2022-08-08T05:54:17.855372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.decomposition import PCA\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.pipeline import make_pipeline","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:54:20.103289Z","iopub.execute_input":"2022-08-08T05:54:20.103778Z","iopub.status.idle":"2022-08-08T05:54:20.113830Z","shell.execute_reply.started":"2022-08-08T05:54:20.103734Z","shell.execute_reply":"2022-08-08T05:54:20.112336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_num = train.drop(cat, axis = 1)\ntest_num = test.drop(cat, axis = 1)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:54:21.065530Z","iopub.execute_input":"2022-08-08T05:54:21.066550Z","iopub.status.idle":"2022-08-08T05:54:21.077265Z","shell.execute_reply.started":"2022-08-08T05:54:21.066504Z","shell.execute_reply":"2022-08-08T05:54:21.076295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca = make_pipeline(\n    StandardScaler(),\n    PCA(random_state = 0, n_components = None)\n).fit(train_num)\n\npca_df = pd.DataFrame(\n    pca.fit_transform(train_num),\n    columns = [f'PC{i}' for i in range(train_num.shape[1])]\n)\n\nloadings = pd.DataFrame(\n    pca[1].components_,\n    columns = train_num.columns,\n    index = pca_df.columns\n).T\n\nexplained_variance_ratio = pd.Series(\n    pca[1].explained_variance_ratio_,\n    index = pca_df.columns\n)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:54:21.800792Z","iopub.execute_input":"2022-08-08T05:54:21.801270Z","iopub.status.idle":"2022-08-08T05:54:22.010455Z","shell.execute_reply.started":"2022-08-08T05:54:21.801229Z","shell.execute_reply":"2022-08-08T05:54:22.008750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mi_scores = mutual_info_classif(pca_df, y, random_state = 0)\nmi_scores = pd.Series(\n    mi_scores, \n    index = pca_df.columns\n).sort_values(ascending = False)\nmi_scores","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:54:23.794578Z","iopub.execute_input":"2022-08-08T05:54:23.795333Z","iopub.status.idle":"2022-08-08T05:54:28.451775Z","shell.execute_reply.started":"2022-08-08T05:54:23.795290Z","shell.execute_reply":"2022-08-08T05:54:28.450645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loadings.style.background_gradient()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:54:33.452723Z","iopub.execute_input":"2022-08-08T05:54:33.453205Z","iopub.status.idle":"2022-08-08T05:54:33.647063Z","shell.execute_reply.started":"2022-08-08T05:54:33.453165Z","shell.execute_reply":"2022-08-08T05:54:33.645653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = sns.scatterplot(data = train_num, x = 'loading', y = 'measurement_6_isna', hue = y)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:55:46.429323Z","iopub.execute_input":"2022-08-08T05:55:46.429887Z","iopub.status.idle":"2022-08-08T05:55:47.400877Z","shell.execute_reply.started":"2022-08-08T05:55:46.429837Z","shell.execute_reply":"2022-08-08T05:55:47.399519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['loadingxisna'] = train['loading'] * train['measurement_9_isna']\ntest['loadingxisna'] = test['loading'] * test['measurement_9_isna']","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:56:52.727231Z","iopub.execute_input":"2022-08-08T05:56:52.727707Z","iopub.status.idle":"2022-08-08T05:56:52.738074Z","shell.execute_reply.started":"2022-08-08T05:56:52.727657Z","shell.execute_reply":"2022-08-08T05:56:52.736636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = sns.scatterplot(data = pca_df, x = 'PC0', y = 'PC1', hue = y)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:57:05.617215Z","iopub.execute_input":"2022-08-08T05:57:05.618405Z","iopub.status.idle":"2022-08-08T05:57:06.654405Z","shell.execute_reply.started":"2022-08-08T05:57:05.618356Z","shell.execute_reply":"2022-08-08T05:57:06.653262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pca_df = pd.DataFrame(\n    pca.transform(test_num),\n    columns = [f'PC{i}' for i in range(test_num.shape[1])]\n)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:57:13.286165Z","iopub.execute_input":"2022-08-08T05:57:13.286656Z","iopub.status.idle":"2022-08-08T05:57:13.321027Z","shell.execute_reply.started":"2022-08-08T05:57:13.286616Z","shell.execute_reply":"2022-08-08T05:57:13.319181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_num = train_num.join(pca_df)\ntest_num = test_num.join(test_pca_df)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:57:14.061802Z","iopub.execute_input":"2022-08-08T05:57:14.062339Z","iopub.status.idle":"2022-08-08T05:57:14.087569Z","shell.execute_reply.started":"2022-08-08T05:57:14.062296Z","shell.execute_reply":"2022-08-08T05:57:14.086598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_num","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:57:15.663317Z","iopub.execute_input":"2022-08-08T05:57:15.663746Z","iopub.status.idle":"2022-08-08T05:57:15.705586Z","shell.execute_reply.started":"2022-08-08T05:57:15.663710Z","shell.execute_reply":"2022-08-08T05:57:15.704034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_num","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:57:18.095397Z","iopub.execute_input":"2022-08-08T05:57:18.095820Z","iopub.status.idle":"2022-08-08T05:57:18.131714Z","shell.execute_reply.started":"2022-08-08T05:57:18.095789Z","shell.execute_reply":"2022-08-08T05:57:18.130323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.discriminant_analysis import LinearDiscriminantAnalysis","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:57:20.336923Z","iopub.execute_input":"2022-08-08T05:57:20.337810Z","iopub.status.idle":"2022-08-08T05:57:20.355268Z","shell.execute_reply.started":"2022-08-08T05:57:20.337765Z","shell.execute_reply":"2022-08-08T05:57:20.354320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lda = make_pipeline(\n    StandardScaler(),\n    LinearDiscriminantAnalysis(n_components = 1)\n)\nlda_df = pd.DataFrame(\n    lda.fit_transform(train_num, y),\n    columns = ['LD0']\n)\nlda_df","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:57:22.643073Z","iopub.execute_input":"2022-08-08T05:57:22.643777Z","iopub.status.idle":"2022-08-08T05:57:23.162749Z","shell.execute_reply.started":"2022-08-08T05:57:22.643742Z","shell.execute_reply":"2022-08-08T05:57:23.160885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lda_df.corrwith(y)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:57:24.573513Z","iopub.execute_input":"2022-08-08T05:57:24.574760Z","iopub.status.idle":"2022-08-08T05:57:24.587536Z","shell.execute_reply.started":"2022-08-08T05:57:24.574705Z","shell.execute_reply":"2022-08-08T05:57:24.586520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mi_scores = mutual_info_classif(lda_df, y, random_state = 0)\nmi_scores = pd.Series(\n    mi_scores,\n    index = lda_df.columns\n).sort_values(ascending = False)\nmi_scores","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:57:25.964152Z","iopub.execute_input":"2022-08-08T05:57:25.964612Z","iopub.status.idle":"2022-08-08T05:57:26.104469Z","shell.execute_reply.started":"2022-08-08T05:57:25.964578Z","shell.execute_reply":"2022-08-08T05:57:26.103250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_lda_df = pd.DataFrame(\n    lda.transform(test_num),\n    columns = ['LD0']\n)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:57:28.798666Z","iopub.execute_input":"2022-08-08T05:57:28.799178Z","iopub.status.idle":"2022-08-08T05:57:28.843861Z","shell.execute_reply.started":"2022-08-08T05:57:28.799139Z","shell.execute_reply":"2022-08-08T05:57:28.842139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.concat([train, pca_df, lda_df], axis = 1)\ntest = pd.concat([test, test_pca_df, test_lda_df], axis = 1)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:57:32.965691Z","iopub.execute_input":"2022-08-08T05:57:32.967250Z","iopub.status.idle":"2022-08-08T05:57:33.013742Z","shell.execute_reply.started":"2022-08-08T05:57:32.967199Z","shell.execute_reply":"2022-08-08T05:57:33.012720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in cat :\n    print(i, ':', train[i].unique())\n    print(i, ':', test[i].unique())","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:57:33.879975Z","iopub.execute_input":"2022-08-08T05:57:33.881214Z","iopub.status.idle":"2022-08-08T05:57:33.899444Z","shell.execute_reply.started":"2022-08-08T05:57:33.881155Z","shell.execute_reply":"2022-08-08T05:57:33.897856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[cat] = train[cat].apply(lambda x : [c[-1] for c in x]).astype(int)\ntest[cat] = test[cat].apply(lambda x : [c[-1] for c in x]).astype(int)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:57:38.998441Z","iopub.execute_input":"2022-08-08T05:57:38.998893Z","iopub.status.idle":"2022-08-08T05:57:39.070472Z","shell.execute_reply.started":"2022-08-08T05:57:38.998861Z","shell.execute_reply":"2022-08-08T05:57:39.069229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:57:39.970546Z","iopub.execute_input":"2022-08-08T05:57:39.971064Z","iopub.status.idle":"2022-08-08T05:57:40.009762Z","shell.execute_reply.started":"2022-08-08T05:57:39.971024Z","shell.execute_reply":"2022-08-08T05:57:40.008826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:57:48.802104Z","iopub.execute_input":"2022-08-08T05:57:48.802570Z","iopub.status.idle":"2022-08-08T05:57:48.842904Z","shell.execute_reply.started":"2022-08-08T05:57:48.802534Z","shell.execute_reply":"2022-08-08T05:57:48.841663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['attribute_2x3'] = train['attribute_2'] * train['attribute_3']\ntest['attribute_2x3'] = test['attribute_2'] * test['attribute_3']","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:58:12.109807Z","iopub.execute_input":"2022-08-08T05:58:12.110314Z","iopub.status.idle":"2022-08-08T05:58:12.119435Z","shell.execute_reply.started":"2022-08-08T05:58:12.110276Z","shell.execute_reply":"2022-08-08T05:58:12.118047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold, train_test_split\nfrom sklearn.metrics import roc_auc_score\nimport optuna\nfrom sklearn.ensemble import RandomForestClassifier\nfrom xgboost import XGBClassifier\nfrom catboost import CatBoostClassifier\nfrom lightgbm import LGBMClassifier","metadata":{"execution":{"iopub.status.busy":"2022-08-08T06:01:17.321610Z","iopub.execute_input":"2022-08-08T06:01:17.322123Z","iopub.status.idle":"2022-08-08T06:01:17.328504Z","shell.execute_reply.started":"2022-08-08T06:01:17.322084Z","shell.execute_reply":"2022-08-08T06:01:17.327014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xtrain, xval, ytrain, yval = train_test_split(train, y, random_state = 0, test_size = .2)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T06:01:17.854716Z","iopub.execute_input":"2022-08-08T06:01:17.855184Z","iopub.status.idle":"2022-08-08T06:01:17.885043Z","shell.execute_reply.started":"2022-08-08T06:01:17.855147Z","shell.execute_reply":"2022-08-08T06:01:17.883875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def optimize(trial) :\n    param = {\n        'max_depth': trial.suggest_int('max_depth', 2, 15),\n        'subsample': trial.suggest_float('subsample', 0.1, 1.0),\n        'eta': trial.suggest_float('eta', 1e-4, 5e-1),\n        \"colsample_bytree\": trial.suggest_float(\"colsample_bytree\", 0.1, 1.0),\n        'colsample_bylevel' : trial.suggest_float('colsample_bylevel', 0.1, 1.0),\n        'colsample_bynode' : trial.suggest_float('colsample_bynode', 0.1, 1.0),\n        'reg_alpha' : trial.suggest_float('reg_alpha', 1e-8, 100),\n        'reg_lambda' : trial.suggest_float('reg_lambda', 1e-8, 100),\n        \n    }\n    model = XGBClassifier(\n        n_estimators = 5000,\n        tree_method = 'exact',\n        random_state = 0,\n        eval_metric = 'auc',\n        early_stopping_rounds = 400,\n        scale_pos_weight = 4,\n        **param\n    )\n    model.fit(\n        xtrain, ytrain,\n        eval_set = [(xval, yval)],\n        verbose = 500\n    )\n    valpred = model.predict_proba(xval)[:, 1]\n    score = roc_auc_score(yval, valpred)\n    return score","metadata":{"execution":{"iopub.status.busy":"2022-08-08T06:04:43.867689Z","iopub.execute_input":"2022-08-08T06:04:43.868576Z","iopub.status.idle":"2022-08-08T06:04:43.879815Z","shell.execute_reply.started":"2022-08-08T06:04:43.868530Z","shell.execute_reply":"2022-08-08T06:04:43.878314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_params = {\n    'max_depth': 2,\n    'subsample': 0.7171053811540031,\n    'eta': 0.09706062045080699,\n    'colsample_bytree': 0.9496933896620908,\n    'colsample_bylevel': 0.7402087590023215,\n    'colsample_bynode': 0.6891766803611067,\n    'reg_alpha': 6.359827783196938,\n    'reg_lambda': 0.0999149082913462\n}","metadata":{"execution":{"iopub.status.busy":"2022-08-08T06:19:51.362613Z","iopub.execute_input":"2022-08-08T06:19:51.363067Z","iopub.status.idle":"2022-08-08T06:19:51.369513Z","shell.execute_reply.started":"2022-08-08T06:19:51.363032Z","shell.execute_reply":"2022-08-08T06:19:51.368259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = XGBClassifier(\n        n_estimators = 5000,\n        tree_method = 'exact',\n        random_state = 0,\n        eval_metric = 'auc',\n        early_stopping_rounds = 400,\n        scale_pos_weight = 4,\n        **best_params\n)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T06:20:09.558706Z","iopub.execute_input":"2022-08-08T06:20:09.559140Z","iopub.status.idle":"2022-08-08T06:20:09.565466Z","shell.execute_reply.started":"2022-08-08T06:20:09.559104Z","shell.execute_reply":"2022-08-08T06:20:09.564381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kf = StratifiedKFold(n_splits = 5, random_state = 0, shuffle = True)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T06:20:37.281216Z","iopub.execute_input":"2022-08-08T06:20:37.281694Z","iopub.status.idle":"2022-08-08T06:20:37.289204Z","shell.execute_reply.started":"2022-08-08T06:20:37.281655Z","shell.execute_reply":"2022-08-08T06:20:37.287615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc","metadata":{"execution":{"iopub.status.busy":"2022-08-08T06:23:49.669480Z","iopub.execute_input":"2022-08-08T06:23:49.670274Z","iopub.status.idle":"2022-08-08T06:23:49.674775Z","shell.execute_reply.started":"2022-08-08T06:23:49.670236Z","shell.execute_reply":"2022-08-08T06:23:49.674003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores = []\ntest_preds = []\nfor i, (t, v) in enumerate(kf.split(train, y)) :\n    xtrain = train.iloc[t, :]\n    xval = train.iloc[v, :]\n    xtest = test.copy()\n    ytrain = y.iloc[t]\n    yval = y.iloc[v]\n    \n    model = XGBClassifier(\n        n_estimators = 5000,\n        tree_method = 'exact',\n        random_state = 0,\n        eval_metric = 'auc',\n        early_stopping_rounds = 400,\n        scale_pos_weight = 4,\n        **best_params\n    )\n    model.fit(\n        xtrain, ytrain,\n        eval_set = [(xval, yval)],\n        verbose = 100\n    )\n    valpred = model.predict_proba(xval)[:, 1]\n    score = roc_auc_score(yval, valpred)\n    scores.append(score)\n    pred = model.predict_proba(xtest)[:, 1]\n    test_preds.append(pred)\n    print('='*20, f'FOLD {i} : {score}', '='*20)\n    del xtrain, xval, ytrain, yval, xtest, model\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T06:23:49.936352Z","iopub.execute_input":"2022-08-08T06:23:49.937120Z","iopub.status.idle":"2022-08-08T06:24:48.835505Z","shell.execute_reply.started":"2022-08-08T06:23:49.937064Z","shell.execute_reply":"2022-08-08T06:24:48.834407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(np.mean(scores), np.std(scores))","metadata":{"execution":{"iopub.status.busy":"2022-08-08T06:24:58.781754Z","iopub.execute_input":"2022-08-08T06:24:58.782157Z","iopub.status.idle":"2022-08-08T06:24:58.789892Z","shell.execute_reply.started":"2022-08-08T06:24:58.782124Z","shell.execute_reply":"2022-08-08T06:24:58.788049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bestPred = test_preds[0]\nmeanPred = np.mean(test_preds, axis = 0)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T06:26:48.535023Z","iopub.execute_input":"2022-08-08T06:26:48.535968Z","iopub.status.idle":"2022-08-08T06:26:48.541223Z","shell.execute_reply.started":"2022-08-08T06:26:48.535908Z","shell.execute_reply":"2022-08-08T06:26:48.540281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub['failure'] = bestPred\nsub.to_csv('bestPred.csv', index = False)\nsub","metadata":{"execution":{"iopub.status.busy":"2022-08-08T06:27:11.690678Z","iopub.execute_input":"2022-08-08T06:27:11.691179Z","iopub.status.idle":"2022-08-08T06:27:11.754043Z","shell.execute_reply.started":"2022-08-08T06:27:11.691142Z","shell.execute_reply":"2022-08-08T06:27:11.752795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub['failure'] = meanPred\nsub.to_csv('submission.csv', index = False)\nsub","metadata":{"execution":{"iopub.status.busy":"2022-08-08T06:27:33.280630Z","iopub.execute_input":"2022-08-08T06:27:33.281070Z","iopub.status.idle":"2022-08-08T06:27:33.332705Z","shell.execute_reply.started":"2022-08-08T06:27:33.281037Z","shell.execute_reply":"2022-08-08T06:27:33.331907Z"},"trusted":true},"execution_count":null,"outputs":[]}]}