{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-12T16:18:11.882575Z","iopub.execute_input":"2022-08-12T16:18:11.883246Z","iopub.status.idle":"2022-08-12T16:18:12.418660Z","shell.execute_reply.started":"2022-08-12T16:18:11.883155Z","shell.execute_reply":"2022-08-12T16:18:12.417389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pd.read_csv('/kaggle/input/tabular-playground-series-aug-2022/train.csv', index_col='id')\ntrain_data","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:12.424117Z","iopub.execute_input":"2022-08-12T16:18:12.424416Z","iopub.status.idle":"2022-08-12T16:18:12.575927Z","shell.execute_reply.started":"2022-08-12T16:18:12.424388Z","shell.execute_reply":"2022-08-12T16:18:12.574756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.columns","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:12.577612Z","iopub.execute_input":"2022-08-12T16:18:12.577929Z","iopub.status.idle":"2022-08-12T16:18:12.584768Z","shell.execute_reply.started":"2022-08-12T16:18:12.577901Z","shell.execute_reply":"2022-08-12T16:18:12.583784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = pd.read_csv('/kaggle/input/tabular-playground-series-aug-2022/test.csv', index_col='id')\ntest_data","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:12.588964Z","iopub.execute_input":"2022-08-12T16:18:12.589473Z","iopub.status.idle":"2022-08-12T16:18:12.701595Z","shell.execute_reply.started":"2022-08-12T16:18:12.589429Z","shell.execute_reply":"2022-08-12T16:18:12.700229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train = train_data.failure\nX_train = train_data.drop(columns=['failure'], axis=1, inplace=False)\nX_test = test_data","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:12.704255Z","iopub.execute_input":"2022-08-12T16:18:12.705132Z","iopub.status.idle":"2022-08-12T16:18:12.713212Z","shell.execute_reply.started":"2022-08-12T16:18:12.705081Z","shell.execute_reply":"2022-08-12T16:18:12.711997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:12.715509Z","iopub.execute_input":"2022-08-12T16:18:12.715961Z","iopub.status.idle":"2022-08-12T16:18:12.763791Z","shell.execute_reply.started":"2022-08-12T16:18:12.715918Z","shell.execute_reply":"2022-08-12T16:18:12.762772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"code","source":"train_data.info()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:12.765504Z","iopub.execute_input":"2022-08-12T16:18:12.765824Z","iopub.status.idle":"2022-08-12T16:18:12.786367Z","shell.execute_reply.started":"2022-08-12T16:18:12.765796Z","shell.execute_reply":"2022-08-12T16:18:12.785528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.attribute_0.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:12.787842Z","iopub.execute_input":"2022-08-12T16:18:12.788530Z","iopub.status.idle":"2022-08-12T16:18:12.798522Z","shell.execute_reply.started":"2022-08-12T16:18:12.788495Z","shell.execute_reply":"2022-08-12T16:18:12.797298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.attribute_1.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:12.799960Z","iopub.execute_input":"2022-08-12T16:18:12.800305Z","iopub.status.idle":"2022-08-12T16:18:12.813956Z","shell.execute_reply.started":"2022-08-12T16:18:12.800272Z","shell.execute_reply":"2022-08-12T16:18:12.812750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:12.815438Z","iopub.execute_input":"2022-08-12T16:18:12.816198Z","iopub.status.idle":"2022-08-12T16:18:12.924113Z","shell.execute_reply.started":"2022-08-12T16:18:12.816153Z","shell.execute_reply":"2022-08-12T16:18:12.922423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.isna().sum() / train_data.shape[0]","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:12.925489Z","iopub.execute_input":"2022-08-12T16:18:12.926560Z","iopub.status.idle":"2022-08-12T16:18:12.941539Z","shell.execute_reply.started":"2022-08-12T16:18:12.926523Z","shell.execute_reply":"2022-08-12T16:18:12.940261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.heatmap(train_data.isnull(),yticklabels=False,cbar=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:12.943555Z","iopub.execute_input":"2022-08-12T16:18:12.944269Z","iopub.status.idle":"2022-08-12T16:18:13.856404Z","shell.execute_reply.started":"2022-08-12T16:18:12.944186Z","shell.execute_reply":"2022-08-12T16:18:13.855223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.heatmap(train_data.corr(), cmap='icefire')","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:13.861986Z","iopub.execute_input":"2022-08-12T16:18:13.862403Z","iopub.status.idle":"2022-08-12T16:18:14.344122Z","shell.execute_reply.started":"2022-08-12T16:18:13.862368Z","shell.execute_reply":"2022-08-12T16:18:14.342981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(data=train_data, x='attribute_0', hue='failure')\nplt.title('attribute_0')","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:14.345778Z","iopub.execute_input":"2022-08-12T16:18:14.346127Z","iopub.status.idle":"2022-08-12T16:18:14.585135Z","shell.execute_reply.started":"2022-08-12T16:18:14.346094Z","shell.execute_reply":"2022-08-12T16:18:14.583983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(data=train_data, x='attribute_1', hue='failure')\nplt.title('attribute_1')","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:14.586799Z","iopub.execute_input":"2022-08-12T16:18:14.587142Z","iopub.status.idle":"2022-08-12T16:18:14.833985Z","shell.execute_reply.started":"2022-08-12T16:18:14.587109Z","shell.execute_reply":"2022-08-12T16:18:14.832763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.violinplot(data=train_data, x='failure',y='loading')","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:14.835346Z","iopub.execute_input":"2022-08-12T16:18:14.835700Z","iopub.status.idle":"2022-08-12T16:18:15.190586Z","shell.execute_reply.started":"2022-08-12T16:18:14.835669Z","shell.execute_reply":"2022-08-12T16:18:15.189471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_cols = [col for col in X_train.columns if X_train[col].dtype=='object']\ncat_cols","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:15.192122Z","iopub.execute_input":"2022-08-12T16:18:15.192511Z","iopub.status.idle":"2022-08-12T16:18:15.202323Z","shell.execute_reply.started":"2022-08-12T16:18:15.192476Z","shell.execute_reply":"2022-08-12T16:18:15.201124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"float_cols = [col for col in X_train.columns if X_train[col].dtype=='float64']\nfloat_cols","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:15.203630Z","iopub.execute_input":"2022-08-12T16:18:15.204480Z","iopub.status.idle":"2022-08-12T16:18:15.214177Z","shell.execute_reply.started":"2022-08-12T16:18:15.204446Z","shell.execute_reply":"2022-08-12T16:18:15.213214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"int_cols = [col for col in X_train.columns if X_train[col].dtype=='int64']\nint_cols","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:15.215904Z","iopub.execute_input":"2022-08-12T16:18:15.216297Z","iopub.status.idle":"2022-08-12T16:18:15.227641Z","shell.execute_reply.started":"2022-08-12T16:18:15.216262Z","shell.execute_reply":"2022-08-12T16:18:15.226826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_cols = float_cols + int_cols","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:15.229113Z","iopub.execute_input":"2022-08-12T16:18:15.229466Z","iopub.status.idle":"2022-08-12T16:18:15.238058Z","shell.execute_reply.started":"2022-08-12T16:18:15.229436Z","shell.execute_reply":"2022-08-12T16:18:15.236967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's look at relationships of two variables...\n","metadata":{}},{"cell_type":"code","source":"# sns.scatterplot(data=train_data, x='measurement_3', y='measurement_4', hue='failure')","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:15.240729Z","iopub.execute_input":"2022-08-12T16:18:15.241071Z","iopub.status.idle":"2022-08-12T16:18:15.250131Z","shell.execute_reply.started":"2022-08-12T16:18:15.241041Z","shell.execute_reply":"2022-08-12T16:18:15.249321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sns.scatterplot(data=train_data, x='measurement_3', y='measurement_5', hue='failure')","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:15.252341Z","iopub.execute_input":"2022-08-12T16:18:15.252688Z","iopub.status.idle":"2022-08-12T16:18:15.261502Z","shell.execute_reply.started":"2022-08-12T16:18:15.252656Z","shell.execute_reply":"2022-08-12T16:18:15.260332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sns.scatterplot(data=train_data, x='measurement_3', y='measurement_6', hue='failure')","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:15.265148Z","iopub.execute_input":"2022-08-12T16:18:15.265615Z","iopub.status.idle":"2022-08-12T16:18:15.273468Z","shell.execute_reply.started":"2022-08-12T16:18:15.265583Z","shell.execute_reply":"2022-08-12T16:18:15.272264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Hmm, it seems we can't see any meaningful patterns. Perhaps we should explore all the features, but it'll take a lot of time, so let's postpone this task","metadata":{}},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"from sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import OneHotEncoder","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:15.275348Z","iopub.execute_input":"2022-08-12T16:18:15.276544Z","iopub.status.idle":"2022-08-12T16:18:15.367375Z","shell.execute_reply.started":"2022-08-12T16:18:15.276498Z","shell.execute_reply":"2022-08-12T16:18:15.365792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numerical_transformer = SimpleImputer(strategy='mean')\n\n# Preprocessing for categorical data\ncategorical_transformer = Pipeline(steps=[\n    ('imputer', SimpleImputer(strategy='most_frequent')),\n    ('onehot', OneHotEncoder(handle_unknown='ignore'))\n])","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:15.369070Z","iopub.execute_input":"2022-08-12T16:18:15.370011Z","iopub.status.idle":"2022-08-12T16:18:15.375607Z","shell.execute_reply.started":"2022-08-12T16:18:15.369976Z","shell.execute_reply":"2022-08-12T16:18:15.374578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preprocessor = ColumnTransformer(\n    transformers=[\n        ('num', numerical_transformer, num_cols),\n        ('cat', categorical_transformer, cat_cols)\n    ])\n","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:15.376831Z","iopub.execute_input":"2022-08-12T16:18:15.377194Z","iopub.status.idle":"2022-08-12T16:18:15.387154Z","shell.execute_reply.started":"2022-08-12T16:18:15.377146Z","shell.execute_reply":"2022-08-12T16:18:15.386121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = preprocessor.fit_transform(X_train)\nX_train","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:15.389080Z","iopub.execute_input":"2022-08-12T16:18:15.389690Z","iopub.status.idle":"2022-08-12T16:18:15.467719Z","shell.execute_reply.started":"2022-08-12T16:18:15.389645Z","shell.execute_reply":"2022-08-12T16:18:15.466738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:15.469220Z","iopub.execute_input":"2022-08-12T16:18:15.470431Z","iopub.status.idle":"2022-08-12T16:18:15.477754Z","shell.execute_reply.started":"2022-08-12T16:18:15.470387Z","shell.execute_reply":"2022-08-12T16:18:15.476523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = preprocessor.transform(X_test)\nX_test","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:15.479251Z","iopub.execute_input":"2022-08-12T16:18:15.479628Z","iopub.status.idle":"2022-08-12T16:18:15.539430Z","shell.execute_reply.started":"2022-08-12T16:18:15.479595Z","shell.execute_reply":"2022-08-12T16:18:15.538559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nscaler = StandardScaler()\nX_train_scaled = scaler.fit_transform(X_train)\nX_train_scaled","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:15.541092Z","iopub.execute_input":"2022-08-12T16:18:15.541454Z","iopub.status.idle":"2022-08-12T16:18:15.564858Z","shell.execute_reply.started":"2022-08-12T16:18:15.541423Z","shell.execute_reply":"2022-08-12T16:18:15.563269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test_scaled = scaler.transform(X_test)\nX_test_scaled","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:15.566340Z","iopub.execute_input":"2022-08-12T16:18:15.566683Z","iopub.status.idle":"2022-08-12T16:18:15.578050Z","shell.execute_reply.started":"2022-08-12T16:18:15.566653Z","shell.execute_reply":"2022-08-12T16:18:15.576758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_scaled.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:15.579188Z","iopub.execute_input":"2022-08-12T16:18:15.580123Z","iopub.status.idle":"2022-08-12T16:18:15.587989Z","shell.execute_reply.started":"2022-08-12T16:18:15.580090Z","shell.execute_reply":"2022-08-12T16:18:15.586899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test_scaled.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:15.589822Z","iopub.execute_input":"2022-08-12T16:18:15.590619Z","iopub.status.idle":"2022-08-12T16:18:15.600271Z","shell.execute_reply.started":"2022-08-12T16:18:15.590585Z","shell.execute_reply":"2022-08-12T16:18:15.599451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = np.vstack([X_train_scaled, X_test_scaled])\nX","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:15.601471Z","iopub.execute_input":"2022-08-12T16:18:15.602484Z","iopub.status.idle":"2022-08-12T16:18:15.620506Z","shell.execute_reply.started":"2022-08-12T16:18:15.602448Z","shell.execute_reply":"2022-08-12T16:18:15.619628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:15.621593Z","iopub.execute_input":"2022-08-12T16:18:15.622680Z","iopub.status.idle":"2022-08-12T16:18:15.630473Z","shell.execute_reply.started":"2022-08-12T16:18:15.622634Z","shell.execute_reply":"2022-08-12T16:18:15.629087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PCA\nCan we find some useful features using PCA?","metadata":{}},{"cell_type":"code","source":"from sklearn.decomposition import PCA\n\n# Create principal components\npca = PCA()\nX_pca = pca.fit_transform(X)\n\n# Convert to dataframe\npd.DataFrame(X_pca).head()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:15.632369Z","iopub.execute_input":"2022-08-12T16:18:15.633205Z","iopub.status.idle":"2022-08-12T16:18:15.775097Z","shell.execute_reply.started":"2022-08-12T16:18:15.633156Z","shell.execute_reply":"2022-08-12T16:18:15.773882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pca.components_","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:15.776957Z","iopub.execute_input":"2022-08-12T16:18:15.777803Z","iopub.status.idle":"2022-08-12T16:18:15.788996Z","shell.execute_reply.started":"2022-08-12T16:18:15.777751Z","shell.execute_reply":"2022-08-12T16:18:15.784198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"expl_var = pca.explained_variance_ratio_\nexpl_var","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:15.790755Z","iopub.execute_input":"2022-08-12T16:18:15.791534Z","iopub.status.idle":"2022-08-12T16:18:15.802206Z","shell.execute_reply.started":"2022-08-12T16:18:15.791467Z","shell.execute_reply":"2022-08-12T16:18:15.800747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 0\nwhile expl_var[i] > 10 ** (-3):\n    i += 1\nprint(i)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:15.803957Z","iopub.execute_input":"2022-08-12T16:18:15.804540Z","iopub.status.idle":"2022-08-12T16:18:15.815838Z","shell.execute_reply.started":"2022-08-12T16:18:15.804494Z","shell.execute_reply":"2022-08-12T16:18:15.814275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_pca_train = X_pca[:X_train_scaled.shape[0], :i]\nX_pca_test = X_pca[X_train_scaled.shape[0]:, :i]","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:15.826350Z","iopub.execute_input":"2022-08-12T16:18:15.827035Z","iopub.status.idle":"2022-08-12T16:18:15.835393Z","shell.execute_reply.started":"2022-08-12T16:18:15.826984Z","shell.execute_reply":"2022-08-12T16:18:15.834131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#X_train = np.hstack([X_train, X_pca_train])\nX_train = X_pca_train\nX_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:15.837472Z","iopub.execute_input":"2022-08-12T16:18:15.838326Z","iopub.status.idle":"2022-08-12T16:18:15.848113Z","shell.execute_reply.started":"2022-08-12T16:18:15.838278Z","shell.execute_reply":"2022-08-12T16:18:15.847301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#X_test = np.hstack([X_test, X_pca_test])\nX_test = X_pca_test\n","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:15.849921Z","iopub.execute_input":"2022-08-12T16:18:15.850780Z","iopub.status.idle":"2022-08-12T16:18:15.858028Z","shell.execute_reply.started":"2022-08-12T16:18:15.850730Z","shell.execute_reply":"2022-08-12T16:18:15.857131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Logistic Regression","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import KFold, cross_val_score\nfrom sklearn.linear_model import LogisticRegression\nbest_coeff = 0\nbest_score = 0\nkf = KFold(shuffle=True, random_state=42)\nfor reg_coeff in [0.1, 1, 5, 10, 20, 50, 100, 500, 1000]:\n    print(f'Regularization coefficient = {reg_coeff}')    \n    clf = LogisticRegression(penalty='l2', C=reg_coeff, random_state=42)\n    score = round(cross_val_score(clf, X_train_scaled, y_train, cv = kf, scoring='roc_auc').mean(), 3)\n    print(f'ROC-AUC = {score}')\n    if score > best_score:\n        best_score, best_coeff = score, reg_coeff","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:15.859684Z","iopub.execute_input":"2022-08-12T16:18:15.860491Z","iopub.status.idle":"2022-08-12T16:18:19.336870Z","shell.execute_reply.started":"2022-08-12T16:18:15.860434Z","shell.execute_reply":"2022-08-12T16:18:19.335720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_score, best_coeff","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:19.338948Z","iopub.execute_input":"2022-08-12T16:18:19.339781Z","iopub.status.idle":"2022-08-12T16:18:19.347757Z","shell.execute_reply.started":"2022-08-12T16:18:19.339736Z","shell.execute_reply":"2022-08-12T16:18:19.346443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clf_log = LogisticRegression(penalty='l2', C=best_coeff, random_state=42)\nclf_log.fit(X_train_scaled, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:19.349837Z","iopub.execute_input":"2022-08-12T16:18:19.350845Z","iopub.status.idle":"2022-08-12T16:18:19.433332Z","shell.execute_reply.started":"2022-08-12T16:18:19.350801Z","shell.execute_reply":"2022-08-12T16:18:19.431991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"probabilities_log = clf_log.predict_proba(X_test_scaled)\nclf_log.classes_, probabilities_log","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:19.435537Z","iopub.execute_input":"2022-08-12T16:18:19.436375Z","iopub.status.idle":"2022-08-12T16:18:19.451682Z","shell.execute_reply.started":"2022-08-12T16:18:19.436332Z","shell.execute_reply":"2022-08-12T16:18:19.450294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions_log = probabilities_log[:, 1]\npredictions_log","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:19.453875Z","iopub.execute_input":"2022-08-12T16:18:19.454741Z","iopub.status.idle":"2022-08-12T16:18:19.463682Z","shell.execute_reply.started":"2022-08-12T16:18:19.454696Z","shell.execute_reply":"2022-08-12T16:18:19.462343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output_log = pd.DataFrame({'id': test_data.index, 'failure': predictions_log})\noutput_log.to_csv('submission_log.csv', index=False)\nprint(\"Your submission was successfully saved!\")\noutput_log","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:19.465989Z","iopub.execute_input":"2022-08-12T16:18:19.467102Z","iopub.status.idle":"2022-08-12T16:18:19.592953Z","shell.execute_reply.started":"2022-08-12T16:18:19.467056Z","shell.execute_reply":"2022-08-12T16:18:19.591734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# XGBoost","metadata":{}},{"cell_type":"code","source":"from xgboost import XGBClassifier\n\nbest_score = 0\nbest_k = 0\n\nfor k in [100, 300, 500]:  #range(100, 1000, 200):\n    print(f'{k} models')\n    clf = XGBClassifier(n_estimators=k, learning_rate=0.1, n_jobs=4)\n    clf.fit(X_train, y_train)\n    score = round(cross_val_score(clf, X_train, y_train, cv=kf, scoring='roc_auc').mean(), 3)\n    print(f'{k}: ROC-AUC = {score}')\n    if score > best_score:\n        best_score = score\n        best_k = k\n    ","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:18:19.594988Z","iopub.execute_input":"2022-08-12T16:18:19.595825Z","iopub.status.idle":"2022-08-12T16:24:44.202784Z","shell.execute_reply.started":"2022-08-12T16:18:19.595771Z","shell.execute_reply":"2022-08-12T16:24:44.201703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clf_xgb = XGBClassifier(n_estimators=best_k, learning_rate=0.05, n_jobs=4)\nclf_xgb.fit(X_train_scaled, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:24:44.204383Z","iopub.execute_input":"2022-08-12T16:24:44.205279Z","iopub.status.idle":"2022-08-12T16:24:48.313466Z","shell.execute_reply.started":"2022-08-12T16:24:44.205210Z","shell.execute_reply":"2022-08-12T16:24:48.312401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"probabilities_xgb = clf_xgb.predict_proba(X_test_scaled)\nclf_xgb.classes_, probabilities_xgb","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:24:48.317344Z","iopub.execute_input":"2022-08-12T16:24:48.318151Z","iopub.status.idle":"2022-08-12T16:24:48.353051Z","shell.execute_reply.started":"2022-08-12T16:24:48.318113Z","shell.execute_reply":"2022-08-12T16:24:48.352196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions_xgb = probabilities_xgb[:, 1]\npredictions_xgb","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:24:48.357124Z","iopub.execute_input":"2022-08-12T16:24:48.359385Z","iopub.status.idle":"2022-08-12T16:24:48.368141Z","shell.execute_reply.started":"2022-08-12T16:24:48.359344Z","shell.execute_reply":"2022-08-12T16:24:48.366980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output_xgb = pd.DataFrame({'id': test_data.index, 'failure': predictions_xgb})\noutput_xgb.to_csv('submission_xgb.csv', index=False)\nprint(\"Your submission was successfully saved!\")\noutput_xgb","metadata":{"execution":{"iopub.status.busy":"2022-08-12T16:24:48.369631Z","iopub.execute_input":"2022-08-12T16:24:48.370634Z","iopub.status.idle":"2022-08-12T16:24:48.427030Z","shell.execute_reply.started":"2022-08-12T16:24:48.370596Z","shell.execute_reply":"2022-08-12T16:24:48.426283Z"},"trusted":true},"execution_count":null,"outputs":[]}]}