{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd; pd.set_option('display.max_columns', 50) # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-09T11:12:05.316002Z","iopub.execute_input":"2022-08-09T11:12:05.316634Z","iopub.status.idle":"2022-08-09T11:12:05.352717Z","shell.execute_reply.started":"2022-08-09T11:12:05.316499Z","shell.execute_reply":"2022-08-09T11:12:05.351461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Initial Exploration**","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\n## Reading data-files\ntrain = pd.read_csv('/kaggle/input/tabular-playground-series-aug-2022/train.csv')\ntrain = train.drop(columns = ['id'], axis = 1)\n\ntest = pd.read_csv('/kaggle/input/tabular-playground-series-aug-2022/test.csv')\ntest_id = test['id']\ntest = test.drop(columns = ['id'], axis = 1)\n\n## target variable\ntrain['failure'].value_counts() / train.shape[0]","metadata":{"execution":{"iopub.status.busy":"2022-08-09T11:15:23.638612Z","iopub.execute_input":"2022-08-09T11:15:23.639028Z","iopub.status.idle":"2022-08-09T11:15:23.882501Z","shell.execute_reply.started":"2022-08-09T11:15:23.638997Z","shell.execute_reply":"2022-08-09T11:15:23.881267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T11:12:43.609821Z","iopub.execute_input":"2022-08-09T11:12:43.610255Z","iopub.status.idle":"2022-08-09T11:12:43.644385Z","shell.execute_reply.started":"2022-08-09T11:12:43.610219Z","shell.execute_reply":"2022-08-09T11:12:43.643274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Taking a look at the categorical variables \nprint(' product_code in the train dataset\\n')\nprint(train['product_code'].value_counts())\n\nprint('\\n  product_code in the test dataset\\n')\nprint(test['product_code'].value_counts())\n\nprint('\\n attribute_0 in the train dataset\\n')\nprint(train['attribute_0'].value_counts())\n\nprint('\\n attribute_0 in the test dataset\\n')\nprint(test['attribute_0'].value_counts())\n\nprint('\\n attribute_1 in the train dataset\\n')\nprint(train['attribute_1'].value_counts())\n\nprint('\\n attribute_1 in the test dataset\\n')\nprint(test['attribute_1'].value_counts())\n\nprint('\\n attribute_2 in the train dataset\\n')\nprint(train['attribute_2'].value_counts())\n\nprint('\\n attribute_2 in the test dataset\\n')\nprint(test['attribute_2'].value_counts())\n\nprint('\\n attribute_3 in the train dataset\\n')\nprint(train['attribute_3'].value_counts())\n\nprint('\\n attribute_3 in the test dataset\\n')\nprint(test['attribute_3'].value_counts())\n\nprint('\\n measurement_0 in the train dataset\\n')\nprint(train['measurement_0'].value_counts())\n\nprint('\\n measurement_0 in the test dataset\\n')\nprint(test['measurement_0'].value_counts())\n\nprint('\\n measurement_1 in the train dataset\\n')\nprint(train['measurement_1'].value_counts())\n\nprint('\\n measurement_1 in the test dataset\\n')\nprint(test['measurement_1'].value_counts())\n\nprint('\\n measurement_2 in the train dataset\\n')\nprint(train['measurement_2'].value_counts())\n\nprint('\\n measurement_2 in the test dataset\\n')\nprint(test['measurement_2'].value_counts())\n\n#attribute_0 is the only categorical variables that has consistent labels in the train and test datasets.","metadata":{"execution":{"iopub.status.busy":"2022-08-05T23:06:48.885728Z","iopub.execute_input":"2022-08-05T23:06:48.886263Z","iopub.status.idle":"2022-08-05T23:06:48.916775Z","shell.execute_reply.started":"2022-08-05T23:06:48.886228Z","shell.execute_reply":"2022-08-05T23:06:48.915925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Taking a look at the numerical variables\nfig, axes = plt.subplots(16, 2, figsize = (18, 100))\n\nsns.histplot(ax = axes[0, 0], data = train, x = 'loading', color = 'royalblue').set(title = 'Loading in Train dataset')\nsns.histplot(ax = axes[0, 1], data = test, x = 'loading', color = 'darkorange').set(title = 'Loading in Test dataset')\n\nsns.histplot(ax = axes[1, 0], data = train, x = 'measurement_3', color = 'royalblue').set(title = 'measurement_3 in Train dataset')\nsns.histplot(ax = axes[1, 1], data = test, x = 'measurement_3', color = 'darkorange').set(title = 'measurement_3 in Test dataset')\n\nsns.histplot(ax = axes[2, 0], data = train, x = 'measurement_4', color = 'royalblue').set(title = 'measurement_4 in Train dataset')\nsns.histplot(ax = axes[2, 1], data = test, x = 'measurement_4', color = 'darkorange').set(title = 'measurement_4 in Test dataset')\n\nsns.histplot(ax = axes[3, 0], data = train, x = 'measurement_5', color = 'royalblue').set(title = 'measurement_5 in Train dataset')\nsns.histplot(ax = axes[3, 1], data = test, x = 'measurement_5', color = 'darkorange').set(title = 'measurement_5 in Test dataset')\n\nsns.histplot(ax = axes[4, 0], data = train, x = 'measurement_6', color = 'royalblue').set(title = 'measurement_6 in Train dataset')\nsns.histplot(ax = axes[4, 1], data = test, x = 'measurement_6', color = 'darkorange').set(title = 'measurement_6 in Test dataset')\n\nsns.histplot(ax = axes[5, 0], data = train, x = 'measurement_7', color = 'royalblue').set(title = 'measurement_7 in Train dataset')\nsns.histplot(ax = axes[5, 1], data = test, x = 'measurement_7', color = 'darkorange').set(title = 'measurement_7 in Test dataset')\n\nsns.histplot(ax = axes[6, 0], data = train, x = 'measurement_8', color = 'royalblue').set(title = 'measurement_8 in Train dataset')\nsns.histplot(ax = axes[6, 1], data = test, x = 'measurement_8', color = 'darkorange').set(title = 'measurement_8 in Test dataset')\n\nsns.histplot(ax = axes[7, 0], data = train, x = 'measurement_9', color = 'royalblue').set(title = 'measurement_9 in Train dataset')\nsns.histplot(ax = axes[7, 1], data = test, x = 'measurement_9', color = 'darkorange').set(title = 'measurement_9 in Test dataset')\n\nsns.histplot(ax = axes[8, 0], data = train, x = 'measurement_10', color = 'royalblue').set(title = 'measurement_10 in Train dataset')\nsns.histplot(ax = axes[8, 1], data = test, x = 'measurement_10', color = 'darkorange').set(title = 'measurement_10 in Test dataset')\n\nsns.histplot(ax = axes[9, 0], data = train, x = 'measurement_11', color = 'royalblue').set(title = 'measurement_11 in Train dataset')\nsns.histplot(ax = axes[9, 1], data = test, x = 'measurement_11', color = 'darkorange').set(title = 'measurement_11 in Test dataset')\n\nsns.histplot(ax = axes[10, 0], data = train, x = 'measurement_12', color = 'royalblue').set(title = 'measurement_12 in Train dataset')\nsns.histplot(ax = axes[10, 1], data = test, x = 'measurement_12', color = 'darkorange').set(title = 'measurement_12 in Test dataset')\n\nsns.histplot(ax = axes[11, 0], data = train, x = 'measurement_13', color = 'royalblue').set(title = 'measurement_13 in Train dataset')\nsns.histplot(ax = axes[11, 1], data = test, x = 'measurement_13', color = 'darkorange').set(title = 'measurement_13 in Test dataset')\n\nsns.histplot(ax = axes[12, 0], data = train, x = 'measurement_14', color = 'royalblue').set(title = 'measurement_14 in Train dataset')\nsns.histplot(ax = axes[12, 1], data = test, x = 'measurement_14', color = 'darkorange').set(title = 'measurement_14 in Test dataset')\n\nsns.histplot(ax = axes[13, 0], data = train, x = 'measurement_15', color = 'royalblue').set(title = 'measurement_15 in Train dataset')\nsns.histplot(ax = axes[13, 1], data = test, x = 'measurement_15', color = 'darkorange').set(title = 'measurement_15 in Test dataset')\n\nsns.histplot(ax = axes[14, 0], data = train, x = 'measurement_16', color = 'royalblue').set(title = 'measurement_16 in Train dataset')\nsns.histplot(ax = axes[14, 1], data = test, x = 'measurement_16', color = 'darkorange').set(title = 'measurement_16 in Test dataset')\n\nsns.histplot(ax = axes[15, 0], data = train, x = 'measurement_17', color = 'royalblue').set(title = 'measurement_17 in Train dataset')\nsns.histplot(ax = axes[15, 1], data = test, x = 'measurement_17', color = 'darkorange').set(title = 'measurement_17 in Test dataset')\n\n## From the below histograms, we see that numerical features in the train & test datasets have very similar \n## distributions.","metadata":{"execution":{"iopub.status.busy":"2022-08-05T23:07:09.027934Z","iopub.execute_input":"2022-08-05T23:07:09.028277Z","iopub.status.idle":"2022-08-05T23:07:17.103613Z","shell.execute_reply.started":"2022-08-05T23:07:09.028253Z","shell.execute_reply":"2022-08-05T23:07:17.102782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# k-NN Imputation","metadata":{}},{"cell_type":"code","source":"from sklearn.impute import KNNImputer\n\n## Changing labels to dummies\ntrain_dummies = pd.get_dummies(train[['attribute_0']])\ntrain['area'] = train['attribute_2'] * train['attribute_3'] \ntrain = train.drop(columns = ['product_code', 'attribute_0', 'attribute_1', 'attribute_2', 'attribute_3', 'measurement_0', 'measurement_1', 'measurement_2'], axis = 1)\ntrain = pd.concat([train, train_dummies], axis = 1)\n\ntest_dummies = pd.get_dummies(test[['attribute_0']])\ntest['area'] = test['attribute_2'] * test['attribute_3'] \ntest = test.drop(columns = ['product_code', 'attribute_0', 'attribute_1', 'attribute_2', 'attribute_3', 'measurement_0', 'measurement_1', 'measurement_2'], axis = 1)\ntest = pd.concat([test, test_dummies], axis = 1)\n\n## Filling missing values with kNN\nknn_imputer = KNNImputer(n_neighbors = 5, weights = 'distance')\ntrain = pd.DataFrame(knn_imputer.fit_transform(train), columns = train.columns)\ntest = pd.DataFrame(knn_imputer.fit_transform(test), columns = test.columns)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T11:15:28.033457Z","iopub.execute_input":"2022-08-09T11:15:28.033858Z","iopub.status.idle":"2022-08-09T11:16:24.474324Z","shell.execute_reply.started":"2022-08-09T11:15:28.033828Z","shell.execute_reply":"2022-08-09T11:16:24.472584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering ","metadata":{}},{"cell_type":"code","source":"from sklearn.tree import DecisionTreeClassifier, plot_tree\nfrom sklearn.model_selection import train_test_split\n\n## Defining input and target variables\nX = train.drop(columns = ['failure'], axis = 1)\nY = train['failure']\n\n## Splitting the data into train (80%) and test (20%)\nX_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.2, stratify = Y)\n\n## Building the decision tree on the train data-frame\ntree_md = DecisionTreeClassifier(max_depth = 3).fit(X_train, Y_train)\n\n## Visualizing the decision-tree model \nfig = plt.figure(figsize = (25, 15))\nplot_tree(tree_md, feature_names = X.columns, filled = True)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T11:16:35.661122Z","iopub.execute_input":"2022-08-09T11:16:35.661618Z","iopub.status.idle":"2022-08-09T11:16:36.840175Z","shell.execute_reply.started":"2022-08-09T11:16:35.661580Z","shell.execute_reply":"2022-08-09T11:16:36.839257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(8, 2, figsize = (18, 50))\n\ncolor = np.where(train['failure'] == 0, 'royalblue', 'darkorange').tolist()\n\nsns.scatterplot(ax = axes[0, 0], data = train, x = 'loading', y = 'measurement_3', hue = color)\nsns.scatterplot(ax = axes[0, 1], data = train, x = 'loading', y = 'measurement_4', hue = color)\nsns.scatterplot(ax = axes[1, 0], data = train, x = 'loading', y = 'measurement_5', hue = color)\nsns.scatterplot(ax = axes[1, 1], data = train, x = 'loading', y = 'measurement_6', hue = color)\nsns.scatterplot(ax = axes[2, 0], data = train, x = 'loading', y = 'measurement_7', hue = color)\nsns.scatterplot(ax = axes[2, 1], data = train, x = 'loading', y = 'measurement_8', hue = color)\nsns.scatterplot(ax = axes[3, 0], data = train, x = 'loading', y = 'measurement_9', hue = color)\nsns.scatterplot(ax = axes[3, 1], data = train, x = 'loading', y = 'measurement_10', hue = color)\nsns.scatterplot(ax = axes[4, 0], data = train, x = 'loading', y = 'measurement_11', hue = color)\nsns.scatterplot(ax = axes[4, 1], data = train, x = 'loading', y = 'measurement_12', hue = color)\nsns.scatterplot(ax = axes[5, 0], data = train, x = 'loading', y = 'measurement_13', hue = color)\nsns.scatterplot(ax = axes[5, 1], data = train, x = 'loading', y = 'measurement_14', hue = color)\nsns.scatterplot(ax = axes[6, 0], data = train, x = 'loading', y = 'measurement_15', hue = color)\nsns.scatterplot(ax = axes[6, 1], data = train, x = 'loading', y = 'measurement_16', hue = color)\nsns.scatterplot(ax = axes[7, 0], data = train, x = 'loading', y = 'measurement_17', hue = color)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T23:40:57.398328Z","iopub.execute_input":"2022-08-05T23:40:57.398651Z","iopub.status.idle":"2022-08-05T23:41:08.312463Z","shell.execute_reply.started":"2022-08-05T23:40:57.398627Z","shell.execute_reply":"2022-08-05T23:41:08.311027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(7, 2, figsize = (18, 50))\n\nsns.scatterplot(ax = axes[0, 0], data = train, x = 'measurement_3', y = 'measurement_4', hue = color)\nsns.scatterplot(ax = axes[0, 1], data = train, x = 'measurement_3', y = 'measurement_5', hue = color)\nsns.scatterplot(ax = axes[1, 0], data = train, x = 'measurement_3', y = 'measurement_6', hue = color)\nsns.scatterplot(ax = axes[1, 1], data = train, x = 'measurement_3', y = 'measurement_7', hue = color)\nsns.scatterplot(ax = axes[2, 0], data = train, x = 'measurement_3', y = 'measurement_8', hue = color)\nsns.scatterplot(ax = axes[2, 1], data = train, x = 'measurement_3', y = 'measurement_9', hue = color)\nsns.scatterplot(ax = axes[3, 0], data = train, x = 'measurement_3', y = 'measurement_10', hue = color)\nsns.scatterplot(ax = axes[3, 1], data = train, x = 'measurement_3', y = 'measurement_11', hue = color)\nsns.scatterplot(ax = axes[4, 0], data = train, x = 'measurement_3', y = 'measurement_12', hue = color)\nsns.scatterplot(ax = axes[4, 1], data = train, x = 'measurement_3', y = 'measurement_13', hue = color)\nsns.scatterplot(ax = axes[5, 0], data = train, x = 'measurement_3', y = 'measurement_14', hue = color)\nsns.scatterplot(ax = axes[5, 1], data = train, x = 'measurement_3', y = 'measurement_15', hue = color)\nsns.scatterplot(ax = axes[6, 0], data = train, x = 'measurement_3', y = 'measurement_16', hue = color)\nsns.scatterplot(ax = axes[6, 1], data = train, x = 'measurement_3', y = 'measurement_17', hue = color)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T23:48:45.682376Z","iopub.execute_input":"2022-08-05T23:48:45.682788Z","iopub.status.idle":"2022-08-05T23:48:56.281490Z","shell.execute_reply.started":"2022-08-05T23:48:45.682762Z","shell.execute_reply":"2022-08-05T23:48:56.280770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(7, 2, figsize = (18, 50))\n\nsns.scatterplot(ax = axes[0, 0], data = train, x = 'measurement_4', y = 'measurement_5', hue = color)\nsns.scatterplot(ax = axes[0, 1], data = train, x = 'measurement_4', y = 'measurement_6', hue = color)\nsns.scatterplot(ax = axes[1, 0], data = train, x = 'measurement_4', y = 'measurement_7', hue = color)\nsns.scatterplot(ax = axes[1, 1], data = train, x = 'measurement_4', y = 'measurement_8', hue = color)\nsns.scatterplot(ax = axes[2, 0], data = train, x = 'measurement_4', y = 'measurement_9', hue = color)\nsns.scatterplot(ax = axes[2, 1], data = train, x = 'measurement_4', y = 'measurement_10', hue = color)\nsns.scatterplot(ax = axes[3, 0], data = train, x = 'measurement_4', y = 'measurement_11', hue = color)\nsns.scatterplot(ax = axes[3, 1], data = train, x = 'measurement_4', y = 'measurement_12', hue = color)\nsns.scatterplot(ax = axes[4, 0], data = train, x = 'measurement_4', y = 'measurement_13', hue = color)\nsns.scatterplot(ax = axes[4, 1], data = train, x = 'measurement_4', y = 'measurement_14', hue = color)\nsns.scatterplot(ax = axes[5, 0], data = train, x = 'measurement_4', y = 'measurement_15', hue = color)\nsns.scatterplot(ax = axes[5, 1], data = train, x = 'measurement_4', y = 'measurement_16', hue = color)\nsns.scatterplot(ax = axes[6, 0], data = train, x = 'measurement_4', y = 'measurement_17', hue = color)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T23:52:56.767891Z","iopub.execute_input":"2022-08-05T23:52:56.768224Z","iopub.status.idle":"2022-08-05T23:53:06.215612Z","shell.execute_reply.started":"2022-08-05T23:52:56.768200Z","shell.execute_reply":"2022-08-05T23:53:06.213792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(6, 2, figsize = (18, 50))\n\nsns.scatterplot(ax = axes[0, 0], data = train, x = 'measurement_6', y = 'measurement_7', hue = color)\nsns.scatterplot(ax = axes[0, 1], data = train, x = 'measurement_6', y = 'measurement_8', hue = color)\nsns.scatterplot(ax = axes[1, 0], data = train, x = 'measurement_6', y = 'measurement_9', hue = color)\nsns.scatterplot(ax = axes[1, 1], data = train, x = 'measurement_6', y = 'measurement_10', hue = color)\nsns.scatterplot(ax = axes[2, 0], data = train, x = 'measurement_6', y = 'measurement_11', hue = color)\nsns.scatterplot(ax = axes[2, 1], data = train, x = 'measurement_6', y = 'measurement_12', hue = color)\nsns.scatterplot(ax = axes[3, 0], data = train, x = 'measurement_6', y = 'measurement_13', hue = color)\nsns.scatterplot(ax = axes[3, 1], data = train, x = 'measurement_6', y = 'measurement_14', hue = color)\nsns.scatterplot(ax = axes[4, 0], data = train, x = 'measurement_6', y = 'measurement_15', hue = color)\nsns.scatterplot(ax = axes[4, 1], data = train, x = 'measurement_6', y = 'measurement_16', hue = color)\nsns.scatterplot(ax = axes[5, 0], data = train, x = 'measurement_6', y = 'measurement_17', hue = color)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T23:58:32.949154Z","iopub.execute_input":"2022-08-05T23:58:32.949619Z","iopub.status.idle":"2022-08-05T23:58:41.011682Z","shell.execute_reply.started":"2022-08-05T23:58:32.949593Z","shell.execute_reply":"2022-08-05T23:58:41.010996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(5, 2, figsize = (18, 50))\n\nsns.scatterplot(ax = axes[0, 0], data = train, x = 'measurement_7', y = 'measurement_8', hue = color)\nsns.scatterplot(ax = axes[0, 1], data = train, x = 'measurement_7', y = 'measurement_9', hue = color)\nsns.scatterplot(ax = axes[1, 0], data = train, x = 'measurement_7', y = 'measurement_10', hue = color)\nsns.scatterplot(ax = axes[1, 1], data = train, x = 'measurement_7', y = 'measurement_11', hue = color)\nsns.scatterplot(ax = axes[2, 0], data = train, x = 'measurement_7', y = 'measurement_12', hue = color)\nsns.scatterplot(ax = axes[2, 1], data = train, x = 'measurement_7', y = 'measurement_13', hue = color)\nsns.scatterplot(ax = axes[3, 0], data = train, x = 'measurement_7', y = 'measurement_14', hue = color)\nsns.scatterplot(ax = axes[3, 1], data = train, x = 'measurement_7', y = 'measurement_15', hue = color)\nsns.scatterplot(ax = axes[4, 0], data = train, x = 'measurement_7', y = 'measurement_16', hue = color)\nsns.scatterplot(ax = axes[4, 1], data = train, x = 'measurement_7', y = 'measurement_17', hue = color)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T00:02:35.561218Z","iopub.execute_input":"2022-08-06T00:02:35.561582Z","iopub.status.idle":"2022-08-06T00:02:42.875802Z","shell.execute_reply.started":"2022-08-06T00:02:35.561556Z","shell.execute_reply":"2022-08-06T00:02:42.874245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(5, 2, figsize = (18, 50))\n\nsns.scatterplot(ax = axes[0, 0], data = train, x = 'measurement_8', y = 'measurement_9', hue = color)\nsns.scatterplot(ax = axes[0, 1], data = train, x = 'measurement_8', y = 'measurement_10', hue = color)\nsns.scatterplot(ax = axes[1, 0], data = train, x = 'measurement_8', y = 'measurement_11', hue = color)\nsns.scatterplot(ax = axes[1, 1], data = train, x = 'measurement_8', y = 'measurement_12', hue = color)\nsns.scatterplot(ax = axes[2, 0], data = train, x = 'measurement_8', y = 'measurement_13', hue = color)\nsns.scatterplot(ax = axes[2, 1], data = train, x = 'measurement_8', y = 'measurement_14', hue = color)\nsns.scatterplot(ax = axes[3, 0], data = train, x = 'measurement_8', y = 'measurement_15', hue = color)\nsns.scatterplot(ax = axes[3, 1], data = train, x = 'measurement_8', y = 'measurement_16', hue = color)\nsns.scatterplot(ax = axes[4, 0], data = train, x = 'measurement_8', y = 'measurement_17', hue = color)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T00:05:22.788950Z","iopub.execute_input":"2022-08-06T00:05:22.789268Z","iopub.status.idle":"2022-08-06T00:05:29.327868Z","shell.execute_reply.started":"2022-08-06T00:05:22.789242Z","shell.execute_reply":"2022-08-06T00:05:29.326075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(4, 2, figsize = (18, 40))\n\nsns.scatterplot(ax = axes[0, 0], data = train, x = 'measurement_9', y = 'measurement_10', hue = color)\nsns.scatterplot(ax = axes[0, 1], data = train, x = 'measurement_9', y = 'measurement_11', hue = color)\nsns.scatterplot(ax = axes[1, 0], data = train, x = 'measurement_9', y = 'measurement_12', hue = color)\nsns.scatterplot(ax = axes[1, 1], data = train, x = 'measurement_9', y = 'measurement_13', hue = color)\nsns.scatterplot(ax = axes[2, 0], data = train, x = 'measurement_9', y = 'measurement_14', hue = color)\nsns.scatterplot(ax = axes[2, 1], data = train, x = 'measurement_9', y = 'measurement_15', hue = color)\nsns.scatterplot(ax = axes[3, 0], data = train, x = 'measurement_9', y = 'measurement_16', hue = color)\nsns.scatterplot(ax = axes[3, 1], data = train, x = 'measurement_9', y = 'measurement_17', hue = color)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T00:08:38.720243Z","iopub.execute_input":"2022-08-06T00:08:38.720544Z","iopub.status.idle":"2022-08-06T00:08:44.722754Z","shell.execute_reply.started":"2022-08-06T00:08:38.720520Z","shell.execute_reply":"2022-08-06T00:08:44.720627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(4, 2, figsize = (18, 40))\n\nsns.scatterplot(ax = axes[0, 0], data = train, x = 'measurement_10', y = 'measurement_11', hue = color)\nsns.scatterplot(ax = axes[0, 1], data = train, x = 'measurement_10', y = 'measurement_12', hue = color)\nsns.scatterplot(ax = axes[1, 0], data = train, x = 'measurement_10', y = 'measurement_13', hue = color)\nsns.scatterplot(ax = axes[1, 1], data = train, x = 'measurement_10', y = 'measurement_14', hue = color)\nsns.scatterplot(ax = axes[2, 0], data = train, x = 'measurement_10', y = 'measurement_15', hue = color)\nsns.scatterplot(ax = axes[2, 1], data = train, x = 'measurement_10', y = 'measurement_16', hue = color)\nsns.scatterplot(ax = axes[3, 0], data = train, x = 'measurement_10', y = 'measurement_17', hue = color)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T00:09:45.655883Z","iopub.execute_input":"2022-08-06T00:09:45.656228Z","iopub.status.idle":"2022-08-06T00:09:51.749688Z","shell.execute_reply.started":"2022-08-06T00:09:45.656204Z","shell.execute_reply":"2022-08-06T00:09:51.748586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(3, 2, figsize = (18, 30))\n\nsns.scatterplot(ax = axes[0, 0], data = train, x = 'measurement_11', y = 'measurement_12', hue = color)\nsns.scatterplot(ax = axes[0, 1], data = train, x = 'measurement_11', y = 'measurement_13', hue = color)\nsns.scatterplot(ax = axes[1, 0], data = train, x = 'measurement_11', y = 'measurement_14', hue = color)\nsns.scatterplot(ax = axes[1, 1], data = train, x = 'measurement_11', y = 'measurement_15', hue = color)\nsns.scatterplot(ax = axes[2, 0], data = train, x = 'measurement_11', y = 'measurement_16', hue = color)\nsns.scatterplot(ax = axes[2, 1], data = train, x = 'measurement_11', y = 'measurement_17', hue = color)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T00:10:39.452594Z","iopub.execute_input":"2022-08-06T00:10:39.452985Z","iopub.status.idle":"2022-08-06T00:10:43.815408Z","shell.execute_reply.started":"2022-08-06T00:10:39.452959Z","shell.execute_reply":"2022-08-06T00:10:43.814225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(3, 2, figsize = (18, 30))\n\nsns.scatterplot(ax = axes[0, 0], data = train, x = 'measurement_12', y = 'measurement_13', hue = color)\nsns.scatterplot(ax = axes[0, 1], data = train, x = 'measurement_12', y = 'measurement_14', hue = color)\nsns.scatterplot(ax = axes[1, 0], data = train, x = 'measurement_12', y = 'measurement_15', hue = color)\nsns.scatterplot(ax = axes[1, 1], data = train, x = 'measurement_12', y = 'measurement_16', hue = color)\nsns.scatterplot(ax = axes[2, 0], data = train, x = 'measurement_12', y = 'measurement_17', hue = color)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T00:12:21.969589Z","iopub.execute_input":"2022-08-06T00:12:21.969973Z","iopub.status.idle":"2022-08-06T00:12:25.712447Z","shell.execute_reply.started":"2022-08-06T00:12:21.969949Z","shell.execute_reply":"2022-08-06T00:12:25.710809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(2, 2, figsize = (18, 20))\n\nsns.scatterplot(ax = axes[0, 0], data = train, x = 'measurement_13', y = 'measurement_14', hue = color)\nsns.scatterplot(ax = axes[0, 1], data = train, x = 'measurement_13', y = 'measurement_15', hue = color)\nsns.scatterplot(ax = axes[1, 0], data = train, x = 'measurement_13', y = 'measurement_16', hue = color)\nsns.scatterplot(ax = axes[1, 1], data = train, x = 'measurement_13', y = 'measurement_17', hue = color)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T00:15:16.423511Z","iopub.execute_input":"2022-08-06T00:15:16.423879Z","iopub.status.idle":"2022-08-06T00:15:19.442144Z","shell.execute_reply.started":"2022-08-06T00:15:16.423853Z","shell.execute_reply":"2022-08-06T00:15:19.440752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(2, 2, figsize = (18, 20))\n\nsns.scatterplot(ax = axes[0, 0], data = train, x = 'measurement_14', y = 'measurement_15', hue = color)\nsns.scatterplot(ax = axes[0, 1], data = train, x = 'measurement_14', y = 'measurement_16', hue = color)\nsns.scatterplot(ax = axes[1, 0], data = train, x = 'measurement_14', y = 'measurement_17', hue = color)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T00:16:51.788102Z","iopub.execute_input":"2022-08-06T00:16:51.788438Z","iopub.status.idle":"2022-08-06T00:16:54.068027Z","shell.execute_reply.started":"2022-08-06T00:16:51.788414Z","shell.execute_reply":"2022-08-06T00:16:54.067058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 2, figsize = (18, 10))\n\nsns.scatterplot(ax = axes[0], data = train, x = 'measurement_15', y = 'measurement_16', hue = color)\nsns.scatterplot(ax = axes[1], data = train, x = 'measurement_15', y = 'measurement_17', hue = color)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T00:17:52.237235Z","iopub.execute_input":"2022-08-06T00:17:52.237548Z","iopub.status.idle":"2022-08-06T00:17:53.708000Z","shell.execute_reply.started":"2022-08-06T00:17:52.237524Z","shell.execute_reply":"2022-08-06T00:17:53.707151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From the above charts, it is clear that there no clear pattern in the data.","metadata":{}},{"cell_type":"markdown","source":"# Logistic Regression","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import MinMaxScaler\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.linear_model import LogisticRegression\n\n## Defining input and target variables\nX = train.drop(columns = ['failure'], axis = 1)\nY = train['failure']\n\n## Scaling inputs to 0-1\nscaler = MinMaxScaler()\nX = pd.DataFrame(scaler.fit_transform(X), columns = X.columns)\ntest = pd.DataFrame(scaler.fit_transform(test), columns = test.columns)\n\n## Defining the hyper-parameter grid\nlogistic_param_grid = {'penalty': ['l1', 'l2'],\n                       'C': [0.001, 0.01, 0.1, 1, 10, 100],\n                       'solver': ['liblinear', 'sag', 'saga']}\n\n## Performing grid search with 5 folds\nlogistic_grid_search = GridSearchCV(LogisticRegression(), logistic_param_grid, cv = 5, scoring = 'roc_auc', n_jobs = -1, verbose = 1).fit(X, Y)\n\n## Extracting the best parameters\nbest_params = logistic_grid_search.best_params_\nprint('The optimal hyper-parameters are:', best_params)\n\n## Extracting the best score\nbest_score = logistic_grid_search.best_score_\nprint('The best area under the ROC cure is:', best_score)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T11:16:51.876484Z","iopub.execute_input":"2022-08-09T11:16:51.876927Z","iopub.status.idle":"2022-08-09T11:17:30.844572Z","shell.execute_reply.started":"2022-08-09T11:16:51.876895Z","shell.execute_reply":"2022-08-09T11:17:30.842744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Extracting the best model and its estimated parameters\nlogistic_md = logistic_grid_search.best_estimator_\ncoefs =  pd.DataFrame({'feature': X.columns, 'est_coef': abs(logistic_md.coef_.flatten())})\ncoefs = coefs.sort_values(by = 'est_coef', ascending = False).reset_index(drop = True)\ncoefs","metadata":{"execution":{"iopub.status.busy":"2022-08-09T11:17:46.949468Z","iopub.execute_input":"2022-08-09T11:17:46.950020Z","iopub.status.idle":"2022-08-09T11:17:46.972873Z","shell.execute_reply.started":"2022-08-09T11:17:46.949979Z","shell.execute_reply":"2022-08-09T11:17:46.971854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Re-fitting the logistic model with top 5 features\nlogit_md = LogisticRegression(C = 0.1, penalty = 'l1', solver = 'saga', max_iter = 1000).fit(X[coefs['feature'].loc[0:6].values], Y)\nlogit_md_pred = logit_md.predict_proba(X[coefs['feature'].loc[0:6].values])[:, 1]\nlogit_md_score = roc_auc_score(Y, logit_md_pred)\nprint('The area under the ROC curve is:', logit_md_score)\n\n## Predicting on test with best model \nlogit_md_test_pred = logit_md.predict_proba(test[coefs['feature'].loc[0:6].values])[:, 1] \n\n## Defining data-frame to be exported\ndata_out = pd.DataFrame({'id': test_id, 'failure': logit_md_test_pred})\ndata_out.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T11:18:55.287417Z","iopub.execute_input":"2022-08-09T11:18:55.289029Z","iopub.status.idle":"2022-08-09T11:18:55.503992Z","shell.execute_reply.started":"2022-08-09T11:18:55.288976Z","shell.execute_reply":"2022-08-09T11:18:55.502298Z"},"trusted":true},"execution_count":null,"outputs":[]}]}