{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import libraries","metadata":{}},{"cell_type":"markdown","source":"#### Continuation from [logistic regression](https://www.kaggle.com/code/kostiantynlavronenko/tps-08-22-logistic-regression) to test different models on the dataset. ","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as  pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n\nfrom sklearn.neural_network import MLPClassifier, MLPRegressor\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report\nfrom sklearn import preprocessing\nfrom sklearn.impute import SimpleImputer\nfrom imblearn.over_sampling import SMOTE\nfrom collections import Counter\nimport warnings; warnings.filterwarnings(\"ignore\")\nimport missingno as msno\n\nUSE_CATEGORICAL = True\nUSE_CROSS_VALIDATION = True\nUSE_SIMPLE_IMPUTER = True\nUSE_OVERSAMPLING = True\n","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:51:39.003442Z","iopub.execute_input":"2022-08-05T06:51:39.004275Z","iopub.status.idle":"2022-08-05T06:51:40.811947Z","shell.execute_reply.started":"2022-08-05T06:51:39.004154Z","shell.execute_reply":"2022-08-05T06:51:40.810259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:51:40.814035Z","iopub.execute_input":"2022-08-05T06:51:40.814397Z","iopub.status.idle":"2022-08-05T06:51:40.821703Z","shell.execute_reply.started":"2022-08-05T06:51:40.814364Z","shell.execute_reply":"2022-08-05T06:51:40.820724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Look at the data 🔍","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(\"/kaggle/input/tabular-playground-series-aug-2022/train.csv\")\nprint(\"train_df shape:\", train_df.shape)\nprint(f\"Amount of failures {train_df['failure'].sum()} of {train_df.shape[0]}\")\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:51:40.823249Z","iopub.execute_input":"2022-08-05T06:51:40.823904Z","iopub.status.idle":"2022-08-05T06:51:41.055949Z","shell.execute_reply.started":"2022-08-05T06:51:40.823866Z","shell.execute_reply":"2022-08-05T06:51:41.054536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv(\"/kaggle/input/tabular-playground-series-aug-2022/test.csv\")\nprint(\"test_df shape:\", test_df.shape)\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:51:41.059502Z","iopub.execute_input":"2022-08-05T06:51:41.060012Z","iopub.status.idle":"2022-08-05T06:51:41.197450Z","shell.execute_reply.started":"2022-08-05T06:51:41.059961Z","shell.execute_reply":"2022-08-05T06:51:41.196116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Descriptive features","metadata":{}},{"cell_type":"code","source":"print(\"Different output classes:\", *train_df[\"failure\"].unique())\ndescriptive_features = []\nfor i in range(18):\n    descriptive_features.append(\"measurement_\"+str(i))\ndescriptive_features.append(\"loading\")\ndescriptive_features.append(\"attribute_2\")\ndescriptive_features.append(\"attribute_3\")\nprint(\"descriptive_features:\", *descriptive_features)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:51:41.198982Z","iopub.execute_input":"2022-08-05T06:51:41.199475Z","iopub.status.idle":"2022-08-05T06:51:41.210994Z","shell.execute_reply.started":"2022-08-05T06:51:41.199427Z","shell.execute_reply":"2022-08-05T06:51:41.209860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if USE_CATEGORICAL:\n    categorical_descriptive_features = [\"product_code\", \"attribute_0\", \"attribute_1\"]\n    descriptive_features.extend(categorical_descriptive_features)\n    descriptive_features","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:51:41.212850Z","iopub.execute_input":"2022-08-05T06:51:41.213589Z","iopub.status.idle":"2022-08-05T06:51:41.221799Z","shell.execute_reply.started":"2022-08-05T06:51:41.213543Z","shell.execute_reply":"2022-08-05T06:51:41.220667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Test","metadata":{}},{"cell_type":"code","source":"train_data = train_df[descriptive_features]\ntrain_labels = train_df[\"failure\"]\ntest_data = test_df[descriptive_features]","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:51:41.223716Z","iopub.execute_input":"2022-08-05T06:51:41.224471Z","iopub.status.idle":"2022-08-05T06:51:41.240496Z","shell.execute_reply.started":"2022-08-05T06:51:41.224425Z","shell.execute_reply":"2022-08-05T06:51:41.239034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if USE_CATEGORICAL:\n    train_data = pd.get_dummies(train_data, columns = categorical_descriptive_features)\n    test_data = pd.get_dummies(test_data, columns = categorical_descriptive_features)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:51:41.242334Z","iopub.execute_input":"2022-08-05T06:51:41.242988Z","iopub.status.idle":"2022-08-05T06:51:41.284303Z","shell.execute_reply.started":"2022-08-05T06:51:41.242939Z","shell.execute_reply":"2022-08-05T06:51:41.282855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Columns in train_data\", train_data.columns)\nprint(\"Columns in test_data\", test_data.columns)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:51:41.286187Z","iopub.execute_input":"2022-08-05T06:51:41.287723Z","iopub.status.idle":"2022-08-05T06:51:41.295024Z","shell.execute_reply.started":"2022-08-05T06:51:41.286909Z","shell.execute_reply":"2022-08-05T06:51:41.293679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if USE_CATEGORICAL:\n    train_columns_to_drop = [\"product_code_A\", \"product_code_B\", \"product_code_C\", \"product_code_D\", \"product_code_E\", \"attribute_1_material_8\"]\n    train_data = train_data.drop(columns = train_columns_to_drop)\n    test_columns_to_drop = [\"product_code_F\", \"product_code_G\", \"product_code_H\", \"product_code_I\", \"attribute_1_material_7\"]\n    test_data = test_data.drop(columns = test_columns_to_drop)\n    print(\"All columns are equal: \", (train_data.columns == test_data.columns).all())","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:51:41.296983Z","iopub.execute_input":"2022-08-05T06:51:41.297825Z","iopub.status.idle":"2022-08-05T06:51:41.318724Z","shell.execute_reply.started":"2022-08-05T06:51:41.297779Z","shell.execute_reply":"2022-08-05T06:51:41.317045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Handling NaN","metadata":{}},{"cell_type":"code","source":"if USE_SIMPLE_IMPUTER:\n    train_imputer = SimpleImputer(missing_values=np.NaN, strategy = \"mean\")\n    transformed_values_train = train_imputer.fit_transform(train_data)\n    print(\"Train NaN values: \", np.isnan(transformed_values_train).sum())\n    \n    test_imputer = SimpleImputer(missing_values=np.NaN, strategy = \"mean\")\n    transformed_values_test = test_imputer.fit_transform(test_data)\n    print(\"Test NaN values: \",np.isnan(transformed_values_test).sum())\nelse:\n    transformed_values_train = train_data.fillna(train_data.mean(axis=0))\n    transformed_values_test = test_data.fillna(test_data.mean(axis=0))\n    print(\"Shape of train data after fillna\", transformed_values_train.shape)\n    print(\"Shape of test data after fillna\",transformed_values_test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:51:41.323355Z","iopub.execute_input":"2022-08-05T06:51:41.324956Z","iopub.status.idle":"2022-08-05T06:51:41.388517Z","shell.execute_reply.started":"2022-08-05T06:51:41.324907Z","shell.execute_reply":"2022-08-05T06:51:41.386903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Normalize","metadata":{}},{"cell_type":"code","source":"scaler = preprocessing.StandardScaler()\nscaled_train_data = scaler.fit_transform(transformed_values_train)\nscaled_test_data = scaler.fit_transform(transformed_values_test)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:51:41.390344Z","iopub.execute_input":"2022-08-05T06:51:41.391132Z","iopub.status.idle":"2022-08-05T06:51:41.421564Z","shell.execute_reply.started":"2022-08-05T06:51:41.391083Z","shell.execute_reply":"2022-08-05T06:51:41.420155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Imbalanced","metadata":{}},{"cell_type":"code","source":"print(Counter(train_labels))\nplt.subplots(figsize=(8, 8))\nplt.pie(train_labels.value_counts(), startangle=90, wedgeprops={'width':0.3})\nplt.text(0, 0, f\"{train_labels.value_counts()[0] / train_labels.count() * 100:.2f}%\", ha='center', va='center', fontweight='bold', fontsize=42)\nplt.legend(train_labels.value_counts().index, ncol=2, loc='lower center', fontsize=16)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:51:41.423369Z","iopub.execute_input":"2022-08-05T06:51:41.424116Z","iopub.status.idle":"2022-08-05T06:51:41.682109Z","shell.execute_reply.started":"2022-08-05T06:51:41.424067Z","shell.execute_reply":"2022-08-05T06:51:41.681012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if USE_OVERSAMPLING:\n    oversample = SMOTE()\n    scaled_train_data, train_labels = oversample.fit_resample(scaled_train_data, train_labels)\n    scaler = preprocessing.StandardScaler()\n    scaled_train_data = scaler.fit_transform(scaled_train_data)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:51:41.683575Z","iopub.execute_input":"2022-08-05T06:51:41.684561Z","iopub.status.idle":"2022-08-05T06:51:42.672805Z","shell.execute_reply.started":"2022-08-05T06:51:41.684522Z","shell.execute_reply":"2022-08-05T06:51:42.671404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Plot","metadata":{}},{"cell_type":"code","source":"if USE_OVERSAMPLING:\n    print(Counter(train_labels))\n    plt.subplots(figsize=(8, 8))\n    plt.pie(train_labels.value_counts(), startangle=90, wedgeprops={'width':0.3})\n    plt.text(0, 0, f\"{train_labels.value_counts()[0] / train_labels.count() * 100:.2f}%\", ha='center', va='center', fontweight='bold', fontsize=42)\n    plt.legend(train_labels.value_counts().index, ncol=2, loc='lower center', fontsize=16)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:51:42.674791Z","iopub.execute_input":"2022-08-05T06:51:42.675513Z","iopub.status.idle":"2022-08-05T06:51:42.831768Z","shell.execute_reply.started":"2022-08-05T06:51:42.675465Z","shell.execute_reply":"2022-08-05T06:51:42.830459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Neural network 🦾","metadata":{}},{"cell_type":"code","source":"train_x, val_x, train_y, val_y = train_test_split(scaled_train_data, train_labels, test_size=0.05, random_state=42, stratify = train_labels)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:51:42.833349Z","iopub.execute_input":"2022-08-05T06:51:42.833764Z","iopub.status.idle":"2022-08-05T06:51:42.872205Z","shell.execute_reply.started":"2022-08-05T06:51:42.833726Z","shell.execute_reply":"2022-08-05T06:51:42.870878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classifier = MLPClassifier(hidden_layer_sizes=(10,10,10,10,2), verbose = True, early_stopping = True, validation_fraction=0.05, max_iter=200)\nclassifier.fit(train_x, train_y)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:55:22.424353Z","iopub.execute_input":"2022-08-05T06:55:22.424825Z","iopub.status.idle":"2022-08-05T06:55:40.923689Z","shell.execute_reply.started":"2022-08-05T06:55:22.424782Z","shell.execute_reply":"2022-08-05T06:55:40.922089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted = classifier.predict(val_x)\nprint(classification_report(val_y, predicted))","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:55:51.577499Z","iopub.execute_input":"2022-08-05T06:55:51.578391Z","iopub.status.idle":"2022-08-05T06:55:51.614260Z","shell.execute_reply.started":"2022-08-05T06:55:51.578334Z","shell.execute_reply":"2022-08-05T06:55:51.612712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_y = classifier.predict_proba(scaled_test_data)[:,1]","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:55:56.664184Z","iopub.execute_input":"2022-08-05T06:55:56.665694Z","iopub.status.idle":"2022-08-05T06:55:56.700523Z","shell.execute_reply.started":"2022-08-05T06:55:56.665614Z","shell.execute_reply":"2022-08-05T06:55:56.699194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(\"/kaggle/input/tabular-playground-series-aug-2022/sample_submission.csv\")\nsubmission.failure = pred_y\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:55:58.726087Z","iopub.execute_input":"2022-08-05T06:55:58.726549Z","iopub.status.idle":"2022-08-05T06:55:58.750190Z","shell.execute_reply.started":"2022-08-05T06:55:58.726509Z","shell.execute_reply":"2022-08-05T06:55:58.748923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission 🚀","metadata":{}},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index = False)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T06:56:03.237020Z","iopub.execute_input":"2022-08-05T06:56:03.237499Z","iopub.status.idle":"2022-08-05T06:56:03.299821Z","shell.execute_reply.started":"2022-08-05T06:56:03.237460Z","shell.execute_reply":"2022-08-05T06:56:03.298440Z"},"trusted":true},"execution_count":null,"outputs":[]}]}