{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Data Processing\nimport numpy as np\nimport pandas as pd\nimport random\nfrom sklearn import preprocessing\nfrom sklearn.experimental import enable_iterative_imputer\nfrom sklearn.impute import IterativeImputer\n\n# Modeling \nfrom sklearn.ensemble import VotingClassifier\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import cluster, accuracy_score, roc_auc_score\nfrom sklearn.model_selection import cross_validate, GridSearchCV, cross_val_score, StratifiedKFold\nfrom sklearn.preprocessing import LabelEncoder\n\nimport tensorflow as tf\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Input\nfrom tensorflow.keras.layers import Dense\nfrom tensorflow.keras.utils import plot_model\n\n# Other\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T12:54:02.595650Z","iopub.execute_input":"2022-08-05T12:54:02.596158Z","iopub.status.idle":"2022-08-05T12:54:02.605567Z","shell.execute_reply.started":"2022-08-05T12:54:02.596120Z","shell.execute_reply":"2022-08-05T12:54:02.604158Z"},"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h1><center><b>Regression or Classification ???</b></center></h1>\n\n![why not](https://c.tenor.com/odyVsZbC-OYAAAAC/why-not-both-why-not.gif)","metadata":{}},{"cell_type":"markdown","source":"## Reference\n\n[Logistic regression is all u need](https://www.kaggle.com/code/heyspaceturtle/logistic-regression-is-all-u-need/notebook)","metadata":{}},{"cell_type":"markdown","source":"## Feature Engineering","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/tabular-playground-series-aug-2022/train.csv')\ntest = pd.read_csv('../input/tabular-playground-series-aug-2022/test.csv')\nsample_submission = pd.read_csv('../input/tabular-playground-series-aug-2022/sample_submission.csv')\n\ntrain.head()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-05T12:51:28.302108Z","iopub.execute_input":"2022-08-05T12:51:28.302888Z","iopub.status.idle":"2022-08-05T12:51:28.592139Z","shell.execute_reply.started":"2022-08-05T12:51:28.302854Z","shell.execute_reply":"2022-08-05T12:51:28.590937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id_var = ['id']\ntarget= ['failure']\ncat_vars = ['product_code','attribute_0','attribute_1']\nnum_vars = [v for v in test.columns if v not in id_var and v not in cat_vars]\npredictors = cat_vars + num_vars","metadata":{"execution":{"iopub.status.busy":"2022-08-05T12:51:28.593305Z","iopub.execute_input":"2022-08-05T12:51:28.593665Z","iopub.status.idle":"2022-08-05T12:51:28.599228Z","shell.execute_reply.started":"2022-08-05T12:51:28.593603Z","shell.execute_reply":"2022-08-05T12:51:28.598241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"multi_imp = IterativeImputer(max_iter = 9, random_state = 42, verbose = 0, skip_complete = True, n_nearest_features = 10, tol = 0.001)\nmulti_imp.fit(train[num_vars])\ntrain[num_vars] = multi_imp.transform(train[num_vars])\ntest[num_vars] = multi_imp.transform(test[num_vars])","metadata":{"execution":{"iopub.status.busy":"2022-08-05T12:51:28.601469Z","iopub.execute_input":"2022-08-05T12:51:28.601820Z","iopub.status.idle":"2022-08-05T12:51:35.310481Z","shell.execute_reply.started":"2022-08-05T12:51:28.601788Z","shell.execute_reply":"2022-08-05T12:51:35.308842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"attributes = ['attribute_2', 'attribute_3', 'measurement_4', 'measurement_5', 'measurement_6']\ntrain[attributes] = preprocessing.normalize(train[attributes])\ntest[attributes] = preprocessing.normalize(test[attributes])","metadata":{"execution":{"iopub.status.busy":"2022-08-05T12:51:35.312975Z","iopub.execute_input":"2022-08-05T12:51:35.314185Z","iopub.status.idle":"2022-08-05T12:51:35.346920Z","shell.execute_reply.started":"2022-08-05T12:51:35.314100Z","shell.execute_reply":"2022-08-05T12:51:35.345268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = test.drop(['product_code'], axis = 1)\ntrain = train.drop(['product_code'], axis = 1)\ncat_vars.remove('product_code')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T12:51:35.349056Z","iopub.execute_input":"2022-08-05T12:51:35.349884Z","iopub.status.idle":"2022-08-05T12:51:35.366857Z","shell.execute_reply.started":"2022-08-05T12:51:35.349838Z","shell.execute_reply":"2022-08-05T12:51:35.365214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for v in cat_vars:\n    tempdf = pd.get_dummies(train[v], prefix = v)\n    tempdf_test = pd.get_dummies(test[v], prefix = v)\n    train = pd.merge(left = train, right = tempdf, left_index = True, right_index = True)\n    test = pd.merge(left = test, right = tempdf_test, left_index = True, right_index = True)\ntrain = train.drop(cat_vars, axis = 1)\ntest = test.drop(cat_vars, axis = 1)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T12:51:35.369185Z","iopub.execute_input":"2022-08-05T12:51:35.370169Z","iopub.status.idle":"2022-08-05T12:51:35.437126Z","shell.execute_reply.started":"2022-08-05T12:51:35.370120Z","shell.execute_reply":"2022-08-05T12:51:35.436147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictors = [v for v in train.columns if v not in id_var and v not in target]\n\ny_class = LabelEncoder().fit_transform(train[target])\n\n# Train test split\nX_train, X_test, y_train, y_test, y_train_class, y_test_class = train_test_split(train[predictors], train[target], y_class, test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T12:51:35.438875Z","iopub.execute_input":"2022-08-05T12:51:35.439701Z","iopub.status.idle":"2022-08-05T12:51:35.463286Z","shell.execute_reply.started":"2022-08-05T12:51:35.439656Z","shell.execute_reply":"2022-08-05T12:51:35.461989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Classification + Regression Model","metadata":{}},{"cell_type":"code","source":"seed = 42\n\nrandom.seed(seed)\nnp.random.seed(seed)\ntf.random.set_seed(seed)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visible = Input(shape=(X_train.shape[1],))\nlayer = Dense(256, activation='swish', kernel_initializer='he_normal')(visible)\nlayer = Dense(128, activation='swish', kernel_initializer='he_normal')(layer)\nlayer = Dense(64, activation='swish', kernel_initializer='he_normal')(layer)\nlayer = Dense(32, activation='swish', kernel_initializer='he_normal')(layer)\nlayer = Dense(16, activation='swish', kernel_initializer='he_normal')(layer)\nlayer = Dense(8, activation='swish', kernel_initializer='he_normal')(layer)\n\n# regression + classification\nout_reg = Dense(1, activation='linear')(layer)\nout_clas = Dense(2, activation='softmax')(layer)\n\nmodel = Model(inputs=visible, outputs=[out_reg, out_clas])\nmodel.compile(loss=['mse','sparse_categorical_crossentropy'], optimizer='adam')\nplot_model(model, to_file='model.png', show_shapes=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T12:56:56.215985Z","iopub.execute_input":"2022-08-05T12:56:56.216382Z","iopub.status.idle":"2022-08-05T12:56:56.454321Z","shell.execute_reply.started":"2022-08-05T12:56:56.216349Z","shell.execute_reply":"2022-08-05T12:56:56.453147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"callback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=5, restore_best_weights=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T12:56:57.285132Z","iopub.execute_input":"2022-08-05T12:56:57.286028Z","iopub.status.idle":"2022-08-05T12:56:57.293025Z","shell.execute_reply.started":"2022-08-05T12:56:57.285987Z","shell.execute_reply":"2022-08-05T12:56:57.291272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(X_train, [y_train,y_train_class], epochs=150, batch_size=32, callbacks=[callback], verbose=2)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T12:56:58.341731Z","iopub.execute_input":"2022-08-05T12:56:58.342707Z","iopub.status.idle":"2022-08-05T12:58:17.411338Z","shell.execute_reply.started":"2022-08-05T12:56:58.342656Z","shell.execute_reply":"2022-08-05T12:58:17.410013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## The famous Logistic Regression","metadata":{}},{"cell_type":"code","source":"lrgs = LogisticRegression(max_iter = 200, C=0.0001, penalty='l2', solver='newton-cg')\nlrgs.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T12:58:28.189595Z","iopub.execute_input":"2022-08-05T12:58:28.190410Z","iopub.status.idle":"2022-08-05T12:58:29.149523Z","shell.execute_reply.started":"2022-08-05T12:58:28.190369Z","shell.execute_reply":"2022-08-05T12:58:29.147274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submissions","metadata":{}},{"cell_type":"code","source":"test = test.drop('id', axis = 1)\ny_hat = model.predict(test)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T12:59:12.772733Z","iopub.execute_input":"2022-08-05T12:59:12.773161Z","iopub.status.idle":"2022-08-05T12:59:14.162631Z","shell.execute_reply.started":"2022-08-05T12:59:12.773127Z","shell.execute_reply":"2022-08-05T12:59:14.161658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Regression","metadata":{}},{"cell_type":"code","source":"sub1 = sample_submission.copy()\nsub1.failure = y_hat[0]\nsub1.to_csv('submission_nn_regression.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T13:01:17.922271Z","iopub.execute_input":"2022-08-05T13:01:17.922757Z","iopub.status.idle":"2022-08-05T13:01:17.972352Z","shell.execute_reply.started":"2022-08-05T13:01:17.922708Z","shell.execute_reply":"2022-08-05T13:01:17.971432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Classification","metadata":{}},{"cell_type":"code","source":"sub1 = sample_submission.copy()\nsub1.failure = y_hat[1][:,1]\nsub1.to_csv('submission_nn_classification.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T13:01:42.655293Z","iopub.execute_input":"2022-08-05T13:01:42.655727Z","iopub.status.idle":"2022-08-05T13:01:42.702222Z","shell.execute_reply.started":"2022-08-05T13:01:42.655676Z","shell.execute_reply":"2022-08-05T13:01:42.701312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Ensemble Regression + Classification","metadata":{}},{"cell_type":"code","source":"sub1 = sample_submission.copy()\nsub1.failure = (y_hat[0] + y_hat[1][:,1]) / 2\nsub1.to_csv('submission_nn_ensemble.csv', index = False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Logistic Regression","metadata":{}},{"cell_type":"code","source":"sub1 = sample_submission.copy()\nsub1.failure = lrgs.predict_proba(test)[:,1]\nsub1.to_csv('submission_lr.csv', index = False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Ensemble","metadata":{}},{"cell_type":"code","source":"sub1 = sample_submission.copy()\nsub1.failure = (0.3 * lrgs.predict_proba(test)[:,1]) + (0.35 * y_hat[0]) + (0.35 * y_hat[1][:,1]) \nsub1.to_csv('submission_ensemble.csv', index = False)","metadata":{},"execution_count":null,"outputs":[]}]}