{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport numpy as np\nimport pandas as pd\nfrom itertools import product\nimport random\nfrom sklearn import preprocessing\nfrom sklearn.experimental import enable_iterative_imputer\nfrom sklearn.impute import IterativeImputer\n\nfrom sklearn.ensemble import VotingClassifier\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import cluster, accuracy_score, roc_auc_score\nfrom sklearn.model_selection import cross_validate, GridSearchCV, cross_val_score, StratifiedKFold\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import RepeatedStratifiedKFold\nfrom sklearn.exceptions import DataConversionWarning\n\nimport tensorflow as tf\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Input\nfrom tensorflow.keras.layers import Dense\nfrom tensorflow.keras.utils import plot_model\n\n# Other\nimport warnings\nwarnings.filterwarnings('ignore')\nwarnings.filterwarnings(action='ignore', category=DataConversionWarning)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-08-05T14:55:07.597488Z","iopub.execute_input":"2022-08-05T14:55:07.600477Z","iopub.status.idle":"2022-08-05T14:55:07.616094Z","shell.execute_reply.started":"2022-08-05T14:55:07.600420Z","shell.execute_reply":"2022-08-05T14:55:07.614253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<img src='https://www.mihaileric.com/static/number-of-features-too-high-a2f672081c44c4f08df97807478dd72b-232e6.jpg'/>","metadata":{}},{"cell_type":"markdown","source":"## Reference\n\n[Logistic regression is all u need](https://www.kaggle.com/code/heyspaceturtle/logistic-regression-is-all-u-need/notebook)<br>\n[TPS Aug 22 - Regression + Classification Keras](https://www.kaggle.com/code/trixytea/tps-aug-22-regression-classification-keras)","metadata":{}},{"cell_type":"markdown","source":"## Feature Engineering","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/tabular-playground-series-aug-2022/train.csv')\ntest = pd.read_csv('../input/tabular-playground-series-aug-2022/test.csv')\nsample_submission = pd.read_csv('../input/tabular-playground-series-aug-2022/sample_submission.csv')\n\ntrain.head()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-05T14:55:10.466241Z","iopub.execute_input":"2022-08-05T14:55:10.466679Z","iopub.status.idle":"2022-08-05T14:55:10.865075Z","shell.execute_reply.started":"2022-08-05T14:55:10.466644Z","shell.execute_reply":"2022-08-05T14:55:10.863641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id_var = ['id']\ntarget= ['failure']\ncat_vars = ['product_code','attribute_0','attribute_1']\nnum_vars = [v for v in test.columns if v not in id_var and v not in cat_vars]\npredictors = cat_vars + num_vars","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:55:10.867770Z","iopub.execute_input":"2022-08-05T14:55:10.868682Z","iopub.status.idle":"2022-08-05T14:55:10.876823Z","shell.execute_reply.started":"2022-08-05T14:55:10.868627Z","shell.execute_reply":"2022-08-05T14:55:10.875296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"multi_imp = IterativeImputer(max_iter = 9, random_state = 42, verbose = 0, skip_complete = True, n_nearest_features = 10, tol = 0.001)\nmulti_imp.fit(train[num_vars])\ntrain[num_vars] = multi_imp.transform(train[num_vars])\ntest[num_vars] = multi_imp.transform(test[num_vars])","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:55:11.742597Z","iopub.execute_input":"2022-08-05T14:55:11.743066Z","iopub.status.idle":"2022-08-05T14:55:19.397971Z","shell.execute_reply.started":"2022-08-05T14:55:11.743004Z","shell.execute_reply":"2022-08-05T14:55:19.395663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"attributes = ['attribute_2', 'attribute_3', 'measurement_4', 'measurement_5', 'measurement_6']\ntrain[attributes] = preprocessing.normalize(train[attributes])\ntest[attributes] = preprocessing.normalize(test[attributes])","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:55:19.403442Z","iopub.execute_input":"2022-08-05T14:55:19.406551Z","iopub.status.idle":"2022-08-05T14:55:19.470304Z","shell.execute_reply.started":"2022-08-05T14:55:19.406464Z","shell.execute_reply":"2022-08-05T14:55:19.468129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = test.drop(['product_code'], axis = 1)\ntrain = train.drop(['product_code'], axis = 1)\ncat_vars.remove('product_code')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:55:19.474132Z","iopub.execute_input":"2022-08-05T14:55:19.477070Z","iopub.status.idle":"2022-08-05T14:55:19.502466Z","shell.execute_reply.started":"2022-08-05T14:55:19.476966Z","shell.execute_reply":"2022-08-05T14:55:19.501114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for v in cat_vars:\n    tempdf = pd.get_dummies(train[v], prefix = v)\n    tempdf_test = pd.get_dummies(test[v], prefix = v)\n    train = pd.merge(left = train, right = tempdf, left_index = True, right_index = True)\n    test = pd.merge(left = test, right = tempdf_test, left_index = True, right_index = True)\ntrain = train.drop(cat_vars, axis = 1)\ntest = test.drop(cat_vars, axis = 1)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:55:19.505121Z","iopub.execute_input":"2022-08-05T14:55:19.506063Z","iopub.status.idle":"2022-08-05T14:55:19.563700Z","shell.execute_reply.started":"2022-08-05T14:55:19.506012Z","shell.execute_reply":"2022-08-05T14:55:19.562351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictors = [v for v in train.columns if v not in id_var and v not in target]\ny_class = LabelEncoder().fit_transform(train[target])","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:55:19.565786Z","iopub.execute_input":"2022-08-05T14:55:19.566640Z","iopub.status.idle":"2022-08-05T14:55:19.578759Z","shell.execute_reply.started":"2022-08-05T14:55:19.566588Z","shell.execute_reply":"2022-08-05T14:55:19.577058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Selection","metadata":{}},{"cell_type":"code","source":"X, y = train[predictors].values, train[target].values","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:55:19.580689Z","iopub.execute_input":"2022-08-05T14:55:19.581679Z","iopub.status.idle":"2022-08-05T14:55:19.603911Z","shell.execute_reply.started":"2022-08-05T14:55:19.581625Z","shell.execute_reply":"2022-08-05T14:55:19.602109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_cols = X.shape[1]\nbest_subset, best_score = None, 0.0\n\nfor subset in product([True, False], repeat=n_cols):\n    ix = [i for i, x in enumerate(subset) if x]\n    if len(ix) == 0:\n        continue\n    X_new = X[:, ix]\n    model = LogisticRegression(max_iter = 200, C=0.0001, penalty='l2', solver='newton-cg')\n    cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)\n    scores = cross_val_score(model, X_new, y, scoring='accuracy', cv=cv, n_jobs=-1)\n    result = np.mean(scores)\n\n    if best_score is None or result >= best_score:\n        best_subset, best_score = ix, result","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:24:23.626815Z","iopub.execute_input":"2022-08-05T14:24:23.627400Z"},"_kg_hide-output":true,"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('f(%s) = %f' % (best_subset, best_score))","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:55:19.605891Z","iopub.execute_input":"2022-08-05T14:55:19.606764Z","iopub.status.idle":"2022-08-05T14:55:19.614189Z","shell.execute_reply.started":"2022-08-05T14:55:19.606692Z","shell.execute_reply":"2022-08-05T14:55:19.612786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictors = [predictors[i] for i in best_subset]\nprint(predictors)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:56:35.725074Z","iopub.execute_input":"2022-08-05T14:56:35.725608Z","iopub.status.idle":"2022-08-05T14:56:35.732678Z","shell.execute_reply.started":"2022-08-05T14:56:35.725572Z","shell.execute_reply":"2022-08-05T14:56:35.731761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seed = 42\n\n# Train test split\nX_train, X_test, y_train, y_test, y_train_class, y_test_class = train_test_split(train[predictors], train[target], y_class, test_size=0.2, random_state=seed)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Classification + Regression Model","metadata":{}},{"cell_type":"code","source":"random.seed(seed)\nnp.random.seed(seed)\ntf.random.set_seed(seed)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visible = Input(shape=(X_train.shape[1],))\nlayer = Dense(256, activation='swish', kernel_initializer='he_normal')(visible)\nlayer = Dense(128, activation='swish', kernel_initializer='he_normal')(layer)\nlayer = Dense(64, activation='swish', kernel_initializer='he_normal')(layer)\nlayer = Dense(32, activation='swish', kernel_initializer='he_normal')(layer)\nlayer = Dense(16, activation='swish', kernel_initializer='he_normal')(layer)\nlayer = Dense(8, activation='swish', kernel_initializer='he_normal')(layer)\n\n# regression + classification\nout_reg = Dense(1, activation='linear')(layer)\nout_clas = Dense(2, activation='softmax')(layer)\n\nmodel = Model(inputs=visible, outputs=[out_reg, out_clas])\nmodel.compile(loss=['mse','sparse_categorical_crossentropy'], optimizer='adam')\nplot_model(model, to_file='model.png', show_shapes=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T12:56:56.215985Z","iopub.execute_input":"2022-08-05T12:56:56.216382Z","iopub.status.idle":"2022-08-05T12:56:56.454321Z","shell.execute_reply.started":"2022-08-05T12:56:56.216349Z","shell.execute_reply":"2022-08-05T12:56:56.453147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"callback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=5, restore_best_weights=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T12:56:57.285132Z","iopub.execute_input":"2022-08-05T12:56:57.286028Z","iopub.status.idle":"2022-08-05T12:56:57.293025Z","shell.execute_reply.started":"2022-08-05T12:56:57.285987Z","shell.execute_reply":"2022-08-05T12:56:57.291272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(X_train, [y_train,y_train_class], epochs=150, batch_size=32, callbacks=[callback], verbose=2)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T12:56:58.341731Z","iopub.execute_input":"2022-08-05T12:56:58.342707Z","iopub.status.idle":"2022-08-05T12:58:17.411338Z","shell.execute_reply.started":"2022-08-05T12:56:58.342656Z","shell.execute_reply":"2022-08-05T12:58:17.410013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submissions","metadata":{}},{"cell_type":"code","source":"test = test.drop('id', axis = 1)\ny_hat = model.predict(test)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T12:59:12.772733Z","iopub.execute_input":"2022-08-05T12:59:12.773161Z","iopub.status.idle":"2022-08-05T12:59:14.162631Z","shell.execute_reply.started":"2022-08-05T12:59:12.773127Z","shell.execute_reply":"2022-08-05T12:59:14.161658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Regression","metadata":{}},{"cell_type":"code","source":"sub1 = sample_submission.copy()\nsub1.failure = y_hat[0]\nsub1.to_csv('submission_nn_regression.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T13:01:17.922271Z","iopub.execute_input":"2022-08-05T13:01:17.922757Z","iopub.status.idle":"2022-08-05T13:01:17.972352Z","shell.execute_reply.started":"2022-08-05T13:01:17.922708Z","shell.execute_reply":"2022-08-05T13:01:17.971432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Classification","metadata":{}},{"cell_type":"code","source":"sub1 = sample_submission.copy()\nsub1.failure = y_hat[1][:,1]\nsub1.to_csv('submission_nn_classification.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T13:01:42.655293Z","iopub.execute_input":"2022-08-05T13:01:42.655727Z","iopub.status.idle":"2022-08-05T13:01:42.702222Z","shell.execute_reply.started":"2022-08-05T13:01:42.655676Z","shell.execute_reply":"2022-08-05T13:01:42.701312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Ensemble Regression + Classification","metadata":{}},{"cell_type":"code","source":"sub1 = sample_submission.copy()\nsub1.failure = (y_hat[0] + y_hat[1][:,1]) / 2\nsub1.to_csv('submission_nn_ensemble.csv', index = False)","metadata":{},"execution_count":null,"outputs":[]}]}