{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"#  Tensorflow Decision Forest - TPS August 2022","metadata":{}},{"cell_type":"code","source":"!pip install -qq tensorflow_decision_forests","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-08-02T19:22:51.655793Z","iopub.execute_input":"2022-08-02T19:22:51.656192Z","iopub.status.idle":"2022-08-02T19:23:56.462395Z","shell.execute_reply.started":"2022-08-02T19:22:51.656098Z","shell.execute_reply":"2022-08-02T19:23:56.460369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport os\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\n\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.preprocessing import PowerTransformer\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import mean_squared_error\n\nimport tensorflow as tf\ntf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)\n\nimport tensorflow_decision_forests as tfdf\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-08-02T19:23:56.465715Z","iopub.execute_input":"2022-08-02T19:23:56.466046Z","iopub.status.idle":"2022-08-02T19:24:01.752522Z","shell.execute_reply.started":"2022-08-02T19:23:56.466018Z","shell.execute_reply":"2022-08-02T19:24:01.751778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/tabular-playground-series-aug-2022/train.csv')\ntest = pd.read_csv('../input/tabular-playground-series-aug-2022/test.csv')\nsub = pd.read_csv('../input/tabular-playground-series-aug-2022/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-08-02T19:24:01.753714Z","iopub.execute_input":"2022-08-02T19:24:01.754972Z","iopub.status.idle":"2022-08-02T19:24:02.022886Z","shell.execute_reply.started":"2022-08-02T19:24:01.754935Z","shell.execute_reply":"2022-08-02T19:24:02.021266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe().T","metadata":{"execution":{"iopub.status.busy":"2022-08-02T19:24:02.027517Z","iopub.execute_input":"2022-08-02T19:24:02.027836Z","iopub.status.idle":"2022-08-02T19:24:02.142931Z","shell.execute_reply.started":"2022-08-02T19:24:02.027812Z","shell.execute_reply":"2022-08-02T19:24:02.142211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Engineering","metadata":{}},{"cell_type":"code","source":"train.drop(columns=['id'], inplace=True)\ntest.drop(columns=['id'], inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-02T19:24:02.144528Z","iopub.execute_input":"2022-08-02T19:24:02.145178Z","iopub.status.idle":"2022-08-02T19:24:02.158771Z","shell.execute_reply.started":"2022-08-02T19:24:02.145099Z","shell.execute_reply":"2022-08-02T19:24:02.157829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"float_columns = train.select_dtypes(include=[float]).columns\none_hot_columns = set(list(train.columns)) - set(list(float_columns)) - set(list(['failure']))","metadata":{"execution":{"iopub.status.busy":"2022-08-02T19:24:02.160291Z","iopub.execute_input":"2022-08-02T19:24:02.160682Z","iopub.status.idle":"2022-08-02T19:24:02.174453Z","shell.execute_reply.started":"2022-08-02T19:24:02.160655Z","shell.execute_reply":"2022-08-02T19:24:02.173617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoders = {}\n\nfor col in one_hot_columns:\n    enc = OneHotEncoder(handle_unknown='ignore')\n    enc.fit(train[col].values.reshape(-1, 1))\n    encoders[col] = enc","metadata":{"execution":{"iopub.status.busy":"2022-08-02T19:24:02.177041Z","iopub.execute_input":"2022-08-02T19:24:02.177733Z","iopub.status.idle":"2022-08-02T19:24:02.197048Z","shell.execute_reply.started":"2022-08-02T19:24:02.177702Z","shell.execute_reply":"2022-08-02T19:24:02.195599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_df_encoding(data, encoders):\n    \n    df = pd.DataFrame()\n\n    for col in one_hot_columns:\n        header_cols = encoders[col].get_feature_names_out([col])\n        arr = encoders[col].transform(data[col].values.reshape(-1, 1)).toarray()\n        df = pd.concat([df, pd.DataFrame(arr, columns=header_cols)], axis=1)\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-08-02T19:24:02.200175Z","iopub.execute_input":"2022-08-02T19:24:02.200554Z","iopub.status.idle":"2022-08-02T19:24:02.206547Z","shell.execute_reply.started":"2022-08-02T19:24:02.200528Z","shell.execute_reply":"2022-08-02T19:24:02.205690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = pd.concat([train[float_columns], get_df_encoding(train, encoders)], axis=1)\nlabels = train['failure']\n\ntest_prep = pd.concat([test[float_columns], get_df_encoding(test, encoders)], axis=1)\n\nnew_one_hot_columns = set(features.columns) - set(float_columns)","metadata":{"execution":{"iopub.status.busy":"2022-08-02T19:24:02.207578Z","iopub.execute_input":"2022-08-02T19:24:02.208263Z","iopub.status.idle":"2022-08-02T19:24:02.370852Z","shell.execute_reply.started":"2022-08-02T19:24:02.208236Z","shell.execute_reply":"2022-08-02T19:24:02.369735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Verify imbalance","metadata":{}},{"cell_type":"code","source":"x = [len(labels[labels==0]), len(labels[labels==1])]\nplt_labels = ['Negative', 'Positive']\n\nfig, ax = plt.subplots()\nax.pie(x, labels = plt_labels)\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-02T19:24:02.373289Z","iopub.execute_input":"2022-08-02T19:24:02.373538Z","iopub.status.idle":"2022-08-02T19:24:02.509323Z","shell.execute_reply.started":"2022-08-02T19:24:02.373516Z","shell.execute_reply":"2022-08-02T19:24:02.508479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"code","source":"def get_model(num_features=118):\n\n    raw_features = tf.keras.layers.Input(shape=(num_features,))\n\n    preprocessor = tf.keras.layers.Dense(128, activation=tf.nn.relu6)\n    preprocess_features = preprocessor(raw_features)\n\n    # Model #1: NN\n    m1_z1 = tf.keras.layers.Dense(64, activation=tf.nn.relu6)(preprocess_features)\n    m1_z1 = tf.keras.layers.Dense(32, activation=tf.nn.relu6)(m1_z1)\n    m1_z1 = tf.keras.layers.Dense(16, activation=tf.nn.relu6)(m1_z1)\n    m1_z1 = tf.keras.layers.Dense(8, activation=tf.nn.relu6)(m1_z1)\n    m1_pred = tf.keras.layers.Dense(1, activation=tf.nn.sigmoid)(m1_z1)\n\n    # Model #2: NN\n    m2_z1 = tf.keras.layers.Dense(64, activation=tf.nn.relu6)(preprocess_features)\n    m2_z1 = tf.keras.layers.Dense(32, activation=tf.nn.relu6)(m2_z1)\n    m2_z1 = tf.keras.layers.Dense(16, activation=tf.nn.relu6)(m2_z1)\n    m2_z1 = tf.keras.layers.Dense(8, activation=tf.nn.relu6)(m2_z1)\n    m2_pred = tf.keras.layers.Dense(1, activation=tf.nn.sigmoid)(m2_z1)\n\n    # Model #3: DF\n    model_3 = tfdf.keras.RandomForestModel(num_trees=1000, random_seed=1234)\n    m3_pred = model_3(preprocess_features)\n\n    # Model #4: DF\n    model_4 = tfdf.keras.RandomForestModel(\n        num_trees=1000,\n        #split_axis=\"SPARSE_OBLIQUE\", # Uncomment this line to increase the quality of this model\n        random_seed=4567)\n    m4_pred = model_4(preprocess_features)\n\n    mean_nn_only = tf.reduce_mean(tf.stack([m1_pred, m2_pred], axis=0), axis=0)\n    mean_nn_and_df = tf.reduce_mean(\n        tf.stack([m1_pred, m2_pred, m3_pred, m4_pred], axis=0), axis=0)\n    \n    ensemble_nn_only = tf.keras.models.Model(raw_features, mean_nn_only)\n    ensemble_nn_and_df = tf.keras.models.Model(raw_features, mean_nn_and_df)\n\n    return ensemble_nn_only, ensemble_nn_and_df","metadata":{"execution":{"iopub.status.busy":"2022-08-02T19:24:02.510550Z","iopub.execute_input":"2022-08-02T19:24:02.511292Z","iopub.status.idle":"2022-08-02T19:24:02.525819Z","shell.execute_reply.started":"2022-08-02T19:24:02.511258Z","shell.execute_reply":"2022-08-02T19:24:02.524894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"n_splits = 5\nskf = StratifiedKFold(n_splits=n_splits)\noof_preds = np.zeros((n_splits, len(test), 1))\n\nlgb_best_params = []\nxgb_best_params = []\n\nfor i, (train_index, test_index) in enumerate(skf.split(features, labels)):\n    X_train, X_test = features.iloc[train_index], features.iloc[test_index]\n    y_train, y_test = labels.iloc[train_index], labels.iloc[test_index]\n    \n    imp = SimpleImputer(missing_values=np.nan, strategy='mean')\n    X_train = imp.fit_transform(X_train)\n    test_df = imp.transform(test_prep)\n    \n    X_train = pd.DataFrame(X_train, columns=features.columns)\n    test_df = pd.DataFrame(test_df, columns=features.columns)\n    \n    scaler = PowerTransformer().fit(X_train[float_columns])\n    temp_train_float = pd.DataFrame(scaler.transform(X_train[float_columns]), columns=float_columns)\n    temp_valid_float = pd.DataFrame(scaler.transform(X_test[float_columns]), columns=float_columns)\n    temp_test_float = pd.DataFrame(scaler.transform(test_df[float_columns]), columns=float_columns)\n    \n    train_df = np.hstack((X_train[new_one_hot_columns].values, temp_train_float.values))\n    valid_df = np.hstack((X_test[new_one_hot_columns].values, temp_valid_float.values))\n    test_df = np.hstack((test_df[new_one_hot_columns].values, temp_test_float.values))\n    \n    X, y = train_df, y_train.values\n    X_v, y_v = valid_df, y_test.values\n    \n    ensemble_nn_only, ensemble_nn_and_df = get_model()\n    \n    ensemble_nn_only.compile(\n        optimizer=tf.keras.optimizers.Adam(),\n        loss=tf.keras.losses.BinaryCrossentropy(),\n        metrics=[\"accuracy\"])\n\n    ensemble_nn_only.fit(X, y, epochs=20, validation_data=(X_v, y_v), verbose=False)\n\n    evaluation_nn_only = ensemble_nn_only.evaluate(X_v, y_v, return_dict=True)\n    print(\"Accuracy (NN #1 and #2 only): \", evaluation_nn_only[\"accuracy\"])\n\n    ensemble_nn_and_df.compile(\n        loss=tf.keras.losses.BinaryCrossentropy(), metrics=[\"accuracy\"])\n\n    ensemble_nn_and_df.fit(X, y, epochs=20, validation_data=(X_v, y_v), verbose=False)\n    evaluation_nn_and_df = ensemble_nn_and_df.evaluate(\n        X_v, y_v, return_dict=True)\n    print(\"Accuracy (2xNN and 2xDF): \", evaluation_nn_and_df[\"accuracy\"])\n    \n    oof_preds[i] = (ensemble_nn_only(test_df) + ensemble_nn_and_df(test_df)) / 2","metadata":{"execution":{"iopub.status.busy":"2022-08-02T19:24:02.527295Z","iopub.execute_input":"2022-08-02T19:24:02.527882Z","iopub.status.idle":"2022-08-02T19:25:05.203779Z","shell.execute_reply.started":"2022-08-02T19:24:02.527834Z","shell.execute_reply":"2022-08-02T19:25:05.202695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"sub['failure'] = np.mean(oof_preds, axis=0)\nsub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-02T19:25:05.205010Z","iopub.execute_input":"2022-08-02T19:25:05.205269Z","iopub.status.idle":"2022-08-02T19:25:05.258317Z","shell.execute_reply.started":"2022-08-02T19:25:05.205247Z","shell.execute_reply":"2022-08-02T19:25:05.256858Z"},"trusted":true},"execution_count":null,"outputs":[]}]}