{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<img src=\"https://memegenerator.net/img/instances/63895047.jpg\" />","metadata":{}},{"cell_type":"markdown","source":"## References\n\nmaxsarmento (https://www.kaggle.com/code/maxsarmento/lb-0-58978-standing-on-the-shoulder-of-giants/notebook)\n\ndesalegngeb (https://www.kaggle.com/code/desalegngeb/tps08-logisticregression-and-some-fe/notebook?scriptVersionId=102691691)","metadata":{}},{"cell_type":"code","source":"import os\nimport sys\nimport joblib\n\nimport numpy as np \nimport pandas as pd\n\nfrom sklearn.preprocessing import KBinsDiscretizer\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import roc_auc_score, log_loss\n\n!pip install -qq feature_engine\nfrom feature_engine.encoding import WoEEncoder\n\n!git clone --quiet https://github.com/analokmaus/kuma_utils.git\nsys.path.append(\"kuma_utils/\")\nfrom kuma_utils.preprocessing.imputer import LGBMImputer\nfrom sklearn.preprocessing import PowerTransformer\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n!pip install -qq autokeras\n\nimport autokeras as ak\nimport tensorflow as tf\n\nimport gc\ngc.enable()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-08-10T03:58:44.187077Z","iopub.execute_input":"2022-08-10T03:58:44.187793Z","iopub.status.idle":"2022-08-10T04:00:26.204004Z","shell.execute_reply.started":"2022-08-10T03:58:44.187582Z","shell.execute_reply":"2022-08-10T04:00:26.202887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Engineering","metadata":{}},{"cell_type":"code","source":"def missing_cols(train, test):\n    missing_cols = set(test.columns) - set(train.columns)\n    for c in missing_cols:\n        train[c] = np.zeros(len(train))\n    return train[test.columns], test\n\ndef concat_one_hot(X, features):\n    remaining_features_cols = set(X[features].columns) - set(X[features].select_dtypes([int, object]).columns)\n    X_one_hot = pd.get_dummies(X[features].select_dtypes([int, object]).astype(str))\n    return pd.concat([X[remaining_features_cols], X_one_hot], axis=1)\n\ndef preprocessing(df_train, df_test):\n     \n    # Missing indicator\n    for df in [df_train, df_test]:\n        df[\"m_3_missing\"] = df[\"measurement_3\"].isnull().astype(int)\n        df[\"m_5_missing\"] = df[\"measurement_5\"].isnull().astype(int)\n\n        \n    # Missing value imputation\n    imptr = LGBMImputer(n_iter=50)\n    def impute_nans(df_train, df_test):\n        nan_features = [col for col in df_train.columns if df_train[col].isnull().any()]\n        for pc in df_train[\"product_code\"].unique():\n            df_train.loc[df_train[\"product_code\"]==pc, nan_features] = imptr.fit_transform(df_train.loc[df_train[\"product_code\"]==pc, nan_features])\n        \n        nan_features = [col for col in df_test.columns if df_test[col].isnull().any()]\n        for pc in df_test[\"product_code\"].unique():\n            df_test.loc[df_test[\"product_code\"]==pc, nan_features] = imptr.fit_transform(df_test.loc[df_test[\"product_code\"]==pc, nan_features])\n\n        return df_train, df_test\n    \n    df_train, df_test = impute_nans(df_train, df_test)\n    \n    \n    # Area\n    for df in [df_train, df_test]:\n        df[\"attribute_2*3\"] = df[\"attribute_2\"] * df[\"attribute_3\"]\n    \n    \n    # Aggregations\n    meas_gr1_cols = [f\"measurement_{i:d}\" for i in list(range(3, 5)) + list(range(9, 17))]\n    meas_gr2_cols = [f\"measurement_{i:d}\" for i in list(range(5, 9))]\n    \n    for df in [df_train, df_test]:\n        df[\"meas_gr1_avg\"] = np.mean(df[meas_gr1_cols], axis=1)\n        df[\"meas_gr1_std\"] = np.std(df[meas_gr1_cols], axis=1)\n        df[\"meas_gr2_avg\"] = np.mean(df[meas_gr2_cols], axis=1)\n    \n    for df in [df_train, df_test]:\n        df[\"meas17/meas_gr2_avg\"] = df[\"measurement_17\"] / df[\"meas_gr2_avg\"]\n    \n    \n    # WoE encoding\n    woe_encoder = WoEEncoder(variables=[\"attribute_0\"])\n    df_train[\"attribute_0\"] = woe_encoder.fit_transform(df_train[\"attribute_0\"].to_frame(), df_train[\"failure\"])\n    df_test[\"attribute_0\"] = woe_encoder.transform(df_test[\"attribute_0\"].to_frame())\n    \n    \n    features = [\"attribute_0\", \"measurement_0\", \"measurement_1\", \"measurement_2\", \"m_3_missing\", \"m_5_missing\",\n               \"meas_gr1_avg\", \"meas_gr1_std\", \"attribute_2*3\", \"loading\", \"measurement_17\", \"meas17/meas_gr2_avg\"] \\\n\n    \n    labels = df_train['failure']\n    product_codes_train = df_train['product_code']\n    product_codes_test = df_test['product_code']\n    \n    df_train = concat_one_hot(df_train, features)\n    df_test = concat_one_hot(df_test, features)\n    \n    df_train, df_test = missing_cols(df_train, df_test)\n    \n    df_train['product_code'] = product_codes_train\n    \n    return df_train, df_test, labels","metadata":{"execution":{"iopub.status.busy":"2022-08-10T04:00:26.206623Z","iopub.execute_input":"2022-08-10T04:00:26.207383Z","iopub.status.idle":"2022-08-10T04:00:26.226081Z","shell.execute_reply.started":"2022-08-10T04:00:26.207344Z","shell.execute_reply":"2022-08-10T04:00:26.225042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/tabular-playground-series-aug-2022/train.csv')\ntest = pd.read_csv('../input/tabular-playground-series-aug-2022/test.csv')\nsub = pd.read_csv('../input/tabular-playground-series-aug-2022/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-08-10T04:00:26.230906Z","iopub.execute_input":"2022-08-10T04:00:26.231580Z","iopub.status.idle":"2022-08-10T04:00:26.474555Z","shell.execute_reply.started":"2022-08-10T04:00:26.231537Z","shell.execute_reply":"2022-08-10T04:00:26.473503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train, test, labels = preprocessing(train, test)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T04:00:26.477010Z","iopub.execute_input":"2022-08-10T04:00:26.477383Z","iopub.status.idle":"2022-08-10T04:00:58.393392Z","shell.execute_reply.started":"2022-08-10T04:00:26.477344Z","shell.execute_reply":"2022-08-10T04:00:58.392396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"product_codes = ['A','B','C','D','E']\noof_preds = np.zeros((5, len(test)))\n\nfor i,code in enumerate(product_codes):\n    \n    oof_temp_preds = np.zeros((3, len(test)))\n    skf = StratifiedKFold(n_splits=3, shuffle=True, random_state=i)\n    \n    X = train[train['product_code'] == code].drop(columns=['product_code'])\n    y = labels.iloc[train[train['product_code'] == code].index]\n    \n    X.reset_index(inplace=True, drop=True)\n    y = pd.DataFrame(y).reset_index(drop=True)\n    \n    higher_cols = X[X.columns[~X[X > 1.].isna().all()]].columns\n    rest_cols = set(X.columns) - set(higher_cols)\n    all_df = pd.concat([X[higher_cols], test[higher_cols]], axis=0)\n    \n    scaler = PowerTransformer()\n    scaler.fit(all_df)\n    \n    X_higher_cols = scaler.transform(X[higher_cols])\n    test_power = scaler.transform(test[higher_cols])\n    \n    X = pd.DataFrame(np.hstack([X_higher_cols, X[rest_cols]]), columns=set(higher_cols).union(rest_cols))\n    test_power = pd.DataFrame(np.hstack([test_power, test[rest_cols]]), columns=set(higher_cols).union(rest_cols))\n\n    for i,(train_index, test_index) in enumerate(skf.split(X, y)):\n        X_train, X_test = X.iloc[train_index], X.iloc[test_index]\n        y_train, y_test = y.iloc[train_index], y.iloc[test_index]\n\n        reg = ak.StructuredDataRegressor(max_trials=10, overwrite=True)\n        reg.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=100)\n\n        oof_temp_preds[i] = reg.predict(test_power).reshape(-1)\n        \n    oof_preds[i] = np.mean(oof_temp_preds, axis=0)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T04:03:50.597574Z","iopub.execute_input":"2022-08-10T04:03:50.598009Z","iopub.status.idle":"2022-08-10T04:06:45.731618Z","shell.execute_reply.started":"2022-08-10T04:03:50.597974Z","shell.execute_reply":"2022-08-10T04:06:45.729818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"sub['failure'] = np.mean(oof_preds, axis=0)\nsub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T04:02:01.772445Z","iopub.status.idle":"2022-08-10T04:02:01.772932Z","shell.execute_reply.started":"2022-08-10T04:02:01.772675Z","shell.execute_reply":"2022-08-10T04:02:01.772701Z"},"trusted":true},"execution_count":null,"outputs":[]}]}