{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import copy\nimport pathlib\nimport random\nfrom typing import Dict, List, Tuple, Union\n\nfrom IPython.display import display\nimport lightgbm as lgbm\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler\n\nnp.random.seed(0)\n\npd.set_option(\"display.max_columns\", 50)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T11:51:24.716335Z","iopub.execute_input":"2022-07-17T11:51:24.717265Z","iopub.status.idle":"2022-07-17T11:51:26.226166Z","shell.execute_reply.started":"2022-07-17T11:51:24.717135Z","shell.execute_reply":"2022-07-17T11:51:26.225083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dir_path = pathlib.Path(\"../input/spaceship-titanic\")\n\ndf_train = pd.read_csv(dir_path.joinpath(\"train.csv\"))\ndf_test = pd.read_csv(dir_path.joinpath(\"test.csv\"))\n\ndisplay(df_train.head(20))\ndisplay(df_train.isna().sum())\ndisplay(df_train.dtypes)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T11:51:26.228281Z","iopub.execute_input":"2022-07-17T11:51:26.228738Z","iopub.status.idle":"2022-07-17T11:51:26.362112Z","shell.execute_reply.started":"2022-07-17T11:51:26.228694Z","shell.execute_reply":"2022-07-17T11:51:26.361017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def split_data(\n    df_: pd.DataFrame,\n    features: List[str],\n    label_encoders: Dict[str, object],\n    train_label_encoders: bool = True\n) -> Union[\n    Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame],\n    Tuple[pd.DataFrame, pd.DataFrame]\n]:\n    x = df_[features].copy()\n    id_ = df_[\"PassengerId\"].copy()\n    \"\"\"\n    if \"Age\" in features:\n        age = x[\"Age\"].copy()\n        alpha1 = (age - age.mean())**2\n        alpha2 = (age - age.mean())**3\n        alpha1.rename(\"alpha1\", inplace=True)\n        alpha2.rename(\"alpha2\", inplace=True)\n        x = pd.concat([x, alpha1, alpha2], axis=1)\n    \"\"\"\n    if \"HomePlanet\" in features:\n        if train_label_encoders:\n            label_encoders[\"HomePlanet\"].fit(x[\"HomePlanet\"].copy())\n        hp = label_encoders[\"HomePlanet\"].transform(x.pop(\"HomePlanet\"))\n        x = pd.concat([x, pd.Series(hp, name=\"HomePlanet\")], axis=1)\n        \n    if \"CryoSleep\" in features:\n        #x = x.assign(CryoSleep=x[\"CryoSleep\"].copy().fillna(-1).astype(int))\n        if train_label_encoders:\n            label_encoders[\"CryoSleep\"].fit(x[\"CryoSleep\"].copy())\n        cs = label_encoders[\"CryoSleep\"].transform(x.pop(\"CryoSleep\"))\n        x = pd.concat([x, pd.Series(cs, name=\"CryoSleep\")], axis=1)\n    \n    if \"Cabin\" in features:\n        cabin = x.pop(\"Cabin\").str.split(\"/\", expand=True)\n        cabin.columns = [\"deck\", \"num\", \"side\"]\n        if train_label_encoders:\n            label_encoders[\"deck\"].fit(cabin[\"deck\"].copy())\n            label_encoders[\"side\"].fit(cabin[\"side\"].copy())\n        deck = label_encoders[\"deck\"].transform(cabin.pop(\"deck\"))\n        side = label_encoders[\"side\"].transform(cabin.pop(\"side\"))\n        x = pd.concat([x, pd.Series(deck, name=\"deck\"), cabin[\"num\"].astype(float),\n                       pd.Series(side, name=\"side\")], axis=1)\n    \n    if \"Destination\" in features:\n        if train_label_encoders:\n            label_encoders[\"Destination\"].fit(x[\"Destination\"].copy())\n        dst = label_encoders[\"Destination\"].transform(x.pop(\"Destination\"))\n        x = pd.concat([x, pd.Series(dst, name=\"Destination\")], axis=1)\n        \n    if \"VIP\" in features:\n        #x = x.assign(VIP=x[\"VIP\"].copy().fillna(-1).astype(int))\n        if train_label_encoders:\n            label_encoders[\"VIP\"].fit(x[\"VIP\"].copy())\n        vip = label_encoders[\"VIP\"].transform(x.pop(\"VIP\"))\n        x = pd.concat([x, pd.Series(vip, name=\"VIP\")], axis=1)\n        \n    if \"PassengerId\" in features:\n        pid = x.pop(\"PassengerId\").str.split(\"_\", expand=True)\n        pid.columns = [\"gggg\", \"pp\"]\n        \"\"\"\n        if train_label_encoders:\n            label_encoders[\"pp\"].fit(pid[\"pp\"].copy())\n        pp = pd.Series(label_encoders[\"pp\"].transform(pid[\"pp\"]), name=\"pp\")\n        \"\"\"\n        x = pd.concat([x, pid[\"gggg\"]], axis=1)\n        \n        def _fillna(xx):\n            num_na = xx.isna().sum().sum()\n            if num_na == 0:\n                return xx\n            elif num_na != len(xx):\n                xx.fillna(xx.median(), inplace=True)\n            elif num_na == len(xx):\n                xx.fillna(0, inplace=True)\n            return xx\n            \n        x = x.groupby(\"gggg\").transform(lambda xx: _fillna(xx))\n    \n    x.fillna(x.median(), inplace=True)\n    \n    if \"Transported\" in df_.columns:\n        y = df_[\"Transported\"].astype(int)\n        return x, id_, y\n    \n    return x, id_","metadata":{"execution":{"iopub.status.busy":"2022-07-17T11:51:26.363671Z","iopub.execute_input":"2022-07-17T11:51:26.364484Z","iopub.status.idle":"2022-07-17T11:51:26.384552Z","shell.execute_reply.started":"2022-07-17T11:51:26.364450Z","shell.execute_reply":"2022-07-17T11:51:26.383143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = [\n    \"PassengerId\",\n    \"HomePlanet\",\n    \"CryoSleep\",\n    \"Cabin\",\n    \"Destination\",\n    \"Age\",\n    \"VIP\",\n    \"RoomService\",\n    \"FoodCourt\",\n    \"ShoppingMall\",\n    \"Spa\",\n    \"VRDeck\",\n]\n\nlabel_encoders = {\n    \"CryoSleep\": LabelEncoder(),\n    \"VIP\": LabelEncoder(),\n    \"HomePlanet\": LabelEncoder(),\n    \"Destination\": LabelEncoder(),\n    \"deck\": LabelEncoder(),\n    \"side\": LabelEncoder(),\n    #\"pp\": LabelEncoder(),\n}   \n\nx_train, id_train, y_train = split_data(df_train, features, label_encoders, True)\nx_test, id_test = split_data(df_test, features, label_encoders, False)\n\nprint(x_train.shape, id_train.shape, y_train.shape)\nprint(x_test.shape, id_test.shape)\n\ndisplay(x_train.head(20))\ndisplay(x_train.isna().sum())\ndisplay(x_train.dtypes)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T11:51:26.566258Z","iopub.execute_input":"2022-07-17T11:51:26.566623Z","iopub.status.idle":"2022-07-17T11:52:12.206038Z","shell.execute_reply.started":"2022-07-17T11:51:26.566592Z","shell.execute_reply":"2022-07-17T11:52:12.205020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"categorical_indices = []\ncategorical_dims = []\nfor i, f in enumerate(x_train.columns):\n    if f in label_encoders.keys():\n        categorical_indices.append(i)\n        categorical_dims.append(len(label_encoders[f].classes_))\nprint(categorical_indices)\nprint(categorical_dims)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T11:52:12.208151Z","iopub.execute_input":"2022-07-17T11:52:12.209165Z","iopub.status.idle":"2022-07-17T11:52:12.216159Z","shell.execute_reply.started":"2022-07-17T11:52:12.209129Z","shell.execute_reply":"2022-07-17T11:52:12.215068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_data(\n    x: Union[pd.DataFrame, np.ndarray],\n    y: Union[pd.DataFrame, np.ndarray],\n    use_y: bool = True\n) -> None:\n    \n    nrows = x.shape[1] // 3 if x.shape[1] % 3 == 0 else x.shape[1] // 3 + 1\n    _, axs = plt.subplots(nrows, 3, figsize=(16, 4*nrows))\n    axs = axs.reshape(-1, 3)\n    if isinstance(x, pd.DataFrame):\n        is_df = True\n    elif isinstance(x, np.ndarray):\n        is_df = False\n    for i in range(x.shape[1]):\n        data = x.iloc[:, i] if is_df else x[:, i]\n        if use_y == False:\n            axs[i//3, i%3].hist(data, bins=50, histtype=\"step\")\n        else:\n            _, bins, _ = axs[i//3, i%3].hist(data[y==0], bins=50, histtype=\"step\")\n            axs[i//3, i%3].hist(data[y==1], bins=bins, histtype=\"step\")\n        if is_df:\n            axs[i//3, i%3].set_xlabel(data.name)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-17T11:52:12.217380Z","iopub.execute_input":"2022-07-17T11:52:12.217788Z","iopub.status.idle":"2022-07-17T11:52:12.230992Z","shell.execute_reply.started":"2022-07-17T11:52:12.217755Z","shell.execute_reply":"2022-07-17T11:52:12.230063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_data(x_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T11:52:12.234660Z","iopub.execute_input":"2022-07-17T11:52:12.235190Z","iopub.status.idle":"2022-07-17T11:52:13.858679Z","shell.execute_reply.started":"2022-07-17T11:52:12.235150Z","shell.execute_reply":"2022-07-17T11:52:13.857786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def take_ratio(df_: pd.DataFrame, ratio_features: List[str]) -> pd.DataFrame:\n    ratio = df_[ratio_features].copy()\n    ratio = ratio.div(ratio.sum(axis=1), axis=0)\n    ratio.fillna(ratio.median(), inplace=True)\n    ratio.rename(columns={k: f\"ratio of \"+str(k) for k in ratio_features}, inplace=True)\n    return pd.concat([df_, ratio], axis=1)\n\n\ndef take_logarithm(df_: pd.DataFrame, log_features: List[str]) -> None:\n    df_[log_features] = df_[log_features] + 1\n    df_[log_features] = df_[log_features].apply(np.log)\n    df_.rename(columns={k: f\"log(1+\"+str(k)+\")\" for k in log_features}, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T11:52:13.859785Z","iopub.execute_input":"2022-07-17T11:52:13.860586Z","iopub.status.idle":"2022-07-17T11:52:13.869098Z","shell.execute_reply.started":"2022-07-17T11:52:13.860549Z","shell.execute_reply":"2022-07-17T11:52:13.867823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ratio_features = [\n    \"RoomService\",\n    \"FoodCourt\",\n    \"ShoppingMall\",\n    \"Spa\",\n    \"VRDeck\",\n]\n\nlog_features = [\n    \"RoomService\",\n    \"FoodCourt\",\n    \"ShoppingMall\",\n    \"Spa\",\n    \"VRDeck\",\n]\n\nx_train = take_ratio(x_train, ratio_features)\nx_test = take_ratio(x_test, ratio_features)\n\ntake_logarithm(x_train, log_features)\ntake_logarithm(x_test, log_features)\n\nprint(x_train.shape, id_train.shape, y_train.shape)\nprint(x_test.shape, id_test.shape)\n\ndisplay(x_train.head(20))","metadata":{"execution":{"iopub.status.busy":"2022-07-17T11:52:13.870470Z","iopub.execute_input":"2022-07-17T11:52:13.870856Z","iopub.status.idle":"2022-07-17T11:52:13.931095Z","shell.execute_reply.started":"2022-07-17T11:52:13.870822Z","shell.execute_reply":"2022-07-17T11:52:13.929749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(x_train.isna().sum())","metadata":{"execution":{"iopub.status.busy":"2022-07-17T11:52:13.932876Z","iopub.execute_input":"2022-07-17T11:52:13.933301Z","iopub.status.idle":"2022-07-17T11:52:13.944506Z","shell.execute_reply.started":"2022-07-17T11:52:13.933260Z","shell.execute_reply":"2022-07-17T11:52:13.943740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for u, c in zip(*np.unique(y_train.values, return_counts=True)):\n    print(f\"{u}: {c}\")","metadata":{"execution":{"iopub.status.busy":"2022-07-17T11:52:13.945930Z","iopub.execute_input":"2022-07-17T11:52:13.946246Z","iopub.status.idle":"2022-07-17T11:52:13.958896Z","shell.execute_reply.started":"2022-07-17T11:52:13.946215Z","shell.execute_reply":"2022-07-17T11:52:13.957770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_data(x_train, y_train)\nplot_data(x_test, y_train, False)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T11:52:13.960060Z","iopub.execute_input":"2022-07-17T11:52:13.960610Z","iopub.status.idle":"2022-07-17T11:52:17.971366Z","shell.execute_reply.started":"2022-07-17T11:52:13.960577Z","shell.execute_reply":"2022-07-17T11:52:17.970160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train LightGBM Models","metadata":{}},{"cell_type":"code","source":"def train_lgbm_fold(\n    x: np.ndarray,\n    y: np.ndarray,\n    kfold_params: Dict[str, object],\n    lgbm_params: Dict[str, object],\n    categorical_index: List[int],\n    callbacks: List[object]=None,\n) -> object:\n    \n    kfold = StratifiedKFold(**kfold_params)\n    models = []\n    standardize_index = [i for i in range(x.shape[1]) if i not in categorical_index]\n    \n    for k, (indices_train, indices_valid) in enumerate(kfold.split(x, y)):\n        print(f\"fold: {k+1} / {kfold.n_splits}\")\n        print(len(indices_train), len(indices_valid))\n        model = lgbm.LGBMClassifier(**lgbm_params)\n        scaler = StandardScaler()\n        \n        xx_train = x[indices_train]\n        yy_train = y[indices_train]\n        xx_train = standardize(scaler, xx_train, categorical_index, True)\n        \n        xx_valid = x[indices_valid]\n        yy_valid = y[indices_valid]\n        xx_valid = standardize(scaler, xx_valid, categorical_index, False)\n        \n        model.fit(\n            X=xx_train,\n            y=yy_train,\n            eval_set=[(xx_train, yy_train), (xx_valid, yy_valid)],\n            eval_metric=[\"logloss\", \"auc\"],\n            categorical_feature=categorical_index,\n            callbacks=callbacks\n        )\n        models.append((scaler, model))\n    return models","metadata":{"execution":{"iopub.status.busy":"2022-07-17T11:52:17.974297Z","iopub.execute_input":"2022-07-17T11:52:17.974882Z","iopub.status.idle":"2022-07-17T11:52:17.985040Z","shell.execute_reply.started":"2022-07-17T11:52:17.974848Z","shell.execute_reply":"2022-07-17T11:52:17.984236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def standardize(\n    scaler: object,\n    x: np.ndarray,\n    categorical_index: List[int],\n    train_scaler: bool = True\n) -> np.ndarray:\n    \n    standardize_index = [i for i in range(x.shape[1]) if i not in categorical_index]\n    if train_scaler:\n        scaler.fit(x[:, standardize_index])\n    standardized_x = scaler.transform(x[:, standardize_index])\n    x = np.concatenate([standardized_x, x[:, categorical_index]], axis=-1)\n    return x[:, np.argsort(standardize_index+categorical_index)]","metadata":{"execution":{"iopub.status.busy":"2022-07-17T11:52:17.986416Z","iopub.execute_input":"2022-07-17T11:52:17.986962Z","iopub.status.idle":"2022-07-17T11:52:18.004354Z","shell.execute_reply.started":"2022-07-17T11:52:17.986915Z","shell.execute_reply":"2022-07-17T11:52:18.003182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kfold_params = {\n    \"n_splits\": 5,\n    \"shuffle\": True,\n    \"random_state\": 0\n}\n\ngbdt_params = {\n    \"boosting_type\": \"gbdt\",\n    \"learning_rate\": 0.1,\n    \"n_estimators\": 100,\n    \"objective\" : \"binary\",\n    \"num_leaves\": 50,\n    \"min_data_in_leaf\": 10,\n    \"min_sum_hessian_in_leaf\": 1e-01,\n    \"max_depth\": 5,\n    \"reg_lambda\": 0.1,\n    \"colsample_bytree\": 0.9,\n}\n\ncallbacks = [\n    lgbm.early_stopping(20),\n]\n\ngbdt_models = train_lgbm_fold(\n    x_train.values,\n    y_train.values,\n    kfold_params,\n    gbdt_params,\n    categorical_indices,\n    callbacks\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T11:52:18.006368Z","iopub.execute_input":"2022-07-17T11:52:18.006878Z","iopub.status.idle":"2022-07-17T11:52:19.166841Z","shell.execute_reply.started":"2022-07-17T11:52:18.006831Z","shell.execute_reply":"2022-07-17T11:52:19.165751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_, axs = plt.subplots(len(gbdt_models), 2, figsize=(14, 18))\nfor i, (_, model) in enumerate(gbdt_models):\n    lgbm.plot_metric(model, metric=\"binary_logloss\", ax=axs[i, 0], title=\"binary_logloss\")\n    lgbm.plot_metric(model, metric=\"auc\", ax=axs[i, 1], title=\"auc\")\nplt.subplots_adjust(hspace=0.5)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-17T11:52:19.169261Z","iopub.execute_input":"2022-07-17T11:52:19.170446Z","iopub.status.idle":"2022-07-17T11:52:20.366735Z","shell.execute_reply.started":"2022-07-17T11:52:19.170405Z","shell.execute_reply":"2022-07-17T11:52:20.365742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def inference_lgbm_models(models, x, categorical_indices):\n    if isinstance(x, pd.DataFrame):\n        x = x.values\n    probas = []\n    for scaler, model in models:\n        xx = standardize(scaler, x, categorical_indices, False)\n        probas.append(model.predict_proba(xx))\n    probas = np.stack(probas, axis=-1)\n    probas = np.mean(probas, axis=-1)\n    return probas","metadata":{"execution":{"iopub.status.busy":"2022-07-17T11:52:20.368140Z","iopub.execute_input":"2022-07-17T11:52:20.368478Z","iopub.status.idle":"2022-07-17T11:52:20.376095Z","shell.execute_reply.started":"2022-07-17T11:52:20.368447Z","shell.execute_reply":"2022-07-17T11:52:20.374902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_lgbm = inference_lgbm_models(gbdt_models, x_train, categorical_indices)\npred_lgbm = np.argmax(pred_lgbm, axis=-1) \nprint(f\"{sum(pred_lgbm==y_train) / len(y_train):.4f}\")","metadata":{"execution":{"iopub.status.busy":"2022-07-17T11:52:20.377545Z","iopub.execute_input":"2022-07-17T11:52:20.377937Z","iopub.status.idle":"2022-07-17T11:52:20.491818Z","shell.execute_reply.started":"2022-07-17T11:52:20.377906Z","shell.execute_reply":"2022-07-17T11:52:20.490732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_lgbm = inference_lgbm_models(gbdt_models, x_test, categorical_indices)\npred = np.argmax(pred_lgbm, axis=-1)\n\nsubmission = pd.DataFrame({\n    \"PassengerId\": id_test,\n    \"Transported\": pred.astype(bool),\n})\n\ndisplay(submission.head(20))","metadata":{"execution":{"iopub.status.busy":"2022-07-17T11:52:20.493300Z","iopub.execute_input":"2022-07-17T11:52:20.493879Z","iopub.status.idle":"2022-07-17T11:52:20.557618Z","shell.execute_reply.started":"2022-07-17T11:52:20.493846Z","shell.execute_reply":"2022-07-17T11:52:20.556757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T11:52:20.559170Z","iopub.execute_input":"2022-07-17T11:52:20.559474Z","iopub.status.idle":"2022-07-17T11:52:20.573063Z","shell.execute_reply.started":"2022-07-17T11:52:20.559446Z","shell.execute_reply":"2022-07-17T11:52:20.571739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}