{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import copy\nimport pathlib\nimport random\nfrom typing import Dict, List, Tuple, Union\n\nfrom IPython.display import display\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler\nimport torch\nfrom torch import nn\nfrom torch.utils.data import TensorDataset, DataLoader\nfrom torch.utils.data.dataset import Subset\n\nnp.random.seed(0)\ntorch.manual_seed(0)\n\npd.set_option(\"display.max_columns\", 50)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T12:10:01.655811Z","iopub.execute_input":"2022-07-17T12:10:01.656262Z","iopub.status.idle":"2022-07-17T12:10:01.667533Z","shell.execute_reply.started":"2022-07-17T12:10:01.656226Z","shell.execute_reply":"2022-07-17T12:10:01.665899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dir_path = pathlib.Path(\"../input/spaceship-titanic\")\n\ndf_train = pd.read_csv(dir_path.joinpath(\"train.csv\"))\ndf_test = pd.read_csv(dir_path.joinpath(\"test.csv\"))\n\ndisplay(df_train.head(20))\ndisplay(df_train.isna().sum())\ndisplay(df_train.dtypes)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T12:10:01.669933Z","iopub.execute_input":"2022-07-17T12:10:01.670313Z","iopub.status.idle":"2022-07-17T12:10:01.784532Z","shell.execute_reply.started":"2022-07-17T12:10:01.670279Z","shell.execute_reply":"2022-07-17T12:10:01.783365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def split_data(\n    df_: pd.DataFrame,\n    features: List[str],\n    label_encoders: Dict[str, object],\n    train_label_encoders: bool = True\n) -> Union[\n    Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame],\n    Tuple[pd.DataFrame, pd.DataFrame]\n]:\n    x = df_[features].copy()\n    id_ = df_[\"PassengerId\"].copy()\n    \"\"\"\n    if \"Age\" in features:\n        age = x[\"Age\"].copy()\n        alpha1 = (age - age.mean())**2\n        alpha2 = (age - age.mean())**3\n        alpha1.rename(\"alpha1\", inplace=True)\n        alpha2.rename(\"alpha2\", inplace=True)\n        x = pd.concat([x, alpha1, alpha2], axis=1)\n    \"\"\"\n    if \"HomePlanet\" in features:\n        if train_label_encoders:\n            label_encoders[\"HomePlanet\"].fit(x[\"HomePlanet\"].copy())\n        hp = label_encoders[\"HomePlanet\"].transform(x.pop(\"HomePlanet\"))\n        x = pd.concat([x, pd.Series(hp, name=\"HomePlanet\")], axis=1)\n        \n    if \"CryoSleep\" in features:\n        #x = x.assign(CryoSleep=x[\"CryoSleep\"].copy().fillna(-1).astype(int))\n        if train_label_encoders:\n            label_encoders[\"CryoSleep\"].fit(x[\"CryoSleep\"].copy())\n        cs = label_encoders[\"CryoSleep\"].transform(x.pop(\"CryoSleep\"))\n        x = pd.concat([x, pd.Series(cs, name=\"CryoSleep\")], axis=1)\n    \n    if \"Cabin\" in features:\n        cabin = x.pop(\"Cabin\").str.split(\"/\", expand=True)\n        cabin.columns = [\"deck\", \"num\", \"side\"]\n        if train_label_encoders:\n            label_encoders[\"deck\"].fit(cabin[\"deck\"].copy())\n            label_encoders[\"side\"].fit(cabin[\"side\"].copy())\n        deck = label_encoders[\"deck\"].transform(cabin.pop(\"deck\"))\n        side = label_encoders[\"side\"].transform(cabin.pop(\"side\"))\n        x = pd.concat([x, pd.Series(deck, name=\"deck\"), cabin[\"num\"].astype(float),\n                       pd.Series(side, name=\"side\")], axis=1)\n    \n    if \"Destination\" in features:\n        if train_label_encoders:\n            label_encoders[\"Destination\"].fit(x[\"Destination\"].copy())\n        dst = label_encoders[\"Destination\"].transform(x.pop(\"Destination\"))\n        x = pd.concat([x, pd.Series(dst, name=\"Destination\")], axis=1)\n        \n    if \"VIP\" in features:\n        #x = x.assign(VIP=x[\"VIP\"].copy().fillna(-1).astype(int))\n        if train_label_encoders:\n            label_encoders[\"VIP\"].fit(x[\"VIP\"].copy())\n        vip = label_encoders[\"VIP\"].transform(x.pop(\"VIP\"))\n        x = pd.concat([x, pd.Series(vip, name=\"VIP\")], axis=1)\n        \n    if \"PassengerId\" in features:\n        pid = x.pop(\"PassengerId\").str.split(\"_\", expand=True)\n        pid.columns = [\"gggg\", \"pp\"]\n        \"\"\"\n        if train_label_encoders:\n            label_encoders[\"pp\"].fit(pid[\"pp\"].copy())\n        pp = pd.Series(label_encoders[\"pp\"].transform(pid[\"pp\"]), name=\"pp\")\n        \"\"\"\n        x = pd.concat([x, pid[\"gggg\"]], axis=1)\n        \n        def _fillna(xx):\n            num_na = xx.isna().sum().sum()\n            if num_na == 0:\n                return xx\n            elif num_na != len(xx):\n                xx.fillna(xx.median(), inplace=True)\n            elif num_na == len(xx):\n                xx.fillna(0, inplace=True)\n            return xx\n            \n        x = x.groupby(\"gggg\").transform(lambda xx: _fillna(xx))\n    \n    x.fillna(x.median(), inplace=True)\n    \n    if \"Transported\" in df_.columns:\n        y = df_[\"Transported\"].astype(int)\n        return x, id_, y\n    \n    return x, id_","metadata":{"execution":{"iopub.status.busy":"2022-07-17T12:10:01.786019Z","iopub.execute_input":"2022-07-17T12:10:01.786469Z","iopub.status.idle":"2022-07-17T12:10:01.812381Z","shell.execute_reply.started":"2022-07-17T12:10:01.786432Z","shell.execute_reply":"2022-07-17T12:10:01.811185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = [\n    \"PassengerId\",\n    \"HomePlanet\",\n    \"CryoSleep\",\n    \"Cabin\",\n    \"Destination\",\n    \"Age\",\n    \"VIP\",\n    \"RoomService\",\n    \"FoodCourt\",\n    \"ShoppingMall\",\n    \"Spa\",\n    \"VRDeck\",\n]\n\nlabel_encoders = {\n    \"CryoSleep\": LabelEncoder(),\n    \"VIP\": LabelEncoder(),\n    \"HomePlanet\": LabelEncoder(),\n    \"Destination\": LabelEncoder(),\n    \"deck\": LabelEncoder(),\n    \"side\": LabelEncoder(),\n    #\"pp\": LabelEncoder(),\n}   \n\nx_train, id_train, y_train = split_data(df_train, features, label_encoders, True)\nx_test, id_test = split_data(df_test, features, label_encoders, False)\n\nprint(x_train.shape, id_train.shape, y_train.shape)\nprint(x_test.shape, id_test.shape)\n\ndisplay(x_train.head(20))\ndisplay(x_train.isna().sum())\ndisplay(x_train.dtypes)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T12:10:01.816198Z","iopub.execute_input":"2022-07-17T12:10:01.817385Z","iopub.status.idle":"2022-07-17T12:10:52.500046Z","shell.execute_reply.started":"2022-07-17T12:10:01.817329Z","shell.execute_reply":"2022-07-17T12:10:52.498445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"categorical_indices = []\ncategorical_dims = []\nfor i, f in enumerate(x_train.columns):\n    if f in label_encoders.keys():\n        categorical_indices.append(i)\n        categorical_dims.append(len(label_encoders[f].classes_))\nprint(categorical_indices)\nprint(categorical_dims)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T12:10:52.503179Z","iopub.execute_input":"2022-07-17T12:10:52.504474Z","iopub.status.idle":"2022-07-17T12:10:52.513428Z","shell.execute_reply.started":"2022-07-17T12:10:52.504418Z","shell.execute_reply":"2022-07-17T12:10:52.512185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_data(\n    x: Union[pd.DataFrame, np.ndarray],\n    y: Union[pd.DataFrame, np.ndarray],\n    use_y: bool = True\n) -> None:\n    \n    nrows = x.shape[1] // 3 if x.shape[1] % 3 == 0 else x.shape[1] // 3 + 1\n    _, axs = plt.subplots(nrows, 3, figsize=(16, 4*nrows))\n    axs = axs.reshape(-1, 3)\n    if isinstance(x, pd.DataFrame):\n        is_df = True\n    elif isinstance(x, np.ndarray):\n        is_df = False\n    for i in range(x.shape[1]):\n        data = x.iloc[:, i] if is_df else x[:, i]\n        if use_y == False:\n            axs[i//3, i%3].hist(data, bins=50, histtype=\"step\")\n        else:\n            _, bins, _ = axs[i//3, i%3].hist(data[y==0], bins=50, histtype=\"step\")\n            axs[i//3, i%3].hist(data[y==1], bins=bins, histtype=\"step\")\n        if is_df:\n            axs[i//3, i%3].set_xlabel(data.name)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-17T12:10:52.515393Z","iopub.execute_input":"2022-07-17T12:10:52.515786Z","iopub.status.idle":"2022-07-17T12:10:52.528989Z","shell.execute_reply.started":"2022-07-17T12:10:52.515751Z","shell.execute_reply":"2022-07-17T12:10:52.527917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_data(x_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T12:10:52.531032Z","iopub.execute_input":"2022-07-17T12:10:52.531500Z","iopub.status.idle":"2022-07-17T12:10:54.602065Z","shell.execute_reply.started":"2022-07-17T12:10:52.531464Z","shell.execute_reply":"2022-07-17T12:10:54.600594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def take_ratio(df_: pd.DataFrame, ratio_features: List[str]) -> pd.DataFrame:\n    ratio = df_[ratio_features].copy()\n    ratio = ratio.div(ratio.sum(axis=1), axis=0)\n    ratio.fillna(ratio.median(), inplace=True)\n    ratio.rename(columns={k: f\"ratio of \"+str(k) for k in ratio_features}, inplace=True)\n    return pd.concat([df_, ratio], axis=1)\n\n\ndef take_logarithm(df_: pd.DataFrame, log_features: List[str]) -> None:\n    df_[log_features] = df_[log_features] + 1\n    df_[log_features] = df_[log_features].apply(np.log)\n    df_.rename(columns={k: f\"log(1+\"+str(k)+\")\" for k in log_features}, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T12:10:54.605344Z","iopub.execute_input":"2022-07-17T12:10:54.605717Z","iopub.status.idle":"2022-07-17T12:10:54.615839Z","shell.execute_reply.started":"2022-07-17T12:10:54.605685Z","shell.execute_reply":"2022-07-17T12:10:54.614196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ratio_features = [\n    \"RoomService\",\n    \"FoodCourt\",\n    \"ShoppingMall\",\n    \"Spa\",\n    \"VRDeck\",\n]\n\nlog_features = [\n    \"RoomService\",\n    \"FoodCourt\",\n    \"ShoppingMall\",\n    \"Spa\",\n    \"VRDeck\",\n]\n\nx_train = take_ratio(x_train, ratio_features)\nx_test = take_ratio(x_test, ratio_features)\n\ntake_logarithm(x_train, log_features)\ntake_logarithm(x_test, log_features)\n\nprint(x_train.shape, id_train.shape, y_train.shape)\nprint(x_test.shape, id_test.shape)\n\ndisplay(x_train.head(20))","metadata":{"execution":{"iopub.status.busy":"2022-07-17T12:10:54.617707Z","iopub.execute_input":"2022-07-17T12:10:54.618219Z","iopub.status.idle":"2022-07-17T12:10:54.690876Z","shell.execute_reply.started":"2022-07-17T12:10:54.618169Z","shell.execute_reply":"2022-07-17T12:10:54.689510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(x_train.isna().sum())","metadata":{"execution":{"iopub.status.busy":"2022-07-17T12:10:54.692496Z","iopub.execute_input":"2022-07-17T12:10:54.692873Z","iopub.status.idle":"2022-07-17T12:10:54.706095Z","shell.execute_reply.started":"2022-07-17T12:10:54.692836Z","shell.execute_reply":"2022-07-17T12:10:54.704479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for u, c in zip(*np.unique(y_train.values, return_counts=True)):\n    print(f\"{u}: {c}\")","metadata":{"execution":{"iopub.status.busy":"2022-07-17T12:10:54.708052Z","iopub.execute_input":"2022-07-17T12:10:54.708679Z","iopub.status.idle":"2022-07-17T12:10:54.715683Z","shell.execute_reply.started":"2022-07-17T12:10:54.708630Z","shell.execute_reply":"2022-07-17T12:10:54.714420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_data(x_train, y_train)\nplot_data(x_test, y_train, False)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T12:10:54.717264Z","iopub.execute_input":"2022-07-17T12:10:54.717761Z","iopub.status.idle":"2022-07-17T12:10:59.350081Z","shell.execute_reply.started":"2022-07-17T12:10:54.717715Z","shell.execute_reply":"2022-07-17T12:10:59.348799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Pytorch Models","metadata":{}},{"cell_type":"code","source":"class ResMLP(nn.Module):\n    def __init__(self, blocks: List[List[int]], max_cate: int) -> None:\n        super().__init__()\n        self.emb = nn.Sequential(\n            nn.Embedding(max_cate, 20),\n            nn.Linear(20, 1),\n            nn.LeakyReLU(),\n        )\n        \n        self.layers = nn.ModuleList()\n        for units in blocks:\n            self.layers.append(ResidualBlock(units))\n        \n    def forward(self, x: torch.Tensor, c: torch.Tensor) -> torch.Tensor:\n        c = self.emb(c)\n        x = torch.cat([x, c.squeeze(dim=-1)], dim=-1)\n        for layer in self.layers:\n            x = layer(x)\n        return x\n    \n\nclass ResidualBlock(nn.Module):\n    def __init__(self, units: List[str]) -> None:\n        super().__init__()\n        input_output_dim = list(zip(units[:-1], units[1:]))\n        self.res = self._residual(units[0], units[-1])\n        self.layers = nn.ModuleList()\n        for i, (input_dim, output_dim) in enumerate(input_output_dim):\n            self.layers.extend([\n                nn.BatchNorm1d(input_dim),\n                nn.LeakyReLU(),\n                nn.Linear(input_dim, output_dim),\n            ])\n            if i % 2 == 0:\n                self.layers.append(nn.Dropout(0.2))\n    \n    def forward(self, x: torch.Tensor) -> torch.Tensor:\n        r = self.res(x)\n        for layer in self.layers:\n            x = layer(x)\n        return x + r\n    \n    def _residual(self, input_dim: int, output_dim: int) -> object:\n        if input_dim == output_dim:\n            return lambda x: x\n        else:\n            return nn.Linear(input_dim, output_dim)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T12:10:59.351755Z","iopub.execute_input":"2022-07-17T12:10:59.352979Z","iopub.status.idle":"2022-07-17T12:10:59.373572Z","shell.execute_reply.started":"2022-07-17T12:10:59.352923Z","shell.execute_reply":"2022-07-17T12:10:59.372191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_nn_fold(\n    x: np.ndarray,\n    y: np.ndarray,\n    kfold_params: Dict[str, object],\n    dataloader_params: Dict[str, object],\n    mlp_params: Dict[str, Union[List[List[int]], int]],\n    categorical_index: List[int],\n) -> object:\n    \n    kfold = StratifiedKFold(**kfold_params)\n    models = []\n    standardize_index = [i for i in range(x.shape[1]) if i not in categorical_index]\n    \n    for k, (indices_train, indices_valid) in enumerate(kfold.split(x, y)):\n        print(f\"fold: {k+1} / {kfold.n_splits}\")\n        print(len(indices_train), len(indices_valid))\n        model = ResMLP(**mlp_params).to(device)\n        if k == 0:\n            print(model)\n        loss_fn = torch.nn.CrossEntropyLoss()\n        optimizer = torch.optim.RAdam(model.parameters(), lr=1e-03)\n        scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(\n            optimizer,\n            T_max=10,\n            eta_min=1e-08\n        )\n        scaler = StandardScaler()\n        \n        xx_train = x[indices_train]\n        yy_train = y[indices_train]\n        xx_train, cate_train = standardize4nn(scaler, xx_train,\n                                              categorical_index, True)\n        dataloader_train = build_dataloader(xx_train, cate_train,\n                                            yy_train, **dataloader_params)\n        \n        xx_valid = x[indices_valid]\n        yy_valid = y[indices_valid]\n        xx_valid, cate_valid = standardize4nn(scaler, xx_valid,\n                                              categorical_index, False)\n        dataloader_valid = build_dataloader(xx_valid, cate_valid,\n                                            yy_valid, 128, False, True)\n        \n        best_acc = 0\n        for e in range(epochs):\n            print(f\" epoch: {e+1}/{epochs}\")\n            train_nn(dataloader_train, model, loss_fn, optimizer)\n            \n            loss_train, acc_train, auc_train = test_nn(dataloader_train, model, loss_fn)\n            loss_valid, acc_valid, auc_valid = test_nn(dataloader_valid, model, loss_fn)\n            \n            scheduler.step()\n                \n            print(f\"  current lr = {scheduler.get_last_lr()[0]:.4e}\")\n            print(f\"  train: loss = {loss_train:.4f}, acc = {acc_train:.4f}, auc = {auc_train:.4f}\")\n            print(f\"  valid: loss = {loss_valid:.4f}, acc = {acc_valid:.4f}, auc = {auc_valid:.4f}\")\n            \n            if best_acc < acc_valid and 0 < acc_train - acc_valid < 0.1:\n                best_scaler = copy.deepcopy(scaler)\n                best_model = copy.deepcopy(model)\n                best_acc = acc_valid\n        if best_acc == 0:\n            best_scaler = scaler\n            best_model = model\n        \n        print(f\" best acc = {best_acc:.4f}\")\n        models.append((best_scaler, best_model))\n    return models\n    \n    \ndef build_dataloader(\n    x: np.ndarray,\n    c: np.ndarray,\n    y: np.ndarray,\n    batch_size: int,\n    shuffle: bool,\n    drop_last: bool,\n    worker_init_fn: object = None,\n    generator: object = None\n) -> object:\n    \n    tensor_x = torch.from_numpy(x).float().clone()\n    tensor_c = torch.from_numpy(c).long().clone()\n    tensor_y = torch.from_numpy(y).long().clone()\n    dataset = TensorDataset(tensor_x, tensor_c, tensor_y)\n    dataloader = DataLoader(\n        dataset,\n        batch_size=batch_size,\n        shuffle=shuffle,\n        drop_last=drop_last,\n        worker_init_fn=worker_init_fn,\n        generator=generator\n    )\n    return dataloader","metadata":{"execution":{"iopub.status.busy":"2022-07-17T12:10:59.376029Z","iopub.execute_input":"2022-07-17T12:10:59.376667Z","iopub.status.idle":"2022-07-17T12:10:59.403703Z","shell.execute_reply.started":"2022-07-17T12:10:59.376614Z","shell.execute_reply":"2022-07-17T12:10:59.402607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def standardize4nn(\n    scaler: object,\n    x: np.ndarray,\n    categorical_index: List[int],\n    train_scaler: bool = True\n) -> np.ndarray:\n    \n    standardize_index = [i for i in range(x.shape[1]) if i not in categorical_index]\n    if train_scaler:\n        scaler.fit(x[:, standardize_index])\n    standardized_x = scaler.transform(x[:, standardize_index])\n    return standardized_x, x[:, categorical_index]","metadata":{"execution":{"iopub.status.busy":"2022-07-17T12:10:59.405116Z","iopub.execute_input":"2022-07-17T12:10:59.405706Z","iopub.status.idle":"2022-07-17T12:10:59.421733Z","shell.execute_reply.started":"2022-07-17T12:10:59.405669Z","shell.execute_reply":"2022-07-17T12:10:59.420620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_nn(dataloader, model, loss_fn, optimizer):\n    model.train()\n    for x, c, y in dataloader:\n        x = x.to(device)\n        c = c.to(device)\n        y = y.to(device)\n        \n        y_hat = model(x, c)\n        loss = loss_fn(y_hat, y)\n        \n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        \n\ndef test_nn(dataloader, model, loss_fn):\n    model.eval()\n    loss = 0\n    acc = 0\n    probas = []\n    truth = []\n    with torch.no_grad():\n        for batch, (x, c, y) in enumerate(dataloader):\n            x = x.to(device)\n            c = c.to(device)\n            y = y.to(device)\n            \n            y_hat = model(x, c)\n            loss += loss_fn(y_hat, y).item()\n            proba = torch.nn.functional.softmax(y_hat, dim=1)\n            probas.append(proba[:, 1])\n            truth.append(y)\n            acc += (proba.argmax(dim=1)==y).float().sum().item()\n    probas = torch.cat(probas)\n    truth = torch.cat(truth)\n    auc = calc_auc(probas, truth)\n    return loss / len(dataloader), acc / len(dataloader.dataset), auc\n\n\ndef calc_auc(y_hat, y):\n    a = y.detach().numpy().copy()\n    b = y_hat.detach().numpy().copy()\n    return roc_auc_score(a, b)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T12:10:59.423015Z","iopub.execute_input":"2022-07-17T12:10:59.424192Z","iopub.status.idle":"2022-07-17T12:10:59.442193Z","shell.execute_reply.started":"2022-07-17T12:10:59.424110Z","shell.execute_reply":"2022-07-17T12:10:59.440552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_worker(worker_id):\n    # copied from https://pytorch.org/docs/stable/notes/randomness.html\n    worker_seed = torch.initial_seed() % 2**32\n    np.random.seed(worker_seed)\n    random.seed(worker_seed)\n\ntorch.backends.cudnn.benchmark = False\ng = torch.Generator()\ng.manual_seed(0)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T12:10:59.446084Z","iopub.execute_input":"2022-07-17T12:10:59.447185Z","iopub.status.idle":"2022-07-17T12:10:59.463116Z","shell.execute_reply.started":"2022-07-17T12:10:59.447107Z","shell.execute_reply":"2022-07-17T12:10:59.461657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nepochs = 50\n\nkfold_params = {\n    \"n_splits\": 5,\n    \"shuffle\": True,\n    \"random_state\": 0\n}\n\ndataloader_params = {\n    \"batch_size\": 64,\n    \"shuffle\": True,\n    \"drop_last\": True,\n    \"worker_init_fn\": seed_worker,\n    \"generator\": g\n}\n\nmodel_params = {\n    \"blocks\": [\n        [x_train.shape[1], 64, 64, 64, 32],\n        [32, 32, 32, 32, 32],\n        #[32, 32, 32, 32, 32],\n        [32, 16, 16, 16, 2],\n    ],\n    \"max_cate\": max(categorical_dims)\n}\n\nnn_models = train_nn_fold(\n    x_train.values,\n    y_train.values,\n    kfold_params,\n    dataloader_params,\n    model_params,\n    categorical_indices,\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T12:10:59.465494Z","iopub.execute_input":"2022-07-17T12:10:59.465985Z","iopub.status.idle":"2022-07-17T12:16:27.772441Z","shell.execute_reply.started":"2022-07-17T12:10:59.465935Z","shell.execute_reply":"2022-07-17T12:16:27.770958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def inference_nn_models(models, x, categorical_indices):\n    if isinstance(x, pd.DataFrame):\n        x = x.values\n    probas = []\n    for scaler, model in models:\n        xx, cc = standardize4nn(scaler, x, categorical_indices, False)\n        xx = torch.from_numpy(xx).float().clone()\n        cc = torch.from_numpy(cc).long().clone()\n        y_hat = model(xx, cc)\n        proba = torch.nn.functional.softmax(y_hat, dim=1)\n        probas.append(proba.detach().numpy().copy())\n    probas = np.stack(probas, axis=-1)\n    probas = np.mean(probas, axis=-1)\n    return probas","metadata":{"execution":{"iopub.status.busy":"2022-07-17T12:16:27.774518Z","iopub.execute_input":"2022-07-17T12:16:27.775705Z","iopub.status.idle":"2022-07-17T12:16:27.785469Z","shell.execute_reply.started":"2022-07-17T12:16:27.775654Z","shell.execute_reply":"2022-07-17T12:16:27.784446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_nn = inference_nn_models(nn_models, x_train, categorical_indices)\npred_nn = np.argmax(pred_nn, axis=-1)\nprint(f\"{sum(pred_nn==y_train) / len(y_train):.4f}\")","metadata":{"execution":{"iopub.status.busy":"2022-07-17T12:16:27.787106Z","iopub.execute_input":"2022-07-17T12:16:27.787492Z","iopub.status.idle":"2022-07-17T12:16:27.871979Z","shell.execute_reply.started":"2022-07-17T12:16:27.787460Z","shell.execute_reply":"2022-07-17T12:16:27.871038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_nn = inference_nn_models(nn_models, x_test, categorical_indices)\npred = np.argmax(pred_nn, axis=-1)\n\nsubmission = pd.DataFrame({\n    \"PassengerId\": id_test,\n    \"Transported\": pred.astype(bool),\n})\n\ndisplay(submission.head(20))","metadata":{"execution":{"iopub.status.busy":"2022-07-17T12:16:27.873394Z","iopub.execute_input":"2022-07-17T12:16:27.874246Z","iopub.status.idle":"2022-07-17T12:16:27.933174Z","shell.execute_reply.started":"2022-07-17T12:16:27.874203Z","shell.execute_reply":"2022-07-17T12:16:27.931673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T12:16:27.936622Z","iopub.execute_input":"2022-07-17T12:16:27.937597Z","iopub.status.idle":"2022-07-17T12:16:27.952766Z","shell.execute_reply.started":"2022-07-17T12:16:27.937543Z","shell.execute_reply":"2022-07-17T12:16:27.951721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}