{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-09-23T12:41:16.87938Z","iopub.execute_input":"2022-09-23T12:41:16.879828Z","iopub.status.idle":"2022-09-23T12:41:16.914536Z","shell.execute_reply.started":"2022-09-23T12:41:16.87974Z","shell.execute_reply":"2022-09-23T12:41:16.913492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nimport pandas as pd\nimport numpy as np\nimport os\nfrom tqdm import tqdm\nimport datetime\nimport time\nimport argparse\nfrom sklearn.model_selection import  StratifiedKFold\nimport lightgbm as lgb\nfrom torch.utils.data import DataLoader\nimport torch.cuda.amp as amp\nimport torch.nn as nn\nimport torch\nimport argparse\nimport torch.optim as optim\nfrom shutil import move","metadata":{"execution":{"iopub.status.busy":"2022-09-23T12:41:16.916394Z","iopub.execute_input":"2022-09-23T12:41:16.916866Z","iopub.status.idle":"2022-09-23T12:41:20.637532Z","shell.execute_reply.started":"2022-09-23T12:41:16.916827Z","shell.execute_reply":"2022-09-23T12:41:20.636419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nclass TaskDataset:\n    def __init__(self,df_series,df_feature,uidxs,df_y=None):\n        self.df_series = df_series\n        self.df_feature = df_feature\n        self.df_y = df_y\n        self.uidxs = uidxs\n\n    def __len__(self):\n        return (len(self.uidxs))\n\n    def __getitem__(self, index):\n        i1,i2,idx = self.uidxs[index]\n        #change from 1 to 2 to get rid of customer_ID column\n        series = self.df_series.iloc[i1:i2+1,2:].values\n\n        if len(series.shape) == 1:\n            series = series.reshape((-1,)+series.shape[-1:])\n        series_ = series.copy()\n        series_[series_!=0] = 1.0 - series_[series_!=0] + 0.001\n        feature = self.df_feature.loc[idx].values[1:]\n        feature_ = feature.copy()\n        feature_[feature_!=0] = 1.0 - feature_[feature_!=0] + 0.001\n        if self.df_y is not None:\n            label = self.df_y.iloc[idx,1]\n            return {\n                    'SERIES': series,#np.concatenate([series,series_],axis=1),\n                    'FEATURE': np.concatenate([feature,feature_]),\n                    'LABEL': label,\n                    }\n        else:\n            return {\n                    'SERIES': series,#np.concatenate([series,series_],axis=1),\n                    'FEATURE': np.concatenate([feature,feature_]),\n                    }\n\n    def collate_fn(self, batch):\n        \"\"\"\n        Padding to same size.\n        \"\"\"\n\n        batch_size = len(batch)\n        batch_series = torch.zeros((batch_size, 13, batch[0]['SERIES'].shape[1]))\n        batch_mask = torch.zeros((batch_size, 13))\n        batch_feature = torch.zeros((batch_size, batch[0]['FEATURE'].shape[0]))\n        batch_y = torch.zeros((batch_size, 1))\n\n        for i, item in enumerate(batch):\n            v = item['SERIES']\n            batch_series[i, :v.shape[0], :] = torch.tensor(v).float()\n            batch_mask[i,:v.shape[0]] = 1.0\n            v = item['FEATURE'].astype(np.float32)\n            batch_feature[i] = torch.tensor(v).float()\n            if self.df_y is not None:\n                v = item['LABEL'].astype(np.float32)\n                batch_y[i] = torch.tensor(v).float()\n\n        return {'batch_series':batch_series,'batch_mask':batch_mask,'batch_feature':batch_feature,'batch_y':batch_y}\n\n","metadata":{"execution":{"iopub.status.busy":"2022-09-23T12:41:20.639546Z","iopub.execute_input":"2022-09-23T12:41:20.640228Z","iopub.status.idle":"2022-09-23T12:41:20.655623Z","shell.execute_reply.started":"2022-09-23T12:41:20.640191Z","shell.execute_reply":"2022-09-23T12:41:20.654566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nclass SchedulerBase(object):\n    def __init__(self):\n        self._is_load_best_weight = True\n        self._is_load_best_optim = True\n        self._is_freeze_bn=False\n        self._is_adjust_lr = True\n        self._lr = 0.01\n        self._cur_optimizer = None\n\n    def schedule(self, net, epoch, epochs, **kwargs):\n        raise Exception('Did not implemented')\n\n    def step(self, net, epoch, epochs):\n        optimizer, lr = self.schedule(net, epoch, epochs)\n        for param_group in optimizer.param_groups:\n            param_group['lr'] = lr\n\n        lr_list = []\n        for param_group in optimizer.param_groups:\n            lr_list += [param_group['lr']]\n        return lr_list\n\n    def is_load_best_weight(self):\n        return self._is_load_best_weight\n\n    def is_load_best_optim(self):\n        return self._is_load_best_optim\n\n    def is_freeze_bn(self):\n        return self._is_freeze_bn\n\n    def reset(self):\n        self._is_load_best_weight = True\n        self._load_best_optim = True\n        self._is_freeze_bn = False\n\n    def is_adjust_lr(self):\n        return self._is_adjust_lr\nclass Adam12(SchedulerBase):\n    def __init__(self, params_list=None):\n        super().__init__()\n        self._lr = 3e-4\n        self._cur_optimizer = None\n        self.params_list=params_list\n\n    def schedule(self, net, epoch, epochs, **kwargs):\n        lr = 100e-5\n        if epoch > 4:\n            lr = 10e-5\n        if epoch > 8:\n            lr = 1e-5\n        # if epoch > 9:\n        #     lr = 1e-5\n        # if epoch > 12:\n        #     lr = 1e-5\n        self._lr = lr\n        if self._cur_optimizer is None:\n            self._cur_optimizer = optim.Adam(net.parameters(), lr=lr)#, eps=1e-5, weight_decay=0.001\n        return self._cur_optimizer, self._lr\n","metadata":{"execution":{"iopub.status.busy":"2022-09-23T12:41:20.658467Z","iopub.execute_input":"2022-09-23T12:41:20.659113Z","iopub.status.idle":"2022-09-23T12:41:20.672995Z","shell.execute_reply.started":"2022-09-23T12:41:20.659076Z","shell.execute_reply":"2022-09-23T12:41:20.671871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Amodel(nn.Module):\n    def __init__(self, series_dim, feature_dim, target_num, hidden_num, hidden_dim, drop_rate=0.5, use_series_oof=False):\n\n        super(Amodel, self).__init__()\n        self.series_dim = series_dim\n        self.feature_dim = feature_dim\n        self.use_series_oof = use_series_oof\n        self.input_series_block = nn.Sequential(\n                                        nn.Linear(series_dim, hidden_dim)\n                                        ,nn.LayerNorm(hidden_dim)\n                                        )\n        self.input_feature_block = nn.Sequential(\n                                        nn.Linear(feature_dim, hidden_dim)\n                                        ,nn.BatchNorm1d(hidden_dim)\n                                        ,nn.LeakyReLU()\n                                        )\n        self.gru_series = nn.GRU(hidden_dim, hidden_dim, batch_first=True, bidirectional=True)\n        self.hidden_feature_block = []\n        for h in range(hidden_num-1):\n            self.hidden_feature_block.extend([\n                                     nn.Linear(hidden_dim, hidden_dim)\n                                     ,nn.BatchNorm1d(hidden_dim)\n                                     ,nn.Dropout(drop_rate)\n                                     ,nn.LeakyReLU()\n                                     ])\n        self.hidden_feature_block = nn.Sequential(*self.hidden_feature_block)\n\n        self.output_block = nn.Sequential(\n                                         nn.Linear(3*hidden_dim if use_series_oof else 2*hidden_dim, 1*hidden_dim)\n                                         ,nn.LeakyReLU()\n\n                                         ,nn.Linear(1*hidden_dim, 1*hidden_dim)\n                                         ,nn.LeakyReLU()\n                                         \n                                         ,nn.Linear(1*hidden_dim, target_num)\n                                         ,nn.Sigmoid()\n                                         )\n\n    def batch_gru(self,series,mask):\n        node_num = mask.sum(dim=-1).detach().cpu()\n        pack = nn.utils.rnn.pack_padded_sequence(series, node_num, batch_first=True, enforce_sorted=False)\n        message,hidden = self.gru_series(pack)\n        pooling_feature = []\n\n        for i,n in enumerate(node_num.numpy()):\n            n = int(n)\n            bi = 0\n\n            si = message.unsorted_indices[i]\n            for k in range(n):\n\n                if k == n-1:\n                    sample_feature = message.data[bi+si]\n                bi = bi + message.batch_sizes[k]\n\n            pooling_feature.append(sample_feature)\n        return torch.stack(pooling_feature,0)\n\n    def forward(self, data):\n        x1 = self.input_series_block(data['batch_series'])\n\n        x1 = self.batch_gru(x1,data['batch_mask'])\n        if self.use_series_oof:\n            x2 = self.input_feature_block(data['batch_feature'])\n            x2 = self.hidden_feature_block(x2)\n            x = torch.cat([x1,x2],axis=1)\n            y = self.output_block(x)\n        else:\n            y = self.output_block(x1)\n        return y\n","metadata":{"execution":{"iopub.status.busy":"2022-09-23T12:41:20.674882Z","iopub.execute_input":"2022-09-23T12:41:20.675143Z","iopub.status.idle":"2022-09-23T12:41:20.692923Z","shell.execute_reply.started":"2022-09-23T12:41:20.675102Z","shell.execute_reply":"2022-09-23T12:41:20.691724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class model_amex:\n    def __init__(self):\n        # to speed up set this to true and then only train data will be used to similuate both train andtest and then compute the out-of-sample score using the simultaed test data\n        self.DOWN_SAMPLING = True\n        # when building the model set this to false so that I can compute the score using simulated test data. real test data from Kaggle do not have label.\n        self.USE_TEST_DATA = False\n        # number of the data (for amex, userid's that will be sampled\n        # self.SIZE_TRAIN = 10000\n        # self.SIZE_TEST = 10000\n        # self.SIZE_TRAIN = 1000000000\n        # self.SIZE_TEST = 500000\n        self.TOTAL_SAMPLING_PCNT = 0.02\n        self.TEST_PCNT = 0.1\n\n        # folder where data are located such as train and test data\n#         self._DIR_DATA = self._DIR_DATA_DOWN_SAMPLED = r'../mine/'\n#         self._DIR_DATA_INPUT = r'../mine/'\n        self._DIR_DATA_INPUT = r'/kaggle/input/amex-default-prediction/'\n        self._DIR_DATA = r'/kaggle/working/'\n\n        self._DIR_DATA_DOWN_SAMPLED = r'/kaggle/input/downsampleddata/'\n        self._FILE_TRAIN = 'train_data.csv'\n        self._FILE_TEST = 'test_data.csv'\n        self._FILE_TRAIN_LABEL = 'train_labels.csv'\n        self._FILE_TRAIN_FEATHER_ORIG = 'train_orig.feather'\n        self._FILE_TEST_FEATHER_ORIG = 'test_orig.feather'\n        self._DIR_OUTPUT = './output/'\n\n        # list columns that are not numerical, we will need to encode these.\n        self.NON_NUM_COLS = ['D_63', 'D_64']\n        self.cat_features = [\"B_30\", \"B_38\", \"D_114\", \"D_116\", \"D_117\", \"D_120\", \"D_126\", \"D_63\", \"D_64\", \"D_66\",\n                             \"D_68\"]\n\n        self.COL_UID = \"customer_ID\"\n        self.COL_TIME = 'S_2'\n        self.COL_LABEL = 'target'\n\n        self.seed = 42\n        parser = argparse.ArgumentParser()\n        parser.add_argument(\"--root\", type=str, default=self._DIR_DATA_INPUT + '')\n        parser.add_argument(\"--save_dir\", type=str, default='tmp')\n        parser.add_argument(\"--use_apm\", action='store_true', default=False)\n        parser.add_argument(\"--num_workers\", type=int, default=16)\n        parser.add_argument(\"--do_train\", action='store_true', default=False)\n        parser.add_argument(\"--test\", action='store_true', default=False)\n        parser.add_argument(\"--seed\", type=int, default=42)\n        parser.add_argument(\"--remark\", type=str, default='')\n\n        self.args, unknown = parser.parse_known_args()\n\n    def get_path_train_orig(self):\n        return self._DIR_DATA_INPUT + '/' + self._FILE_TRAIN\n\n    def get_path_train(self):\n        if self.DOWN_SAMPLING:\n            return self._DIR_DATA_DOWN_SAMPLED + '/' + 'train_down_sampling_' + str(self.TOTAL_SAMPLING_PCNT) + '_' + str(self.TEST_PCNT) + '.feather'\n        else:\n            return self._DIR_DATA + '/' + self._FILE_TRAIN_FEATHER_ORIG\n\n        # not used\n\n    def get_path_test(self):\n        if self.USE_TEST_DATA:\n            return self._DIR_DATA + '/' + self._FILE_TEST_FEATHER_ORIG\n        else:\n            return self._DIR_DATA_DOWN_SAMPLED + '/' + 'test_down_sampling_from_train_' + str(self.TOTAL_SAMPLING_PCNT) + '_' + str(self.TEST_PCNT) + '.feather'\n\n    if False:\n        # not used\n\n        # not used\n        def build_label_encoder(self):\n            label_encoder_model = dict()\n            # manually check to see which file (test or train) contains the full set of values. in amex this is train\n            df = pd.read_csv(self.get_path_train_orig(), usecols=self.NON_NUM_COLS)\n            print('finished reading the NON_NUM_COLS from train_data')\n\n            for c in self.NON_NUM_COLS:\n                label_encoder_model[c] = LabelEncoder().fit(df[c])\n                np.save(self._DIR_DATA + 'label_encoder_model_' + c + '.npy', label_encoder_model[c].classes_)\n\n        # not used\n        def get_label_encoder(self):\n            label_encoder_model = dict()\n            if not os.path.exists(self._DIR_DATA + 'label_encoder_model_' + self.NON_NUM_COLS[0] + '.npy'):\n                self.build_label_encoder()\n            np_load_old = np.load\n            # modify the default parameters of np.load\n            np.load = lambda *a, **k: np_load_old(*a, allow_pickle=True, **k)\n\n            for c in self.NON_NUM_COLS:\n                _file_path = self._DIR_DATA + 'label_encoder_model_' + c + '.npy'\n                label_encoder_model[c] = LabelEncoder()\n\n                label_encoder_model[c].classes_ = np.load(_file_path)\n            # restore np.load for future normal usage\n            np.load = np_load_old\n            return label_encoder_model\n\n    def S1_denoise(self):\n        def denoise(df):\n            df['D_63'] = df['D_63'].apply(lambda t: {'CR': 0, 'XZ': 1, 'XM': 2, 'CO': 3, 'CL': 4, 'XL': 5}[t]).astype(\n                np.int8)\n            df['D_64'] = df['D_64'].apply(lambda t: {np.nan: -1, 'O': 0, '-1': 1, 'R': 2, 'U': 3}[t]).astype(np.int8)\n            for col in tqdm(df.columns):\n                if col not in [self.COL_UID, self.COL_TIME] + self.NON_NUM_COLS:\n                    df[col] = np.floor(df[col] * 100)\n            return df\n\n        # chunksize = 10 ** 6\n\n        files = [self.get_path_train_orig(), self._DIR_DATA_INPUT + self._FILE_TEST]\n        files_out = [self._DIR_DATA + self._FILE_TRAIN_FEATHER_ORIG, self._DIR_DATA + self._FILE_TEST_FEATHER_ORIG]\n        for file_in, file_out in zip(files, files_out):\n            # FIRST_TIME = True\n            # for chunk in pd.read_csv(file_in, chunksize=chunksize):\n            chunk = pd.read_csv(file_in)\n            # train = pd.read_csv(self.get_path_train_orig())\n            chunk = denoise(chunk)\n            chunk.to_feather(file_out)\n            #     if FIRST_TIME:\n            #         chunk.to_csv(file_out, index=False)\n            #         FIRST_TIME = False\n            #     else:\n            #         chunk.to_csv(file_out, index=False, mode='a')\n\n        # del train\n        # this fails the 2nd time, will need to process by batch\n        # test = pd.read_csv(self._DIR_DATA + self._FILE_TEST)\n        # test = denoise(test)\n        # test.to_feather()\n\n    def S1pnt1_down_sample(self):\n        if self.DOWN_SAMPLING:\n            df_orig = pd.read_feather(self._DIR_DATA + self._FILE_TRAIN_FEATHER_ORIG)\n            customer_ID_samples = df_orig.groupby(self.COL_UID).count()[self.COL_TIME].to_frame()\n            _len = len(customer_ID_samples)\n            # rescale self.SIZE_TRAIN + self.SIZE_TEST\n            # if _len < self.SIZE_TRAIN + self.SIZE_TEST:\n            # self.SIZE_TRAIN, self.SIZE_TEST = int(self.SIZE_TRAIN * _len / (self.SIZE_TRAIN + self.SIZE_TEST)), int(self.SIZE_TEST * _len / (self.SIZE_TRAIN + self.SIZE_TEST))\n            customer_ID_samples = customer_ID_samples.sample(\n                frac=self.TOTAL_SAMPLING_PCNT)\n            msk = np.random.rand(len(customer_ID_samples)) < (1 - self.TEST_PCNT)\n            customer_ID_samples_train = customer_ID_samples[msk]\n            customer_ID_samples_test = customer_ID_samples[~msk]\n            train = df_orig.merge(customer_ID_samples_train, left_on=self.COL_UID, right_index=True,\n                                  suffixes=('', '_y')).drop(\n                self.COL_TIME + '_y', axis=1)\n\n            train.reset_index().to_feather(self.get_path_train())\n\n            test = df_orig.merge(customer_ID_samples_test, left_on=self.COL_UID, right_index=True,\n                                 suffixes=('', '_y')).drop(\n                self.COL_TIME + '_y', axis=1)\n\n            test.reset_index().to_feather(self.get_path_test())\n\n    def cat_feature(self, df, lastk):\n        one_hot_features = [col for col in df.columns if 'oneHot' in col]\n        if lastk is None:\n            num_agg_df = df.groupby(self.COL_UID, sort=False)[one_hot_features].agg(['mean', 'std', 'sum', 'last'])\n        else:\n            num_agg_df = df.groupby(self.COL_UID, sort=False)[one_hot_features].agg(['mean', 'std', 'sum'])\n        num_agg_df.columns = ['_'.join(x) for x in num_agg_df.columns]\n\n        if lastk is None:\n            cat_agg_df = df.groupby(self.COL_UID, sort=False)[self.cat_features].agg(['last', 'nunique'])\n        else:\n            cat_agg_df = df.groupby(self.COL_UID, sort=False)[self.cat_features].agg(['nunique'])\n        cat_agg_df.columns = ['_'.join(x) for x in cat_agg_df.columns]\n\n        count_agg_df = df.groupby(self.COL_UID, sort=False)[[self.COL_TIME]].agg(['count'])\n        count_agg_df.columns = ['_'.join(x) for x in count_agg_df.columns]\n        df = pd.concat([num_agg_df, cat_agg_df, count_agg_df], axis=1).reset_index()\n        print('cat feature shape after engineering', df.shape)\n\n        return df\n\n    def one_hot_encoding(self, df, cols, is_drop=True):\n        for col in cols:\n            print('one hot encoding:', col)\n            dummies = pd.get_dummies(pd.Series(df[col]), prefix='oneHot_%s' % col)\n            df = pd.concat([df, dummies], axis=1)\n        if is_drop:\n            df.drop(cols, axis=1, inplace=True)\n        return df\n\n    def num_feature(self, df, num_features, lastk):\n        if num_features[0][:5] == 'rank_':\n            num_agg_df = df.groupby(self.COL_UID, sort=False)[num_features].agg(['last'])\n        else:\n            if lastk is None:\n                num_agg_df = df.groupby(self.COL_UID, sort=False)[num_features].agg(\n                    ['mean', 'std', 'min', 'max', 'sum', 'last'])\n            else:\n                num_agg_df = df.groupby(self.COL_UID, sort=False)[num_features].agg(\n                    ['mean', 'std', 'min', 'max', 'sum'])\n        num_agg_df.columns = ['_'.join(x) for x in num_agg_df.columns]\n        if num_features[0][:5] != 'rank_':\n            for col in num_agg_df.columns:\n                num_agg_df[col] = num_agg_df[col] // 0.01\n        df = num_agg_df.reset_index()\n        print('num feature shape after engineering', df.shape)\n\n        return df\n\n    def diff_feature(self, df, num_features, lastk):\n        diff_num_features = [f'diff_{col}' for col in num_features]\n        cids = df[self.COL_UID].values\n        df = df.groupby(self.COL_UID)[num_features].diff().add_prefix('diff_')\n        df.insert(0, self.COL_UID, cids)\n        if lastk is None:\n            num_agg_df = df.groupby(self.COL_UID, sort=False)[diff_num_features].agg(\n                ['mean', 'std', 'min', 'max', 'sum', 'last'])\n        else:\n            num_agg_df = df.groupby(self.COL_UID, sort=False)[diff_num_features].agg(\n                ['mean', 'std', 'min', 'max', 'sum'])\n        num_agg_df.columns = ['_'.join(x) for x in num_agg_df.columns]\n        for col in num_agg_df.columns:\n            num_agg_df[col] = num_agg_df[col] // 0.01\n\n        df = num_agg_df.reset_index()\n        print('diff feature shape after engineering', df.shape)\n\n        return df\n\n    def S2_manial_feature(self):\n        n_cpu = 16\n        transform = [['', 'rank_', 'ym_rank_'], [''], ['']]\n        if False:\n            len(df_train)\n        for li, lastk in enumerate([None, 3, 6]):\n            for prefix in transform[li]:\n\n                # df = df_orig.copy()\n\n                df = pd.read_feather(self.get_path_train()).append(\n                    pd.read_feather(self.get_path_test())).reset_index(drop=True)\n                df[self.COL_TIME] = pd.to_datetime(df[self.COL_TIME])\n                all_cols = [c for c in list(df.columns) if c not in [self.COL_UID, self.COL_TIME]]\n                num_features = [col for col in all_cols if col not in self.cat_features]\n\n                # special coded for amex prefix\n                for col in [col for col in df.columns if 'S_' in col or 'P_' in col]:\n                    if col != self.COL_TIME:\n                        df[col] = df[col].fillna(0)\n\n                if lastk is not None:\n                    prefix = f'last{lastk}_' + prefix\n                    print('all df shape', df.shape)\n                    df['rank'] = df.groupby(self.COL_UID)[self.COL_TIME].rank(ascending=False)\n                    df = df.loc[df['rank'] <= lastk].reset_index(drop=True)\n                    df = df.drop(['rank'], axis=1)\n                    print(f'last {lastk} shape', df.shape)\n\n                if prefix == 'rank_':\n                    cids = df[self.COL_UID].values\n                    df = df.groupby(self.COL_UID)[num_features].rank(pct=True).add_prefix('rank_')\n                    df.insert(0, self.COL_UID, cids)\n                    num_features = [f'rank_{col}' for col in num_features]\n\n                if prefix == 'ym_rank_':\n                    cids = df[self.COL_UID].values\n                    old = df[self.COL_TIME]\n                    df[self.COL_TIME] = df[self.COL_TIME].map(str)\n                    df['ym'] = df[self.COL_TIME].apply(lambda x: x[:7])\n                    df[self.COL_TIME] = old\n                    df = df.groupby('ym')[num_features].rank(pct=True).add_prefix('ym_rank_')\n                    num_features = [f'ym_rank_{col}' for col in num_features]\n                    df.insert(0, self.COL_UID, cids)\n\n                if prefix in ['', 'last3_']:\n                    df = self.one_hot_encoding(df, self.cat_features, False)\n\n                vc = df[self.COL_UID].value_counts(sort=False).cumsum()\n                batch_size = int(np.ceil(len(vc) / n_cpu))\n                dfs = []\n                start = 0\n                for i in range(min(n_cpu, int(np.ceil(len(vc) / batch_size)))):\n                    vc_ = vc[i * batch_size:(i + 1) * batch_size]\n                    dfs.append(df[start:vc_[-1]])\n                    start = vc_[-1]\n\n                # for debugging turn of parallel\n                if False:\n                    pool = ThreadPool(n_cpu)\n\n                if prefix in ['', 'last3_']:\n                    if False:\n                        cat_feature_df = pd.concat(pool.map(cat_feature, tqdm(dfs, desc='cat_feature'))).reset_index(\n                            drop=True)\n                    cat_feature_df = self.cat_feature(df, lastk).reset_index(drop=True)\n\n                    cat_feature_df.to_feather(self._DIR_DATA + f'{prefix}cat_feature.feather')\n\n                if prefix in ['', 'last3_', 'last6_', 'rank_', 'ym_rank_']:\n                    if False:\n                        num_feature_df = pd.concat(pool.map(num_feature, tqdm(dfs, desc='num_feature'))).reset_index(\n                            drop=True)\n                    num_feature_df = self.num_feature(df, num_features, lastk).reset_index(drop=True)\n                    num_feature_df.to_feather(self._DIR_DATA + f'{prefix}num_feature.feather')\n\n                if prefix in ['', 'last3_']:\n                    if False:\n                        diff_feature_df = pd.concat(pool.map(diff_feature, tqdm(dfs, desc='diff_feature'))).reset_index(\n                            drop=True)\n                    diff_feature_df = self.diff_feature(df, num_features, lastk).reset_index(drop=True)\n                    diff_feature_df.to_feather(self._DIR_DATA + f'{prefix}diff_feature.feather')\n\n                # for debugging turn of parallel\n                if False:\n                    pool.close()\n\n    def Write_log(self, logFile, text, isPrint=True):\n        if isPrint:\n            print(text)\n        logFile.write(text)\n        logFile.write('\\n')\n        return None\n\n    def amex_metric_mod(self, y_true, y_pred):\n        labels = np.transpose(np.array([y_true, y_pred]))\n        labels = labels[labels[:, 1].argsort()[::-1]]\n        weights = np.where(labels[:, 0] == 0, 20, 1)\n        cut_vals = labels[np.cumsum(weights) <= int(0.04 * np.sum(weights))]\n        top_four = np.sum(cut_vals[:, 0]) / np.sum(labels[:, 0])\n\n        gini = [0, 0]\n        for i in [1, 0]:\n            labels = np.transpose(np.array([y_true, y_pred]))\n            labels = labels[labels[:, i].argsort()[::-1]]\n            weight = np.where(labels[:, 0] == 0, 20, 1)\n            weight_random = np.cumsum(weight / np.sum(weight))\n            total_pos = np.sum(labels[:, 0] * weight)\n            cum_pos_found = np.cumsum(labels[:, 0] * weight)\n            lorentz = cum_pos_found / total_pos\n            gini[i] = np.sum((lorentz - weight_random) * weight)\n\n        return 0.5 * (gini[1] / gini[0] + top_four)\n\n    def Metric(self, labels, preds):\n        return self.amex_metric_mod(labels, preds)\n\n    def Lgb_train_and_predict(self, train, test, config, gkf=False, aug=None, output_root=None, run_id=None):\n        if output_root is None:\n            output_root = self._DIR_OUTPUT\n        if not run_id:\n            run_id = 'run_lgb_' + datetime.datetime.now().strftime('%Y%m%d_%H%M%S')\n            while os.path.exists(output_root + run_id + '/'):\n                time.sleep(1)\n                run_id = 'run_lgb_' + datetime.datetime.now().strftime('%Y%m%d_%H%M%S')\n            output_path = output_root + f'{self.args.save_dir}/'\n        else:\n            output_path = output_root + run_id + '/'\n        if not os.path.exists(output_path):\n            os.makedirs(output_path)\n\n        os.system(f'copy ./*.py {output_path}'.replace('/', \"\\\\\"))\n        os.system(f'copy ./*.sh {output_path}'.replace('/', \"\\\\\"))\n        config['lgb_params']['seed'] = config['seed']\n        oof, sub = None, None\n        if train is not None:\n            log = open(output_path + '/train.log', 'w', buffering=1)\n            log.write(str(config) + '\\n')\n            features = config['feature_name']\n            params = config['lgb_params']\n            rounds = config['rounds']\n            verbose = config['verbose_eval']\n            early_stopping_rounds = config['early_stopping_rounds']\n            folds = config['folds']\n            seed = config['seed']\n            oof = train[[self.COL_UID]]\n            oof[self.COL_LABEL] = 0\n\n            all_valid_metric, feature_importance = [], []\n            if gkf:\n                tmp = train[[self.COL_UID, self.COL_LABEL]].drop_duplicates(self.COL_UID).reset_index(drop=True)\n                skf = StratifiedKFold(n_splits=folds, shuffle=True, random_state=seed)\n                split = skf.split(tmp, tmp[self.COL_LABEL])\n                new_split = []\n                for trn_index, val_index in split:\n                    trn_uids = tmp.loc[trn_index, self.COL_UID].values\n                    val_uids = tmp.loc[val_index, self.COL_UID].values\n                    new_split.append((train.loc[train[self.COL_UID].isin(trn_uids)].index,\n                                      train.loc[train[self.COL_UID].isin(val_uids)].index))\n                split = new_split\n\n                # skf = GroupKFold(n_splits=folds)\n                # split = skf.split(train,train[self.COL_LABEL],train[self.COL_UID])\n            else:\n                skf = StratifiedKFold(n_splits=folds, shuffle=True, random_state=seed)\n                split = skf.split(train, train[self.COL_LABEL])\n            for fold, (trn_index, val_index) in enumerate(split):\n                evals_result_dic = {}\n                train_cids = train.loc[trn_index, self.COL_UID].values\n                if aug:\n                    train_aug = aug.loc[aug[self.COL_UID].isin(train_cids)]\n                    trn_data = lgb.Dataset(train.loc[trn_index, features].append(train_aug[features]),\n                                           label=train.loc[trn_index, self.COL_LABEL].append(train_aug[self.COL_LABEL]))\n                else:\n                    trn_data = lgb.Dataset(train.loc[trn_index, features], label=train.loc[trn_index, self.COL_LABEL])\n\n                val_data = lgb.Dataset(train.loc[val_index, features], label=train.loc[val_index, self.COL_LABEL])\n                model = lgb.train(params,\n                                  train_set=trn_data,\n                                  num_boost_round=rounds,\n                                  valid_sets=[trn_data, val_data],\n                                  evals_result=evals_result_dic,\n                                  early_stopping_rounds=early_stopping_rounds,\n                                  verbose_eval=verbose\n                                  )\n                model.save_model(output_path + '/fold%s.ckpt' % fold)\n\n                valid_preds = model.predict(train.loc[val_index, features], num_iteration=model.best_iteration)\n                oof.loc[val_index, self.COL_LABEL] = valid_preds\n\n                for i in range(len(evals_result_dic['valid_1'][params['metric']]) // verbose):\n                    self.Write_log(log, ' - %i round - train_metric: %.6f - valid_metric: %.6f\\n' % (\n                        i * verbose, evals_result_dic['training'][params['metric']][i * verbose],\n                        evals_result_dic['valid_1'][params['metric']][i * verbose]))\n                all_valid_metric.append(self.Metric(train.loc[val_index, self.COL_LABEL], valid_preds))\n                self.Write_log(log, '- fold%s valid metric: %.6f\\n' % (fold, all_valid_metric[-1]))\n\n                importance_gain = model.feature_importance(importance_type='gain')\n                importance_split = model.feature_importance(importance_type='split')\n                feature_name = model.feature_name()\n                feature_importance.append(pd.DataFrame(\n                    {'feature_name': feature_name, 'importance_gain': importance_gain,\n                     'importance_split': importance_split}))\n\n            feature_importance_df = pd.concat(feature_importance)\n            feature_importance_df = feature_importance_df.groupby(['feature_name']).mean().reset_index()\n            feature_importance_df = feature_importance_df.sort_values(by=['importance_gain'], ascending=False)\n            feature_importance_df.to_csv(output_path + '/feature_importance.csv', index=False)\n\n            mean_valid_metric = np.mean(all_valid_metric)\n            global_valid_metric = self.Metric(train[self.COL_LABEL].values, oof[self.COL_LABEL].values)\n            self.Write_log(log,\n                           'all valid mean metric:%.6f, global valid metric:%.6f' % (\n                               mean_valid_metric, global_valid_metric))\n\n            oof.to_csv(output_path + '/oof.csv', index=False)\n\n            log.close()\n            if os.path.exists(output_path + '/train_%.6f.log' % mean_valid_metric):\n                remove(output_path + '/train_%.6f.log' % mean_valid_metric)\n            move(output_path + '/train.log', output_path + '/train_%.6f.log' % mean_valid_metric)\n\n            log_df = pd.DataFrame({'run_id': [run_id], 'mean metric': [round(mean_valid_metric, 6)],\n                                   'global metric': [round(global_valid_metric, 6)], 'remark': [self.args.remark]})\n            if not os.path.exists(output_root + '/experiment_log.csv'):\n                log_df.to_csv(output_root + '/experiment_log.csv', index=False)\n            else:\n                log_df.to_csv(output_root + '/experiment_log.csv', index=False, header=None, mode='a')\n\n        if test is not None:\n            sub = test[[self.COL_UID]]\n            sub['prediction'] = 0\n            for fold in range(folds):\n                model = lgb.Booster(model_file=output_path + '/fold%s.ckpt' % fold)\n                test_preds = model.predict(test[features], num_iteration=model.best_iteration)\n                sub['prediction'] += (test_preds / folds)\n            sub[[self.COL_UID, 'prediction']].to_csv(output_path + '/submission.csv.zip', compression='zip',\n                                                     index=False)\n        if self.args.save_dir in output_path:\n            os.rename(output_path, output_root + run_id + '/')\n        return oof, sub, (mean_valid_metric, global_valid_metric)\n\n    def S3_series_feature(self):\n        train = pd.read_feather(self.get_path_train())\n\n        test = pd.read_feather(self.get_path_test())\n\n        eps = 1e-3\n\n        train_y = pd.read_csv(self._DIR_DATA_INPUT + self._FILE_TRAIN_LABEL)\n        train = train.merge(train_y, how='left', on=self.COL_UID)\n\n        print(train.shape, test.shape)\n\n        lgb_config = {\n            'lgb_params': {\n                'objective': 'binary',\n                'metric': 'binary_logloss',\n                'boosting': 'dart',\n                'max_depth': -1,\n                'num_leaves': 64,\n                'learning_rate': 0.035,\n                'bagging_freq': 5,\n                'bagging_fraction': 0.7,\n                'feature_fraction': 0.7,\n                'min_data_in_leaf': 256,\n                'max_bin': 63,\n                'min_data_in_bin': 256,\n                # 'min_sum_heassian_in_leaf': 10,\n                'tree_learner': 'serial',\n                'boost_from_average': 'false',\n                'lambda_l1': 0.1,\n                'lambda_l2': 30,\n                'num_threads': 24,\n                'verbosity': 1,\n            },\n            'feature_name': [col for col in train.columns if col not in [self.COL_UID, self.COL_LABEL, self.COL_TIME]],\n            'rounds': 4500,\n            'early_stopping_rounds': 100,\n            'verbose_eval': 50,\n            'folds': 5,\n            'seed': self.seed\n        }\n\n        self.Lgb_train_and_predict(train, test, lgb_config, gkf=True, aug=None, run_id='LGB_with_series_feature')\n\n    def compute_score(self):\n        train_y_orig = pd.read_csv(self._DIR_DATA + self._FILE_TRAIN_LABEL)\n        # merged = False\n        for run_id in ['LGB_with_series_feature', 'LGB_with_manual_feature', 'LGB_with_manual_feature_and_series_oof',\n                       'NN_with_series', 'NN_with_series_and_all_feature']:\n            # LGB_with_series_feature\n            _dir = self._DIR_OUTPUT + run_id + '/'\n            submission = pd.read_csv(_dir + 'submission.csv.zip')\n            submission.to_csv(_dir + 'submission.csv', index=False)\n            submission = submission.groupby(self.COL_UID).mean()['prediction'].reset_index()\n            train_y = train_y_orig.merge(submission, on=self.COL_UID)\n            score = self.amex_metric_mod(train_y[self.COL_LABEL], train_y['prediction'])\n            print(run_id + ' score: ' + str(score))\n\n    def GreedyFindBin(self, distinct_values, counts, num_distinct_values, max_bin, total_cnt, min_data_in_bin=3):\n        # INPUT:\n        #   distinct_values 保存特征取值的数组，特征取值单调递增\n        #   counts 特征的取值对应的样本数目\n        #   num_distinct_values 特征取值的数量\n        #   max_bin 分桶的最大数量\n        #   total_cnt 样本数量\n        #   min_data_in_bin 桶包含的最小样本数\n\n        # bin_upper_bound就是记录桶分界的数组\n        bin_upper_bound = list();\n        assert (max_bin > 0)\n\n        # 特征取值数比max_bin数量少，直接取distinct_values的中点放置\n        if num_distinct_values <= max_bin:\n            cur_cnt_inbin = 0\n            for i in range(num_distinct_values - 1):\n                cur_cnt_inbin += counts[i]\n                # 若一个特征的取值比min_data_in_bin小，则累积下一个取值，直到比min_data_in_bin大，进入循环。\n                if cur_cnt_inbin >= min_data_in_bin:\n                    # 取当前值和下一个值的均值作为该桶的分界点bin_upper_bound\n                    bin_upper_bound.append((distinct_values[i] + distinct_values[i + 1]) / 2.0)\n                    cur_cnt_inbin = 0\n            # 对于最后一个桶的上界则为无穷大\n            cur_cnt_inbin += counts[num_distinct_values - 1];\n            bin_upper_bound.append(float('Inf'))\n            # 特征取值数比max_bin来得大，说明几个特征值要共用一个bin\n        else:\n            if min_data_in_bin > 0:\n                max_bin = min(max_bin, total_cnt // min_data_in_bin)\n                max_bin = max(max_bin, 1)\n            # mean size for one bin\n            mean_bin_size = total_cnt / max_bin\n            rest_bin_cnt = max_bin\n            rest_sample_cnt = total_cnt\n            # 定义is_big_count_value数组：初始设定特征每一个不同的值的数量都小（false）\n            is_big_count_value = [False] * num_distinct_values\n            # 如果一个特征值的数目比mean_bin_size大，那么这些特征需要单独一个bin\n            for i in range(num_distinct_values):\n                # 如果一个特征值的数目比mean_bin_size大，则设定这个特征值对应的is_big_count_value为真。。\n                if counts[i] >= mean_bin_size:\n                    is_big_count_value[i] = True\n                    rest_bin_cnt -= 1\n                    rest_sample_cnt -= counts[i]\n            # 剩下的特征取值的样本数平均每个剩下的bin：mean size for one bin\n            mean_bin_size = rest_sample_cnt / rest_bin_cnt\n            upper_bounds = [float('Inf')] * max_bin\n            lower_bounds = [float('Inf')] * max_bin\n\n            bin_cnt = 0\n            lower_bounds[bin_cnt] = distinct_values[0]\n            cur_cnt_inbin = 0\n            # 重新遍历所有的特征值（包括数目大和数目小的）\n            for i in range(num_distinct_values - 1):\n                # 如果当前的特征值数目是小的\n                if not is_big_count_value[i]:\n                    rest_sample_cnt -= counts[i]\n                cur_cnt_inbin += counts[i]\n\n                # 若cur_cnt_inbin太少，则累积下一个取值，直到满足条件，进入循环。\n                # need a new bin 当前的特征如果是需要单独成一个bin，或者当前几个特征计数超过了mean_bin_size，或者下一个是需要独立成桶的\n                if is_big_count_value[i] or cur_cnt_inbin >= mean_bin_size or \\\n                        is_big_count_value[i + 1] and cur_cnt_inbin >= max(1.0, mean_bin_size * 0.5):\n                    upper_bounds[bin_cnt] = distinct_values[i]  # 第i个bin的最大就是 distinct_values[i]了\n                    bin_cnt += 1\n                    lower_bounds[bin_cnt] = distinct_values[i + 1]  # 下一个bin的最小就是distinct_values[i + 1]，注意先++bin了\n                    if bin_cnt >= max_bin - 1:\n                        break\n                    cur_cnt_inbin = 0\n                    if not is_big_count_value[i]:\n                        rest_bin_cnt -= 1\n                        mean_bin_size = rest_sample_cnt / rest_bin_cnt\n            #             bin_cnt+=1\n            # update bin upper bound 与特征取值数比max_bin数量少的操作类似，取当前值和下一个值的均值作为该桶的分界点\n            for i in range(bin_cnt - 1):\n                bin_upper_bound.append((upper_bounds[i] + lower_bounds[i + 1]) / 2.0)\n            bin_upper_bound.append(float('Inf'))\n        return bin_upper_bound\n\n    def S4_feature_combined(self):\n        def pad_target(x):\n            t = np.zeros(13)\n            t[:-len(x)] = np.nan\n            t[-len(x):] = x\n            return list(t)\n\n        root = self.args.root\n\n        oof = pd.read_csv('./output/LGB_with_series_feature/oof.csv')\n        sub = pd.read_csv('./output/LGB_with_series_feature/submission.csv.zip')\n\n        tmp1 = oof.groupby(self.COL_UID, sort=False)['target'].agg(lambda x: pad_target(x))\n        tmp2 = sub.groupby(self.COL_UID, sort=False)['prediction'].agg(lambda x: pad_target(x))\n\n        tmp = tmp1.append(tmp2)\n\n        tmp = pd.DataFrame(data=tmp.tolist(), columns=['target%s' % i for i in range(1, 14)])\n\n        df = []\n        for fn in ['cat', 'num', 'diff', 'rank_num', 'last3_cat', 'last3_num', 'last3_diff', 'last6_num',\n                   'ym_rank_num']:\n            if len(df) == 0:\n                df.append(pd.read_feather(f'{self._DIR_DATA}/{fn}_feature.feather'))\n            else:\n                df.append(pd.read_feather(f'{self._DIR_DATA}/{fn}_feature.feather').drop([self.COL_UID], axis=1))\n            if 'last' in fn:\n                df[-1] = df[-1].add_prefix('_'.join(fn.split('_')[:-1]) + '_')\n\n        df.append(tmp)\n\n        df = pd.concat(df, axis=1)\n        print(df.shape)\n        df.to_feather(f'{self._DIR_DATA}/all_feature.feather')\n\n        del df\n\n        df = pd.read_feather(self.get_path_train()).append(\n            pd.read_feather(self.get_path_test())).reset_index(drop=True)\n        df = df.drop(['S_2'], axis=1)\n        df = self.one_hot_encoding(df, self.cat_features, True)\n        for col in tqdm(df.columns):\n            if col not in ['customer_ID', 'S_2']:\n                df[col] /= 100\n            df[col] = df[col].fillna(0)\n\n        df.to_feather(self._DIR_DATA + 'nn_series.feather')\n\n        eps = 1e-3\n\n        dfs = []\n        for fn in ['cat', 'num', 'diff', 'rank_num', 'last3_cat', 'last3_num', 'last3_diff', 'last6_num',\n                   'ym_rank_num']:\n            if len(dfs) == 0:\n                dfs.append(pd.read_feather(f'{self._DIR_DATA}/{fn}_feature.feather'))\n            else:\n                dfs.append(pd.read_feather(f'{self._DIR_DATA}/{fn}_feature.feather').drop([self.COL_UID], axis=1))\n\n            if 'last' in fn:\n                dfs[-1] = dfs[-1].add_prefix('_'.join(fn.split('_')[:-1]) + '_')\n\n        for df in dfs:\n            for col in tqdm(df.columns):\n                if col not in ['customer_ID', 'S_2']:\n                    # v_min = df[col].min()\n                    # v_max = df[col].max()\n                    # df[col] = (df[col]-v_min+eps) / (v_max-v_min+eps)\n                    vc = df[col].value_counts().sort_index()\n                    if len(vc) > 0:\n                        bins = self.GreedyFindBin(vc.index.values, vc.values, len(vc), 255, vc.sum())\n                        df[col] = np.digitize(df[col], [-np.inf] + bins)\n                        df.loc[df[col] == len(bins) + 1, col] = 0\n                        df[col] = df[col] / df[col].max()\n\n        tmp = tmp.fillna(0)\n        dfs.append(tmp)\n        df = pd.concat(dfs, axis=1)\n\n        df.to_feather(self._DIR_DATA + 'nn_all_feature.feather')\n\n    def S5_LGB_main(self):\n        root = self.args.root\n        seed = self.args.seed\n\n        df = pd.read_feather(f'{self._DIR_DATA}/all_feature.feather')\n\n        train_y = pd.read_csv(f'{self._DIR_DATA_INPUT}/' + self._FILE_TRAIN_LABEL)\n\n        # todo: change this to use train_feather count\n        train_feather = pd.read_feather(self.get_path_train())\n        train_feather = train_feather.groupby(self.COL_UID).count()[self.COL_TIME]\n\n        train = df[:train_feather.shape[0]]\n        train = train.merge(train_y, left_on=self.COL_UID, right_on=self.COL_UID)\n        test = df[train_feather.shape[0]:].reset_index(drop=True)\n        del df\n\n        print(train.shape, test.shape)\n\n        lgb_config = {\n            'lgb_params': {\n                'objective': 'binary',\n                'metric': 'binary_logloss',\n                'boosting': 'dart',\n                'max_depth': -1,\n                'num_leaves': 64,\n                'learning_rate': 0.035,\n                'bagging_freq': 5,\n                'bagging_fraction': 0.75,\n                'feature_fraction': 0.05,\n                'min_data_in_leaf': 256,\n                'max_bin': 63,\n                'min_data_in_bin': 256,\n                # 'min_sum_heassian_in_leaf': 10,\n                'tree_learner': 'serial',\n                'boost_from_average': 'false',\n                'lambda_l1': 0.1,\n                'lambda_l2': 30,\n                'num_threads': 24,\n                'verbosity': 1,\n            },\n            'feature_name': [],\n            'rounds': 4500,\n            'early_stopping_rounds': 100,\n            'verbose_eval': 50,\n            'folds': 5,\n            'seed': seed\n        }\n\n        lgb_config = {\n            'lgb_params': {\n                'objective': 'binary',\n                'metric': 'binary_logloss',\n                'boosting': 'dart',\n                'max_depth': -1,\n                'num_leaves': 64,\n                'learning_rate': 0.035,\n                'bagging_freq': 5,\n                'bagging_fraction': 0.75,\n                'feature_fraction': 0.05,\n                'min_data_in_leaf': 256,\n                'max_bin': 63,\n                'min_data_in_bin': 256,\n                # 'min_sum_heassian_in_leaf': 10,\n                'tree_learner': 'serial',\n                'boost_from_average': 'false',\n                'lambda_l1': 0.1,\n                'lambda_l2': 30,\n                'num_threads': 24,\n                'verbosity': 1,\n            },\n            'feature_name': [col for col in train.columns if col not in [self.COL_UID, self.COL_LABEL,\n                                                                         self.COL_TIME] and 'skew' not in col and 'kurt' not in col and 'sub_mean' not in col and 'div_mean' not in col],\n            'rounds': 4500,\n            'early_stopping_rounds': 100,\n            'verbose_eval': 50,\n            'folds': 5,\n            'seed': seed\n        }\n        lgb_config['feature_name'] = [col for col in train.columns if\n                                      col not in [self.COL_UID, self.COL_LABEL, 'S_2'] and 'target' not in col]\n        self.Lgb_train_and_predict(train, test, lgb_config, aug=None, run_id='LGB_with_manual_feature')\n\n        lgb_config['feature_name'] = [col for col in train.columns if\n                                      col not in [self.COL_UID, self.COL_LABEL, self.COL_TIME]]\n        self.Lgb_train_and_predict(train, test, lgb_config, aug=None, run_id='LGB_with_manual_feature_and_series_oof')\n\n    def use_cuda(self):\n        return True\n\n    def NN_train_and_predict(self, train, test, model_class, config, use_series_oof, logit=False,\n                             output_root='./output/',\n                             run_id=None):\n        if not run_id:\n            run_id = 'run_nn_' + datetime.datetime.now().strftime('%Y%m%d_%H%M%S')\n            while os.path.exists(output_root + run_id + '/'):\n                time.sleep(1)\n                run_id = 'run_nn_' + datetime.datetime.now().strftime('%Y%m%d_%H%M%S')\n            output_path = output_root + f'{self.args.save_dir}/'\n        else:\n            output_path = output_root + run_id + '/'\n        if not os.path.exists(output_path):\n            os.makedirs(output_path)\n        os.system(f'copy ./*.py {output_path}'.replace('/', \"\\\\\"))\n        feature_name = config['feature_name']\n        obj_max = config['obj_max']\n        epochs = config['epochs']\n        smoothing = config['smoothing']\n        patience = config['patience']\n        lr = config['lr']\n        batch_size = config['batch_size']\n        folds = config['folds']\n        seed = config['seed']\n        if train is not None:\n            train_series, train_feature, train_y, train_series_idx = train\n\n            oof = train_y[[self.COL_UID]]\n            oof['fold'] = -1\n            oof[self.COL_LABEL] = 0.0\n            oof[self.COL_LABEL] = oof[self.COL_LABEL].astype(np.float32)\n        else:\n            oof = None\n\n        if train is not None:\n            log = open(output_path + 'train.log', 'w', buffering=1)\n            log.write(str(config) + '\\n')\n\n            all_valid_metric = []\n\n            skf = StratifiedKFold(n_splits=folds, shuffle=True, random_state=seed)\n\n            model_num = 0\n            train_folds = []\n\n            for fold, (trn_index, val_index) in enumerate(skf.split(train_y, train_y[self.COL_LABEL])):\n\n                train_dataset = TaskDataset(train_series, train_feature, [train_series_idx[i] for i in trn_index],\n                                            train_y)\n                train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, drop_last=True,\n                                              collate_fn=train_dataset.collate_fn, num_workers=self.args.num_workers,\n                                              pin_memory=True)\n                valid_dataset = TaskDataset(train_series, train_feature, [train_series_idx[i] for i in val_index],\n                                            train_y)\n                valid_dataloader = DataLoader(valid_dataset, batch_size=batch_size, shuffle=False, drop_last=False,\n                                              collate_fn=valid_dataset.collate_fn, num_workers=self.args.num_workers)\n\n                model = model_class(223, (6375 + 13) * 2, 1, 3, 128, use_series_oof=use_series_oof)\n                if run_id == 'NN_with_series_and_all_feature':\n                    model = model_class(223, 12820, 1, 3, 128, use_series_oof=use_series_oof)\n\n                scheduler = Adam12()\n\n                model.cuda()\n                if self.args.use_apm:\n                    scaler = amp.GradScaler()\n                optimizer = scheduler.schedule(model, 0, epochs)[0]\n\n                # optimizer = torch.optim.Adam(model.parameters(), lr=1e-2, weight_decay=1e-8)\n                # scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e5,\n                #                                                 max_lr=1e-2, epochs=epochs, steps_per_epoch=len(train_dataloader))\n                # torch.optim.Adam(model.parameters(),betas=(0.9, 0.99), lr=lr, weight_decay=0.00001,eps=1e-5)\n                gpus = list(range(torch.cuda.device_count()))\n                if len(gpus) > 1:\n                    model = nn.DataParallel(model, device_ids=gpus, output_device=gpus[0])\n\n                loss_tr = nn.BCELoss()\n                loss_tr1 = nn.BCELoss(reduction='none')\n                if obj_max == 1:\n                    best_valid_metric = 0\n                else:\n                    best_valid_metric = 1e9\n                not_improve_epochs = 0\n                if self.args.do_train:\n                    for epoch in range(epochs):\n                        # if epoch <= 13:\n                        #     continue\n                        np.random.seed(666 * epoch)\n                        train_loss = 0.0\n                        train_num = 0\n                        scheduler.step(model, epoch, epochs)\n                        model.train()\n                        bar = tqdm(train_dataloader)\n                        for data in bar:  # train_dataloader:\n                            optimizer.zero_grad()\n                            if self.use_cuda():\n                                for k in data:\n                                    data[k] = data[k].cuda()\n                            y = data['batch_y']\n                            if self.args.use_apm:\n                                with amp.autocast():\n                                    outputs = model(data)\n                                    # loss_series = loss_tr1(series_outputs,y.repeat(1,13))\n                                    # loss_series = (loss_series * data['batch_mask']).sum() / data['batch_mask'].sum()\n                                    # if epoch < 30:\n                                    #     loss = loss_series\n                                    # else:\n                                    loss = loss_tr(outputs,\n                                                   y)  # + loss_series # 0.5 * (loss_tr(outputs,y) + loss_feature(feature,y))\n                                if str(loss.item()) == 'nan': continue\n                                scaler.scale(loss).backward()\n                                torch.nn.utils.clip_grad_norm(model.parameters(), clipnorm)\n                                scaler.step(optimizer)\n                                scaler.update()\n                            else:\n                                outputs = model(data)\n                                loss = loss_tr(outputs, y)\n                                loss.backward()\n                                optimizer.step()\n                            # scheduler.step()\n                            train_num += data['batch_feature'].shape[0]\n                            train_loss += data['batch_feature'].shape[0] * loss.item()\n                            bar.set_description('loss: %.4f' % (loss.item()))\n\n                        train_loss /= train_num\n\n                        # eval\n                        model.eval()\n                        valid_preds = []\n                        for data in tqdm(valid_dataloader):\n\n                            # for data in valid_dataloader:\n                            if self.use_cuda():\n                                for k in data:\n                                    data[k] = data[k].cuda()\n                            with torch.no_grad():\n                                if logit:\n                                    outputs = model(data).sigmoid()\n                                    # feature,outputs = model(data)\n                                    # outputs = outputs.sigmoid()\n                                else:\n                                    outputs = model(data)\n                                    # feature,outputs = model(data)\n                            valid_preds.append(outputs.detach().cpu().numpy())\n\n                        valid_preds = np.concatenate(valid_preds).reshape(-1)\n                        valid_Y = train_y.iloc[val_index][self.COL_LABEL].values  # oof train\n                        valid_mean = np.mean(valid_preds)\n                        valid_metric = self.Metric(valid_Y, valid_preds)\n\n                        if obj_max * (valid_metric) > obj_max * best_valid_metric:\n                            if len(gpus) > 1:\n                                torch.save(model.module.state_dict(), output_path + 'fold%s.ckpt' % fold)\n                            else:\n                                torch.save(model.state_dict(), output_path + 'fold%s.ckpt' % fold)\n                            not_improve_epochs = 0\n                            best_valid_metric = valid_metric\n                            self.Write_log(log,\n                                           '[epoch %s] lr: %.6f, train_loss: %.6f, valid_metric: %.6f, valid_mean:%.6f' % (\n                                               epoch, optimizer.param_groups[0]['lr'], train_loss, valid_metric,\n                                               valid_mean))\n                        else:\n                            not_improve_epochs += 1\n                            self.Write_log(log,\n                                           '[epoch %s] lr: %.6f, train_loss: %.6f, valid_metric: %.6f, valid_mean:%.6f, NIE +1 ---> %s' % (\n                                               epoch, optimizer.param_groups[0]['lr'], train_loss, valid_metric,\n                                               valid_mean,\n                                               not_improve_epochs))\n                            if not_improve_epochs >= patience:\n                                break\n\n                # state_dict = torch.load(output_path + 'fold%s.ckpt'%fold, torch.device('cuda' if torch.cuda.is_available() else 'cpu') )\n                state_dict = torch.load(output_path + 'fold%s.ckpt' % fold,\n                                        torch.device('cuda' if self.use_cuda() else 'cpu'))\n\n                model = model_class(223, (6375 + 13) * 2, 1, 3, 128, use_series_oof=use_series_oof)\n                if run_id == 'NN_with_series_and_all_feature':\n                    model = model_class(223, 12820, 1, 3, 128, use_series_oof=use_series_oof)\n                if self.use_cuda():\n                    model.cuda()\n                model.load_state_dict(state_dict)\n                if len(gpus) > 1:\n                    model = nn.DataParallel(model, device_ids=gpus, output_device=gpus[0])\n\n                model.eval()\n\n                valid_preds = []\n                valid_Y = []\n                for data in tqdm(valid_dataloader):\n\n                    # for data in valid_dataloader:\n                    if self.use_cuda():\n                        for k in data:\n                            data[k] = data[k].cuda()\n                    with torch.no_grad():\n                        if logit:\n                            outputs = model(data).sigmoid()\n                            # feature,outputs = model(data)\n                            # outputs = outputs.sigmoid()\n                        else:\n                            outputs = model(data)\n                            # feature,outputs = model(data)\n                    valid_preds.append(outputs.detach().cpu().numpy())\n                    valid_Y.append(y.detach().cpu().numpy())\n\n                valid_preds = np.concatenate(valid_preds).reshape(-1)\n                valid_Y = train_y.iloc[val_index][self.COL_LABEL].values  # oof train\n                valid_mean = np.mean(valid_preds)\n                valid_metric = self.Metric(valid_Y, valid_preds)\n                self.Write_log(log,\n                               '[fold %s] best_valid_metric: %.6f, best_valid_mean: %.6f' % (\n                                   fold, valid_metric, valid_mean))\n\n                all_valid_metric.append(valid_metric)\n                oof.iloc[val_index, oof.columns.get_loc(self.COL_LABEL)] = valid_preds\n                oof.iloc[val_index, oof.columns.get_loc('fold')] = fold\n                train_folds.append(fold)\n\n            mean_valid_metric = np.mean(all_valid_metric)\n            self.Write_log(log, 'all valid mean metric:%.6f' % (mean_valid_metric))\n            oof.loc[oof['fold'].isin(train_folds)].to_csv(output_path + 'oof.csv', index=False)\n\n            if test is None:\n                log.close()\n                # os.rename(output_path + 'train.log', output_path + 'train_%.6f.log' % mean_valid_metric)\n                if os.path.exists(output_path + '/train_%.6f.log' % mean_valid_metric):\n                    remove(output_path + '/train_%.6f.log' % mean_valid_metric)\n                move(output_path + '/train.log', output_path + '/train_%.6f.log' % mean_valid_metric)\n\n\n            log_df = pd.DataFrame(\n                {'run_id': [run_id], 'folds': folds, 'metric': [round(mean_valid_metric, 6)], 'lb': [np.nan],\n                 'remark': [config['remark']]})\n            if not os.path.exists(output_root + 'experiment_log.csv'):\n                log_df.to_csv(output_root + 'experiment_log.csv', index=False)\n            else:\n                log_df.to_csv(output_root + 'experiment_log.csv', index=False, mode='a', header=None)\n\n        if test is not None:\n            if train is None:\n                log = open(output_path + 'test.log', 'w', buffering=1)\n                self.Write_log(log, str(config) + '\\n')\n            test_series, test_feature, test_series_idx = test\n\n            sub = test_feature[-len(test_series_idx):][[self.COL_UID]].reset_index(drop=True)\n            sub['prediction'] = 0\n\n            test_dataset = TaskDataset(test_series, test_feature, test_series_idx)\n            test_dataloader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, drop_last=False,\n                                         collate_fn=test_dataset.collate_fn, num_workers=self.args.num_workers)\n            models = []\n            for fold in range(folds):\n                if not os.path.exists(output_path + 'fold%s.ckpt' % fold):\n                    continue\n                model = model_class(223, (6375 + 13) * 2, 1, 3, 128, use_series_oof=use_series_oof)\n                if run_id == 'NN_with_series_and_all_feature':\n                    model = model_class(223, 12820, 1, 3, 128, use_series_oof=use_series_oof)\n                if self.use_cuda():\n                    model.cuda()\n                    state_dict = torch.load(output_path + 'fold%s.ckpt' % fold, torch.device('cuda'))\n                else:\n                    state_dict = torch.load(output_path + 'fold%s.ckpt' % fold, torch.device('cpu'))\n\n                model.load_state_dict(state_dict)\n                if len(gpus) > 1:\n                    model = nn.DataParallel(model, device_ids=gpus, output_device=gpus[0])\n\n                model.eval()\n                models.append(model)\n            print('model count:', len(models))\n            test_preds = []\n            with torch.no_grad():\n                for data in tqdm(test_dataloader):\n\n                    # for data in test_dataloader:\n                    if self.use_cuda():\n\n                        for k in data:\n                            data[k] = data[k].cuda()\n\n                    if logit:\n                        # outputs = model(data).sigmoid()\n                        outputs = torch.stack([m(data).sigmoid() for m in models], 0).mean(0)\n                        # feature,outputs = model(data)\n                        # outputs = outputs.sigmoid()\n                    else:\n                        # outputs = model(data)\n                        outputs = torch.stack([m(data) for m in models], 0).mean(0)\n                        # feature,outputs = model(data)\n                    test_preds.append(outputs.cpu().detach().numpy())\n            test_preds = np.concatenate(test_preds).reshape(-1)\n            test_mean = np.mean(test_preds)\n            self.Write_log(log, 'test_mean: %.6f' % (test_mean))\n            sub['prediction'] = test_preds\n            sub.to_csv(output_path + 'submission.csv.zip', index=False, compression='zip')\n        else:\n            sub = None\n\n        if self.args.save_dir in output_path:\n            os.rename(output_path, output_root + run_id + '/')\n        return oof, sub\n\n    def S6_NN_main(self):\n        x = datetime.datetime.now()\n        print('start: ', x)\n        df = pd.read_feather(self._DIR_DATA + 'nn_series.feather')\n        y = pd.read_csv(self._DIR_DATA_INPUT + 'train_labels.csv')\n\n        # todo: change this to use train_feather count\n        if True:\n            train_feather = pd.read_feather(self.get_path_train())\n            train_feather = train_feather.groupby('customer_ID').count()['S_2']\n        # df_uid = df.groupby(self.COL_UID).count()['index']\n        y = y.merge(train_feather, left_on=self.COL_UID, right_index=True).drop(columns='S_2')\n\n\n        f = pd.read_feather(self._DIR_DATA + 'nn_all_feature.feather')\n        df['idx'] = df.index\n        series_idx = df.groupby(self.COL_UID, sort=False).idx.agg(['min', 'max'])\n        series_idx['feature_idx'] = np.arange(len(series_idx))\n        df = df.drop(['idx'], axis=1)\n        print(f.head())\n        nn_config = {\n            'id_name': self.COL_UID,\n            'feature_name': [],\n            'label_name': self.COL_LABEL,\n            'obj_max': 1,\n            'epochs': 10,\n            'smoothing': 0.001,\n            'clipnorm': 1,\n            'patience': 100,\n            'lr': 3e-4,\n            'batch_size': 256,\n            'folds': 5,\n            'seed': self.args.seed,\n            'remark': self.args.remark\n        }\n        if True:\n            self.args.do_train = True\n            self.args.batch_size = 512\n            # https://github.com/pytorch/pytorch/issues/2341\n            self.args.num_workers = 1\n        if True:\n            self.NN_train_and_predict([df, f, y, series_idx.values[:y.shape[0]]],\n                                      [df, f, series_idx.values[y.shape[0]:]],\n                                      Amodel, nn_config, use_series_oof=False, run_id='NN_with_series')\n\n        self.NN_train_and_predict([df, f, y, series_idx.values[:y.shape[0]]], [df, f, series_idx.values[y.shape[0]:]],\n                                  Amodel, nn_config, use_series_oof=True, run_id='NN_with_series_and_all_feature')\n        x = datetime.datetime.now()\n        print('end: ', x)\n\n\n","metadata":{"execution":{"iopub.status.busy":"2022-09-23T12:41:20.695762Z","iopub.execute_input":"2022-09-23T12:41:20.696393Z","iopub.status.idle":"2022-09-23T12:41:20.887233Z","shell.execute_reply.started":"2022-09-23T12:41:20.696359Z","shell.execute_reply":"2022-09-23T12:41:20.886295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"    model = model_amex()\n    if False:\n        model.S1_denoise()\n        model.S1pnt1_down_sample()\n\n    if True:\n        model.S2_manial_feature()\n\n        model.S3_series_feature()\n        model.S4_feature_combined()\n        model.S5_LGB_main()\n    if True:\n\n\n        \n        model.S6_NN_main()\n        model.compute_score()","metadata":{"execution":{"iopub.status.busy":"2022-09-23T12:41:20.888586Z","iopub.execute_input":"2022-09-23T12:41:20.888951Z","iopub.status.idle":"2022-09-23T18:06:38.555175Z","shell.execute_reply.started":"2022-09-23T12:41:20.888917Z","shell.execute_reply":"2022-09-23T18:06:38.553249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}