{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"We want to share method We used to extract very usefull set of features. This notebook creates 400-500 features and in all our models at least 6-7 of them were in top 10 features in any model. We call it SequentialEncoder and it produces 1 feature per 1 raw feature. \nFor example, `P_2_seq` feature much more informative than `P_2_last`.\n\n\nThe classic way of extracting features from a sequence (in our case sequence of length 13) is to use reduce functions such as mean, min, max, std, first, last etc and then feed them into a LGBM and in many cases it's good enough. But this method does not takes into consideration the time dynamics of the sequence.\n\n\nOne of the methods to extract it is LSTM. One can simply feed a univariate feature into the LSTM model and using k-Fold extract prediction for each sample. Than use those predictions as a feature in any tabular model. You do not have to use LSTM - there is plenty other architectures, such as many variations of 1D-CNN. Those models are great for extracting signal from time-series based models. \n\n\nUnfortunately, in our tests those models was not able to achieve decent performance, probably due to short length of the sequences. Instead of this, we chose to use tree-based version of that approach. We extracted many features from each sequence (602 features in total) which are averages of different parts of the sequence (different positions and different lengths), also we subtracted averages from other averages to catch the derivatives of the sequence. For example `np.mean(P_2[7:12]) - np.mean(P_2[3:5])`. Then we feed all those features into lightgbm with 5 folds and used OOF as a stand-alone feature.\n\n\nWe extracted those features for every raw feature and also for different interaction of features (example - `P_2-B_9`)\n\n---\n\n**Important** to use this features in addition to other features and not as substitution: This feature manage to capture lot of signal, but lose all cross-feature information.\n\n\nI also want to apologize for not polishing the notebook. You cant run it here, so it doesn't make sence :) Just sharing ideas with the best community ever :)\n\nThis is not the full solution - only part. More to come... ","metadata":{}},{"cell_type":"code","source":"import gc\nimport warnings\nimport pathlib\nimport pandas as pd\nimport random\nimport numpy as np\nimport os\nfrom sklearn.preprocessing import LabelEncoder\nimport plotly.express as px\n\nimport joblib\nimport pathlib\nimport lightgbm as lgb\nimport gc\nfrom functools import reduce\nimport re\nimport itertools\nfrom tqdm import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"warnings.filterwarnings('ignore')\npd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\npd.set_option('display.width', 1000)\n\ndataset_name = 'final'\nroot_folder = pathlib.Path('choose_your_own_path')\ninput_folder = root_folder / 'input'\nfeatures_folder = root_folder / 'features' / dataset_name\n\nfeatures_folder.mkdir(parents=True, exist_ok=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    seed = 16\n    n_folds = 5\n    target = 'target'\n\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    try:\n        import torch\n        torch.manual_seed(seed)\n        torch.cuda.manual_seed(seed)\n        torch.backends.cudnn.deterministic = True\n    except Exception as e:\n        print('torch was not seeded!')\n\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_payment_rank(df):\n    df.S_2 = pd.to_datetime(df.S_2)\n    df['rank'] = df.groupby('customer_ID').S_2.rank()\n    df['total_rows'] = df['customer_ID'].map(df.groupby('customer_ID')['rank'].max())\n    df['rank'] = df['rank'] - df['total_rows'] + 13\n    df['month'] = pd.to_datetime(df.S_2.dt.year.astype(str) + '-' + df.S_2.dt.month.astype(str) + '-01')\n    df['cohort'] = df.customer_ID.map(df.groupby('customer_ID').month.max())\n    month_dict = {cohort: df.loc[lambda dx: dx.cohort.eq(cohort)].loc[lambda dx: dx.total_rows.eq(13)].loc[:,\n                          ['rank', 'month']].drop_duplicates().set_index('month').to_dict()['rank'] for cohort in\n                  df.cohort.unique()}\n    for cohort in df.cohort.unique():\n        df.loc[lambda dx: dx.cohort.eq(cohort), 'rank'] = df.loc[lambda dx: dx.cohort.eq(cohort)].month.map(\n            month_dict[cohort])\n\n    df['first_rank'] = df.customer_ID.map(df.groupby(['customer_ID'])['rank'].min())\n    df['rank2'] = df['rank'] - df['first_rank'] + 1\n    df = df.set_index('customer_ID')\n    df['len_period'] = df.groupby('customer_ID')['rank2'].max()\n    return df.drop(columns=['month', 'first_rank'])\n  \n    \ndef numeric_feature_eng(df, num_features, funcs):\n    df_num_agg = df.groupby(\"customer_ID\")[num_features].agg(funcs)\n    df_num_agg.columns = ['_'.join(x) for x in df_num_agg.columns]\n    return df_num_agg\n\n\ndef categorical_feature_eng(df, cat_features, funcs):\n    df_cat_agg = df.groupby(\"customer_ID\")[cat_features].agg(funcs)\n    df_cat_agg.columns = ['_'.join(x) for x in df_cat_agg.columns]\n    return df_cat_agg\n\n\ndef pre_flattening(df):\n    for bcol in ['B_11', 'B_14', 'B_17', 'D_39','D_131', 'S_16', 'S_23']:\n        for pcol in ['P_2','P_3']:\n            if bcol in df.columns:\n                df[f'{bcol}-{pcol}'] = df[bcol] - df[pcol]\n    return df\n\ndef check_input(arr):\n    \"\"\"\n    Check the input\n    \"\"\"\n    if type(arr) is pd.DataFrame:\n        arr = arr[arr.columns[0]]\n        \n    if type(arr) is pd.Series:\n        arr = arr.values\n        \n    if len(arr.shape) > 1:\n        arr = arr[:, 0]\n        \n    return arr\n\n\ndef gini(cs_0, cs_1, sum_0, sum_1):\n    \"\"\"\n    Gini part\n    \"\"\"\n    auc_ = (cs_0 - sum_0 / 2) * sum_1\n    tot = cs_0[-1] * cs_1[-1]\n\n    return 2 * float(auc_.sum() / tot) - 1\n\n\ndef recall_at4(cs_0, cs_1, sum_1):\n    \"\"\"\n    Recall part\n    \"\"\"\n    cs_tot = cs_0 + cs_1\n    th = cs_tot[-1] * 0.96\n    \n    return float(sum_1[cs_tot >= th].sum() / cs_1[-1])\n    \n    \ndef _amex_score(y_true, y_pred):\n    \"\"\"\n    Faster NumPy metric implementation\n    \"\"\"\n    y_true = check_input(y_true)\n    y_pred = check_input(y_pred)\n\n    sum_1 = y_true[y_pred.argsort()]\n    sum_0 = (1 - sum_1) \n    sum_0 *= 20\n    \n    cs_0, cs_1 = np.cumsum(sum_0, dtype=np.float64), np.cumsum(sum_1, dtype=np.float64)\n    \n    g = gini(cs_0, cs_1, sum_0, sum_1)\n    d = recall_at4(cs_0, cs_1, sum_1)\n    \n    return (g + d) / 2\n  \ndef amex_score(y_true, y_pred):\n    return np.max([_amex_score(y_true, y_pred), _amex_score(y_true, -y_pred)])\n  \n    \ndef generate_features(data: pd.DataFrame, feature_name: str):\n    dt = data.set_index('rank', append=True).loc[:, feature_name].unstack()\n    metadata = dict()\n\n    for idx, col in enumerate(dt.columns, start=1):\n        metadata[col] = dict(min_ts=idx, max_ts=idx)\n    df = []\n    for e in range(1, 14):\n        for s in range(1, e):\n            if e - s < 3:\n                continue\n            name = f'avg({s}:{e})'\n            df.append(dt.iloc[:, s:e].mean(1).to_frame(name))\n            metadata[name] = dict(min_ts=s, max_ts=e)\n\n    df = pd.concat(df, axis=1)\n    df = df.join(dt)\n    for col1, col2 in itertools.permutations(df.columns, 2):\n        if metadata[col1]['min_ts'] <= metadata[col2]['max_ts']:\n            continue\n        df[f'sub({col1}, {col2})'] = df[col1] - df[col2]\n        all_ts = [metadata[col1]['min_ts'], metadata[col1]['max_ts'], metadata[col2]['min_ts'],\n                  metadata[col2]['max_ts']]\n\n        metadata[f'sub({col1}, {col2})'] = dict(min_ts=np.min(all_ts), max_ts=np.max(all_ts))\n    df = df.add_prefix(f'{feature_name}_')\n    print(f'shape: {df.shape}')\n\n    df['fold_id'] = df.reset_index().customer_ID.str[-16:].apply(lambda x: int(x, 16) % CFG.n_folds).values\n\n    df = df.rename(columns=lambda x: re.sub('[^A-Za-z0-9_]+', '_', x))\n    return df\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seed_everything(CFG.seed)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SequentialTransformer:\n    def __init__(self, seed, n_folds, target, feature_name):\n        self.models = []\n        self.seed = seed\n        self.n_folds = n_folds\n        self.target = target\n        self.feature_name = feature_name\n        self.features = []\n\n    def fit(self, X, y):\n        df = X.join(y)\n        self.features = [col for col in df.columns if col not in ['fold_id', 'target']]\n\n        params = {\n            'objective': 'binary',\n            'metric': \"binary_logloss\",\n            'seed': self.seed\n        }\n\n        print(f'numerical features ({len(self.features)}): ')\n        for i in range(0, len(self.features), 10):\n            print(f'\\t{self.features[i:i + 10]}')\n        oof = []\n        for fold_id in range(self.n_folds):\n            print(' ')\n            print('-' * 50)\n            print(f'Training fold {fold_id} with {len(self.features)} features...')\n\n            x_trn = df.loc[lambda dx: dx.fold_id.ne(fold_id), self.features]\n            x_val = df.loc[lambda dx: dx.fold_id.eq(fold_id), self.features]\n            y_trn = df.loc[lambda dx: dx.fold_id.ne(fold_id), self.target]\n            y_val = df.loc[lambda dx: dx.fold_id.eq(fold_id), self.target]\n\n            lgb_train = lgb.Dataset(x_trn, y_trn)\n            lgb_valid = lgb.Dataset(x_val, y_val)\n\n            model = lgb.train(params=params,\n                              train_set=lgb_train,\n                              num_boost_round=100000,\n                              valid_sets=[lgb_train, lgb_valid],\n                              early_stopping_rounds=50,\n                              verbose_eval=100\n                              )\n            val_pred = model.predict(x_val)\n            score = amex_score(y_val, val_pred)\n            print(f'Our fold {fold_id} CV score is {score}')\n            oof.append(pd.DataFrame(val_pred, index=x_val.index, columns=[self.feature_name]))\n            self.models.append(model)\n        oof = pd.concat(oof)\n        r = oof.join(y)\n        print(amex_score(r.target, r[self.feature_name]))\n        return self\n\n    def predict(self, X):\n        predictions = []\n        for model in self.models:\n            predictions.append(pd.DataFrame(model.predict(X.loc[:, self.features]),\n                                            index=X.index,\n                                            columns=[self.feature_name]))\n        predictions = pd.concat(predictions).groupby(level=0).mean()\n        return predictions","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_parquet(input_folder / 'train.parquet').pipe(prepare_payment_rank)\ntrain_labels = pd.read_csv(input_folder / 'train_labels.csv').set_index('customer_ID')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_features = ['D_51-R_7', 'P_2-D_72', 'D_41-B_33', 'P_2-S_17', 'B_2-R_1', 'B_2-R_2', 'D_51-S_23', 'R_2-B_6', 'B_5-S_22', 'D_41-B_10', 'P_2-D_109',\n                'P_2-B_22', 'P_2-R_17', 'R_5-D_115', 'P_2-D_111', 'P_2-D_41', 'P_2-S_16', 'P_2-B_28', 'P_2-R_8', 'D_44-D_47', 'D_51-D_65', 'P_2-R_10',\n                'B_2-D_58', 'B_12-D_131', 'B_2-B_9', 'P_2-B_3', 'D_81-D_115', 'D_71-R_7', 'P_2-R_18', 'P_2-B_26', 'D_51-S_5', 'P_2-B_1', 'R_4-D_115',\n                'R_6-B_33', 'D_45-R_4', 'P_2-S_23', 'P_2-D_93', 'B_6-B_9', 'S_23-P_3', 'B_25-D_112', 'D_41-D_45', 'P_2-R_7', 'P_2-D_87', 'P_2-B_31',\n                'B_5-S_24', 'D_45-D_84', 'P_2-B_21', 'D_52-D_65', 'P_2-R_15', 'P_2-D_133', 'R_2-S_25', 'B_2-D_41', 'D_47-R_4', 'S_5-B_12', 'S_23-P_2',\n                'R_2-D_52', 'D_51-B_26', 'P_2-B_9', 'S_16-P_2', 'D_78-B_33', 'D_45-R_5', 'D_51-R_6', 'D_71-R_8', 'D_45-B_22', 'S_16-P_3', 'R_2-D_71',\n                'B_2-D_84', 'D_47-B_22', 'P_2-D_83', 'P_2-R_22', 'P_2-D_78', 'P_2-R_25', 'P_2-R_2', 'R_1-D_52', 'P_2-R_19', 'D_47-R_5', 'P_2-D_140',\n                'P_2-B_32', 'B_2-D_44', 'D_51-D_79', 'B_8-D_112', 'P_2-S_20', 'P_2-D_138', 'B_12-S_24', 'D_71-D_81', 'R_2-B_10', 'D_45-D_72', 'D_52-R_5',\n                'D_52-R_6', 'B_9-D_52', 'D_44-B_33', 'P_2-B_37', 'D_52-S_23', 'D_41-D_47', 'P_2-B_27', 'R_2-D_47', 'B_17-P_2', 'P_2-R_24', 'D_51-R_5',\n                'P_2-R_5', 'D_47-D_78', 'D_39-P_3', 'B_5-B_9', 'B_9-B_12', 'B_14-P_2', 'D_71-D_72', 'D_51-R_10', 'D_51-D_84', 'B_17-P_3', 'D_45-D_65',  \n                'D_131-P_2', 'R_1-D_47',  'D_71-D_84', 'D_45-B_9', 'D_45-R_2', 'D_52-B_22',  'D_52-D_84', 'D_84-B_33', 'D_41-B_18', 'D_51-B_14', 'B_9-D_54', \n                'S_5-S_13', 'R_5-D_71','B_2-S_23', 'B_9-B_10', 'D_39-P_2',  'D_45-R_10', 'D_51-S_24', 'B_22-D_112',   'D_51-D_81', 'R_8-D_115', 'B_12-D_79', \n                'R_1-B_6',  'D_51-R_4',  'B_9-B_18', 'B_12-B_14',  'R_4-D_71', 'D_45-R_7',  'S_8-S_22', 'B_33-S_23', 'B_11-P_3', 'S_13-S_24',   'R_2-D_51', \n                'R_1-B_33','S_8-S_24', 'R_2-B_18',  'D_51-D_72',   'D_52-B_28', 'D_51-R_26', 'D_47-D_65', 'R_2-B_33', 'D_72-D_115',  'R_1-B_18', 'R_8-D_121',  \n                'D_51-B_9', 'B_11-P_2', 'D_131-P_3', 'D_41-D_52', 'D_52-D_78', 'R_1-D_45', 'D_47-S_23', 'D_45-R_6',  'D_47-R_6', 'D_47-D_84', 'D_45-D_78', \n                'D_47-B_9', 'R_1-B_10', 'D_71-R_10', 'B_9-B_33', 'D_47-R_10',   'D_45-S_23', 'D_45-D_81', 'B_14-P_3', 'D_45-B_26',  'P_2-B_11']","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_new_seq_set(df):\n    new_df = [df.loc[:, ['rank']]]\n    for col in new_features:\n        print(col)\n        c1, c2 = col.split('-')\n        new_df.append((df[c1] - df[c2]).to_frame(col))\n    new_df = pd.concat(new_df, axis=1)\n    print(new_df.shape)\n    return new_df","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = create_new_seq_set(train).join(train_labels)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for idx, feature_name in enumerate(train.columns):\n    try:\n        print(f'starting {feature_name} {idx}/{len(train.columns)}')\n        model_name = feature_name\n        model_path = root_folder / 'models' / 'transactional' / model_name / 'model.pkl'\n        if model_path.exists():\n            continue\n\n        df = generate_features(data=train, feature_name=feature_name)\n        df = df.join(train_labels)\n        X = df.drop(columns=[CFG.target])\n        y = df.loc[:, CFG.target]\n\n        model = SequentialTransformer(seed=42, n_folds=CFG.n_folds, target=CFG.target, feature_name=feature_name)\n        model.fit(X, y)\n\n\n        models_folder = root_folder / 'models' / 'transactional' / model_name\n        oof_folder = root_folder / 'oof' / 'transactional' / model_name\n        predictions_folder = root_folder / 'predictions' / 'transactional' / model_name\n\n        models_folder.mkdir(parents=True, exist_ok=True)\n        oof_folder.mkdir(parents=True, exist_ok=True)\n        predictions_folder.mkdir(parents=True, exist_ok=True)\n        joblib.dump(model, model_path)\n        gc.collect()\n    except Exception as e:\n        print(f'failed to process {feature_name}')\n        print(e)\n        pass","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for idx, feature_name in enumerate(train.columns):\n    try:\n        print(f'starting {feature_name} {idx}/{len(train.columns)}')\n        model_name = feature_name\n\n        models_folder = root_folder / 'models' / 'transactional' / model_name\n        oof_folder = root_folder / 'oof' / 'transactional' / model_name\n\n        if (oof_folder / 'oof.parquet').exists():\n            continue\n        model = joblib.load(models_folder / 'model.pkl')\n        X = generate_features(data=train, feature_name=feature_name)\n\n\n        oof = []\n        for fold_id, m in enumerate(model.models):\n            x_val = X.loc[lambda dx: dx.fold_id.eq(fold_id), model.features]\n            val_pred = m.predict(x_val)\n            oof.append(pd.DataFrame(val_pred, index=x_val.index, columns=[feature_name]))\n        oof = pd.concat(oof)\n        oof.to_parquet(oof_folder / 'oof.parquet')\n    except Exception as e:\n        print(f'failed to predict {feature_name}')\n        print(e)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = (pd.concat([pd.read_parquet(input_folder / f'test_chunk_{chunk_id}.parquet') \n                   for chunk_id in list('0123456789abcdef')])\n        .pipe(prepare_payment_rank)\n       )\n\ntest = create_new_seq_set(test)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for idx, feature_name in enumerate(test.columns[1:]):\n    break\nprint(f'starting {feature_name} {idx}/{len(test.columns)}')\nmodel_name = feature_name\n\nmodels_folder = root_folder / 'models' / 'transactional' / model_name\noof_folder = root_folder / 'oof' / 'transactional' / model_name\npredictions_folder = root_folder / 'predictions' / 'transactional' / model_name\n\nmodels_folder.mkdir(parents=True, exist_ok=True)\noof_folder.mkdir(parents=True, exist_ok=True)\npredictions_folder.mkdir(parents=True, exist_ok=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for idx, feature_name in enumerate(test.columns):\n    print(f'starting {feature_name} {idx}/{len(test.columns)}')\n    try:\n        model_name = feature_name\n        models_folder = root_folder / 'models' / 'transactional' / model_name\n        oof_folder = root_folder / 'oof' / 'transactional' / model_name\n        predictions_folder = root_folder / 'predictions' / 'transactional' / model_name\n        savepath = predictions_folder / 'predictions.parquet'\n        if savepath.exists():\n            continue\n        models_folder.mkdir(parents=True, exist_ok=True)\n        oof_folder.mkdir(parents=True, exist_ok=True)\n        predictions_folder.mkdir(parents=True, exist_ok=True)\n        model = joblib.load(models_folder / 'model.pkl')\n        x_tst = generate_features(data=test, feature_name=feature_name)\n        predictions = pd.DataFrame(model.predict(x_tst), index=x_tst.index, columns=[feature_name])\n        predictions.reset_index().to_parquet(savepath)\n    except Exception as e:\n        print(f'failed to predict {feature_name}')\n        print(e)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof_folder = root_folder / 'oof' / 'transactional'\noof = [pd.read_parquet(p) for p in tqdm(oof_folder.rglob('*.parquet'))]\noof = pd.concat(oof, axis=1).add_suffix('_seq')\noof.to_parquet(features_folder / 'train_seq.parquet')\n\n# ##\n\npredictions_folder = root_folder / 'predictions' / 'transactional'\npredictions = [pd.read_parquet(p).set_index('customer_ID') for p in tqdm(predictions_folder.rglob('*.parquet'))]\npredictions = pd.concat(predictions, axis=1).add_suffix('_seq')\npredictions.to_parquet(features_folder / 'test_seq.parquet')","metadata":{},"execution_count":null,"outputs":[]}]}