{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"scrolled":true},"cell_type":"code","source":"%matplotlib inline\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nimport os\nimport gc\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\n\nDATA_DIR = '../input/avito-demand-prediction/'\ntextdata_path = '../input/adp-prepare-kfold-text/textdata.csv'\ntarget_col = 'deal_probability'\nos.listdir(DATA_DIR)","execution_count":19,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"collapsed":true},"cell_type":"code","source":"usecols = ['user_id', #'item_id',\n           'region', 'city', 'parent_category_name', 'category_name', \n           'param_1', 'param_2', 'param_3', \n           'activation_date',\n           'title', 'description', \n           'price', 'item_seq_number', \n           'user_type', \n           'image_top_1', 'image']\neval_sets = pd.read_csv(textdata_path, usecols=['eval_set'])['eval_set'].values\ntrain_num = (eval_sets!=10).sum()\neval_sets = eval_sets[:train_num]\ntrain = pd.read_csv(DATA_DIR+'train.csv', usecols=usecols+[target_col])\ntest = pd.read_csv(DATA_DIR+'test.csv', usecols=usecols)","execution_count":20,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ec2b0b119b870d084226cb6dfd3d8b61d6181e1"},"cell_type":"code","source":"def get_dow(df):\n    f = lambda x:pd.to_datetime(x).dayofweek\n    unq = df['activation_date'].unique().tolist()\n    d = dict([u, f(u)] for u in unq)\n    df['dow'] = df['activation_date'].map(d.get)\n    return df\ntrain = get_dow(train)\ntest = get_dow(test)\ndel train['activation_date'], test['activation_date']; gc.collect()","execution_count":21,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"284150e83adf5d084363cef5f0304e37df068bd8"},"cell_type":"code","source":"len(set(train['user_id'].values.tolist()) & set(test['user_id'].values.tolist())), len(set(test['user_id'].values.tolist()))","execution_count":22,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"70a837911b84fc6734325f590e85a8347cface4b"},"cell_type":"code","source":"common_indexes = set(train['user_id'].values.tolist()) & set(test['user_id'].values.tolist())\ncommon_indexes = list(common_indexes)\ntrain['user_common'] = 0\ntest['user_common'] = 0\ntrain = train.set_index('user_id')\ntest = test.set_index('user_id')\ntrain.loc[common_indexes, 'user_common'] = 1\ntest.loc[common_indexes, 'user_common'] = 1\ntrain = train.reset_index()\ntest = test.reset_index()\ndel common_indexes; gc.collect()","execution_count":23,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"6966eb97051715b69a3f25579c550f6cf30b1a9f"},"cell_type":"code","source":"train['user_id_common'] = train['user_id'].values\ntrain.loc[train['user_common']==0, 'user_id_common'] = 'unknown'\ntest['user_id_common'] = test['user_id'].values\ntest.loc[test['user_common']==0, 'user_id_common'] = 'unknown'","execution_count":24,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef2f848be55b191fde5d4c6004817cd531dbd1c2"},"cell_type":"code","source":"train.head(3).T","execution_count":25,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d439f9f9500a5831806dd6435823dc9d68e7a0ef"},"cell_type":"code","source":"train_num == len(train)","execution_count":26,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b8fca9e67c9729ed74a37480cce0afd5e7eb4434"},"cell_type":"code","source":"y = train[target_col].values\ndel train[target_col]; gc.collect()\ntrain_num = len(train)\ndf = pd.concat([train, test], ignore_index=True)\ndel train, test; gc.collect()","execution_count":27,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"901801c179b83a37bb22e51f47baee46ed03c795"},"cell_type":"code","source":"df['image'].isnull().sum()","execution_count":28,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"a2605cc08df180d21fd606a6d7211afd76757794"},"cell_type":"code","source":"df['image'] = (~df['image'].isnull()).astype('int8')","execution_count":29,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"3d98ecfba2194930034d370642a84c45864cae76"},"cell_type":"code","source":"del df['user_common']; gc.collect();","execution_count":30,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2da64aeaf0949a2c92855f7ed75a649d41cbbd08"},"cell_type":"code","source":"df['image_top_1'].isnull().sum()","execution_count":31,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c1ef71596234dc70c87e5647b0adb97e62f286d9"},"cell_type":"code","source":"df['image_top_1'].min(), df['image_top_1'].max()","execution_count":32,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a2382c044a334689bfae5b171f097cd13ec2d8e1"},"cell_type":"code","source":"df.head(3).T","execution_count":33,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7d08dac6bf0123639fd358560c220147bebd134a","collapsed":true},"cell_type":"code","source":"df['price_bin'] = pd.cut(np.log1p(df['price']), 256, labels=np.arange(256))\ndf['price_bin'] = df['price_bin'].astype('float').fillna(-1)\ndf['price_bin'] = df['price_bin'].astype('int')","execution_count":34,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"664f5570f1a1b681e3a45b19a058070d48bc7eb4"},"cell_type":"code","source":"df['item_seq_bin'] = pd.cut(np.log1p(df['item_seq_number']), 512, labels=np.arange(512))\ndf['item_seq_bin'] = df['item_seq_bin'].astype('float').fillna(-1)\ndf['item_seq_bin'] = df['item_seq_bin'].astype('int')","execution_count":35,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fc88ce257f8e5d7a0ddb2f6552fe1fc4ad558e84"},"cell_type":"code","source":"enc_cols = ['user_id', 'user_id_common',\n            'region', 'city', 'parent_category_name', 'category_name', \n            'param_1', 'param_2', 'param_3', \n            'user_type', #'price_bin',\n            'image_top_1']\n\nenc_dict = {}\nfor i, c in enumerate(enc_cols):\n    print('label encoding', i, c)\n    values, names = pd.factorize(df[c].fillna('unknown'))\n    df[c] = values\n    enc_dict[c] = pd.DataFrame(names.values, columns=['lbe'])\n    #enc_dict[c].to_csv(c+'_enc.csv', index=False)","execution_count":36,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"546e1baca38a44f3ee4a226a052f0c4b9a9029eb"},"cell_type":"code","source":"df.head(3).T","execution_count":37,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7f059d0fb804f8fdfd9a26c5eae5b8cec77c80d3"},"cell_type":"code","source":"del df['title'], df['description']; gc.collect();\ndf.info()","execution_count":38,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"13f60f75142dd558fa815acf69f1b9c4395de4f3","scrolled":false},"cell_type":"code","source":"def reduce_memory(df):\n    for c in df.columns:\n        if df[c].dtype=='int':\n            if df[c].min()<0:\n                if df[c].abs().max()<2**7:\n                    df[c] = df[c].astype('int8')\n                elif df[c].abs().max()<2**15:\n                    df[c] = df[c].astype('int16')\n                elif df[c].abs().max()<2**31:\n                    df[c] = df[c].astype('int32')\n                else:\n                    continue\n            else:\n                if df[c].max()<2**8:\n                    df[c] = df[c].astype('uint8')\n                elif df[c].max()<2**16:\n                    df[c] = df[c].astype('uint16')\n                elif df[c].max()<2**32:\n                    df[c] = df[c].astype('uint32')\n                else:\n                    continue\n    return df\ndf = reduce_memory(df)\nprint(df.info())","execution_count":39,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f44e7b81dd8234a3d313e8e7656bb2fc9d7c01b6","collapsed":true},"cell_type":"code","source":"cols = ['user_id',\n        'user_id_common',\n        'region',\n        'city',\n        'parent_category_name',\n        'category_name',\n        'param_1',\n        'param_2',\n        'param_3',\n        'price', 'price_bin',\n        'item_seq_number', 'item_seq_bin',\n        'user_type',\n        'image',\n        'image_top_1',\n        'dow']\ndf = df[cols]","execution_count":41,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"9451146317a9382ee96baf0719cf9266f23622fe"},"cell_type":"code","source":"#df.to_csv('data_lbe.csv', index=False)","execution_count":42,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"683a363bfef90545061a2e1eb611ca7b2624e83e"},"cell_type":"code","source":"df.shape","execution_count":43,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7770fd6c702ada9817974fef667a25378d3ef164"},"cell_type":"code","source":"train = df[:train_num].reset_index(drop=True)\ntest  = df[train_num:].reset_index(drop=True)\ntrain[target_col] = y\ntrain.head(2).T","execution_count":44,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3f7c63770e6855cbc57ca10d0fe0c9a6743fd5da","_kg_hide-output":false},"cell_type":"code","source":"ts_folds = pd.read_csv(DATA_DIR+'train.csv', usecols=['activation_date'])\nts_folds.groupby('activation_date').size()","execution_count":45,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dfba48e7f5336bf2bc1fbe64cd5616229051dd12","collapsed":true},"cell_type":"code","source":"ts_folds = ts_folds['activation_date'].values\nts_folds[ts_folds>'2017-03-28'] = '2017-03-28'","execution_count":46,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9c880518aa611a8de37fcdf799a423bc24cb4629"},"cell_type":"code","source":"pd.DataFrame(ts_folds).groupby(0).size()","execution_count":47,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fd93d1dc076c8b0083086acbac3b48cbad23a250"},"cell_type":"code","source":"ts_valid_fold_list = sorted(np.unique(ts_folds))\nts_valid_fold_list","execution_count":48,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7c0b7fcc5869120be9689187c50b618d646d2f94"},"cell_type":"code","source":"# valid_fold = 0\n# mask_val = eval_sets==valid_fold\n# mask_tr  = ~mask_val\ncross_valid_fold_list = list(range(10))\ncross_valid_fold_list","execution_count":49,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"4513cb222cb0a2aeffc4fbeb029155db7db5e41b"},"cell_type":"code","source":"## https://zhuanlan.zhihu.com/p/26308272\nclass MeanEncoder(object):\n    def __init__(self, \n                 categorical_features,\n                 is_class_target, \n                 cross_valid_set, \n                 timeseries_valid_set=None, \n                 only_return_me_features=True,\n                 verbose=True,\n                 prior_weight_func=None):\n        \"\"\"MeanEncoder Class\n        :param categorical_features: list of str, the name of the categorical columns to encode\n        :param is_class_target: True for {classification} False for {regression}\n        :param *_valid_set: 'tuple'-> ('list': valid_fold_names, 'list/np.array/series': valid_fold_values)\n        :param prior_weight_func:\n            a function that takes in the number of observations, and outputs prior weight\n            when a dict is passed, the default exponential decay function will be used:\n            k: the number of observations needed for the posterior to be weighted equally as the prior\n            f: larger f --> smaller slope\n        \"\"\"\n        self.categorical_features = categorical_features\n        if is_class_target:\n            self.target_values = []\n        else:\n            self.target_values = None\n        self.is_class_target = is_class_target\n        self.verbose = verbose\n        self.only_return_me_features = only_return_me_features\n        self.learned_stats = {}\n        \n        assert isinstance(cross_valid_set[0], list)\n        if isinstance(cross_valid_set[1], list):\n            cross_valid_set[1] = np.array(cross_valid_set[1]).copy()\n        elif hasattr(cross_valid_set[1], 'values'):\n            cross_valid_set[1] = cross_valid_set[1].values.copy()\n        self.cv_names, self.cv_values = cross_valid_set\n        \n        if timeseries_valid_set is not None:\n            assert isinstance(timeseries_valid_set[0], list)\n            if isinstance(timeseries_valid_set[1], list):\n                timeseries_valid_set[1] = np.array(timeseries_valid_set[1]).copy()\n            elif hasattr(timeseries_valid_set[1], 'values'):\n                timeseries_valid_set[1] = timeseries_valid_set[1].values.copy()\n            self.tscv_names, self.tscv_values = timeseries_valid_set\n        else:\n            self.tscv_names, self.tscv_values = None, None\n        \n        if isinstance(prior_weight_func, dict):\n            self.prior_weight_func = eval('lambda x: 1 / (1 + np.exp((x - k) / f))', \n                                          dict(prior_weight_func, np=np))\n        elif callable(prior_weight_func):\n            self.prior_weight_func = prior_weight_func\n        else:\n            print('Set prior_weight_func to default: k=2, f=1')\n            print('   lambda x: 1 / (1 + np.exp((x - 2) / 1))')\n            self.prior_weight_func = lambda x: 1 / (1 + np.exp((x - 2) / 1))\n        \n    @staticmethod\n    def mean_encode_subroutine(X_train, y_train, X_test, variable, target, prior_fn):\n        X_train = X_train[[variable]].copy()\n        X_test = X_test[[variable]].copy()\n        if target is not None:\n            nf_name = '{}_pred_{}'.format(variable, target)\n            X_train['pred_temp'] = (y_train == target).astype(int)  # classification\n        else:\n            nf_name = '{}_pred'.format(variable)\n            X_train['pred_temp'] = y_train  # regression\n        prior = X_train['pred_temp'].mean()\n        col_avg_y = X_train.groupby(\n            by=variable, sort=False, axis=0\n        )['pred_temp'].agg(\n            {'mean': 'mean', 'beta': 'size'}\n        )\n        col_avg_y['beta'] = prior_fn(col_avg_y['beta'])\n        col_avg_y[nf_name] = col_avg_y['beta'] * prior + (1-col_avg_y['beta']) * col_avg_y['mean']\n        col_avg_y.drop(['beta', 'mean'], axis=1, inplace=True)\n        nf_train = X_train.join(col_avg_y, on=variable)[nf_name].values\n        nf_valid = X_test.join(col_avg_y, on=variable).fillna(prior)[nf_name].values\n        return nf_train, nf_valid, prior, col_avg_y\n    \n    def fit_transform(self, X, y):\n        \"\"\"\n        :param X: pandas DataFrame, (n_samples * n_features)\n        :param y: pandas Series or numpy array: (n_samples,)\n        return\n        X_new: the transformed pandas DataFrame containing mean-encoded categorical features\n        \"\"\"\n        X_new = X.copy()\n        new_feat_names = []\n        if self.is_class_target:\n            self.target_values = sorted(set(y))\n            self.learned_stats = {'{}_pred_{}'.format(variable, target): [] \\\n                                  for variable, target in product(\n                                      self.categorical_features, self.target_values)}\n            for variable, target in product(self.categorical_features, self.target_values):\n                nf_name = '{}_pred_{}'.format(variable, target)\n                new_feat_names.append(nf_name)\n                X_new.loc[:, nf_name] = np.nan\n                if self.verbose:\n                    gen = tqdm(enumerate(self.cv_names), ascii=True, desc=nf_name, total=len(self.cv_names))\n                else:\n                    gen = enumerate(self.cv_names)\n                for cv_idx, cv_name in gen:\n                    mask_valid = self.cv_values==cv_idx\n                    mask_train = self.cv_values!=cv_idx\n                    if self.tscv_names is None:\n                        #print('cv', np.sum(mask_valid), np.sum(mask_train)) #debug\n                        nf_train, nf_valid, prior, col_avg_y = MeanEncoder.mean_encode_subroutine(\n                            X_new.iloc[mask_train], y[mask_train], X_new.iloc[mask_valid], \n                            variable, target, self.prior_weight_func)\n                        X_new.loc[mask_valid, nf_name] = nf_valid\n                        self.learned_stats[nf_name].append((prior, col_avg_y))\n                    else:\n                        mask_expanding = False\n                        tsgen = enumerate(self.tscv_names)\n                        for tscv_idx, tscv_name in tsgen:\n                            mask_current = self.tscv_values==tscv_name\n                            mask_valid = (self.cv_values==cv_idx) & mask_current\n                            mask_expanding |= mask_current\n                            mask_train = (self.cv_values!=cv_idx) & mask_expanding\n                            #print('cv+ts', np.sum(mask_valid), np.sum(mask_train)) #debug\n                            nf_train, nf_valid, prior, col_avg_y = MeanEncoder.mean_encode_subroutine(\n                                X_new.iloc[mask_train], y[mask_train], X_new.iloc[mask_valid], \n                                variable, target, self.prior_weight_func)\n                            X_new.loc[mask_valid, nf_name] = nf_valid\n                        ####\n                        self.learned_stats[nf_name].append((prior, col_avg_y))\n        else:\n            self.learned_stats = {'{}_pred'.format(variable): [] for variable in self.categorical_features}\n            for variable in self.categorical_features:\n                nf_name = '{}_pred'.format(variable)\n                new_feat_names.append(nf_name)\n                X_new.loc[:, nf_name] = np.nan\n                if self.verbose:\n                    gen = tqdm(enumerate(self.cv_names), ascii=True, desc=nf_name, total=len(self.cv_names))\n                else:\n                    gen = enumerate(self.cv_names)\n                for cv_idx, cv_name in gen:\n                    mask_valid = self.cv_values==cv_idx\n                    mask_train = self.cv_values!=cv_idx\n                    if self.tscv_names is None:\n                        #print('cv', np.sum(mask_valid), np.sum(mask_train)) #debug\n                        nf_train, nf_valid, prior, col_avg_y = MeanEncoder.mean_encode_subroutine(\n                            X_new.iloc[mask_train], y[mask_train], X_new.iloc[mask_valid], \n                            variable, None, self.prior_weight_func)\n                        X_new.loc[mask_valid, nf_name] = nf_valid\n                        self.learned_stats[nf_name].append((prior, col_avg_y))\n                    else:\n                        mask_expanding = False\n                        tsgen = enumerate(self.tscv_names)\n                        for tscv_idx, tscv_name in tsgen:\n                            mask_current = self.tscv_values==tscv_name\n                            mask_valid = (self.cv_values==cv_idx) & mask_current\n                            mask_expanding |= mask_current\n                            mask_train = (self.cv_values!=cv_idx) & mask_expanding\n                            #print('cv+ts', np.sum(mask_valid), np.sum(mask_train)) #debug\n                            nf_train, nf_valid, prior, col_avg_y = MeanEncoder.mean_encode_subroutine(\n                                X_new.iloc[mask_train], y[mask_train], X_new.iloc[mask_valid], \n                                variable, None, self.prior_weight_func)\n                            X_new.loc[mask_valid, nf_name] = nf_valid\n                        ####\n                        self.learned_stats[nf_name].append((prior, col_avg_y))\n        if self.only_return_me_features:\n            X_new = X_new[new_feat_names]\n        return X_new\n    \n    def transform(self, X):\n        \"\"\"\n        :param X: pandas DataFrame, n_samples * n_features\n        return \n        :param X_new: the transformed pandas DataFrame containing mean-encoded categorical features\n        \"\"\"\n        X_new = X.copy()\n        new_feat_names = []\n        if self.is_class_target:\n            for variable, target in product(self.categorical_features, self.target_values):\n                nf_name = '{}_pred_{}'.format(variable, target)\n                new_feat_names.append(nf_name)\n                X_new[nf_name] = 0\n                for prior, col_avg_y in self.learned_stats[nf_name]:\n                    X_new[nf_name] += X_new[[variable]].join(col_avg_y, on=variable).fillna(prior)[\n                        nf_name]\n                X_new[nf_name] /= len(self.cv_names)\n        else:\n            for variable in self.categorical_features:\n                nf_name = '{}_pred'.format(variable)\n                new_feat_names.append(nf_name)\n                X_new[nf_name] = 0\n                for prior, col_avg_y in self.learned_stats[nf_name]:\n                    X_new[nf_name] += X_new[[variable]].join(col_avg_y, on=variable).fillna(prior)[\n                        nf_name]\n                X_new[nf_name] /= len(self.cv_names)\n        if self.only_return_me_features:\n            X_new = X_new[new_feat_names]\n        return X_new","execution_count":50,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"38b3a899d817d3c10dec7f36643cfc445a17e729"},"cell_type":"code","source":"# For Target Encoding\ncat_cols = ['user_id',\n            'user_id_common',\n            'region',\n            'city',\n            'parent_category_name',\n            'category_name',\n            'param_1',\n            'param_2',\n            'param_3',\n            'price_bin', #'price',\n            'item_seq_number', 'item_seq_bin',\n            'user_type',\n            'image',\n            'image_top_1',\n            'dow']","execution_count":51,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ad9129710c7d6a25a4992d1bf69c2f8166fe29c7","collapsed":true},"cell_type":"code","source":"def get_me_feats(k, f, with_ts=False, save=True):\n    cv_set = (cross_valid_fold_list, eval_sets)\n    ts_set = (ts_valid_fold_list, ts_folds) if with_ts else None\n    prior_weight_func = dict(k=k, f=f)\n    me = MeanEncoder(cat_cols, False, \n                     cv_set, \n                     ts_set, \n                     prior_weight_func=prior_weight_func)\n    me_tr = me.fit_transform(train, y).fillna(-1).astype('float32')\n    me_te = me.transform(test).fillna(-1).astype('float32')\n    if save:\n        fprefix = 'cvts' if with_ts else 'cv'\n        fprefix = '_me_'+fprefix+'_k_{}_f_{}.csv'.format(k, f)\n        me_tr.to_csv('train'+fprefix, index=False)\n        me_te.to_csv('test'+fprefix, index=False)\n    return me_tr, me_te","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3b563adebc06b392b61e6e3422f2acc7153467fb"},"cell_type":"code","source":"%%time\nsettings = [\n            (20, 2, True)\n           ]\nfor k, f, with_ts in settings:\n    me_tr, me_te = get_me_feats(k, f, with_ts)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}