{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"collapsed":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nfrom sklearn.preprocessing import LabelEncoder","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"## Data Preprocessing"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"5200187886d5aac8337889caa6b1d94431496f4b"},"cell_type":"code","source":"# load data\ncat_cols = [\n    'region', 'city', 'parent_category_name', 'category_name',\n    'param_1', 'param_2', 'param_3', 'image_top_1'\n]\n\ntrain = pd.read_csv('../input/train.csv', usecols=cat_cols+['deal_probability'])\ntest = pd.read_csv('../input/test.csv', usecols=cat_cols)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"6fe607a73067607f3751a06c52e0504b645325e4"},"cell_type":"code","source":"# fill in missing\ntrain['image_top_1'] = train['image_top_1'].astype(str)\ntest['image_top_1'] = test['image_top_1'].astype(str)\ntrain.fillna('', inplace=True)\ntest.fillna('', inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"e4a40193e6cdd11cf007f72ca6c4170e1d87a0ce"},"cell_type":"code","source":"# label encoding\nfor col in cat_cols:\n    le = LabelEncoder()\n    le.fit(np.concatenate([train[col], test[col]]))\n    train[col] = le.transform(train[col])\n    test[col] = le.transform(test[col])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"35a621afd98627b2c14d7ebaf7dbee10a00e6846","collapsed":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ecf760fbc92ff6ef2990df412a676717b3ca7359","collapsed":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7454452c674af7085c20dc2fdc2513e84c90e4ee"},"cell_type":"markdown","source":"## Target Encoding"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"99f4bb4a350193cccee581e488ced64b66fcfa1b"},"cell_type":"code","source":"class BetaEncoder(object):\n        \n    def __init__(self, group):\n        \n        self.group = group\n        self.stats = None\n        \n    # get counts from df\n    def fit(self, df, target_col):\n        self.prior_mean = np.mean(df[target_col])\n        stats = df[[target_col, self.group]].groupby(self.group)\n        stats = stats.agg(['sum', 'count'])[target_col]    \n        stats.rename(columns={'sum': 'n', 'count': 'N'}, inplace=True)\n        stats.reset_index(level=0, inplace=True)           \n        self.stats = stats\n        \n    # extract posterior statistics\n    def transform(self, df, stat_type, N_min=1):\n        \n        df_stats = pd.merge(df[[self.group]], self.stats, how='left')\n        n = df_stats['n'].copy()\n        N = df_stats['N'].copy()\n        \n        # fill in missing\n        nan_indexs = np.isnan(n)\n        n[nan_indexs] = self.prior_mean\n        N[nan_indexs] = 1.0\n        \n        # prior parameters\n        N_prior = np.maximum(N_min-N, 0)\n        alpha_prior = self.prior_mean*N_prior\n        beta_prior = (1-self.prior_mean)*N_prior\n        \n        # posterior parameters\n        alpha = alpha_prior + n\n        beta =  beta_prior + N-n\n        \n        # calculate statistics\n        if stat_type=='mean':\n            num = alpha\n            dem = alpha+beta\n                    \n        elif stat_type=='mode':\n            num = alpha-1\n            dem = alpha+beta-2\n            \n        elif stat_type=='median':\n            num = alpha-1/3\n            dem = alpha+beta-2/3\n        \n        elif stat_type=='var':\n            num = alpha*beta\n            dem = (alpha+beta)**2*(alpha+beta+1)\n                    \n        elif stat_type=='skewness':\n            num = 2*(beta-alpha)*np.sqrt(alpha+beta+1)\n            dem = (alpha+beta+2)*np.sqrt(alpha*beta)\n\n        elif stat_type=='kurtosis':\n            num = 6*(alpha-beta)**2*(alpha+beta+1) - alpha*beta*(alpha+beta+2)\n            dem = alpha*beta*(alpha+beta+2)*(alpha+beta+3)\n            \n        # replace missing\n        value = num/dem\n        value[np.isnan(value)] = np.nanmedian(value)\n        return value","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"4135d4cb904866ec7417027ebd97feec8886197a"},"cell_type":"code","source":"N_min = 1000\nfeature_cols = []    \n\n# encode variables\nfor c in cat_cols:\n\n    # fit encoder\n    be = BetaEncoder(c)\n    be.fit(train, 'deal_probability')\n\n    # mean\n    feature_name = f'{c}_mean'\n    train[feature_name] = be.transform(train, 'mean', N_min)\n    test[feature_name]  = be.transform(test,  'mean', N_min)\n    feature_cols.append(feature_name)\n\n    # mode\n    feature_name = f'{c}_mode'\n    train[feature_name] = be.transform(train, 'mode', N_min)\n    test[feature_name]  = be.transform(test,  'mode', N_min)\n    feature_cols.append(feature_name)\n    \n    # median\n    feature_name = f'{c}_median'\n    train[feature_name] = be.transform(train, 'median', N_min)\n    test[feature_name]  = be.transform(test,  'median', N_min)\n    feature_cols.append(feature_name)    \n\n    # var\n    feature_name = f'{c}_var'\n    train[feature_name] = be.transform(train, 'var', N_min)\n    test[feature_name]  = be.transform(test,  'var', N_min)\n    feature_cols.append(feature_name)        \n    \n    # skewness\n    feature_name = f'{c}_skewness'\n    train[feature_name] = be.transform(train, 'skewness', N_min)\n    test[feature_name]  = be.transform(test,  'skewness', N_min)\n    feature_cols.append(feature_name)    \n    \n    # kurtosis\n    feature_name = f'{c}_kurtosis'\n    train[feature_name] = be.transform(train, 'kurtosis', N_min)\n    test[feature_name]  = be.transform(test,  'kurtosis', N_min)\n    feature_cols.append(feature_name)    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f18954d14e7816810a1cda9b74d0c980354e6d2a","collapsed":true},"cell_type":"code","source":"train[['deal_probability']+feature_cols].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8608ebe9c163e74a5a15d269e6f124e456437bb8","collapsed":true},"cell_type":"code","source":"test[feature_cols].head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}