{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input/elo-merchant-category-recommendation/\"))\n# Any results you write to the current directory are saved as output.\n\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e89bbc846ed6d6df389b3c75225140c3dd662290"},"cell_type":"code","source":"import gc\ngc.enable()\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4133f67c8e460dd2f49a71053f0d54e30da20a6c"},"cell_type":"markdown","source":"## Data import and cleaning"},{"metadata":{"trusted":true,"_uuid":"99ff8b22b72622ee96c382ff0ddd15fd89863722"},"cell_type":"code","source":"#Reduce the memory usage, from \"Elo Merchant Category Recommendation by [team Tour_de_Force]\"\ndef reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = reduce_mem_usage(pd.read_csv('../input/elo-merchant-category-recommendation/train.csv'))\ntest = reduce_mem_usage(pd.read_csv('../input/elo-merchant-category-recommendation/test.csv'))\nmerchants = reduce_mem_usage(pd.read_csv('../input/elo-merchant-category-recommendation/merchants.csv'))\nnew_trans = reduce_mem_usage(pd.read_csv('../input/elo-merchant-category-recommendation/new_merchant_transactions.csv'))\nhistorical_trans = reduce_mem_usage(pd.read_csv('../input/elo-merchant-category-recommendation/historical_transactions.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0fcdd65eb2106f99bafd445234c17eab23c68755"},"cell_type":"code","source":"#historical_trans.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"32584280f287683ce5b9c572b1e8509317953192"},"cell_type":"code","source":"#new_trans.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"83fed2b4c8b1115663d0f7d2fc4a60b4aa7cabdb"},"cell_type":"code","source":"one_hot_cols = ['category_1','category_2', 'category_3',]\nbool_cols= ['authorized_flag']\nnumerical_cols = ['purchase_amount', 'installments', 'month_lag',] #'purchase_date',  <-- this has to be processed \ncategorical_cols = ['city_id', 'state_id', 'merchant_category_id','subsector_id',]\nmerchant_id = ['merchant_id']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2b42e35788a2f6e27c3e908598401eee32760a6e"},"cell_type":"code","source":"# purchase_date cleaning and conversion TBD","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f7cbb122b85015bc3b9bcfac2437a0348d8d373f"},"cell_type":"code","source":"for c in bool_cols:\n    new_trans[c] = new_trans[c].apply(lambda x: True if x=='Y' else False).astype(bool)\n    historical_trans[c] = historical_trans[c].apply(lambda x: True if x=='Y' else False).astype(bool)\n\n#print(historical_trans[bool_cols].describe())\n#print()\n#print(new_trans[bool_cols].describe())\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d4dcbf070f2d6932200c6f27791c6c8d10322ca0"},"cell_type":"code","source":"for c in categorical_cols:\n    historical_trans[c] = historical_trans[c].astype('category')\n    new_trans[c] = new_trans[c].astype('category')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3d949a06eaa271eb2352bfffd2b4dee86d47df59"},"cell_type":"code","source":"for c in numerical_cols:\n    print(historical_trans[c].describe())\n    print(new_trans[c].describe())\n    print(sum(pd.isna(historical_trans[c])), sum(pd.isna(new_trans[c])))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6b8ea6113f4d0fd7b2e818ddac55618df6eea234"},"cell_type":"code","source":"for c in one_hot_cols:\n    historical_trans[c] = historical_trans[c].astype('category')\n    new_trans[c] = new_trans[c].astype('category')\n    '''\n    print('column name: ', c)\n    print(historical_trans[c].value_counts())\n    print(sum(pd.isna(historical_trans[c])))\n    print()\n    print(new_trans[c].value_counts())\n    print(sum(pd.isna(new_trans[c])))\n    print()\n    '''","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"95394ddf3674c9c45c188c884f507680f0882c10"},"cell_type":"markdown","source":"## Data frame merges and feature prep"},{"metadata":{"trusted":true,"_uuid":"1bfcd0e73bec14708c9898ccbf78c75e6edef6aa"},"cell_type":"code","source":"one_hot_new = pd.get_dummies(new_trans[one_hot_cols], dummy_na=True)\none_hot_hist = pd.get_dummies(historical_trans[one_hot_cols], dummy_na=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"592979a5379e68f1d0e5003f1fb5bbf4442f6781"},"cell_type":"code","source":"historical_trans = pd.concat([historical_trans, one_hot_hist], axis = 1)\nnew_trans = pd.concat([new_trans, one_hot_hist], axis =1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"74ad9137ab75d5c296a18c15726bb8afce997820"},"cell_type":"code","source":"agg_function = {\n    'purchase_amount' : ['mean', 'sum', 'std', 'nunique', 'max', 'min'],\n    'installments': ['mean', 'sum', 'std', 'nunique', 'max', 'min'],\n    'month_lag' : ['mean', 'std', 'nunique', 'max', 'min'],\n    \n    'city_id': ['count', 'nunique'],\n    'state_id': ['count', 'nunique'],\n    'merchant_category_id': ['count', 'nunique'],\n    'subsector_id': ['count', 'nunique'],\n    \n   \n    'authorized_flag': ['sum', 'count'],   \n    'category_1_N': ['mean', 'sum'],\n    'category_1_Y': ['mean', 'sum'], \n    'category_1_nan': ['mean', 'sum'],\n    'category_2_1.0': ['mean', 'sum'], \n    'category_2_2.0': ['mean', 'sum'], \n    'category_2_3.0': ['mean', 'sum'],\n    'category_2_4.0': ['mean', 'sum'], \n    'category_2_5.0': ['mean', 'sum'], \n    'category_2_nan': ['mean', 'sum'],\n    'category_3_A': ['mean', 'sum'], \n    'category_3_B': ['mean', 'sum'], \n    'category_3_C': ['mean', 'sum'], \n    'category_3_nan': ['mean', 'sum'],\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7cf5c4a3658dffbaa90bb9a784a96e8bdc89a56b"},"cell_type":"code","source":"new_trans_by_card_id = new_trans.groupby(['card_id']).agg(agg_function)\nhistorical_trans_by_card_id = historical_trans.groupby(['card_id']).agg(agg_function)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3275b7c30b5a7505321c68279857ba13b67ecbdb"},"cell_type":"code","source":"trans_by_card = historical_trans_by_card_id.merge(\n    new_trans_by_card_id,how='left', left_index=True, right_index=True).reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c8d502ec6fb96a4ca7d0bb1b574cacb55efcbc4a"},"cell_type":"code","source":"l_org = trans_by_card.columns.values\nl = [p[0] + '_' + p[1] for p in l_org]\nl[0] = 'card_id'\ntrans_by_card.columns = l","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"357b15b03450afaf88b3a52cfb7b1bf8d2059607"},"cell_type":"code","source":"trans_by_card.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"90b5bda6d18b04cd065f9372c68cbbb791e26b0c"},"cell_type":"code","source":"del [historical_trans, new_trans ]\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"18b2d5a7118f3352d91e2dabb5cbbdfe42cfcad0"},"cell_type":"code","source":"trans_by_card.to_csv('trans_by_card.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f31e4cda7d0d58b6cb36dd15ecd507a3f03ae8eb"},"cell_type":"code","source":"#np.log(historical_trans[historical_trans['authorized_flag']=='Y'].card_id.value_counts()).hist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"676d14727e6635d547061f8284aa1502dbb9a7ae"},"cell_type":"markdown","source":"## Models"},{"metadata":{"trusted":true,"_uuid":"bc4546c385c71789f59f6ca90606d06e5939dba3"},"cell_type":"code","source":"trans_by_card.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2dfef06801ea56fcb96a13bbf62bbc31e5adb757"},"cell_type":"code","source":"df = pd.read_csv('../input/trans_by_card.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cb702c9d13c707c708637768e8d94f5ac908e10e"},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"178f62188fcea9bbd3d82e178bf421ccb44079da"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}