{"cells":[{"metadata":{"_uuid":"0dc2a247ee22ad677ed4e0fc9e6c50ef7b3438ad"},"cell_type":"markdown","source":"# Data Exploration"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false},"cell_type":"code","source":"## Load Package\nimport pandas as pd # package for high-performance, easy-to-use data structures and data analysis\nimport numpy as np # fundamental package for scientific computing with Python\nimport matplotlib\nimport matplotlib.pyplot as plt # for plotting\n%matplotlib inline\nimport seaborn as sns # for making plots with seaborn\ncolor = sns.color_palette()\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nfrom plotly.offline import init_notebook_mode, iplot\ninit_notebook_mode(connected=True)\nimport plotly.graph_objs as go\nimport plotly.offline as offline\noffline.init_notebook_mode()\nimport plotly.tools as tls\nimport squarify\nfrom mpl_toolkits.basemap import Basemap\nfrom numpy import array\nfrom matplotlib import cm\n\nfrom sklearn import preprocessing\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# Print all rows and columns\npd.set_option('display.max_columns', None)\npd.set_option('display.max_rows', None)\n\nfrom nltk.corpus import stopwords\nfrom textblob import TextBlob\nimport datetime as dt\nimport warnings\nimport string\nimport time\n# stop_words = []\nstop_words = list(set(stopwords.words('russian')))\nwarnings.filterwarnings('ignore')\npunctuation = string.punctuation\nimport gc\n\n# Plotting Decision tree\nfrom sklearn import tree\nfrom IPython.display import Image as PImage\nfrom subprocess import check_call\nfrom PIL import Image, ImageDraw, ImageFont\nimport re\n\n# Venn diagram\nfrom matplotlib_venn import venn2\n\nimport os\nprint(os.listdir(\"../input\"))","execution_count":1,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"trusted":false},"cell_type":"code","source":"df_train = pd.read_csv('../input/train.csv')\ndf_per_train = pd.read_csv('../input/periods_train.csv')\ndf_test = pd.read_csv('../input/test.csv')\ndf_per_test      = pd.read_csv('../input/periods_test.csv')\nsample_submission = pd.read_csv('../input/sample_submission.csv')","execution_count":2,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"8ceda2b012ec8cc402c89e7f75a8cab73f77cb7b"},"cell_type":"code","source":"%%time\ndtypes={\n    'price': 'float32',\n    'item_seq_number': 'uint16',\n    'image_top_1':'float32',\n    'deal_probabilty':'float32'\n}\n## train \ndf_train = pd.read_csv('../input/train.csv',parse_dates=['activation_date'],dtype=dtypes)\ndf_per_train = pd.read_csv('../input/periods_train.csv',parse_dates=['activation_date','date_from','date_to'])\ndf_test = pd.read_csv('../input/test.csv',dtype=dtypes,parse_dates=['activation_date'])","execution_count":3,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"48c50f5f71876d480f98a78847e54606ff430112"},"cell_type":"code","source":"%%time\ndtypes1 = {\n    'item_seq_number':'float16',\n    'price':'float32'\n}\n\ndf_act_trn = pd.read_csv('../input/train_active.csv',dtype=dtypes1,parse_dates=['activation_date'],\n                         usecols=['item_id','user_id','city','activation_date']\n                        )","execution_count":4,"outputs":[]},{"metadata":{"_uuid":"d316a3a4deeae488d6adf91b2ada64c3baaa0e9d"},"cell_type":"markdown","source":"## Preprocessing\nFor memory handling and increasing the overall workflow speed from loading data to creating numerical features."},{"metadata":{"trusted":false,"_uuid":"1b45671bdae14853505f8f60c960d19f1d16f0ec"},"cell_type":"code","source":"df_train.info()","execution_count":5,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"6064ccb9ccc799dacf62ee4434db13811776b83f"},"cell_type":"code","source":"df_train.memory_usage(deep=True)*1e-6","execution_count":6,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"afd13412c43cc85c92166c50853073dae60e1117"},"cell_type":"code","source":"def convert_columns_to_catg(df, column_list):\n    for col in column_list:\n        print(\"converting\", col.ljust(30), \"size: \", round(df[col].memory_usage(deep=True)*1e-6,2), end=\"\\t\")\n        df[col] = df[col].astype(\"category\")\n        print(\"->\\t\", round(df[col].memory_usage(deep=True)*1e-6,2))","execution_count":7,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"80547da8bc5ea220255130610bb9d265625beb62"},"cell_type":"code","source":"convert_columns_to_catg(df_train, ['city','region',\"param_1\",\"param_2\",\"param_3\",\"parent_category_name\",\"category_name\", \"user_type\"])","execution_count":8,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"ca482bb162873228a5d46172ff1fefc3af05f4f3"},"cell_type":"code","source":"df_train.memory_usage(deep=True)/(2**20)","execution_count":9,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"9da3e846e05145e5d344c62c9746dcab5240bd15"},"cell_type":"code","source":"cat_cols=['city','region',\"param_1\",\"param_2\",\"param_3\",\"parent_category_name\",\"category_name\", \"user_type\"]\nconvert_columns_to_catg(df_test,cat_cols)","execution_count":10,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"6fadb09f2fc68d679abd6b454ca3cc5f549a742f"},"cell_type":"code","source":"df_train.to_pickle(\"train.pkl\")\ndf_test.to_pickle(\"test.pkl\")\n\n# size is shown in bytes again and needs to be converted to megabytes\nprint(\"train.csv:\", os.stat('../input/train.csv').st_size * 1e-6)\nprint(\"train.pkl:\", os.stat('train.pkl').st_size * 1e-6)\n\nprint(\"test.csv:\", os.stat('../input/test.csv').st_size * 1e-6)\nprint(\"test.pkl:\", os.stat('test.pkl').st_size * 1e-6)","execution_count":11,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"873407e23f1d3504e00647931f01c68df385b897"},"cell_type":"code","source":"%%time \ndf_train = pd.read_pickle('train.pkl')","execution_count":12,"outputs":[]},{"metadata":{"_uuid":"29d56cac6d25912f82a65cd761e718c883cb1da8"},"cell_type":"markdown","source":"## Label Encoding"},{"metadata":{"trusted":false,"_uuid":"ee3ff96d1123dc132f3438460ad01e609cbb252e"},"cell_type":"code","source":"df_train.region.value_counts().tail()","execution_count":13,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"dde14ac6e4e023f5c3113d36985c9642e2d3e127"},"cell_type":"code","source":"df_train.user_id.value_counts().tail()","execution_count":14,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"de377f1f5c83bce7b8c7a9b4d83a6c62186a4dce"},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder","execution_count":15,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"9b44d779b33ce50ebfc3e7c087e47b2a5d571a1b"},"cell_type":"code","source":"def create_label_encoding_with_min_count(df, column, min_count=50):\n    column_counts = df.groupby([column])[column].transform(\"count\").astype(int)\n    column_values = np.where(column_counts >= min_count, df[column], \"\")\n    df[column+\"_label\"] = LabelEncoder().fit_transform(column_values)\n    \n    return df[column+\"_label\"]","execution_count":16,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"87b449d584d314f641f5e4eb3d0ee1f9cef928a5"},"cell_type":"code","source":"print(\"number of unique users      :\", len(df_train[\"user_id\"].unique()))","execution_count":17,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"30cbc500bdaf0f4496fbbb7921b8e96a8354493d"},"cell_type":"code","source":"df_train.loc[df_train[\"city\"]==\"Светлый\", \"region\"].value_counts().head()","execution_count":18,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"449edc0e008d88993f359250f393546b2ddd0fd6"},"cell_type":"code","source":"df_train['region_city'] = df_train.loc[:,['region','city']].apply(lambda s: ' '.join(s),axis=1)","execution_count":19,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"e5f65e473fd01f62fc82e0d0aafc5510afbb0d7f"},"cell_type":"code","source":"print(\"unique:\", len(df_train[\"region_city\"].unique()))\nprint(\"size:  \", df_train[\"region_city\"].memory_usage(deep=True)*1e-6)","execution_count":20,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"9f83b28f727f18a0a41e6f7c604233632276a286"},"cell_type":"code","source":"df_train['region_city2'] = df_train.groupby(['region','city'])['region'].transform(lambda x:np.random.random()) ## faster and encode it correctly!!\ndf_train.region_city2.value_counts().head()","execution_count":21,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"48ea65d41c93981add9a962eafd3b470d919a1c3"},"cell_type":"code","source":"print(\"unique:\", len(df_train[\"region_city2\"].unique()))\nprint(\"size:  \", df_train[\"region_city2\"].memory_usage(deep=True)*1e-6)","execution_count":22,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"e67f0ab4c13aebaabc6b0dc98e0b67e7de64146e"},"cell_type":"code","source":"df_train['region_city2_label']=create_label_encoding_with_min_count(df_train,'region_city2',min_count=50)","execution_count":23,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"b45f22b255d067dd96056d9261ba63f17ab122c4"},"cell_type":"code","source":"df_train.columns","execution_count":24,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"008d34419fd4b2ebf9a8c72254e07642f83d3456"},"cell_type":"code","source":"gc.collect()","execution_count":25,"outputs":[]},{"metadata":{"_uuid":"2a74a7c05b8597b1d2315d4f8cdd57aaa84e6724"},"cell_type":"markdown","source":"### Description, Title, Words, numbers"},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"022b2066043b8bfd7f95632cbfc4fee49c89aeb7"},"cell_type":"code","source":"df_train['title'] = df_train.title.fillna(\" \")\ndf_train['title_len'] = df_train.title.apply(lambda x:len(x.split())).astype('uint8')\ndf_train['title_char'] = df_train.title.apply(len).astype('uint8')","execution_count":26,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"556127bdea5245dd1304e814bd39169721b5187a"},"cell_type":"code","source":"df_train.title_len.value_counts(sort=False).plot(kind='bar')","execution_count":27,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"5995f4f89e7d4df4d909b7d4c9a434e9080168a7"},"cell_type":"code","source":"df_train.title_char.value_counts(sort=False).plot(kind='bar')","execution_count":28,"outputs":[]},{"metadata":{"_uuid":"03b97cb0576f10ba858cdffba324dc3f9ec46626"},"cell_type":"markdown","source":"### Description"},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"5615fda327a21f90be749df6ad9a8a20b05cb2da"},"cell_type":"code","source":"df_train['description'] = df_train.description.fillna(\" \")\ndf_train['description_len'] = df_train.description.apply(lambda x:len(x.split())).astype('uint16')\ndf_train['description_char'] = df_train.description.apply(len).astype('uint16')","execution_count":29,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"5fd423925ffd06038c2c906cec7fe97a62f2d8ee"},"cell_type":"code","source":"ax = df_train.description_len.value_counts(sort=False).plot(kind='bar',log=True)\nax.get_xaxis().set_visible(False)","execution_count":30,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"95c6f82b026450510b04d17a4fb1362a2917c383"},"cell_type":"code","source":"df_train.description_char.value_counts().head().plot(kind='bar',log=True)","execution_count":31,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"618e6e55d41e0a97e77d817e135ec82ac1f9f4fc"},"cell_type":"code","source":"df_train.corr()","execution_count":32,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"431e814aa0a68f27e5c65a8a3ba9695110d44fde"},"cell_type":"code","source":"corr = df_train.corr()\nmask = np.zeros_like(corr, dtype=np.bool)\nmask[np.triu_indices_from(mask)] = True\nsns.set(style=\"white\")\n\ncmap = sns.diverging_palette(30,10,as_cmap=True)\nsns.heatmap(corr,cmap=cmap,center=0,square=True,vmax=.3,linewidths=.1, cbar_kws={\"shrink\": .5});","execution_count":33,"outputs":[]},{"metadata":{"_uuid":"bd3a25d97cfccd138eb4c4445e7a35321c63d8ea"},"cell_type":"markdown","source":"### Encode\n`user_id`, `item_seq_numbers` with `uidx`, `iidx`"},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"448bc983a95059c34f4f857bd69bb5e3ff826b27"},"cell_type":"code","source":"import scipy.sparse as sp","execution_count":34,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"f11a5f32b678de17dc6c49dd86d09721c3e82fa1"},"cell_type":"code","source":"def get_df_matrix_mappings(df, row_name, col_name):\n    # Create mappings\n    rid_to_idx = {}\n    idx_to_rid = {}\n    for (idx, rid) in enumerate(df[row_name].unique().tolist()):\n        rid_to_idx[rid] = idx\n        idx_to_rid[idx] = rid\n\n\n    cid_to_idx = {}\n    idx_to_cid = {}\n    for (idx, cid) in enumerate(df[col_name].unique().tolist()):\n        cid_to_idx[cid] = idx\n        idx_to_cid[idx] = cid\n\n\n    return rid_to_idx, idx_to_rid, cid_to_idx, idx_to_cid","execution_count":35,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"77a7cc62e74221ecc5210c0419ea393facc75c06"},"cell_type":"code","source":"rid_to_idx, idx_to_rid, cid_to_idx, idx_to_cid = get_df_matrix_mappings(df_train,'user_id','item_seq_number')","execution_count":36,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"226d6bcd787dbc39a365489812b993a4257bbd57"},"cell_type":"code","source":"df_trn_uidx = pd.DataFrame()\ndf_trn_uidx['uidx']= df_train.user_id.map(rid_to_idx)\ndf_trn_uidx['iidx']= df_train.item_seq_number.map(cid_to_idx)\ndf_trn_uidx['uid'] = df_train.user_id\ndf_trn_uidx['iid'] = df_train.item_seq_number","execution_count":37,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"71c826acc59538a64480e42176b32bfdba7e6017"},"cell_type":"code","source":"df_trn_uidx.head()","execution_count":38,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"e123762e71db22e64621ece844d07394e5b89eea"},"cell_type":"code","source":"I = df_trn_uidx.uidx.as_matrix()\nJ = df_trn_uidx.iidx.as_matrix()\nV = np.ones(df_trn_uidx.shape[0])\n\nui_trn_sp = sp.coo_matrix((V,(I,J)),dtype='uint8')","execution_count":39,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"6b821e24ea7dc3aa0b797c479751d2dace7b9fb3"},"cell_type":"code","source":"ui_trn_sp.shape","execution_count":40,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"27150f16ebb46308abdcf07d2218719c67c5d64d"},"cell_type":"code","source":"ui_trn_csr =ui_trn_sp.tocsr()","execution_count":41,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"1d6cc4313fe036dc262af33724af64c8033ffe4e"},"cell_type":"code","source":"plt.spy(ui_trn_csr,markersize=0.5,aspect='auto')","execution_count":42,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"655fd370276fbf5459e0943d8386c719eceb99ee"},"cell_type":"code","source":"plt.plot(np.array(ui_trn_csr.sum(axis=1)).flatten())","execution_count":43,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"d77084cbdc7acb329ec11f35fa536d095969f8be"},"cell_type":"code","source":"df_train['iidx'] = df_train.item_seq_number.map(cid_to_idx).astype('uint16')\ndf_train['uidx'] = df_train.user_id.map(rid_to_idx).astype('uint32')","execution_count":44,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"a3f87a3cd8930f74fa035f350292812a4c2954b8"},"cell_type":"code","source":"df_train.groupby(['uidx','iidx']).size().value_counts()","execution_count":45,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"212a0fa19de2382ed8bf275ecbee2ae3ff957957"},"cell_type":"code","source":"df_train.columns","execution_count":46,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"d47f69d62c04d5f4e39b48bf78da926460453eed"},"cell_type":"code","source":"df_trn_uidx = df_trn_uidx.merge(df_train[['uidx','iidx','deal_probability']] , how ='left',on=['uidx','iidx'])","execution_count":47,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"2b0736074f2acdad0d13c0ca25daf1425e85dded"},"cell_type":"code","source":"I = df_trn_uidx.uidx.as_matrix()\nJ = df_trn_uidx.iidx.as_matrix()\nVp = df_trn_uidx.deal_probability\nui_trn_deal = sp.coo_matrix((Vp,(I,J)),dtype='float32')","execution_count":48,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"1c5ecab45885fb5e0be7f60626f554bf0057c70d"},"cell_type":"code","source":"plt.spy(ui_trn_deal,markersize=0.5,aspect='auto')","execution_count":49,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"29afcb8fd631d94b8984e23e27e104b3e89bbe5a"},"cell_type":"code","source":"data = ui_trn_deal.tocsc() # sparse operations are more efficient on csc\nN, M = data.shape\ns, t = 100, 1000           # decimation factors for y and x directions\nT = sp.csc_matrix((np.ones((M,)), np.arange(M), np.r_[np.arange(0, M, t), M]), (M, (M-1) // t + 1))\nS = sp.csr_matrix((np.ones((N,)), np.arange(N), np.r_[np.arange(0, N, s), N]), ((N-1) // s + 1, N))\nresult = S @ data @ T     # downsample by binning into s x t rectangles\nresult = result.todense() # ready for plotting\nplt.imshow(result,cmap='gray_r',aspect='auto')","execution_count":50,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"a071c94c34eaa9860c1017fbf3de97cea82b1957"},"cell_type":"code","source":"df_train.groupby('uidx')['deal_probability'].mean().rolling(1000).mean().plot()","execution_count":51,"outputs":[]},{"metadata":{"_uuid":"27a102f4191a90736d53bd819c34baef7e080c8d"},"cell_type":"markdown","source":"`uidx` in some way sort `ad_cnt_by_user`, this cnt numbers affect deal probability\n"},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"74f48deb79f7cc60bf7d547d48fad403aa55f571"},"cell_type":"code","source":"tmp = df_train.groupby('uidx').size().to_frame().reset_index().rename(columns={0:'ads_cnt_by_uid'})","execution_count":52,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"6c8247e5927581bec63b4169ddf2381d054b6453"},"cell_type":"code","source":"tmp.head()","execution_count":53,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"8b9f61030a904764c828f3e8ccd1d06a8a46449d"},"cell_type":"code","source":"print('doing add cnt by user_id...')\ntmp = df_train.groupby('uidx').size().to_frame().reset_index().rename(columns={0:'ads_cnt_by_uid'})\ntmp['ads_cnt_by_uid'] = tmp.ads_cnt_by_uid.astype('uint32')\ndf_train = df_train.merge(tmp,how='left' ,on='uidx')\n\nprint('doing add cnt by iidx(item_seq_number)...')\ntmp = df_train.groupby('iidx').size().to_frame().reset_index().rename(columns={0:'ads_cnt_by_iid'})\ntmp['ads_cnt_by_iid'] = tmp.ads_cnt_by_iid.astype('uint32')\ndf_train =  df_train.merge(tmp,how='left' ,on='iidx')\nprint('done')\ndel tmp; gc.collect()","execution_count":54,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"027178885e93172f8186d2bf0982c0c2aefc87ef"},"cell_type":"code","source":"usecols = ['uidx','iidx','ads_cnt_by_uid','ads_cnt_by_iid','image_top_1','deal_probability']\n\ncorr = df_train[usecols].corr()\nmask = np.zeros_like(corr, dtype=np.bool)\nmask[np.triu_indices_from(mask)] = True\nsns.set(style=\"white\")\n\ncmap = sns.diverging_palette(30,10,as_cmap=True)\nsns.heatmap(corr,cmap=cmap,center=0,square=True,vmax=.3,linewidths=.1, cbar_kws={\"shrink\": .5});","execution_count":55,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"3ca3b65bfd33523afb332e5cd6ea6a862b6fb6fd"},"cell_type":"code","source":"df_train.groupby('ads_cnt_by_iid')['deal_probability'].mean().plot()","execution_count":56,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"91b4f1ab982a0ac7516b1e3d04a5d2bd6b735259"},"cell_type":"code","source":"df_train.groupby('iidx')['deal_probability'].mean().rolling(100).mean().plot()","execution_count":57,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"bd25538ad76c7a3f6412d2e90c88987c4c0eb937"},"cell_type":"code","source":"df_train.user_id.nunique()","execution_count":58,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"69d01c88a1fd717905da6a3660cb49d0e24a059a"},"cell_type":"code","source":"df_test.user_id.nunique()","execution_count":59,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"ddd272fc18a7b498f5fd0b6fc34cc6d55be24465"},"cell_type":"code","source":"uid_in_trn_test = np.intersect1d(df_test.user_id,df_train.user_id) # overlap # of user 67,929\npd.Series(uid_in_trn_test).nunique()","execution_count":60,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"119e37b6f1c154821ee074989d661faee1c94176"},"cell_type":"code","source":"test_uid = df_test.user_id.unique()\ntrain_uid = df_train.user_id.unique()","execution_count":61,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"255c899405615f5f8a5050f11a4cc4f69bd76771"},"cell_type":"code","source":"train_itemid = df_train.item_id.unique()\ntest_itemid = df_test.item_id.unique()","execution_count":62,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"bc843df2b15f564ce88a2a933509b1d452994408"},"cell_type":"code","source":"train_itemid.size","execution_count":63,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"83647a5c09d9b2889e08baa42ddfffa80d7802ab"},"cell_type":"code","source":"test_itemid.size","execution_count":64,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"1e014a573998ce430049743a0b7c5223f6948ffd"},"cell_type":"code","source":"itmid_in_trn_test = np.intersect1d(train_itemid,test_itemid)","execution_count":65,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"7bd5b5887c497f9b68497b922e05c9d8cbdb4ee1"},"cell_type":"code","source":"itmid_in_trn_test.size","execution_count":66,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"409ce29c7c67fb500649b5b200f73454ef139455"},"cell_type":"code","source":"num_items_by_user = np.array(ui_trn_csr.sum(axis=1).flatten())[0]\npd.Series(num_items_by_user).value_counts().head()","execution_count":67,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"43e09e34f97d08934082b274663217bbcdc7771d"},"cell_type":"code","source":"print('max of df_train.activation_date',df_train.activation_date.max())\nprint('min of df_train.activation_date',df_train.activation_date.min())","execution_count":68,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"6c26d2ba41e8fc00fc23eaa204863c21e5626e24"},"cell_type":"code","source":"df_train.activation_date.value_counts()","execution_count":69,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"abd7902d90e6637bc894aa276ef38987621cab9b"},"cell_type":"code","source":"print('max of df_per_train.activation_date',df_per_train.activation_date.max())\nprint('min of df_per_train.activation_date',df_per_train.activation_date.min())\n\nprint('max of df_per_train.date_from',df_per_train.date_from.max())\nprint('min of df_per_train.date_from',df_per_train.date_from.min())\n\nprint('max of df_per_train.date_to',df_per_train.date_to.max())\nprint('min of df_per_train.date_to',df_per_train.date_to.min())","execution_count":70,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"ec258be1412bfc6d4271ed9b128ddec2c380704a"},"cell_type":"code","source":"df_act_trn.activation_date.value_counts()","execution_count":71,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"e0b873bc9f21d79114496d053b5856f30acfd13e"},"cell_type":"code","source":"df_per_train.shape","execution_count":72,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"cc74d75fc84b40c0f1722aa61e2f959a461a0f09"},"cell_type":"code","source":"df_act_trn.shape","execution_count":73,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"d531e68698b27b4f4a96560c4b00bb0e02fd18dd"},"cell_type":"code","source":"df_per_train.activation_date.value_counts()","execution_count":74,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"6ba247b529a1e3ad8955d67895c1b39173098c5d"},"cell_type":"code","source":"df_per_train.date_from.value_counts()","execution_count":75,"outputs":[]},{"metadata":{"_uuid":"e76139db027989203bea85107ae92fa8e2c71435"},"cell_type":"markdown","source":"`train_active`: Supplemental data from ads that were displayed during the same period as train.csv. Same schema as the train data minus `deal_probability`, `image`, and `image_top_1`.\n\n`periods_train`: Supplemental data showing the dates when the ads from `train_active.csv` were activated and when they where displayed.\n\n- item_id         :  map to \n- activation_date :　date ad was placed\n- date_from       :  first date ad was displayed\n- date_to         :  last date ad was displayed"},{"metadata":{"trusted":false,"_uuid":"9be3750afcc04fe417d958d679c1adcf6599087e"},"cell_type":"code","source":"print('# of item_id : train active ',df_act_trn.item_id.nunique())\nprint('# of item_id : periods train', df_per_train.item_id.nunique())","execution_count":76,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"6df4e5efa7e2e696bb3410751c8eacde54d6d6c8"},"cell_type":"code","source":"print('# of item_id: train ',df_train.item_id.nunique())","execution_count":77,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"cc3c934b5ee65b997febc9aefb06a07e3fc88710"},"cell_type":"code","source":"df_per_train.head()","execution_count":78,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"46103cbd7149bf0d711059ec387c224bf3f4d350"},"cell_type":"code","source":"df_act_trn.head()","execution_count":79,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"4ebe08183f5f3b715e8b43cdf9d7533f889526eb"},"cell_type":"code","source":"df_train.merge(df_per_train,how='inner',on=['item_id'])","execution_count":80,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"72eb92ae3255bbadeeaee4a485c1e49fdd872529"},"cell_type":"code","source":"df_train[df_train.user_id.isin(df_act_trn.user_id.head())]","execution_count":81,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"c1285a08dd2798b43a7d9e9f89613d73879cb73f"},"cell_type":"code","source":"act_trn_itemid = df_act_trn.item_id.head(10)\ndf_per_train[df_per_train.item_id.isin(act_trn_itemid)]","execution_count":82,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"4e8499598ac434e14faf9b7c09f573107025e28b"},"cell_type":"code","source":"df_act_trn.head(20).merge(df_per_train,left_on=['item_id','activation_date'],right_on=['item_id','date_from'],how='inner')","execution_count":83,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"22cecdb804ded0a9d0f32eb2a07eed6beff76d54"},"cell_type":"code","source":"df_per_train.head(10).merge(df_act_trn,on='item_id',how='inner')","execution_count":84,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"0210740f0b92b49fec9441d2201bb117d5325e4b"},"cell_type":"code","source":"df_per_train.columns","execution_count":85,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"219fd69288215faac784a14f17b9af1d388dc627"},"cell_type":"code","source":"df_act_trn.columns","execution_count":86,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"bd0f9a3da626233b9296d215714aa4298996b1bb"},"cell_type":"code","source":"merged_trn_sup = df_act_trn[['item_id','activation_date','user_id']].merge(df_per_train,\n                                                                                           how='inner',\n                                                                                           left_on=['item_id','activation_date'],\n                                                                                           right_on=['item_id','date_from'])","execution_count":87,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"d48bb0d0019b9f9d3bbd227138198bb854ed441e"},"cell_type":"code","source":"df_act_trn.shape","execution_count":88,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"3ee0357497ad112e95a30ade5966f04405a46a04"},"cell_type":"code","source":"merged_trn_sup.shape","execution_count":89,"outputs":[]},{"metadata":{"_cell_guid":"d481fb2e-3dfd-466c-a16f-2e66078ae3b9","_uuid":"fbd8b06c2ae20114d295e0f9b6f3f1ad98e160cd","trusted":false},"cell_type":"code","source":"merged_trn_sup.head()","execution_count":90,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"be9f58e8291c931db42198d91d0c83cf80c4c8c0"},"cell_type":"code","source":"merged_trn_sup.date_from.max()","execution_count":91,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"e937e9506bfa08f17477f93e029381c6f95a9dff"},"cell_type":"code","source":"df_train.activation_date.value_counts()","execution_count":92,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"41c00aa7564f1e35d76093c5255fea08361ec2f3"},"cell_type":"code","source":"df_act_trn.activation_date.value_counts()","execution_count":93,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"5dedf47386d9170f457d07abdca96a18b5f3d50b"},"cell_type":"code","source":"merged_trn_sup.date_to.value_counts()","execution_count":94,"outputs":[]},{"metadata":{"_cell_guid":"7e50632b-73b6-43c3-8d50-4a313cff0424","_uuid":"0d96536b21a94a7e21c41f2aa0b394d00aa53fd2","trusted":false},"cell_type":"code","source":"df_train.columns","execution_count":95,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"db1bb8d914df49d021075d49f89ac88d64172609"},"cell_type":"code","source":"df_train.deal_probability.mean()","execution_count":96,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"872a5c1cef3d9de73a118964441fa7a20553a0f1"},"cell_type":"code","source":"df_train.user_type.value_counts()","execution_count":97,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"f4060ea917f3a26151453c2c4f846d3180798bcb"},"cell_type":"code","source":"df_train.parent_category_name.value_counts()","execution_count":98,"outputs":[]},{"metadata":{"_cell_guid":"8497684a-396d-40db-97a1-078193fcc93d","_uuid":"09bc17cbc806b8d686c8f5f3fef80203e22ffac4","trusted":false},"cell_type":"code","source":"df_train.groupby('parent_category_name').deal_probability.mean()","execution_count":99,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"4d351e66db9a9d98e1fe7de1cac559d5700b72a5"},"cell_type":"code","source":"df_train.groupby('category_name').deal_probability.mean()","execution_count":100,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"663029cad549fd3ea777b001f691ffd743fefad9"},"cell_type":"code","source":"df_train.shape[0] == np.sum(df_train.user_type.value_counts())","execution_count":101,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"520e48a5ce8a70f8aee1d0850e4729a8d80e41f3"},"cell_type":"code","source":"df_train.groupby('user_type').deal_probability.mean()","execution_count":102,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"c15f7353c7858eb3d2671ad6554cbe87ce731ab2"},"cell_type":"code","source":"df_train.groupby('user_id').size().sort_values(ascending=False).head()","execution_count":103,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"0ec4037d7b6ddb64ead11ee844463e67b88bfd51"},"cell_type":"code","source":"df_train.groupby('image_top_1').deal_probability.mean()","execution_count":104,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"fd1c58e73269d862498b27fcfea4587cc9aee3d9"},"cell_type":"code","source":"df_train.groupby('category_name').deal_probability.mean()","execution_count":105,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"377e963d2ab80d46ae2dcd11d39c7f5aa94c9a2a"},"cell_type":"code","source":"df_train.groupby('item_seq_number').deal_probability.mean()","execution_count":106,"outputs":[]},{"metadata":{"_uuid":"a037ea1d217e63a37061a8f1f0597f986844713b"},"cell_type":"markdown","source":"## Handle Text features\n\n -   tfidf + tsvd\n -   binarize tfidf + tsvd\n -   hashing + tsvd\n -   binarize + tsvd\n\n-   constraint on ngram=(1,2), max_features = 10^5\n-   tsvd to dim =5\n\n"},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"e272285a7f39ba85064528b508123f0ece0393bd"},"cell_type":"code","source":"title_text_raw = df_train.title.append(df_test.title)\ntitle_text_raw.reset_index(drop=True,inplace=True)","execution_count":107,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"e852f9c22fb1023561746d74b6c2399e09b14291"},"cell_type":"code","source":"title_text_raw.shape","execution_count":108,"outputs":[]},{"metadata":{"_uuid":"be9d730b84af3e90045cb745a1544d1bc8a5e186"},"cell_type":"markdown","source":"## TF-IDF + TSVD"},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"67c52c1f657ae13555d4065c551b120aa27b2e83"},"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.decomposition import TruncatedSVD","execution_count":109,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"cb3ab4b3f13645abe963dbba2b69633d60f823dc"},"cell_type":"code","source":"## tfidf \ntv = TfidfVectorizer(lowercase=False,ngram_range=(1,2),max_features=100000)\ntv_feats = tv.fit_transform(title_text_raw)\nprint('shape of tfidf Vectorizer:{}'.format(tv_feats.shape))","execution_count":110,"outputs":[]},{"metadata":{"_cell_guid":"d4e3d6ea-4de2-4878-8c4e-0a58b11919e3","_uuid":"b5a5b3ad30af99674ac6b851b5f36f5544c625ce","trusted":false},"cell_type":"code","source":"svd = TruncatedSVD(n_components=5, random_state=0)\ntv_svd_feats = svd.fit_transform(tv_feats)\nprint('shape of tv_svd_feats:',tv_svd_feats.shape)","execution_count":111,"outputs":[]},{"metadata":{"_cell_guid":"fc7abd3f-0eb7-4d4e-9c1d-a6309f7cba2d","_uuid":"af8d5d1207366cb1884e6409b9f929ac25ec86a8","trusted":false},"cell_type":"code","source":"print(svd.explained_variance_ratio_)\nprint(np.cumsum(svd.explained_variance_ratio_))","execution_count":112,"outputs":[]},{"metadata":{"_cell_guid":"77d1ddb7-e5ad-43ea-b336-a0dbcae4a76e","_uuid":"8d4bb712799dad5c0bb285e6b87dc39a185de5ff","trusted":false},"cell_type":"code","source":"tv_svd_df = pd.DataFrame(tv_svd_feats).iloc[:df_train.shape[0]]\ntv_svd_df['y'] = df_train.deal_probability\n\ntv_svd_df.corr()['y']","execution_count":113,"outputs":[]},{"metadata":{"_cell_guid":"5ee8df66-a5c0-4385-ae48-3061d82ef5a1","_uuid":"971b48935f2026a7bb0f665349904c6eab7703fb","trusted":false},"cell_type":"code","source":"sns.jointplot(x = tv_svd_df[0].values, y=tv_svd_df['y'].values)","execution_count":114,"outputs":[]},{"metadata":{"_cell_guid":"d78f8959-a9da-4426-97e1-9d151490d330","_uuid":"4c167d991a3c616970414e668a851c0e2c5f385a","trusted":false},"cell_type":"code","source":"sns.jointplot(x = tv_svd_df[1].values, y=tv_svd_df['y'].values)","execution_count":115,"outputs":[]},{"metadata":{"_cell_guid":"d10915ae-cab7-4ab7-aa21-6abf275baa83","_uuid":"f7176b00090dfd7a505cca4505b90efa6087b769"},"cell_type":"markdown","source":"### Hashing + TSVD"},{"metadata":{"_cell_guid":"bce932e4-244e-4a9d-a0fd-894eff09a568","_uuid":"edf5ef11f1af0a9cb7338aa3d3762fbe77f8a08e","trusted":false},"cell_type":"code","source":"from sklearn.feature_extraction.text import HashingVectorizer\n\nhv = HashingVectorizer(ngram_range=(1, 2), lowercase=False)\nhv_features = hv.fit_transform(title_text_raw).tocsr()\nprint('shape of hv features:{}'.format(hv_features.shape))\n\nsvd = TruncatedSVD(n_components=5, random_state=0)\nhv_svd_features = svd.fit_transform(hv_features)","execution_count":116,"outputs":[]},{"metadata":{"_cell_guid":"c7719508-bad9-4121-b460-24ccfb9a3de0","_uuid":"8dfcd955141a39de9c223eb3df1eb149776742b4","trusted":false},"cell_type":"code","source":"np.cumsum(svd.explained_variance_ratio_)","execution_count":117,"outputs":[]},{"metadata":{"_cell_guid":"c79bb226-8c83-444a-a0ef-b907cc555599","_uuid":"ffd1dc56039d3756c1f47e666197779b54fe81b7","collapsed":true,"trusted":false},"cell_type":"code","source":"hv_svd_df = pd.DataFrame(hv_svd_features).iloc[:df_train.shape[0]]\nhv_svd_df['y'] = df_train.deal_probability","execution_count":118,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"db1b52f7381919d52b16ca1cbae426849c2aac26"},"cell_type":"code","source":"hv_svd_df.corr().y","execution_count":119,"outputs":[]},{"metadata":{"_uuid":"ab8878ce15d4a81307a6b091cc0642ad23d2405f"},"cell_type":"markdown","source":"### For description"},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"34d15fbe6a925e03ea24dd5b1e7dce69c1e04b49"},"cell_type":"code","source":"#desc_raw = df_train.description.append(df_test.description)\n#desc_raw.fillna('',inplace=True)\n#desc_raw.reset_index(drop=True,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"ba49e1fb2a3b4542917d078429f3a355fa9a6001"},"cell_type":"code","source":"## tfidf  + svd \n#tv = TfidfVectorizer(lowercase=False,ngram_range=(1,2),max_features=100000)\n#tv_feats = tv.fit_transform(desc_raw)\n\n#print('shape of tfidf Vectorizer:{}'.format(tv_feats.shape))\n\n#svd = TruncatedSVD(n_components=5, random_state=0)\n#tv_svd_feats1 = svd.fit_transform(tv_feats)\n#print('shape of tv_svd_feats:',tv_svd_feats.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"473fa89cdca209f96b3310f330f4b322617b25dc"},"cell_type":"code","source":"#svd.explained_variance_ratio_","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"f608e43ecc2514695468da0b095d4ff34d50ef61"},"cell_type":"code","source":"## hashing + svd \n\n#hv = HashingVectorizer(ngram_range=(1, 2), lowercase=False)\n#hv_features = hv.fit_transform(desc_raw).tocsr()\n#print('shape of hv features:{}'.format(hv_features.shape))\n\n#svd = TruncatedSVD(n_components=5, random_state=0)\n#hv_svd_features1 = svd.fit_transform(hv_features)","execution_count":123,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"55b1d308fa519b9c67403a4ccfa535cd36d8ac8e"},"cell_type":"code","source":"#hv_svd_features1.shape\n#tv_svd_feats1.shape","execution_count":124,"outputs":[]},{"metadata":{"_uuid":"5eebe7c68889be3ddcae1aba3261c5a170b91771"},"cell_type":"markdown","source":"#### Kernel crashes if you run these commented cells on Kaggle"},{"metadata":{"_uuid":"4d771c09b718a98cd5750527f648a56540dcb433"},"cell_type":"markdown","source":"## Implementing libFM in Keras"},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"ad355ac0fd0902789d6f618ac93b0a8dd40aeec1"},"cell_type":"code","source":"def init_seeds(seed):\n    os.environ['PYTHONHASHSEED'] = '0'\n\n    # The below is necessary for starting Numpy generated random numbers\n    # in a well-defined initial state.\n\n    np.random.seed(seed)\n\n    # The below is necessary for starting core Python generated random numbers\n    # in a well-defined state.\n\n    rn.seed(seed)\n\n    session_conf = tf.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1)\n\n    from keras import backend as K\n\n    # The below tf.set_random_seed() will make random number generation\n    # in the TensorFlow backend have a well-defined initial state.\n    # For further details, see: https://www.tensorflow.org/api_docs/python/tf/set_random_seed\n\n    tf.set_random_seed(seed)\n\n    sess = tf.Session(graph=tf.get_default_graph(), config=session_conf)\n    K.set_session(sess)\n    return sess","execution_count":123,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"129892133a9d66c40963d62bd99f9b88b978b031"},"cell_type":"code","source":"k_latent = 2\nembedding_reg = 0.0002\nkernel_reg = 0.1\n\ndef get_embed(x_input, x_size, k_latent):\n    if x_size > 0: #category\n        embed = Embedding(x_size, k_latent, input_length=1, \n                          embeddings_regularizer=l2(embedding_reg))(x_input)\n        embed = Flatten()(embed)\n    else:\n        embed = Dense(k_latent, kernel_regularizer=l2(embedding_reg))(x_input)\n    return embed\n\ndef build_model_1(X, f_size):\n    dim_input = len(f_size)\n    \n    input_x = [Input(shape=(1,)) for i in range(dim_input)] \n     \n    biases = [get_embed(x, size, 1) for (x, size) in zip(input_x, f_size)]\n    \n    factors = [get_embed(x, size, k_latent) for (x, size) in zip(input_x, f_size)]\n    \n    s = Add()(factors)\n    \n    diffs = [Subtract()([s, x]) for x in factors]\n    \n    dots = [Dot(axes=1)([d, x]) for d,x in zip(diffs, factors)]\n    \n    x = Concatenate()(biases + dots)\n    x = BatchNormalization()(x)\n    output = Dense(1, activation='relu', kernel_regularizer=l2(kernel_reg))(x)\n    model = Model(inputs=input_x, outputs=[output])\n    opt = Adam(clipnorm=0.5)\n    model.compile(optimizer=opt, loss='mean_squared_error')\n    output_f = factors + biases\n    model_features = Model(inputs=input_x, outputs=output_f)\n    return model, model_features","execution_count":124,"outputs":[]},{"metadata":{"_cell_guid":"42f3c17b-9617-4a70-bce5-fd9b381042fd","_uuid":"46c0e6a8878f9de9efd07fca14b8d0e14257af1a","trusted":false},"cell_type":"code","source":"%%time \ndf_train = pd.read_pickle('train.pkl')","execution_count":125,"outputs":[]},{"metadata":{"_cell_guid":"05d4ae43-c4c8-431d-b61a-27c7e94b59b4","_uuid":"940d11f5bd400a147f1569581b9e9666f2cd7d55","trusted":false},"cell_type":"code","source":"print('build id->idx map ...')\nrid_to_idx, idx_to_rid, cid_to_idx, idx_to_cid = get_df_matrix_mappings(df_train,'user_id','item_seq_number')\n\n\ndf_trn_uidx = pd.DataFrame()\ndf_trn_uidx['uidx']= df_train.user_id.map(rid_to_idx)\ndf_trn_uidx['iidx']= df_train.item_seq_number.map(cid_to_idx)\ndf_trn_uidx['uid'] = df_train.user_id\ndf_trn_uidx['iid'] = df_train.item_seq_number\n\nprint('build iidx, uidx col')\ndf_train['iidx'] = df_train.item_seq_number.map(cid_to_idx).astype('uint16')\ndf_train['uidx'] = df_train.user_id.map(rid_to_idx).astype('uint32')","execution_count":126,"outputs":[]},{"metadata":{"_cell_guid":"a2e18397-4b32-4180-ad04-e03dcc57624f","_uuid":"9859bece52f39de0ad4a4d19e7a133200e094b6d","trusted":false},"cell_type":"code","source":"feats = ['uidx','iidx']\n\ntarget = ['deal_probability']\nfm_data = df_train[feats].copy()\nfm_data.head()","execution_count":128,"outputs":[]},{"metadata":{"_cell_guid":"bb80f476-70d3-4145-bccc-0852524dbcd0","_uuid":"89e4fc32eafc1b7f8eb805486dc47dfe3cca064f","trusted":false},"cell_type":"code","source":"fm_data.info()","execution_count":129,"outputs":[]},{"metadata":{"_cell_guid":"d63d202a-c5c6-452e-87de-a07d26252fa5","_uuid":"7f2d781a226c62f04c551070301ab1f74216c4ca","trusted":false},"cell_type":"code","source":"f_size  = [int(fm_data[f].max()) + 1 for f in feats]\nf_size","execution_count":130,"outputs":[]},{"metadata":{"_cell_guid":"8739bb4d-44fd-46b6-ba80-4bf8692cda7e","_uuid":"8544ffcc4089ed7c5f22ea0429759f8975f867e4","trusted":false},"cell_type":"code","source":"fm_data.dtypes","execution_count":131,"outputs":[]},{"metadata":{"_cell_guid":"47ebd7ae-74da-4a97-85da-caefd445042b","_uuid":"7e8299d883c69addf8ddbcf38000666ade7da403","collapsed":true,"trusted":false},"cell_type":"code","source":"fm_data = fm_data.merge(df_train[['uidx','iidx','deal_probability']], how='left',on=['uidx','iidx'])","execution_count":132,"outputs":[]},{"metadata":{"_cell_guid":"6b59eff4-7e4b-457b-b531-5f9f9591f9b0","_uuid":"c43302e87b7eca2121683875df7621ed84ef1048","trusted":false},"cell_type":"code","source":"fm_data.head()","execution_count":133,"outputs":[]},{"metadata":{"_uuid":"3386923f689999e499eef7e5848390b241b08824"},"cell_type":"markdown","source":"## Stay Tuned! Will update libFM with Keras soon."}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.5"}},"nbformat":4,"nbformat_minor":1}