{"cells":[{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"db10397f-665c-8ada-9295-18f9945f0c8b"},"outputs":[],"source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom collections import Counter\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n\n# Any results you write to the current directory are saved as output."},{"cell_type":"markdown","metadata":{"_cell_guid":"ed4029d8-a6e2-d9a6-e4c7-770677e9c902"},"source":"Understand Users"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"9661e7c2-75c2-374f-fdb0-4c05f7cbf566"},"outputs":[],"source":"#User-Defined-Functions\ndef get_location(x,i):\n    try:\n        return x[i]\n    except:\n        return 'Unknown'\n\n\n\n\ndef get_mode(x):\n    data = Counter(x)\n    return data.most_common(1)[0][0]"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"3a5be187-713b-f015-4c02-dbc60ce18e4f"},"outputs":[],"source":"#read in raw data\npage_views = pd.read_csv(\"../input/page_views_sample.csv\")\nevents = pd.read_csv(\"../input/events.csv\")"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"76adc5b4-d646-a1a6-ebe6-7e2805f2218a"},"outputs":[],"source":"#Filter page_views based on user-id and time of event leading to clicks\nevents_filter = events[['uuid','timestamp']]\npage_views_filter = page_views.merge(events_filter,how='inner',left_on='uuid',right_on='uuid')\npage_views_filter = page_views_filter[page_views_filter['timestamp_x'] <= page_views_filter['timestamp_y']]"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"c47a4401-36b3-0d07-8e59-e442459358ee"},"outputs":[],"source":"#Processing Location variable\npage_views_filter['geo_location'] = page_views_filter['geo_location'].fillna(\"Unknown\")\npage_views_filter['geo_location'] = page_views_filter['geo_location'].apply(lambda x: x.split(\">\"))\npage_views_filter['geo_country'] = page_views_filter['geo_location'].apply(lambda x: get_location(x,0))\npage_views_filter['geo_state'] = page_views_filter['geo_location'].apply(lambda x: get_location(x,1)) \npage_views_filter['geo_dma'] = page_views_filter['geo_location'].apply(lambda x: get_location(x,2)) \npage_views_filter=page_views_filter.drop('geo_location',axis=1)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"03fce3f0-a1cb-27af-88be-622cf97c3551"},"outputs":[],"source":"#Processing platform and traffic source\n#platform (desktop = 1, mobile = 2, tablet =3)\npage_views_filter['platform_desktop'] = page_views_filter['platform'].apply(lambda x: 1 if x == 1 else 0)\npage_views_filter['platform_mobile'] = page_views_filter['platform'].apply(lambda x: 1 if x == 2 else 0)\npage_views_filter['platform_tablet'] = page_views_filter['platform'].apply(lambda x: 1 if x == 3 else 0)\n#traffic_source (internal = 1, search = 2, social = 3)\npage_views_filter['source_internal'] = page_views_filter['traffic_source'].apply(lambda x: 1 if x == 1 else 0)\npage_views_filter['source_search'] = page_views_filter['traffic_source'].apply(lambda x: 1 if x == 2 else 0)\npage_views_filter['source_social'] = page_views_filter['traffic_source'].apply(lambda x: 1 if x == 3 else 0)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"fde129ae-489a-c77b-ea22-73526da5e73c"},"outputs":[],"source":"#Get rid of columns that are not useful\npage_views_filter=page_views_filter.drop(['platform','traffic_source'],axis=1)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"7fe782cb-f16e-a616-20f6-35f6af696e79"},"outputs":[],"source":"t=page_views_filter[0:10]"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"acf06470-4fd0-3c34-5fe6-ce672dcb5103"},"outputs":[],"source":"page_views_fa=page_views_filter.groupby(['uuid']).agg({\n    'timestamp_x': [max,min],\n    'document_id':len,\n    'geo_country':get_mode,\n    'geo_state':get_mode,\n    'geo_dma':get_mode,\n    'platform_desktop':sum,\n    'platform_mobile':sum,\n    'platform_tablet':sum,\n    'source_internal':sum,\n    'source_search':sum,\n    'source_social':sum\n    })"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"e5fd3ce1-4ec7-e5de-8da6-56e9255ab136"},"outputs":[],"source":"page_views_fa.columns = [''.join(col).strip() for col in page_views_fa.columns.values]"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"81772ab9-1de5-f908-32a8-dcb855fc527d"},"outputs":[],"source":"page_views_fa=page_views_fa.reset_index()"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"34129201-58a6-eeff-1ea6-a271abe53895"},"outputs":[],"source":"page_views_fa['time_diff'] = page_views_fa['timestamp_xmax'] - page_views_fa['timestamp_xmin']"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"913081de-ebe8-b650-0594-ef83208abca9"},"outputs":[],"source":"page_views_fa['time_diff_by_min'] = page_views_fa['time_diff'].apply(lambda x: round(x/(1000*60),0))"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"81f24cae-b4a5-6f9b-9617-9086d7490b45"},"outputs":[],"source":"page_views_fa[0:100]"},{"cell_type":"markdown","metadata":{"_cell_guid":"db61385a-e6de-2386-d3b0-5f01a4ad8134"},"source":"Exploratory Analysis And Visualization\n--------------------------------------"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"086666ee-1f28-76b8-9689-de4118afbed7"},"outputs":[],"source":"#Import libraries\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"d4aaa1a3-9072-c916-79c4-a6b036241100"},"outputs":[],"source":"fig,ax = plt.subplots()\nax.set_title(\"The distribution of Time Lag\")\nsns.distplot(page_views_fa['time_diff_by_min'],ax=ax)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"ba5829b4-df0d-b011-c7aa-62193e58aa68"},"outputs":[],"source":"fig,ax = plt.subplots()\nax.set_title(\"The distribution of Time Lag (Number of activities > 1)\")\nsns.distplot(page_views_fa['time_diff_by_min'][page_views_fa['time_diff_by_min'] > 0],ax=ax)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"3ba6daaf-9765-79b3-9366-fd38e7d5b8c6"},"outputs":[],"source":"doc_len_rebin = pd.cut(page_views_fa['document_idlen'],10,retbins=True)\n\nxlabel = doc_len_rebin[0].value_counts().index.tolist()\ny = doc_len_rebin[0].value_counts().values.tolist()\nx = np.arange(len(y))\n\n# add some text for labels, title and axes ticks\nfig, ax = plt.subplots()\nax.bar(x,y)\nax.set_ylabel('Number of users')\nax.set_title('Number of document interacted')\nax.set_xticklabels(xlabel)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"88904b69-470c-75bc-b2ca-b82c244bcaeb"},"outputs":[],"source":"doc_len_rebin = pd.cut(page_views_fa['document_idlen'][page_views_fa['document_idlen']<=20],10,retbins=True)\n\nxlabel = doc_len_rebin[0].value_counts().index.tolist()\ny = doc_len_rebin[0].value_counts().values.tolist()\nx = np.arange(len(y))\nwidth = 0.35       # the width of the bars\n\n# add some text for labels, title and axes ticks\nfig, ax = plt.subplots()\nax.bar(x,y,width)\nax.set_ylabel('Number of users')\nax.set_title('Number of document interacted(<20)')\nax.set_xticks(x + width / 2)\nax.set_xticklabels(xlabel,fontsize='small')\n"}],"metadata":{"_change_revision":0,"_is_fork":false,"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.0"}},"nbformat":4,"nbformat_minor":0}