{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nfrom datetime import datetime\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":7,"outputs":[]},{"metadata":{"_cell_guid":"ca19ac8f-e602-44e4-b329-0c6769f58792","_uuid":"da6a490a1de71c394e7af77b53056b511358f7c8"},"cell_type":"markdown","source":"checking dataset size"},{"metadata":{"_cell_guid":"40f6138c-c6e3-42e8-ae98-109a5351019b","_uuid":"3a6cadc09614a40341374e1cf36a139edeac0c07","trusted":true},"cell_type":"code","source":"! ls -la ../input/\n! wc -l ../input/train.csv\n! wc -l ../input/test.csv","execution_count":1,"outputs":[]},{"metadata":{"_cell_guid":"4ed68591-4420-4aee-a7a6-9e76ce68b98e","_uuid":"01acf1b97ab87bfaea6cdbe5c2829f15e8b94b7b"},"cell_type":"markdown","source":"train and test dataset is split by click_time\n- train: 2017-11-06 - 2017-11-09\n- test : 2017-11-10 - 2017-11-11"},{"metadata":{"_cell_guid":"66e178b4-dd6a-45b3-b07f-85b0db502b05","_uuid":"85e3c72236acd7f7c2ef8cf7071e573b4127a0ae","trusted":true},"cell_type":"code","source":"! head ../input/train.csv\n! tail ../input/train.csv","execution_count":3,"outputs":[]},{"metadata":{"_cell_guid":"2c88e374-12ca-4fcd-8f54-5b79577f13ce","_uuid":"bdce3304d5d717661bf72ad2dc4d007b1ceabc6a","trusted":true},"cell_type":"code","source":"! head ../input/test.csv\n! tail ../input/test.csv","execution_count":4,"outputs":[]},{"metadata":{"_cell_guid":"5362104c-8cbf-4552-98e1-8073e40a9eb4","_uuid":"418727bc54840fc96d88e64a85ea285458855e9d","collapsed":true,"trusted":true},"cell_type":"code","source":"def read_file(train_or_test, is_sample=False, skiprows=None, nrows=None):\n    sample = '_sample' if is_sample else ''\n    df = pd.read_csv(\n                        '../input/{}{}.csv'.format(train_or_test, sample),\n                        dtype={'attributed_time': str, 'click_time': str}, parse_dates=['click_time', 'attributed_time'],\n                        skiprows=skiprows, low_memory=True, nrows=nrows)\n    \n    print(train_or_test, df.shape)\n    display(df.head(2))\n    return df\n\ndef describe_data(df):\n    display(pd.concat((df.describe(), pd.DataFrame({'dtype': df.dtypes, 'nunique': df.nunique(), 'isnull': np.sum(df.isnull())}).T)))","execution_count":5,"outputs":[]},{"metadata":{"_cell_guid":"1e11c3d4-eb01-46b9-b15e-49b8b3d3860c","_uuid":"78467b2414d1fd4375b9cf747eebb11e26e8260f"},"cell_type":"markdown","source":"1. read just 10,000,000 rows which is less than 5 % of train datasets"},{"metadata":{"_cell_guid":"5079fb52-a1b3-4d0a-80d4-680f2e7f0699","_uuid":"464951c673fc9cde84a8a9869129b174298e7fd4","trusted":true},"cell_type":"code","source":"df_tr = read_file('train', is_sample=False, nrows=10000000)\n# df_tr = read_file('train', is_sample=False, skiprows=np.random.randint(0, 1000000, 100000))","execution_count":6,"outputs":[]},{"metadata":{"_cell_guid":"bbfe1093-bc6c-45b2-926d-86ecd806e04d","_uuid":"bade87a8f7a2825f3e9ddffa7e9a193d715ed3ba","collapsed":true,"trusted":true},"cell_type":"code","source":"describe_data(df_tr)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"f037af57-5df4-4dfa-a9d0-37041b031100","_uuid":"c0607c8d1aa31d23442ee9a46a3ba92e090bd760"},"cell_type":"markdown","source":"is_attributed balance"},{"metadata":{"_cell_guid":"9575a781-314e-4aa3-a43c-0b90d3986d89","_uuid":"1fffcbd2ddb2856c17053eb7b4b67c8816a867f4","collapsed":true,"trusted":true},"cell_type":"code","source":"df_tr.is_attributed.value_counts().plot(kind='bar', log=True)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"546c56bd-2f27-4ec2-956b-e42d34dc88e6","_uuid":"bcee32b91ce6b0f19668ea86f641022c31375120","collapsed":true,"trusted":true},"cell_type":"code","source":"df_tr.ip.value_counts().hist(bins=100, normed=False, log=True)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"827152ab-270f-401b-96de-ce6b8ce31660","_uuid":"5c8afc859f6d7f84cc42e5f36972ff81b40284f7","collapsed":true,"trusted":true},"cell_type":"code","source":"df_tr.app.value_counts().hist(bins=100, normed=False, log=True)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"6b4884c9-5b40-40b0-afd6-45f9ca57b8c2","_uuid":"8aa4e56135e45021950a547e301f3ee3b8f8de55","collapsed":true,"trusted":true},"cell_type":"code","source":"df_tr.channel.value_counts().hist(bins=100, normed=False, log=True)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"bd9482f9-2dcd-48f0-ad4d-956247bade87","_uuid":"95a5f0b88b3b1dfe2f8dc4b2da45c7cd2b27fbe0","collapsed":true,"trusted":true},"cell_type":"code","source":"df_tr.groupby('channel', as_index=False)['is_attributed'].mean().plot(kind='bar', x='channel', y='is_attributed', figsize=(24, 4), log=True)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"184726e1-64d1-4c9b-95e9-7aa47f0b89fd","_uuid":"f50da0846a064562709aa055ef159d0fcd565706","collapsed":true,"trusted":true},"cell_type":"code","source":"df_tr.groupby('device', as_index=False)['is_attributed'].mean().plot(kind='bar', x='device', y='is_attributed', figsize=(24, 4), log=True)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"4a2c795d-8d66-433b-b7f4-bbc7f17cb6c9","_uuid":"79ee31bf0cf14f31d74b9460dff9e824b6fe1cd1","collapsed":true,"trusted":true},"cell_type":"code","source":"df_tr.groupby('app', as_index=False)['is_attributed'].mean().plot(kind='bar', x='app', y='is_attributed', figsize=(24, 4), log=True)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"38a80cc5-3532-4e7e-97c6-dd27a42df77a","_uuid":"54a551a0341a69387b37fce71f97b6f76af7ac2f","collapsed":true,"trusted":true},"cell_type":"code","source":"df_tr.groupby('os', as_index=False)['is_attributed'].mean().plot(kind='bar', x='os', y='is_attributed', figsize=(24, 4), log=True)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"d15e501d-66f6-4ebe-98aa-a6e04e6916be","_uuid":"f6767fdb0b2b193d6ba06d7c2c27e8eec922e6f9","collapsed":true,"trusted":true},"cell_type":"code","source":"_df_tr_g = df_tr.groupby(['ip', 'channel'], as_index=False)['is_attributed'].sum()\n# _df_tr_g[_df_tr_g.is_attributed > 0].plot(kind='bar', x=['ip', 'channel'], y='is_attributed', figsize=(24, 4), log=False)\n_df_tr_g[_df_tr_g.is_attributed > 0]","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"ae328953-6628-41f6-be23-018d554d326d","_uuid":"fe2b3dffc263073a2b9be37d6705658acf9e00a7"},"cell_type":"markdown","source":"[](http://)duration from clicking time to attributed time"},{"metadata":{"_cell_guid":"67310c9f-e1c4-43d4-aa06-10bdd1f41a3f","_uuid":"3aca51cd4595ace509d0b293240b5969348e9868","collapsed":true,"scrolled":true,"trusted":true},"cell_type":"code","source":"df_tr_tmp = df_tr[~df_tr.attributed_time.isnull()]\ndf_tr_tmp['time_diff'] = df_tr_tmp['attributed_time'] - df_tr_tmp['click_time']\ndf_tr_tmp['time_diff'] =df_tr_tmp['time_diff'].map(lambda x: x.seconds / 3600.)\ndisplay(df_tr_tmp.head(2))\nplt.title('hours duration')\ndf_tr_tmp.time_diff.hist(bins=100, log=True, figsize=(24, 4))\n\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"77a746ec-8240-4d45-8dab-b1ebe4fd5980","_uuid":"129b1ef232b3e429bf5081fb0f110fad85701df1"},"cell_type":"markdown","source":""},{"metadata":{"_cell_guid":"fa6554d4-42da-4d14-95c7-b4e378405769","_uuid":"ae537fa380ed897b39be648856a2734da841ecfe"},"cell_type":"markdown","source":"There are some cycle of attributed with hour in days"},{"metadata":{"_cell_guid":"16492b75-81f7-4691-bb8e-072f674c4d04","_uuid":"e06fbf5019e5e5271dde70e750d3dc8515787766","collapsed":true,"trusted":true},"cell_type":"code","source":"df_tr_tmp = df_tr[~df_tr.attributed_time.isnull()]\ndf_tr_tmp['attributed_hour'] =df_tr_tmp.attributed_time.dt.hour\ndf_tr_tmp['attributed_date'] =df_tr_tmp.attributed_time.dt.strftime(\"%Y-%m-%d\")\ndisplay(df_tr_tmp.head(2))\ndf_tr_tmp.groupby('attributed_hour')['is_attributed'].sum().plot(kind='bar')  \ndf_tr_tmp.groupby(['attributed_date', 'attributed_hour'])['is_attributed'].sum().plot(kind='bar', log=False, figsize=(24, 4))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"affa3078-e722-4dd8-926e-4e4570f93209","_uuid":"2fa300215558797c9f300dbb05ad81b70e69f2c3","collapsed":true,"trusted":true},"cell_type":"code","source":"df_tr.groupby(['ip', 'is_attributed'])['click_time'].count()\n\npt = pd.pivot_table(df_tr,\n                        index=['ip'], \n                        columns=['is_attributed'],\n                        values=['click_time'],\n                        aggfunc=lambda x : len(x),\n                        fill_value=0).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"7b54214f-cf4a-49b9-bf88-204a1d32830f","_uuid":"d314195a6185ffbe2a6586f3c7c4ac9aa6e45664","collapsed":true,"trusted":true},"cell_type":"code","source":"display(df_tr.groupby(['channel'], as_index=False)['is_attributed'].sum().corr())","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"5d97c1ed-f85f-4156-9568-b7d29ae1b2fa","_uuid":"a85ee5f393c3184ae3f94fd8bd2cd73f89ba3012","collapsed":true,"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}