{"cells":[{"metadata":{"_cell_guid":"9206105f-2ab6-4331-bc6a-45f5f1d75b34","_uuid":"9f1254fa73bc1c9de7e4975ff9eb1795ac01abd1"},"cell_type":"markdown","source":"![](https://media.giphy.com/media/nXOds9I8K8gUg/giphy.gif)\n\n                                                          **Please dont forget to vote for me at the end.**"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false,"collapsed":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"e76dc16f-b8f0-44fa-b8f5-076deb829c09","_uuid":"bb0496d9933ee18a465c16ed3790d101bf2d9700","trusted":false,"collapsed":true},"cell_type":"code","source":"import os\nfor i in os.listdir('../input'):\n#     print(i)\n# print(os.path.getsize('../input/'+i ))\n        print(i +'   ' +  str(round(os.path.getsize('../input/' + i) / 1000000,2)) + 'MB')\n    ","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv',nrows=1000000)\ntest = pd.read_csv('../input/test.csv')\ntrain_sample = pd.read_csv('../input/train_sample.csv')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"f6020f3d-56ec-4d82-ab24-10d611d6ea94","_uuid":"d75b2856456dbb7252a79a904f1677d824e3fe33","trusted":false,"collapsed":true},"cell_type":"code","source":"train.shape,test.shape","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"27fd64d6-3221-423e-8b58-8b56abfa3c88","_uuid":"54a2149bafc023cb65ea1e1d1c51c139cfb6b9d8","trusted":false,"collapsed":true},"cell_type":"code","source":"train.columns","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"35fe4007-3786-4ad8-a4a1-cec70784199c","_uuid":"5fc561a56107d05606332fc632dbd7f4424c3c91","trusted":false,"collapsed":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"6438433b-5937-4df2-b9c0-dd1168d8965f","_uuid":"ce14a7204d853b1bb8b5c6e34f34b297a757f6bd","trusted":false,"collapsed":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"932fe409-ae4c-4cb7-93ad-e48fcc45ff12","_uuid":"fbe99221cd53d7737bfdc013d813aa25e6e36f79"},"cell_type":"markdown","source":"Lets us check are there any null values in train dataset"},{"metadata":{"_cell_guid":"3afa214c-59ac-499f-8040-ab5d7248ec0d","_uuid":"e5f0eae5f347a9fa00a156ee275e65dcca891764","trusted":false,"collapsed":true},"cell_type":"code","source":"train.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"61802c82-d708-463d-b242-1198af507c92","_uuid":"61ad72e856660e459deaf8031d4223a8f4090390"},"cell_type":"markdown","source":"from the above we can say that 998307 rows dont have attribute_time means they are not converted, we will cross check it later 1000000-998307"},{"metadata":{"_cell_guid":"fcf14972-bbee-40c9-a6f6-c50672579378","_uuid":"df49976c796caa4f21b545e767ef1229ed445ebf"},"cell_type":"markdown","source":"In this data set ip is always categorical and unique.\n\napp,device,os,channel ther are all given with encoded digits and they are categorical.\n\nNow we have to convert the above mentioned features into categorical features in both train and test sets"},{"metadata":{"_cell_guid":"e1ec7f9c-e157-41e4-9c28-59ceae2a16c5","collapsed":true,"_uuid":"fe5145fc877faa54fd6729841dda1a3d57422745","trusted":false},"cell_type":"code","source":"variables = ['ip', 'app', 'device', 'os', 'channel']\nfor v in variables:\n    train[v] = train[v].astype('category')\n    test[v]=test[v].astype('category')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"4b74a202-9221-4e7e-aa75-2e29181c0459","collapsed":true,"_uuid":"6f25cb56cc0741de0edc9a816359b0fe8f0738ff","trusted":false},"cell_type":"code","source":"#set click_time and attributed_time as timeseries\ntrain['click_time'] = pd.to_datetime(train['click_time'])\ntrain['attributed_time'] = pd.to_datetime(train['attributed_time'])\ntest['click_time'] = pd.to_datetime(test['click_time'])\n\n#set as_attributed in train as a categorical\ntrain['is_attributed']=train['is_attributed'].astype('category')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"40a42b63-d33a-4b64-98ea-4aca7ddc20a8","_uuid":"295dd3e10ccf1d52b9fc586ba908d6384188a3d8"},"cell_type":"markdown","source":"so now we converted the features accordingly and now we will check how many were converted actually as said above, out of 1000000 values 998307 values dont have attribute time and so attribute value. Lets check."},{"metadata":{"_cell_guid":"1435e261-0013-4234-bc96-4bdf946e44be","_uuid":"4ed2a042e22d19aebc569992bfc85448c9f0eada","trusted":false,"collapsed":true},"cell_type":"code","source":"train[['attributed_time', 'is_attributed']][train['is_attributed']==1].describe()\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"31653acf-85b7-4909-b92d-e8e50ebbfd35","_uuid":"d7b55fe35eb4f0997deedf934a096789f1231b56"},"cell_type":"markdown","source":"with the above table we can say that only 1693 were converted (attribute value==1) out of 1000000 which is so less"},{"metadata":{"_cell_guid":"31c5be8b-c14b-45b1-82be-90acb9691da8","_uuid":"eaf062375abfa9f8e304defe53e1bf8d5c243cd3","trusted":false,"collapsed":true},"cell_type":"code","source":"train['is_attributed'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"26bc338c-6275-4208-ba1b-865923a681fc","_uuid":"aed722a9d61c48d9195002e90de7f09dd14b696a","trusted":false,"collapsed":true},"cell_type":"code","source":"train[train['is_attributed']==1].sum()\n#this is the blunder mistakes beginners do, actually attribute value==1 it sums up to 1693 and also it sums up remaining features values also which gives wrong\n#results so beware","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"415e855f-fbf0-4ce7-bf95-4a6a51332f9e","_uuid":"cda6b034295c04bb7cd19ac11c834815e7640db9","trusted":false,"collapsed":true},"cell_type":"code","source":"train.describe()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0538a7eaa48898bae65a043a0b7faa8d60d9080c","_cell_guid":"47716531-24a0-4f29-b8cb-840aa79edb01","scrolled":true,"trusted":false,"collapsed":true},"cell_type":"code","source":"test['click_id']=test['click_id'].astype('category')\ntest.describe()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"2731b5f8-9364-4eb3-9e43-fe60faa018ca","_uuid":"42a0dfa21e27b3b071ac83ee771d7bc0828be144"},"cell_type":"markdown","source":"Now lets check are there any null values in test set."},{"metadata":{"_cell_guid":"cb425428-b9cd-4907-a32d-f0088178da74","_uuid":"a1eb9f911d35ed95b9c617b388ece43ed47f9c9d","trusted":false,"collapsed":true},"cell_type":"code","source":"test.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"c793143c-0289-4884-a675-f42c3cdcb110","_uuid":"473ed1380356a4317b59db0059d1c3e802110302"},"cell_type":"markdown","source":"we will check how many unique ips are there actually."},{"metadata":{"_cell_guid":"477858b8-d9ca-4c72-9131-aa4bcc44e5ed","_uuid":"d45025a77abe362c22559c93c2282ba9622317e9","trusted":false,"collapsed":true},"cell_type":"code","source":"temp = train['ip'].value_counts().reset_index(name='counts')\ntemp.columns = ['ip', 'counts']\ntemp[:10]","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"ccfa5a1a-78b5-4744-bb3b-113027a35213","_uuid":"123ae7974b469857d71948e23ccf93110a318425","trusted":false,"collapsed":true},"cell_type":"code","source":"train['timegap'] = train.attributed_time - train.click_time\ntrain['timegap'].describe()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"ec4a5070-04bd-46be-a1ea-a3076673822e","_uuid":"18bfad294089063f732331e2f2f95e686eec5478"},"cell_type":"markdown","source":"Here timegap is the gap between the click time and atrrtibuted time. "},{"metadata":{"_cell_guid":"b3a135c8-5526-4b64-a4ac-9f175000f72c","_uuid":"cf18c2f2035593f44aa55e88e6a97ead29c2780e","trusted":false,"collapsed":true},"cell_type":"code","source":"train['timegap'].value_counts().sort_values(ascending=False)[:10]","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"c89ef9ff-e006-442d-9c71-0223d73b6417","_uuid":"92e158fedee5007c66620b9eb1124b61347f37ae"},"cell_type":"markdown","source":"It took 57secs to get converted ie is_atrributed / downloaded the app. Like this there are 11 downloads. This table shows number of downloads for a given amount of time."},{"metadata":{"_cell_guid":"de8f6370-938b-44ff-9ef3-0d674cf2a310","_uuid":"ad538b2302f84d5cc0e973a697a0ba60da0e8ea9","trusted":false,"collapsed":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"c7f3adff-1aed-4c9a-a178-e70bde058ad5","_uuid":"2635843ecaaf17e85e1b5744370f9d92a9af05e2","trusted":false,"collapsed":true},"cell_type":"code","source":"plt.figure(figsize=(10, 6))\ncols = ['ip', 'app', 'device', 'os', 'channel']\nuniques = [len(train[col].unique()) for col in cols]\n# sns.set(font_scale=1.2)\nax = sns.barplot(cols, uniques, log=True)\nax.set(xlabel='Feature', ylabel='log(unique count)', title='Number of unique values per feature (from 10,000,000 samples)')\nfor p, uniq in zip(ax.patches, uniques):\n    height = p.get_height()\n    ax.text(p.get_x()+p.get_width()/2.,\n            height + 10,\n            uniq,\n            ha=\"center\")","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"e2a5140d-c963-4a0f-9399-12884693ac67","_uuid":"b5b96e00e5a4d27cfb726e493a0004d93b380aac"},"cell_type":"markdown","source":"**More coming, Plase vote for me to do more.**\n\n**Thank you.**"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}