{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport matplotlib.pyplot as plt\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport seaborn as sns\n%matplotlib inline\n\nfrom dateutil import parser\n\nimport os\nfile_list = os.listdir(\"../\")\n\n#size\n!ls \"../input/\" -al\n\n# Any results you write to the current directory are saved as output.","execution_count":1,"outputs":[]},{"metadata":{"_uuid":"52fd9e47b6e36a1b0bb541188769274a9edad2f6","_cell_guid":"0cfceac4-11f2-4166-9ec1-ffd54a6bedbc","trusted":true},"cell_type":"code","source":"file_list = os.listdir(\"../input/\")\nfiles= ['../input/' + s for s in file_list]\nprint(files)\nnum_lines = [sum(1 for line in open(f)) for  f in files]\nprint(num_lines)","execution_count":2,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"trdf= pd.read_csv('../input/train.csv', nrows=10000000, )\ntest = pd.read_csv('../input/test.csv')\ntrdf.head(10)\n\n","execution_count":3,"outputs":[]},{"metadata":{"_uuid":"ca5a2c1d487f7d49ff953660dbcb0f699821f207","_cell_guid":"da92d8bd-2152-43f8-bb73-4af78e2b432e","trusted":true,"collapsed":true},"cell_type":"code","source":"cols = trdf.columns.values.tolist()\ncols_best = cols[0:5]\nfor c in cols_best:\n    trdf[c] = trdf[c].astype('category')\n    test[c]= test[c].astype('category')\ntrdf['is_attributed']=trdf['is_attributed'].astype('category')\n","execution_count":4,"outputs":[]},{"metadata":{"_uuid":"51d5a115f7bef60fb014765d84788b6bb686af2a","_cell_guid":"1b91e8b9-c813-46d9-952c-463a75aae0fc","trusted":true},"cell_type":"code","source":"trdf.describe()","execution_count":5,"outputs":[]},{"metadata":{"_cell_guid":"580daa84-862d-4fae-82fb-1a3a2a89bfef","_uuid":"f30fef828d47381473dc4258260bd1cd1df010f2","scrolled":true,"trusted":true},"cell_type":"code","source":"trdf[trdf.is_attributed == 1].head()","execution_count":6,"outputs":[]},{"metadata":{"_uuid":"ed49f3587b674ad3de4ae3544816cf9b87ad9279","_cell_guid":"5e68fe41-0692-4ef3-8619-9ce72e03099c","trusted":true},"cell_type":"code","source":"trdf.app.unique()","execution_count":7,"outputs":[]},{"metadata":{"_uuid":"f2bf4560e8f8190cb5085a03d4e6899449d77ca2","_cell_guid":"68421cfa-e421-4b74-acb3-54a7a2441bba","trusted":true},"cell_type":"code","source":"trdf.app.nunique()","execution_count":8,"outputs":[]},{"metadata":{"_uuid":"8efa88c559bb2dfa11c5ede7f4013b28bcca9bd5","_cell_guid":"490f9489-f311-4424-af41-958e7636dc10","trusted":true},"cell_type":"code","source":"trdf.app.value_counts().head()","execution_count":9,"outputs":[]},{"metadata":{"_uuid":"bd46df8448894b318d92b4a04e42065ee7f64853","_cell_guid":"bd4db566-8173-43a0-9315-e3dedc86302d","trusted":true},"cell_type":"code","source":"trdf.device.nunique()","execution_count":10,"outputs":[]},{"metadata":{"_uuid":"32e52c4faf1c49e83554a8b4310256455514659b","_cell_guid":"c02d9bfa-901c-4e16-93f8-5ee0b3e2a815","trusted":true},"cell_type":"code","source":"trdf.device.value_counts().head()","execution_count":11,"outputs":[]},{"metadata":{"_uuid":"1f1be29835d02a35b4aa57073d1ede43694aa51a","_cell_guid":"5b815459-9047-4309-839e-1ffa718af959","trusted":true},"cell_type":"code","source":"plt.figure(figsize=(12, 6))\nplt.hist(trdf.device, log = True)\nplt.show()","execution_count":12,"outputs":[]},{"metadata":{"_uuid":"bd08be96d77697f334c4b5b10df9a5be3c4cfc05","_cell_guid":"9beffb08-3020-42f4-b618-200555e0947d","trusted":true},"cell_type":"code","source":"plt.figure(figsize=(12, 6))\nplt.hist(trdf.app, log = True)\nplt.show()","execution_count":13,"outputs":[]},{"metadata":{"_uuid":"34ea42bc8569ab9d4d596bee9f06b97385450d59","_cell_guid":"113e7c17-1bbb-4873-9fc5-87b056703485","trusted":true},"cell_type":"code","source":"plt.figure(figsize=(12, 6))\ncols = trdf.columns.values.tolist()\ncols_best = cols[0:4]\ncounts = [trdf[col].nunique() for col in cols_best]\n\nsns.set(font_scale=1.5)\nax = sns.barplot(cols_best, counts, palette=sns.color_palette(), log=True)\nax.set(xlabel='features', ylabel='log(counts)')","execution_count":14,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"ddddb200446660a29d041e0a5ef66aeaf41a9c68"},"cell_type":"code","source":"total = trdf.shape[0]\nvalues= trdf.is_attributed.value_counts()\ncat = trdf.is_attributed.unique()","execution_count":43,"outputs":[]},{"metadata":{"_uuid":"e9cddc2b90251c69b020178352035cb4f2b93283","_cell_guid":"1a3ac853-1e4b-476e-804c-75f076787852","trusted":true},"cell_type":"code","source":"sns.set(font_scale=1.5)\nax = sns.barplot(cat, values, palette=sns.color_palette(), log=False)\nax.set(xlabel='values', ylabel='counts')","execution_count":44,"outputs":[]},{"metadata":{"_uuid":"64973022a0ffe940f4d4bebfff52318ea4d9fe5e","_cell_guid":"6dc36096-2ebd-4674-ad10-9cf59ff3191c","trusted":true},"cell_type":"code","source":"pct_imbalance = (values[1]/values[0])*100\nprint(round(pct_imbalance,2))","execution_count":46,"outputs":[]},{"metadata":{"collapsed":true,"_cell_guid":"d5e29d6e-e07a-4848-9bfa-c98a784c76bf","_uuid":"45b296143045b3be5fe89aaeddc085454d1cd05b","trusted":true},"cell_type":"code","source":"trdf.dtypes\ntrdf[\"attributed_time\"] = pd.to_datetime(trdf[\"attributed_time\"])\ntrdf[\"click_time\"] = pd.to_datetime(trdf[\"click_time\"])","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_cell_guid":"142db3a0-7d90-4acc-b2d2-6105174655d1","_uuid":"6a8492c4eca38fcb2413b09c04ef6fa43f461ebc","trusted":true},"cell_type":"code","source":"#add new col\ntrdf['diff_time'] = trdf.attributed_time - trdf.click_time","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_cell_guid":"790e16ae-6b1e-4bb2-bd35-db7547e5443d","_uuid":"6d0e606f12ae1c0dec957aba8e4a01519d980f7a","trusted":true},"cell_type":"code","source":"trdf.dtypes\n\ntrdf['diff_time_hr'] = np.round(trdf['diff_time'] / np.timedelta64(1, 'h'), 2)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ac7003a7f55f20b2c91c7b254e3c3b5b70eb00c0","_cell_guid":"210ba476-9dce-4455-8a8c-884c11e720e1","trusted":true,"collapsed":true},"cell_type":"code","source":"trdf.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7772ba1c9400695b70b6e9506e21bea4ff70ba1f","_cell_guid":"02d66f23-63b8-4365-9cf4-ed6f94f7b9b9","trusted":true,"collapsed":true},"cell_type":"code","source":"trdf['diff_time_hr'].describe()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0f80e1efbd67114eb9d9f06a3c886bfa74edd3c9","_cell_guid":"cdc9ffc5-b765-4c0b-8357-efff5ae9d69b","trusted":true,"collapsed":true},"cell_type":"code","source":"trdf.diff_time_hr.value_counts()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}