{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":false,"collapsed":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nfrom matplotlib.gridspec import GridSpec\nimport seaborn as sns\n\nimport gc\nfrom IPython.core.interactiveshell import InteractiveShell\nInteractiveShell.ast_node_interactivity = \"all\"\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.\n\n%matplotlib inline\npd.set_option('max_rows', 10)\npal = sns.color_palette()","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"1b678740b696422faf62eb00d13693fa0e5da76e","_cell_guid":"a6c63b3e-d194-4061-9eb3-664c4b7b8faf"},"cell_type":"markdown","source":"## **1. Descrição dos arquivos e estrutura de dados**\nSource: https://www.kaggle.com/anokas/talkingdata-adtracking-eda\n"},{"metadata":{"_uuid":"8776b3e0c10afc2f9b282581d6d7eff98b63448c","scrolled":true,"_cell_guid":"30d06f84-4625-45d3-a1fd-5fc925461e79","trusted":false,"collapsed":true},"cell_type":"code","source":"datapath = '../input/'\n\nprint('# File sizes')\nfor f in os.listdir(datapath):\n    if 'zip' not in f:\n        print(f.ljust(30) + str(round(os.path.getsize(datapath + f) / 1000000, 2)) + 'MB')\n\n        \nimport subprocess\n\nprint('\\n# Line count:')\nfor file in ['train.csv', 'test.csv', 'test_supplement.csv', 'train_sample.csv']:\n    lines = subprocess.run(['wc', '-l', os.path.join(datapath, file)], stdout=subprocess.PIPE).stdout.decode('utf-8')\n    print(lines, end='', flush=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b37a0b2ea37c94de89fdcc30adc46d9ff2d45d2a","_cell_guid":"b86889ae-9857-4fd9-ab09-ac7e0dd1776f","trusted":false,"collapsed":true},"cell_type":"code","source":"#datapath = '/dados/Dados/Kaggle'\ndatapath = '../input/'\ntrain_kaggle_sample = pd.read_csv(os.path.join(datapath, 'train_sample.csv'), parse_dates=['attributed_time', 'click_time'])\ntrain_kaggle_sample.info()\nfor col in train_kaggle_sample.select_dtypes(include=np.number):\n    train_kaggle_sample[col] = train_kaggle_sample[col].astype('category')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a1edb59811af3583f7ffee26b4706cd94e0d599a","scrolled":true,"_cell_guid":"50525f0a-dbae-4ab5-a583-7c6e1ceb79f5","trusted":false,"collapsed":true},"cell_type":"code","source":"dtypes = {\n        'ip'            : 'uint32',\n        'app'           : 'uint16',\n        'device'        : 'uint16',\n        'os'            : 'uint16',\n        'channel'       : 'uint16',\n        'is_attributed' : 'uint8',\n        }\n\ncols = ['ip', 'app', 'device', 'os', 'channel', 'is_attributed']\n\nprint('Loading the training data...')\ntrain = pd.read_csv(os.path.join(datapath, 'train.csv'), usecols=cols, dtype=dtypes)\nprint('End loading training data...\\n')\n# checking types\ntrain.info()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"24596fe99a42a8d08744e634bb0365f1a847bc27","_cell_guid":"f66cd4f5-5e12-4ad1-9240-3e501055edf1"},"cell_type":"markdown","source":"train_head = pd.read_csv(os.path.join(datapath, 'train.csv'), nrows=10000, dtype=dtypes)\ntrain_tail = pd.read_csv(os.path.join(datapath, 'train.csv'), skiprows=range(1, len(train)-10000 ), nrows=10000, dtype=dtypes)\ntrain_head\ntrain_tail"},{"metadata":{"_uuid":"12022c6ba0e6bede2a8efd67723854cdda836362","_cell_guid":"17d8e2ab-4f22-49d3-9771-cdb3d9b3e6c8","trusted":false,"collapsed":true},"cell_type":"code","source":"print('Loading the test data...')\ntest = pd.read_csv(os.path.join(datapath, 'test.csv'), dtype=dtypes, parse_dates=['click_time'])\nprint('End loading test data...\\n')\ntest.info()\ntest","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1d8e7e089c6a09bb3464e5048973d0bcaf698f2e","_cell_guid":"f960cecb-7c31-4643-a2de-5555d2b0506e"},"cell_type":"markdown","source":"del train_head\ndel train_tail\ngc.collect()"},{"metadata":{"collapsed":true,"_uuid":"edf27c107823c445c6c6412346ef3cffeaa5b03e","_cell_guid":"57a41824-ff90-4a87-ab8e-8c46a6773892"},"cell_type":"markdown","source":"## **2. Descrição simples dos dados e missing values**\nSource: https://www.kaggle.com/anokas/talkingdata-adtracking-eda"},{"metadata":{"collapsed":true,"_uuid":"d9796d9d6858c36f67a2abb6382b6e177349eab3","_cell_guid":"673ba29a-b8e1-4644-8cdd-af61e7ab2a17","trusted":false},"cell_type":"code","source":"train.isnull().any()\ntest.isnull().any()\ntrain_kaggle_sample.isnull().any()","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"136eaaea41bd1bb7345729f18dc40ee2deff177c","_cell_guid":"01ce08e3-ecc3-4470-8ed9-a1c2f9242661","trusted":false},"cell_type":"code","source":"train_kaggle_sample.info()\ntrain_kaggle_sample[['attributed_time', 'is_attributed']].loc[train_kaggle_sample.is_attributed == 1].describe(include='all')\ntrain_kaggle_sample[['attributed_time', 'is_attributed']].loc[train_kaggle_sample.is_attributed == 0].describe(include='all')","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"601f7730709d0f57b27625f29afa0d3aeddb226f","_cell_guid":"2e742fb9-6712-4b7e-ae32-ea4344539963","trusted":false},"cell_type":"code","source":"size=10000000\nall_rows = len(train)\nnum_parts = all_rows//size + 1","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"36805da6bd93ef5fb612367f723ea5b58c3f82bf","_cell_guid":"bc72a328-ca30-46db-8231-c063ca5015bd","trusted":false,"collapsed":true},"cell_type":"code","source":"#generate the first batch\nchunk = train[0:size]\n\nip_gb = chunk[['ip', 'is_attributed']].groupby('ip').is_attributed.agg([sum, len])\napp_gb = chunk[['app', 'is_attributed']].groupby('app').is_attributed.agg([sum, len])\ndevice_gb = chunk[['device', 'is_attributed']].groupby('device').is_attributed.agg([sum, len])\nos_gb = chunk[['os', 'is_attributed']].groupby('os').is_attributed.agg([sum, len])\nchannel_gb = chunk[['channel', 'is_attributed']].groupby('channel').is_attributed.agg([sum, len])\n\ndfs_gb = [ip_gb, app_gb, device_gb, os_gb, channel_gb]\n\n#add remaining batches\nfor p in range(1,num_parts):\n    start = p*size\n    end = p*size + size\n    \n    if end < all_rows:\n        chunk = train[start:end]#[['ip', 'is_attributed']].groupby('ip', as_index=False).count()\n    else:\n        chunk = train[start:]#[['ip', 'is_attributed']].groupby('ip', as_index=False).count()\n    \n    ip_c = chunk[['ip', 'is_attributed']].groupby('ip').is_attributed.agg([sum, len])\n    app_c = chunk[['app', 'is_attributed']].groupby('app').is_attributed.agg([sum, len])\n    device_c = chunk[['device', 'is_attributed']].groupby('device').is_attributed.agg([sum, len])\n    os_c = chunk[['os', 'is_attributed']].groupby('os').is_attributed.agg([sum, len])\n    channel_c = chunk[['channel', 'is_attributed']].groupby('channel').is_attributed.agg([sum, len])\n    \n    dfs_c = [ip_c, app_c, device_c, os_c, channel_c]\n    \n    dfs_gb[:] = [(df_gb\n                   .join(df_c, how='outer', lsuffix='_gb', rsuffix='_c')\n                   .assign(sum=lambda df: np.nansum((df['sum_gb'], df['sum_c']), axis = 0), len=lambda df: np.nansum((df['len_gb'], df['len_c']), axis = 0))\n                   .drop(columns=['sum_gb', 'len_gb', 'sum_c', 'len_c'])) for df_gb, df_c in zip(dfs_gb, dfs_c)]\n    \n    print(\"Finalizou chunk {}\".format(p))\n    \nip_gb, app_gb, device_gb, os_gb, channel_gb = dfs_gb[:]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e31a9487c60c7c03844ca7b0d6d9cc4cfea4cb92","_cell_guid":"0da62d20-b9f2-469c-b8a0-42974de568d7","trusted":false,"collapsed":true},"cell_type":"code","source":"sns.set()\nsns.set(font_scale=1.2)\nfig = plt.figure(figsize=(24,20))\ngs = GridSpec(2, 2)\n\ncols = ['ip', 'app', 'device', 'os', 'channel']\nuniques = [len(df) for df in dfs_gb]\nuniques_test = [len(test[col].unique()) for col in cols]\nuniques_total = [len(df.join(test[col].value_counts(), how='outer')) for df, col in zip(dfs_gb, cols)]\n\nax0 = plt.subplot(gs[0,:])\nax0 = sns.barplot(cols, uniques_total, palette=pal, log=True)\nsettings = ax0.set(ylabel='log(unique)', title='Quantidade de valores únicos por Feature (Train + Test)')\nfor p, value in zip(ax0.patches, uniques_total):\n    height = p.get_height()\n    text = ax0.text(p.get_x()+p.get_width()/2., height + 10, value, ha=\"center\")\n\nax1 = plt.subplot(gs[1,0])\nax1 = sns.barplot(cols, uniques, palette=pal, log=True)\nsettings = ax1.set(title='Quantidade de valores únicos por Feature (Train)') \nfor p, value in zip(ax1.patches, uniques):\n    height = p.get_height()\n    text = ax1.text(p.get_x()+p.get_width()/2., height + 10, value, ha=\"center\")\n    \nax2 = plt.subplot(gs[1,1], sharey=ax1)\nax2 = sns.barplot(cols, uniques_test, palette=pal, log=True)\nsettings = ax2.set(title='Quantidade de valores únicos por Feature (Test)') \nfor p, value in zip(ax2.patches, uniques_test):\n    height = p.get_height()\n    text = ax2.text(p.get_x()+p.get_width()/2., height + 10, value, ha=\"center\")   ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e185d0dbebae3845ea0982de0fff0ad30af5bb69","_cell_guid":"ddb1fd81-6384-4061-8efa-a738e897f382","trusted":false,"collapsed":true},"cell_type":"code","source":"cols = ['ip', 'app', 'device', 'os', 'channel']\nexclusivo_train = [x-y for x, y in zip(uniques_total, uniques_test)]\nexclusivo_test = [x-y for x, y in zip(uniques_total, uniques)]\nboth_train_test = [x+y-z for x, y, z in zip(uniques, uniques_test, uniques_total)]\npct_uniques = pd.DataFrame(columns=cols, index=['Interseção Train e Test', 'Exclusivo Train', 'Exclusivo Test'], \n                           data=[[round(100*x/y, 2) for x, y in zip(both_train_test, uniques_total)],\n                                 [round(100*x/y, 2) for x, y in zip(exclusivo_train, uniques_total)],\n                                 [round(100*x/y, 2) for x, y in zip(exclusivo_test, uniques_total)]])\n\nsns.set()\nsns.set(font_scale=1.4)\nsns.set_style(\"white\")\nfig = plt.figure(figsize=(20,10))\nax0 = plt.subplot(111)\nax0 = pct_uniques.T.plot(kind='bar', stacked=True, ax=ax0, legend=True, rot=0)\nsettings = ax0.set(ylim=(0,115), ylabel='porcentagem de valores exclusivos e compatilhados dos datasets', xlabel='Features', title='Porcentagem de valores exclusivos e compartilhados dos datasets Train e Test por Feature na composição Total dos dados')\nfor p, v0, v1, v2  in zip(ax0.patches, pct_uniques.iloc[0], pct_uniques.iloc[1], pct_uniques.iloc[2]):\n    height0 = v0 / 2\n    height1 = v1 / 2 + v0\n    height2 = v2 / 2 + v0 + v1\n    for value, height in zip ([v0, v1, v2],[height0, height1, height2]):\n        if value > 0:\n            text = ax0.text(p.get_x()+p.get_width()/2., height, \"{}%\".format(value), ha=\"center\")\nax0 = sns.despine()\n            \nsns.set()\nsns.set(font_scale=1.2)\nfig, [ax1, ax2] = plt.subplots(figsize=(20,10), nrows=1, ncols=2)\n\nax1 = plt.subplot(121)\nax1 = sns.barplot(cols, exclusivo_train, palette=pal, log=True)\nsettings = ax1.set(title='Quantidade de valores únicos exclusivos de Train')\nfor p, value in zip(ax1.patches, exclusivo_train):\n    height = p.get_height() + 0.1*10**np.log10(value+1)\n    text = ax1.text(p.get_x()+p.get_width()/2., height, value, ha=\"center\")\n    \nax2 = plt.subplot(122, sharey=ax1)\nax2 = sns.barplot(cols, exclusivo_test, palette=pal, log=True)\nsettings = ax2.set(title='Quantidade de valores únicos exclusivos de Test')\nfor p, value in zip(ax2.patches, exclusivo_test):\n    height = p.get_height() + 0.1*10**np.log10(value+1)\n    if height < 10: height = height + 16\n    text = ax2.text(p.get_x()+p.get_width()/2., height, value, ha=\"center\")   ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"044b398a4da74140817ec0646208ef0374d0720c","_cell_guid":"b2afc6a6-7d1d-417b-ba1b-49d5d259305c","trusted":false,"collapsed":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9f06cdf9b03a3f0f6016b06c74937beb1a922898","_cell_guid":"f8e5324c-5cd3-42bc-95af-e50d0739bc2d","trusted":false,"collapsed":true},"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(10, 10))\ndownload_pct = app_gb['sum'].sum() / app_gb['len'].sum()\nsettings = ax.set(xlabel='Target Value', ylabel='Probability', title='Target value distribution')\nax = sns.barplot(['Dowloaded (1)', 'Not Dowloaded (0)'], [download_pct, 1-download_pct], palette=pal)\nfor p, value in zip(ax.patches, [download_pct, 1-download_pct]):\n    height = p.get_height()\n    text = ax.text(p.get_x()+p.get_width()/2., height+0.01, '{}%'.format(round(value * 100, 2)), ha=\"center\") ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8991dd4dba335b17bf1dd3e8e257d0e9f5783456","_cell_guid":"f5ee3895-b9ed-4035-8326-a2324e2cf836"},"cell_type":"markdown","source":"## Relação de cada Feature com Target\n### - IP"},{"metadata":{"_uuid":"97a93e9693e7fbf21807f2f8a70d281f7b84dc5a","_cell_guid":"9bf06f96-b1a5-4129-92fa-336096feab6a","trusted":false,"collapsed":true},"cell_type":"code","source":"ip_gb['ip_pct'] = ip_gb['sum'] / ip_gb['len']\nip_gb.sort_values(by=['len'], ascending=False).iloc[:100]\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b221d97ebf5479ee4e27e45d8055644d0daecc1d","_cell_guid":"5335d700-14c4-41fa-b715-6331cfe1db0a"},"cell_type":"markdown","source":"### - App"},{"metadata":{"_uuid":"46121037f1b01683240d104b8e108a060d8e1b4f","_cell_guid":"08adec38-7632-46b5-b062-7df4a878e5fb","trusted":false,"collapsed":true},"cell_type":"code","source":"app_gb_copy = app_gb.copy()\napp_gb_copy['Download_pct'] = app_gb_copy['sum'] / app_gb_copy['len']\ndata = app_gb_copy.sort_values(by=['len'], ascending=False)[:100].reset_index().rename(columns={'len':'Count'})\n\nfig, ax = plt.subplots(figsize=(10, 10))\nax = data.Count.plot(ax=ax, logy=True, legend=True)\nsettings = ax.set(ylabel='log Count of clicks')\nax = data.Download_pct.plot(secondary_y=True, ax=ax, legend=True)\nsettings = ax.set(xlabel='Target Value', ylabel='Probability', title='Conversion Rates over Counts of 100 Most Popular Apps')\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2179e2929b63198847484301cefb22dc25ebf417","_cell_guid":"55b31fff-14ab-498c-a032-87dbd0e6ff8c","trusted":false,"collapsed":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"eb6da3426d890ad60418c2f58eecce3c691647b7","_cell_guid":"e7704762-6982-4c57-917b-431ddb69e62d","trusted":false,"collapsed":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a4c7c630e9cd8d8bf4e32e052b82ecc7fa6793ad","_cell_guid":"1e2833e8-a133-4d87-844e-c72faadfd135","trusted":false,"collapsed":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"51bc99f2797a189dbd63e9d1f3c7409583a9de95","_cell_guid":"3e949b1d-ba24-40f3-bebd-69fa6820e73f","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"name":"python","pygments_lexer":"ipython3","mimetype":"text/x-python","file_extension":".py","version":"3.6.4","nbconvert_exporter":"python","codemirror_mode":{"name":"ipython","version":3}},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}