{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","collapsed":true,"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import gc, os\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom collections import defaultdict\nfrom tqdm import tnrange, tqdm_notebook\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.linear_model import SGDClassifier\ntrain_fpath = \"../input/train.csv\"\ntest_fpath = \"../input/test.csv\"","execution_count":1,"outputs":[]},{"metadata":{"_uuid":"64017200fa027247a3594d73036a8d9fa1c9868f","_cell_guid":"8b16831f-e4da-457f-bb9a-9f85579343f5"},"cell_type":"markdown","source":"## Helper Functions"},{"metadata":{"_uuid":"77ffb52e2dd3c4ef16d0922d26963796c9fa82a6","collapsed":true,"_cell_guid":"1a319374-9e2d-4715-bd2e-109389812bf3","trusted":true},"cell_type":"code","source":"sample_counts_on_each_day = {\n    \"2017-11-06\": 9308568,\n    \"2017-11-07\": 59633310,\n    \"2017-11-08\": 62945075,\n    \"2017-11-09\": 53016937\n}\ncolumns = ['ip', 'app', 'device', 'os', 'channel', 'click_time', 'attributed_time', 'is_attributed']\ndtypes = {\n    'ip': 'uint32',\n    'app': 'uint16',\n    'device': 'uint16',\n    'os': 'uint16',\n    'channel': 'uint16',\n    'is_attributed': 'uint8',\n    'click_id': 'uint32'\n}\ndef get_samples_by_date(at):\n    nrows = 0\n    skiprows = 1\n    for date, count in sample_counts_on_each_day.items():\n        if date == at:\n            nrows = count\n        elif date < at:\n            skiprows += count\n    return pd.read_csv(train_fpath, skiprows=skiprows, nrows=nrows, names=columns, dtype=dtypes, parse_dates=[\"click_time\"])\ndef ratio(x):\n    return x.shape[0] / sample_counts_on_each_day[sample_date]","execution_count":2,"outputs":[]},{"metadata":{"_uuid":"e5c0a2859954f99aed28503708b2e03c391220d8","_cell_guid":"8ceb7fc2-af07-4707-8d61-96785078e801"},"cell_type":"markdown","source":"## The ratio of is_attributed on each day"},{"metadata":{"_uuid":"8bde78430bc16448f614c9a7b1b53450bfc708c5","collapsed":true,"_cell_guid":"1896c6dc-5382-4eee-a0f6-b13562c00e56","trusted":true},"cell_type":"code","source":"df_Test = pd.read_csv(test_fpath, dtype=dtypes, parse_dates=[\"click_time\"])","execution_count":3,"outputs":[]},{"metadata":{"_uuid":"64cd04c4c478be759b525b45aa96972e33c0b45a","_cell_guid":"69d305f2-b253-47c6-9167-f60e522c3ce5","trusted":true},"cell_type":"code","source":"for sample_date in [\"2017-11-06\", \"2017-11-07\", \"2017-11-08\", \"2017-11-09\"]:\n    df_Samples = get_samples_by_date(sample_date)\n    print(sample_date)\n    display(df_Samples.groupby(\"is_attributed\").agg({\"ip\": [\"count\", ratio]}))\n    gc.collect()","execution_count":9,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab5be3189f989f73db51b453c07182b0fb7d58c7"},"cell_type":"code","source":"df_Test.head()","execution_count":10,"outputs":[]},{"metadata":{"_uuid":"6de4aa071441511136eee7d8e9424e7359854ae4"},"cell_type":"markdown","source":"## Does training data cover all values in testing data"},{"metadata":{"_uuid":"f1b877c99758ff14b492ff9f755070bcf503d8ba","_cell_guid":"304933d6-3e1a-4150-9a6e-462c3d3977a3","trusted":true,"scrolled":true},"cell_type":"code","source":"testing_data_vals = {}\nfor col in [\"ip\", \"app\", \"device\", \"os\", \"channel\"]:\n    testing_data_vals[col] = set(df_Test[col].unique())\nfor sample_date in tqdm_notebook([\"2017-11-06\", \"2017-11-07\", \"2017-11-08\", \"2017-11-09\"]):\n    df_Samples = get_samples_by_date(sample_date)\n    for col in [\"ip\", \"app\", \"device\", \"os\", \"channel\"]:\n        testing_data_vals[col] = testing_data_vals[col] - set(df_Samples[col].unique())\n    gc.collect()","execution_count":11,"outputs":[]},{"metadata":{"_uuid":"dfab96dd1cc43c4bedd7fac7ed42d7279513e229","_cell_guid":"373d8693-d364-412f-ae9b-5caa97025bae","trusted":true},"cell_type":"code","source":"for col, vals in testing_data_vals.items():\n    print(\"{}: There are {} values don't exist in training data.\".format(col, len(vals)))","execution_count":12,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"588c78c84c7152b148bb64b385f20897f0718b3b"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}