{"cells":[{"metadata":{"_cell_guid":"af0aab66-fb71-44fc-b80c-dd41050cdd2d","collapsed":true,"_uuid":"40833634550fe5318580f4a61b6c614c209ff271"},"cell_type":"markdown","source":"Inspired by great kernels:\n\nhttps://www.kaggle.com/pkhomchuk/artifacts-in-the-training-data\n\nhttps://www.kaggle.com/jtrotman/scatter-plots-of-ip-over-time-per-channel-part-1\n\nhttps://www.kaggle.com/jtrotman/scatter-plots-of-ip-over-time-per-channel-part-2"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","collapsed":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_kg_hide-input":false,"trusted":true},"cell_type":"code","source":"%matplotlib inline\nimport os\nimport gc\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nimport time\n\n#sample_data = pd.read_csv( '../input/train_sample.csv' )\ndtype = {\n        'ip'               : 'uint32',\n        'app'              : 'uint16',\n        'device'           : 'uint16',\n        'os'               : 'uint16',\n        'channel'          : 'uint16',\n        'is_attributed'    : 'uint8',\n        #'click_id'        : 'uint32',\n        'click_hour'       : 'uint8',\n        'click_minute'     : 'uint8',\n        'click_second'     : 'uint8'        \n        }\nDATA_DIR = '../input/'\ntarget_col = 'is_attributed'\n\ndef read_by_chunk(path, chunksize=10**6, subsample=0.2, read_csv_params={}):\n    df = pd.DataFrame()\n    for chunk in tqdm(pd.read_csv(path, chunksize=chunksize, **read_csv_params)):\n        if 0.0<subsample<1.0:\n            size = int(subsample*len(chunk))\n            chunk = chunk.sample(frac=1).iloc[:size]\n        df = pd.concat([df, chunk], ignore_index=True, axis=0)\n    return df","execution_count":3,"outputs":[]},{"metadata":{"_cell_guid":"202b2c65-5864-410c-be81-e91723f6c8c3","_uuid":"313656b1999414c83e69fbca2f6eb5f1235c2702","trusted":true},"cell_type":"code","source":"key_li = ['ip', 'app', 'device', 'os', 'channel', 'click_time', 'is_attributed']\nread_csv_params = {'dtype':dtype, 'usecols':key_li}\nsubsample = 0.08\ndf_train = read_by_chunk(DATA_DIR+'train.csv', subsample=subsample, read_csv_params=read_csv_params)","execution_count":4,"outputs":[]},{"metadata":{"_cell_guid":"d67d2095-6b28-47fe-a924-e8d20006f167","_uuid":"9d6eb1f5b5282a3ecaa1d704d6ce1b1f514a33bf","trusted":true},"cell_type":"code","source":"print(df_train.shape)\ndf_train.head()","execution_count":5,"outputs":[]},{"metadata":{"_cell_guid":"5020c9e4-25e8-4382-b4cc-b6003174c126","_uuid":"87e6644fa2be5e939adb817faaaf7fdd4b95424b","trusted":true},"cell_type":"code","source":"key_li = ['ip', 'app', 'device', 'os', 'channel', 'click_time']\nread_csv_params = {'dtype':dtype, 'usecols':key_li}\ndf_test = read_by_chunk(DATA_DIR+'test.csv', subsample=subsample, read_csv_params=read_csv_params)\nprint(df_test.shape)\ndf_test.head()","execution_count":6,"outputs":[]},{"metadata":{"_cell_guid":"79a7c727-fd33-425e-bf31-623f6f1fdb3a","_uuid":"a8a6fe6adaf4079c47cf2d66c70bc0709a7c7b09","trusted":true},"cell_type":"code","source":"df_test[target_col] = -1\ndf_test = df_test[key_li+[target_col]]\nsample_data = pd.concat([df_train, df_test], axis=0, ignore_index=True)\ndel df_train, df_test; gc.collect()","execution_count":7,"outputs":[]},{"metadata":{"_cell_guid":"2bad37dc-c410-442d-8ce3-8526649d5888","_uuid":"ff775ad7d1ea5ccfd0db0e7a9bfd0abac49c1fda","trusted":true},"cell_type":"code","source":"print(sample_data.shape)\nsample_data.head()","execution_count":8,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"sample_data[ 'click_time' ] = pd.to_datetime( sample_data[ 'click_time' ] )\nclicktime = np.array( [ t.timestamp() for t in sample_data[ 'click_time' ] ] )\n#clicktime -= np.min( clicktime )\nclicktime -= pd.to_datetime( '2017-11-07 00:00:00' ).timestamp()\n\n#idx_download = sample_data.index[ sample_data[ 'is_attributed' ] == 1 ]\n#idx_test = sample_data.index[ sample_data[ 'is_attributed' ] == -1 ]\nsample_data[ 'time' ] = clicktime.astype( int )\nsample_data.sort_values( by = [ 'time' ], inplace = True )\nsample_data.reset_index( drop = True, inplace = True )\n\ndel clicktime; gc.collect()","execution_count":9,"outputs":[]},{"metadata":{"_cell_guid":"e933f92e-6474-48f8-aace-e4f5f33eef26","collapsed":true,"_uuid":"da4804569802b9b8abeb0edfec9d9a16f5b292a8","trusted":true},"cell_type":"code","source":"def plot_attribute_vs_time( sdata, attr_name ):\n    idx = sdata['is_attributed'] == 1\n    idx_test = sdata['is_attributed'] == -1\n    plt.figure(figsize=(128, 48))\n    plt.plot( sdata[ 'time' ] / 3600, sdata[ attr_name ], 'b.', alpha = 0.1, label = 'is_attributed = 0' )\n    plt.plot( sdata[ 'time' ][idx] / 3600, sdata[ attr_name ][idx], 'ro', alpha = 0.6, label = 'is_attributed = 1' )\n    plt.plot( sdata[ 'time' ][idx_test] / 3600, sdata[ attr_name ][idx_test], 'g.', alpha = 0.6, label = 'test' )\n    plt.xlabel( 'Time [hours]', fontsize = 16 )\n    plt.ylabel( attr_name, fontsize = 16 )\n    leg  = plt.legend( fontsize = 16 )\n    for l in leg.get_lines():\n        l.set_alpha( 1 )\n        l.set_marker( '.' )\n    plt.show()","execution_count":10,"outputs":[]},{"metadata":{"_cell_guid":"480628cb-2a1a-4b65-9b97-a956c2803966","_uuid":"45fd9684b57d9b96eb567c6924019ee04c9d3ada","trusted":true,"scrolled":false},"cell_type":"code","source":"for c in ['ip', 'app', 'device', 'os', 'channel']:\n    print('plotting', c, '...')\n    plot_attribute_vs_time(sample_data, c)","execution_count":11,"outputs":[]},{"metadata":{"_cell_guid":"cd9399e4-2f22-497f-8aa3-db8ab4a709ff","_uuid":"9789f70a0031e01f510d12d83f4930cc8bbad61e","trusted":true},"cell_type":"code","source":"CUT = {}\nCUT['ip'] = sample_data['ip'][sample_data['is_attributed']==-1].max()+1\nCUT['app'] = 300\nCUT['device'] = 500\nCUT['os'] = 200\nCUT['channel'] = sample_data['channel'].max()+1 #NO CUT","execution_count":13,"outputs":[]},{"metadata":{"_cell_guid":"134e4315-dd48-4bcd-9e5b-3f2b80669993","_uuid":"431a42621351483ff8233cb75c2033200054936b","trusted":true},"cell_type":"code","source":"for c in ['ip', 'app', 'device', 'os']:\n    print('plotting cut', c, '...')\n    plot_attribute_vs_time(sample_data[sample_data[c]<CUT[c]], c)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}