{"cells":[{"metadata":{"_uuid":"4b42ca92118e1b285ca81a4a739bf1a94a6fca1a","_cell_guid":"bb36066e-bb1d-4da4-9e88-e7cec976d69b"},"cell_type":"markdown","source":"This is a quick analysis of the duplicates in the data, which I believe are clicks that were separated by less than 1 second, which would explain why duplicates are not always neighbors in the data."},{"metadata":{"collapsed":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# MATHEMATICS\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport random\nimport scipy\n\n# SYSTEM\nimport os \nimport gc\n\n#VIZUALISATION\n%matplotlib inline\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nsns.set()\n#from ipywidgets import Layout\n#import plotly.offline as py\n#py.init_notebook_mode(connected=True)\n\n#from IPython.core.display import display, HTML\n#display(HTML(\"<style>.container { width:100% !important; }</style>\"))\n\n#w = catboost.CatboostIpythonWidget('')\n#w.update_widget()\n\n# set-up\nKaggle_kernel=False\nlocal_path='input/'\nkaggle_path='../input/'\noriginal_features=['app','channel','ip','device','os']","execution_count":2,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_kg_hide-output":true,"_kg_hide-input":true,"trusted":false},"cell_type":"code","source":"def load_data(name,rows=None):\n    ''' Load the csv files into a TimeSeries dataframe with minimal data types to reduce the used RAM space. \n    It also saves the files in parquet file to reduce loading time by a factor of ~10.\n\n    Arg:\n    \n        -name (str): ante_day, last_day, train, train_sample or test\n\n    Returns:\n        pd.DataFrame, with int index equal to 'click_id'\n    '''\n\n    # Defining dtypes\n    types = {\n            'ip':np.uint32,\n            'app': np.uint16,\n            'os': np.uint16,\n            'device': np.uint16,\n            'channel':np.uint16,\n            'click_time': object\n            }\n\n    if name=='test':\n        types['click_id']= np.uint32\n    else:\n        types['is_attributed']='bool'\n\n    # Defining csv file reading parameters\n    read_args={\n        'nrows':rows,\n        'parse_dates':['click_time'],\n        'infer_datetime_format':True,\n        'index_col':'click_time',\n        'usecols':list(types.keys()),\n        'dtype':types,\n        'engine':'c',\n        'sep':','\n        }\n\n    # Setting file path\n    file_path='{}{}'.format(kaggle_path,name)\n\n    with open('{}.csv'.format(file_path),'rb') as File:\n        data=(pd\n            .read_csv(File,**read_args)\n            .tz_localize('UTC')\n            .tz_convert('Asia/Shanghai')\n            .reset_index()\n        )\n\n    # Sorting frames\n    if name=='test': # making sure index == click_id\n        data=data.sort_values(by=['click_id']).reset_index(drop=True)\n    elif name=='train_sample': # sorting time randomized by sampling\n        data=data.sort_values(by=['click_time']).reset_index(drop=True)\n\n    return data\n\ndef actor(dataframe,maxima):\n    result=pd.Series(data=0,index=dataframe.index,dtype=np.uint64)\n    for i in original_features:\n        result=(dataframe[i]+1)+(maxima[i]+1)*result\n    return result","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"82d9ff8b6ae0d0b047e6c42720dbb7df94f8d256","_cell_guid":"7fe19f88-1ce1-418d-8343-53bbe1cfacc4","_kg_hide-input":true,"_kg_hide-output":true,"trusted":false},"cell_type":"code","source":"df=load_data('train')\nmaxima=df.describe().loc['max',:].astype(np.uint32)\ndf.info()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_uuid":"abcdc3fa56eb04230a6408de9f9a2fe9e9520aed","_cell_guid":"53b034c6-252b-4c7c-8394-01dd00cbcbcb"},"cell_type":"markdown","source":"For convenience, I replaced the 5 original features by a unique identifier named 'actor'  for each quintuple. Here are the first five lines of the training data, in the Shanghai timezone."},{"metadata":{"_kg_hide-input":true,"_uuid":"c147e2a46df825791a4e046ac9dd1bc306a889fc","collapsed":true,"_cell_guid":"5f972ad3-0128-4e8a-9f8c-49c4094c59b1","trusted":false},"cell_type":"code","source":"gc.collect()\ndf=df.assign(actor=lambda x: actor(x,maxima)).drop(original_features,axis=1)\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c3b7887c200a0b24a3f14c401e48e9aca507afd0","_cell_guid":"4fad8bd7-b647-4a2d-b169-9e979c954c25"},"cell_type":"markdown","source":"Let us now get the statistics that interest us:"},{"metadata":{"collapsed":true,"_uuid":"bd114674a838b303f79bcfdf1e704eb4dc1cfb94","_cell_guid":"feeae700-d1ed-4d65-a2c8-7cad95123a3b","_kg_hide-output":true,"_kg_hide-input":true,"trusted":false},"cell_type":"code","source":"gc.collect()\nduplicates=df.loc[df.duplicated(subset=['click_time','actor'],keep=False),:]","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"_uuid":"a62d59c0980b878ac0b6224757c24b2382a180a6","collapsed":true,"_cell_guid":"396d2166-ca3e-431c-b5ad-17b226316a5f","trusted":false},"cell_type":"code","source":"gc.collect()\nprint('The proportion of duplicated rows in the training data is : {0:.1f}%'.format(df.duplicated(subset=['click_time','actor'],keep=False).mean()*100))\nprint('The attribution rate of the duplicates is {:.3f}% while the full set is {:.3f}%:'.format(duplicates.is_attributed.mean()*100,df.is_attributed.mean()*100))\nprint('The attribution rate for first elements of duplicates is {:.3f}%'.format(duplicates.loc[duplicates.duplicated(subset=['click_time','actor'],keep='first'),'is_attributed'].mean()*100))\nprint('The attribution rate for last elements of duplicates is {:.3f}%'.format(duplicates.loc[duplicates.duplicated(['click_time','actor'],keep='last'),'is_attributed'].mean()*100))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1e4d8fa4cf222145aae756c548ab49c0a49bfe2c","_cell_guid":"51225b4d-b947-480e-82d0-1d4939cc1af1"},"cell_type":"markdown","source":"So duplicates are half as likely to be positives within duplicates  than within the whole set. Moreover they are also half as likely to be positives if they are first than if they are last of the duplicates. That is interesting. Duplicates might come in more than two though and we need to investigate a little further.\n\nThe table below shows the number of cases by number of duplicates and the mean of their respective attribution rates:"},{"metadata":{"collapsed":true,"_uuid":"42da33cae6675b093d6ff3983063071a88276a5f","_cell_guid":"fb5f2fd4-dd2b-4351-b262-8f2c8751715c","_kg_hide-input":true,"_kg_hide-output":false,"trusted":false},"cell_type":"code","source":"duplicates_dist=(duplicates\n                 .groupby(['click_time','actor'])\n                 .is_attributed.agg(['count','mean'])\n                 .groupby('count')\n                 .agg(['count','mean'])\n                )\nduplicates_dist=(duplicates_dist.rename(columns={'mean':'training'},level=0)\n                 .rename(columns={'mean':'avg attr rate'},level=1)\n                 .rename_axis('')\n                )\ndel(df,duplicates)\ngc.collect()\nduplicates_dist","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c5ca990137ba5b8f6277a24ad99a5a9759e792c3","_cell_guid":"d501fee5-a573-492f-bdae-498a7211c460"},"cell_type":"markdown","source":"Let us now look at the statistics and distribution of duplicates in the test data."},{"metadata":{"collapsed":true,"_uuid":"fe2f4b2d3d93dc2653d4d417749cd8c5cfb1a140","_cell_guid":"7ecf6556-14ff-404b-9696-87b18309d074","_kg_hide-input":true,"_kg_hide-output":true,"trusted":false},"cell_type":"code","source":"df=load_data('test')\nmaxima=df.describe().loc['max',:].astype(np.uint32)\ndf.info()","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"6f5a100ebef84575d07d16e95c663f95984f4062","_cell_guid":"4c69cacd-b0ef-4d5f-980b-e77b480e104d","_kg_hide-input":true,"_kg_hide-output":true,"trusted":false},"cell_type":"code","source":"gc.collect()\ndf=df.assign(actor=lambda x: actor(x,maxima)).drop(original_features,axis=1)\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"54bb7c38d30b2dcbc46dbece035b10d8f54a4249","_cell_guid":"ed535f08-373d-4f53-ba5f-04d71d4e41f0","trusted":true},"cell_type":"code","source":"gc.collect()\nprint('The proportion of duplicated rows is : {0:.1f}%'.format(df.duplicated(subset=['click_time','actor'],keep=False).mean()*100))\nduplicates=df.loc[df.duplicated(subset=['click_time','actor'],keep=False),:]\nprint('The table below shows statistics by number of duplicates:')\nduplicates_dist[('test','count')]=duplicates.groupby(['click_time','actor']).click_id.count().value_counts()\nduplicates_dist[('test','count')]=duplicates_dist[('test','count')].fillna(0).astype(int)\nduplicates_dist","execution_count":3,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}