{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":2,"outputs":[]},{"metadata":{"_cell_guid":"6085e331-717d-42db-876e-7cebdc169719","_uuid":"11899b157199e11d66fef638c8cccf5be43e3d9d"},"cell_type":"markdown","source":"***Get some basic idea of the Data***"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"collapsed":true},"cell_type":"code","source":"# --- Read Data ---\ntrainSample = pd.read_csv('../input/train_sample.csv')\n#testSplmnt  = pd.read_csv('../input/test_supplement.csv')\n#train       = pd.read_csv('../input/train.csv')\ntest        = pd.read_csv('../input/test.csv') \nsampleSubmission = pd.read_csv('../input/sample_submission.csv')","execution_count":25,"outputs":[]},{"metadata":{"_cell_guid":"a0f7a8c5-a933-4883-83d3-aab78345279d","_uuid":"5a087fa43ba9de3e088e9d067d393daa88865d2b","trusted":false,"collapsed":true},"cell_type":"code","source":"# --- Info ---\ntrainSample.info()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"7612a5ad-4910-4ba9-98ad-2682d1914004","_uuid":"900757ba18bea242266900b578d8e36400158c94","trusted":false,"collapsed":true},"cell_type":"code","source":"test.info()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"9d5860c0-e2fa-4e2a-9eb0-ea0167f34ca0","_uuid":"6d379de6b3c9bb99505fc6380a4ff602d3513dd5","trusted":false,"collapsed":true},"cell_type":"code","source":"# --- View Sample Data ---\nprint(trainSample.head())\nprint(test.head())","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"9d295072-159d-422c-b325-89fb5825060f","_uuid":"ecad310d5ece87e6daecc403009bece74375e238","trusted":false,"collapsed":true},"cell_type":"code","source":"# --- is_attributed ---\nprint('Total Attribution :\\n', trainSample['is_attributed'].sum())\nprint('Distribution of is_attributed :\\n', trainSample['is_attributed'].value_counts())","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"f7b9ded5-1b3f-48b0-b5e8-2abac46dea70","_uuid":"c4a3b4a1d6d75afd71ccd96c4ee0c944d640e443","trusted":true},"cell_type":"code","source":"# --- Rest of the Variable ---\nfor col in trainSample.columns:\n    if col != 'is_attributed':\n        print('\\n --- Getting Stats for ' + col + ' --- \\n')\n        print('Unique Count :', trainSample[col].nunique())\n        print('Distribution \\n', trainSample[col].value_counts()[:5])\n        tmpDf = pd.DataFrame(trainSample.groupby(col)['is_attributed'].sum().reset_index())\n        tmpDf.columns = [col, 'isAttributed']\n        tmpDf.sort_values('isAttributed', ascending = False, inplace = True)\n        print('Top Attributed Values \\n', tmpDf.head())\n        print('Reverse Distribution \\n', tmpDf.groupby('isAttributed')[col].count())","execution_count":92,"outputs":[]},{"metadata":{"_cell_guid":"bac44758-4d4f-4a30-a47e-572d074d2e58","_uuid":"0ef8b95e5d7fb4c72e23b4765946838f45669de6","trusted":true,"collapsed":true},"cell_type":"code","source":"# --- Bar Plot ---\nfrom matplotlib import pyplot as plt\nimport seaborn as sns\n\nsns.set(color_codes=True)\nsns.set_style('darkgrid')\n\n\ndef barPlot(df, yAxis, xAxis):\n    xTicks = df[xAxis]\n    yTicks = df[yAxis]\n    plt.bar(df[xAxis].index, df[yAxis])\n    plt.xticks(df[xAxis].index, xTicks)\n    #plt.yticks(yTicks)\n    plt.xlabel(xAxis)\n    plt.ylabel(yAxis)\n    plt.show()","execution_count":90,"outputs":[]},{"metadata":{"_cell_guid":"1e5caf60-9fee-4fe5-b138-72e94091af22","_uuid":"3d7112e37c37907dfe8633d97ca9b4417506e039","collapsed":true,"trusted":true},"cell_type":"code","source":"# --- Attributed Data Only ---\n\nattributedOnly = None\nfor col in trainSample.columns:\n    attributedOnly = trainSample.query('is_attributed==1')\n    if col not in ['is_attributed', 'attributed_time', 'click_time'] :\n        print('\\n --- Getting Stats for ' + col + ' --- \\n')\n        \n        print('Unique Count :', attributedOnly[col].nunique())\n        \n        \n        print('Distribution of Count for ', col, ': \\n')\n        tmpDf = attributedOnly[col].value_counts()[:5].reset_index()\n        tmpDf.columns = [col, 'appInstallCount']\n        tmpDf[col] = tmpDf[col].map(str)\n        print(tmpDf)\n        barPlot(tmpDf, 'appInstallCount', col)\n        \n        \n        print('Distribution of Sum of is Attributed for :', col , '\\n')\n        tmpDf = pd.DataFrame(attributedOnly.groupby(col)['is_attributed'].sum().reset_index())\n        tmpDf.columns = [col, 'isAttributed']\n        tmpDf.sort_values('isAttributed', ascending = False, inplace = True)\n        headTmpDf = tmpDf.head().reset_index()\n        print('Top Attributed Values \\n', headTmpDf)\n        barPlot(headTmpDf, 'isAttributed', col)\n        \n        \n        print('Reverse Distribution \\n')\n        tmpDf = tmpDf.groupby('isAttributed')[col].count().reset_index()\n        tmpDf.columns = ['isAttributed', 'sumIsAttributed']\n        tmpDf.sort_values('sumIsAttributed', inplace = True, ascending = False)\n        headTmpDf = tmpDf.head().reset_index()\n        del headTmpDf['index']\n        print(headTmpDf)\n        barPlot(headTmpDf, 'sumIsAttributed', 'isAttributed')        ","execution_count":91,"outputs":[]},{"metadata":{"_cell_guid":"f399950d-0c58-4aaf-85cc-849456ea0ec2","_uuid":"ee557bf993042f4e6ece0ac2af6f7929e9f504e1"},"cell_type":"markdown","source":"Few points to note:\n* There are only two ips which have been attributed more than once.\n* only .7% of Ip has been attributed\n* 65% of attribution has ome from top 5 apps\n* Top 5 device account for 99.7% of data\n* Device 1 accounts for 94.2% of data and is attributed 67% of total attribution\n* **Device 0 accounts for .56% of data and is attributed 24% of total attribution****\n* Top 5 OS is attributed 57% of total attribution\n* Top 5 Channel is attributed 64% of total attribution\n*  Date need to be analyzed in some other way\n\nThat is a lot of points from a small section of code."},{"metadata":{"_cell_guid":"2a1c707f-d4c1-498c-892c-82a2f17a34d3","_uuid":"94aca79372e94d375a6cc1015f57f67c94445c69","collapsed":true},"cell_type":"markdown","source":"Analysis of Date Field"},{"metadata":{"_cell_guid":"8c04641e-06a1-4c67-bcf7-e9b70b913b6c","_uuid":"3cb2394dc153ee39f15cc981fc46f8563c654dd1","trusted":false,"collapsed":true},"cell_type":"code","source":"trainSample['click_time'] = pd.to_datetime(trainSample['click_time'])\ntrainSample['clickDate'] = trainSample['click_time'].dt.date\ntrainSample['clickHour'] = trainSample['click_time'].dt.hour\ntrainSample['clickWOD'] = trainSample['click_time'].dt.dayofweek","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"c78da034-59e4-4e1d-b724-02370ab2e586","_uuid":"32f1f55865394968804c28b00bc0cbe36bfe42fb","trusted":false,"collapsed":true},"cell_type":"code","source":"# --- Distribution by Day of Week ---\ntmpDf = trainSample['clickWOD'].value_counts().reset_index()\ntmpDf.columns = ['WOD', 'count']\nprint(tmpDf)\nbarPlot(tmpDf, 'count', 'WOD')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"831211e6-4365-42ba-96de-ae94ea0ed547","_uuid":"6e8bcba2056a35118ec1e38bdd8e07b984caf7a1","trusted":false,"collapsed":true},"cell_type":"code","source":"# --- Distribution by Hour of Day ---\ntmpDf = trainSample['clickHour'].value_counts().reset_index()\ntmpDf.columns = ['HOD', 'count']\nprint(tmpDf)\nbarPlot(tmpDf, 'count', 'HOD')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"7ad1efa2-9bc6-4d1b-b818-58f09049cbd3","_uuid":"ee5f2a6d5e0da2e7b612bf010460593c787ec901"},"cell_type":"markdown","source":"***Lets Derive some features***"},{"metadata":{"_cell_guid":"a1d8ba54-0b0f-4ef9-8c5b-aca5dffa16c0","_uuid":"4436ef58eca28c227a204e7f3b0e17e17adc3102","trusted":true},"cell_type":"code","source":"# --- Count Level Features ---\ntrainSampleDict = trainSample.to_dict(orient = 'records')\n\ndictList = []\ntmpDict = {}\n\ndef getCount(val):\n    \n    if val in tmpDict:\n        tmpDict[val] += 1\n    else:\n        tmpDict[val] = 1\n        \n    return tmpDict\n \n    \nfor thisRow in trainSampleDict:\n    \n    getCount('ip' + str(thisRow['ip']))\n    getCount('ap' + str(thisRow['app']))\n    getCount('dv' + str(thisRow['device']))\n    getCount('os' + str(thisRow['os']))\n    getCount('ch' + str(thisRow['channel']))\n\n    \nfor thisRow in trainSampleDict:\n    \n    thisRow['ipCount']      = tmpDict['ip' + str(thisRow['ip'])]\n    thisRow['appCount']     = tmpDict['ap' + str(thisRow['app'])]\n    thisRow['deviceCount']  = tmpDict['dv' + str(thisRow['device'])]\n    thisRow['osCount']      = tmpDict['os' + str(thisRow['os'])]\n    thisRow['channelCount'] = tmpDict['ch' + str(thisRow['channel'])]\n    dictList.append(thisRow)\n","execution_count":60,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"029cf22a292b5ba10b19e63d985a5cda7791e5eb"},"cell_type":"code","source":"df = pd.DataFrame(dictList)\ndf.head(3)","execution_count":65,"outputs":[]},{"metadata":{"_cell_guid":"25eb603e-dd4b-4c25-bd75-ab1c8eeb2635","_uuid":"56ae5a0e3acd263ca5aaeb047c88e6e02d8c39df","trusted":false},"cell_type":"markdown","source":"***Ip Level Features***"},{"metadata":{"_cell_guid":"731f44ae-7b4a-46a6-b62b-b6321b89f233","_uuid":"77c40de092fb8344a6dd354da937c47e7fbdc780","trusted":true},"cell_type":"code","source":"# 1.#App in use from an IP\n# 2.#Device in USe from an IP\n# 3.#OS from an IP\n# 3.Time spent between first and last click from an IP\n\ntmpDict   = {}\nappSet    = set()\ndeviceSet = set()\nosSet     = set()\n\ndictList = []\nfor thisRow in df.to_dict(orient = 'records'):\n    if thisRow['app'] in appSet:\n        tmpDict['ap' + str(thisRow['ip'])] += 1\n    else:\n        tmpDict['ap' + str(thisRow['ip'])] = 1\n    \n    if thisRow['device'] in deviceSet:\n        tmpDict['dv' + str(thisRow['ip'])] += 1\n    else:\n        tmpDict['dv' + str(thisRow['ip'])] = 1   \n        \n    if thisRow['os'] in deviceSet:\n        tmpDict['os' + str(thisRow['ip'])] += 1\n    else:\n        tmpDict['os' + str(thisRow['ip'])] = 1     \n    \n    appSet.add(str(thisRow['app']))\n    deviceSet.add(str(thisRow['device']))\n    osSet.add(str(thisRow['os']))\n    \n    \nfor thisRow in df.to_dict(orient = 'records'):\n    thisRow['appOnIp'] = tmpDict['ap' + str(thisRow['ip'])]\n    thisRow['deviceOnIp'] = tmpDict['dv' + str(thisRow['ip'])]\n    thisRow['osOnIp'] = tmpDict['os' + str(thisRow['ip'])]\n    dictList.append(thisRow)\n    ","execution_count":98,"outputs":[]},{"metadata":{"_cell_guid":"a775df94-d46a-4e58-b0bb-d5be4840c1e6","_uuid":"424527e355d7a1414fc8635cc23cff33ca0637ed","trusted":true},"cell_type":"code","source":"df = pd.DataFrame(dictList)\ntrainSample.head()","execution_count":100,"outputs":[]},{"metadata":{"_cell_guid":"bf40bff8-850b-4740-a283-80c9fa8e2e78","_uuid":"4b216444607cadd0c248cadcb332db0fb3e9ebb3","trusted":true},"cell_type":"code","source":"# Feature - How many OS on a single device\n","execution_count":96,"outputs":[]},{"metadata":{"_uuid":"c45af4f3e5568126facd217a321d253422df3500"},"cell_type":"markdown","source":""},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"b6cca54a65de3f94ea0e5872b8e2322fa94d27ef"},"cell_type":"code","source":"# Time Based Feature\n","execution_count":101,"outputs":[]},{"metadata":{"_uuid":"c7058be5bea46a5fe1872c38844ea67e774ed937"},"cell_type":"markdown","source":"I will keep adding features and will keep posting the updated notebook. Enjoy :) "},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"790318d0f614d44bd05b5dc7dd8e6b982c105899"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}