{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","collapsed":true,"trusted":false},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false,"collapsed":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport os\n\n\nprint(\"show data file\")\n\nfor f in os.listdir(\"../input\"):\n    if 'zip' not in f:\n        print(f.ljust(30) + str(round(os.path.getsize('../input/' + f) / 1000000, 2)) + 'MB')\n        \ndf_train = pd.read_csv('../input/train.csv', nrows=100000)\ndf_test = pd.read_csv('../input/test.csv', nrows=100000)\n\nprint(\"Head of train:\")\ndf_train.head(20)\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"49d1fead-abbd-4064-aa0a-acbbd6b4a6dc","_uuid":"6c3ccbb94eaa0e8797155763e417c9bf1dd7614c","trusted":false,"collapsed":true},"cell_type":"code","source":"\nprint(\"Head of test:\")\ndf_test.head(20)\n\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"89f49095-814a-4807-b86d-0636de87fb27","_uuid":"d32600ebc1bf82db6797ca15b079f373eff288b2","scrolled":true,"trusted":false,"collapsed":true},"cell_type":"code","source":"print(df_train.nunique())\ndf_train.describe()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"ea61db08-e9a2-4815-9d82-5b58d59142c6","_uuid":"9d81d54fc0757f8aebe79453f7fd45a9ea016173","scrolled":false,"trusted":false,"collapsed":true},"cell_type":"code","source":"print(df_test.nunique())\ndf_test.describe()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"f2d099d9-d593-4a3e-8de6-740d33fefb52","_uuid":"c441d939f7b8a94f5ec26b90e815489e6999a3d2"},"cell_type":"markdown","source":"train和test数据unique之后，分析数据发现一个比较奇怪的现象，click_time相对还是比较集中的，并不像ip那样分布很广\n\n下面分析几个集中的数据（app,device,os,channel,click_time）"},{"metadata":{"_cell_guid":"97922456-8544-4256-9258-e40b704221bb","_uuid":"0495617289bbff662861f6d185b56639cc51cf44","trusted":false,"collapsed":true},"cell_type":"code","source":"app = df_train.app.value_counts(normalize=True)\ndevice = df_train.device.value_counts(normalize=True)\nos = df_train.os.value_counts(normalize=True)\n\nplt.figure(figsize=(12,15))\n\nplt.subplot(311)\ng1 = sns.barplot(x=app.index[:20], y =app.values[:20])\ng1.set_title(\"App Dist\", fontsize=15)\ng1.set_xlabel(\"App ID\")\ng1.set_ylabel(\"count normalize\", fontsize=12)\n\nplt.subplot(312)\ng2 = sns.barplot(x=device.index[:20], y=device.values[:20])\ng2.set_title(\"Device Dist\", fontsize=15)\ng2.set_xlabel(\"Device ID\")\ng2.set_ylabel(\"count normalize\", fontsize=12)\n\nplt.subplot(313)\ng3 = sns.barplot(x=os.index[:20], y=os.values[:20])\ng3.set_title(\"OS Dist\", fontsize=15)\ng3.set_xlabel(\"OS ID\")\ng3.set_ylabel(\"count normalize\", fontsize=12)\n\nplt.subplots_adjust(hspace=0.3)\nplt.show()\n\nprint(app.loc[app.values > 0.05])\nprint(device[:3])\nprint(os[:5])","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"f7eabff4-1f50-4af1-8257-71876c2d6a1c","_uuid":"d88b17f20acb7e2f3eb5d95beafd60768ce1cd8a"},"cell_type":"markdown","source":"分析app可以看到占比大于0.05的app只有(9,12,15,8,3,2,18,1)，相对并不是特别集中\n分析device，可以明显看到94.4%的device集中在id 1上，非常集中\n分析os，os也相对比较集中在id 19和13上\n这三维数据都是id与实物对应，理论上应该相对比较集中的。实际数据展示，app集中度一般，device特别集中（不应该呀，与预期不一样...前10W行数据的原因？）"},{"metadata":{"_cell_guid":"f555e702-94f6-45f5-bcc2-c3640149928f","_uuid":"54a895ba4b5e0c42fd08021d268fa7fbd322ce3e","trusted":false,"collapsed":true},"cell_type":"code","source":"channel = df_train.channel.value_counts(normalize=True)\nclick_t = df_train.click_time.value_counts(normalize=True)\n\nplt.figure(figsize=(12,10))\n\nplt.subplot(211)\ng1 = sns.barplot(x=channel.index[:20], y=channel.values[:20])\ng1.set_title(\"channel dist\", fontsize=15)\ng1.set_xlabel(\"channel id\")\ng1.set_ylabel(\"count normalize\")\n\nplt.subplot(212)\ng2 = sns.barplot(x=click_t.index[:20], y=click_t.values[:20])\ng2.set_xticklabels(g2.get_xticklabels(), rotation=90)\ng2.set_title(\"click time dist\", fontsize=15)\ng2.set_xlabel(\"click time\")\ng2.set_ylabel(\"count normalize\")\n\nplt.subplots_adjust(hspace=0.3)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"92e36157-ac7a-481f-b181-de4fe9f2a87f","_uuid":"5f218091de269d3162f66b6cb9f3ab89d852bcfd"},"cell_type":"markdown","source":"channel id与app类似，主要集中在134,145,245，但是集中度一般\n\nclick_time的分布相对均匀，比较奇怪的是click_time的所有时间点比想象中的少很多（前10W数据的原因？）"},{"metadata":{"_cell_guid":"ef56e4fb-5423-458b-ac23-1517223f615a","_uuid":"47e39d751e5c7eb4dc6eff568709768d61aef965","collapsed":true},"cell_type":"markdown","source":"接下来，我们把click_time拆分后看一下分布"},{"metadata":{"_cell_guid":"891a8bbb-7690-465d-a4d4-16687651d2da","_uuid":"86fed6f2b28c5bb715abddccadba724a89833609","scrolled":true,"trusted":false,"collapsed":true},"cell_type":"code","source":"df_train['click_time'] = pd.to_datetime(df_train['click_time'])\n\ndf_train['click_time_D'] = df_train['click_time'].dt.to_period(\"H\")\nprint(\"click_time limit to hour: \")\nprint(df_train['click_time_D'].unique())\n#print(df_train['click_time_D'][:10])\ndf_train['click_time_m'] = df_train['click_time'].dt.to_period(\"min\")\n# print(df_train['click_time_m'][:10])\nprint(\"click_time limit to min, size: \", len(df_train['click_time_m'].unique()))\n\ndf_test['click_time'] = pd.to_datetime(df_test['click_time'])\n\ndf_test['click_time_D'] = df_test['click_time'].dt.to_period(\"H\")\nprint(\"click_time limit to hour: \")\nprint(df_test['click_time_D'].unique())\n#print(df_train['click_time_D'][:10])\ndf_test['click_time_m'] = df_test['click_time'].dt.to_period(\"min\")\n# print(df_train['click_time_m'][:10])\nprint(\"click_time limit to min, size: \", len(df_test['click_time_m'].unique()))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"652939cb-e860-47db-9320-6bc0e5322eea","_uuid":"66717567d40000015817a42cd97a0ba3ef9127d7"},"cell_type":"markdown","source":"可以看到，把click_time限制到小时级，只有三个时间点（why？），限制到分钟级，也只有75个\n\n而测试集则是另外一个时间点\n\n猜测，这里训练集和测试机都只是统计了部分时段的数据，所以click_time没有参考价值"},{"metadata":{"_cell_guid":"ae53f1d6-1249-49ae-9cb7-bbb9030f8dd8","_uuid":"0998fa0accdf8c2a957e379b86c431fed4361530"},"cell_type":"markdown","source":"接下来分析一下，在训练集中app被下载时，各维度数据的分布"},{"metadata":{"_cell_guid":"85d35f1c-70ba-4b51-9cae-bd293cbe0a2a","_uuid":"b2822fe35c0fe59cc7d33e644d32134cedb3aecf","scrolled":false,"trusted":false,"collapsed":true},"cell_type":"code","source":"dw = df_train.loc[df_train['is_attributed'] == 1]\napp_dw = dw.app.value_counts(normalize=True)\ndevice_dw = dw.device.value_counts(normalize=True)\nos_dw = dw.os.value_counts(normalize=True)\n\nplt.figure(figsize=(12,15))\n\nplt.subplot(311)\ng1 = sns.barplot(x=app_dw.index[:20], y =app_dw.values[:20])\ng1.set_title(\"App Dist\", fontsize=15)\ng1.set_xlabel(\"App ID\")\ng1.set_ylabel(\"count normalize\", fontsize=12)\n\nplt.subplot(312)\ng2 = sns.barplot(x=device_dw.index[:20], y=device_dw.values[:20])\ng2.set_title(\"Device Dist\", fontsize=15)\ng2.set_xlabel(\"Device ID\")\ng2.set_ylabel(\"count normalize\", fontsize=12)\n\nplt.subplot(313)\ng3 = sns.barplot(x=os_dw.index[:20], y=os_dw.values[:20])\ng3.set_title(\"OS Dist\", fontsize=15)\ng3.set_xlabel(\"OS ID\")\ng3.set_ylabel(\"count normalize\", fontsize=12)\n\nplt.subplots_adjust(hspace=0.3)\nplt.show()\n\nprint(app[:3])\nprint(app_dw[:3])\nprint(device[:3])\nprint(device_dw[:3])\nprint(os[:3])\nprint(os_dw[:3])","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"e837c062-a2f7-4556-9fda-e182e7daf769","_uuid":"a151316bdc4f01881afab66fca749e2e0b9f43de"},"cell_type":"markdown","source":"对比数据可以明显发现app对是否下载的贡献度很高，其他维度分布基本一致"},{"metadata":{"_cell_guid":"ff765ef4-14e6-4cf1-bfda-dd437b18fbe3","_uuid":"f4e159ea1360d0ec164ee6b229616870738c3f34","trusted":true},"cell_type":"code","source":"plt.figure(figsize=(12,20))\nplt.subplot(411)\nax1 = sns.distplot(df_train['app'][df_train['is_attributed']==0], color='r', label='no download')\nax2 = sns.distplot(df_train['app'][df_train['is_attributed']==1], color='b', label='download')\n\nplt.subplot(412)\nax1 = sns.distplot(df_train['device'][df_train['is_attributed']==0], color='r', label='no download')\nax2 = sns.distplot(df_train['device'][df_train['is_attributed']==1], color='b', label='download')\n\nplt.subplot(413)\nax1 = sns.distplot(df_train['os'][df_train['is_attributed']==0], color='r', label='no download')\nax2 = sns.distplot(df_train['os'][df_train['is_attributed']==1], color='b', label='download')\n\nplt.subplot(414)\nax1 = sns.distplot(df_train['channel'][df_train['is_attributed']==0], color='r', label='no download')\nax2 = sns.distplot(df_train['channel'][df_train['is_attributed']==1], color='b', label='download')\nplt.show()","execution_count":1,"outputs":[]},{"metadata":{"_cell_guid":"276ceb03-676e-45e1-9ad9-46deb0073577","_uuid":"5815d218a302742e67eb2d02b7ab91f174e7bf9c"},"cell_type":"markdown","source":"我们画出这几个主要信号，在app是否被下载的情况下的分布图，可以看到os，channel的分布差别不大，而且app下载量和投放量有一定正相关，但是有个别app下载率很高"},{"metadata":{"_cell_guid":"3bc35098-03c9-40bb-b6b9-19e98a78c551","_uuid":"a3a8a856c9a1e945e41a6c533abc63c5bd3567f8","trusted":true},"cell_type":"code","source":"通过分析可以发现可用的5维数据中，click_time由于数据原因基本是没有贡献度的，剩下的几维数据中，app和device和channel会是相对比较有贡献度的\n\n而ip由于经过了编码，不能做分段处理，贡献度应该也很低。\n\n基本上是一个很少特征的二分类问题，直观上使用xgboost处理。","execution_count":2,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"ea2ce88bdf22c503c83b92b71b04016603625b15"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}