{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false,"collapsed":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nimport seaborn as sns\nimport matplotlib.pyplot as plt\ncolor = sns.color_palette()\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"cde07283-5913-4c8a-8af2-08334954ec6f","_uuid":"10730705a8d1c99e62797cb34ff5ec97afa5d788"},"cell_type":"markdown","source":"**Load training data and see what's inside.**"},{"metadata":{"_cell_guid":"3b4ab322-e133-4f4f-b719-e5911ac4b066","_uuid":"8e16dcb733f723cc36e41236ece4d338c8ad379f","trusted":false,"collapsed":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/train.csv',nrows=10000)\ntrain_df.info(memory_usage='deep')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"8ca0695e-ed2b-4be0-89c6-16db9aedfd4b","_uuid":"4178ae3e7b089c48234640092b2407189eb0d1f3","trusted":false,"collapsed":true},"cell_type":"code","source":"test_df = pd.read_csv('../input/test.csv',nrows=10000)\ntest_df.info(memory_usage='deep')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"f8fc23d3-1f3b-4159-8078-760c86288ef6","_uuid":"5dd1e71723c10455b33b4a0878097962d3944e1e","trusted":false,"collapsed":true},"cell_type":"code","source":"test_df['click_time'] = pd.to_datetime(test_df['click_time'])\ntest_df['day'] = test_df['click_time'].dt.day\ntest_df['day'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"bef38a29-3d7d-4521-a94c-539791da5b2f","_uuid":"3646c69e58a63f586009cf4eacb1fd4732929a3d"},"cell_type":"markdown","source":"* We have 6 parameters types as int64 and 2  parameters types as object in training data, and 6 parameters types as int64 and 1  parameters types as object in testing data.\n* The parameter, attributed_time, is not in the testing data, so let's drop it to save memory.\n* ** Because we have only one day of test data, I using chunk to get one day of training data.**"},{"metadata":{"_cell_guid":"0f237e71-5847-4f65-9bcb-3187283010a8","_uuid":"ef30c737f49f5754e21dde920e47cadd59e246f3","trusted":false,"collapsed":true},"cell_type":"code","source":"del train_df\ndel test_df\ndf = pd.read_csv('../input/train.csv', iterator=True, chunksize=10000,nrows= 3700000, usecols=['ip','app','device','os', 'channel', 'click_time', 'is_attributed'])\ntrain_df = pd.concat([chunk[chunk['click_time'].str.contains(\"2017-11-06\")] for chunk in df])\ntrain_df.info(memory_usage='deep')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"2f48661f-e720-406e-92d5-6ea8142880ec","_uuid":"d17f48d4621ea8876ed8b0c8b1cab838a840a1c6","trusted":false,"collapsed":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"97bf1d08-88df-4cb5-a196-669d41d990b9","_uuid":"d852b2bc3ed7996edd9efca006dd58b4dd95d879"},"cell_type":"markdown","source":"* Now, let's take a look at the distribution of target (is_attributed)."},{"metadata":{"_cell_guid":"dcb4112f-5c81-4155-a2e9-1189afd09d0e","_uuid":"6d30ab7af57eab3f55fae0913f33639c02eb80dd","trusted":false,"collapsed":true},"cell_type":"code","source":"group_df = train_df.is_attributed.value_counts().reset_index()\nk = group_df['is_attributed'].sum()\nplt.figure(figsize = (12,8))\nsns.barplot(group_df['index'], (group_df.is_attributed/k), alpha=0.8, color=color[0])\nprint((group_df.is_attributed/k))\nplt.ylabel('Frequency', fontsize = 12)\nplt.xlabel('Attributed', fontsize = 12)\nplt.title('Frequency of Attributed', fontsize = 16)\nplt.xticks(rotation='vertical')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"28196781-3cd7-465b-8b2c-f11f3ea8dc58","_uuid":"ec3b7b5a5e1039989e1d97174b20353438ab39c7","trusted":false,"collapsed":true},"cell_type":"code","source":"print(train_df.ip.describe())\nplt.figure(figsize=(12, 8))\nsns.kdeplot(train_df.ip, shade=True)\nplt.title('IP distribution', fontsize = 15)\nplt.xlabel('IP', fontsize = 12)\nplt.ylabel('Percent', fontsize = 12)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"83258935-cc54-4ada-9e41-7f0ebade5d38","_uuid":"95fb69a6c1d1318a4938f36f8849e53269036124"},"cell_type":"markdown","source":"* **The distribution of IP data is average.**\n* **After the int64 type parameters have been processed, we need to process the object type parameters.**"},{"metadata":{"_cell_guid":"a234998c-a4e9-45d1-8941-f8d9f8c61f78","_uuid":"c61dc51ea198232166cb094fc2d3aa8d02d9349c","collapsed":true,"trusted":false},"cell_type":"code","source":"train_df['click_time'] = pd.to_datetime(train_df['click_time'])\ntrain_df['hour'] = train_df['click_time'].dt.hour\ntrain_df['minute'] = train_df['click_time'].dt.minute\ntrain_df['second'] = train_df['click_time'].dt.second\ntrain_df=train_df.drop(['click_time'], axis =1)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"e0e0158c-946d-41c6-9be0-15098590bc35","_uuid":"75d34e0b09de3d9da2d86435d82e8ca86a3620de","trusted":false,"collapsed":true},"cell_type":"code","source":"train_df.info(memory_usage='deep')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"8f2fd229-09d4-47dd-af12-3f39d755e463","_uuid":"00a42f30dd53f7eaa46fc230f84ab01a6eef18db","trusted":false,"collapsed":true},"cell_type":"code","source":"import gc\ndel df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"bbcd4cca-0644-4fb3-ab9d-d980fe019f66","_uuid":"a57b2d67f797648d2f1f10708987fa931d9ea495","trusted":false,"collapsed":true},"cell_type":"code","source":"colormap = plt.cm.viridis\nplt.figure(figsize=(16,16))\nplt.title(' The Absolute Correlation Coefficient of Features', y=1.05, size=15)\nsns.heatmap(abs(train_df.astype(float).corr()),linewidths=0.1,vmax=1.0, square=True, cmap=colormap, linecolor='white', annot=True, )\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"7475def1-6b53-46c5-97ec-8dca1ced4e43","_uuid":"f317caac5f5e9ad9de877bb1ce008a7f184c6c76"},"cell_type":"markdown","source":"* **From the picture.**\n*  **We can know more important influence on is_attributed parameter is weekday, hour, ip, and app parameter.**\n* **Thanks for Bryan Arnold, perhaps using the correlation coefficient is not the best way to express the method of targeting non-continuous data.**"},{"metadata":{"_cell_guid":"85ccc436-b759-43c3-9e22-bbb0aa932c98","_uuid":"ec17abdc0276bd8ab6daf5449764cead2be3558b"},"cell_type":"markdown","source":"**Let's group by some parameters**"},{"metadata":{"_cell_guid":"4b6a80cd-093e-4b24-b5cd-2d77b63abfd1","_kg_hide-output":true,"scrolled":true,"_uuid":"a30845e859d21cbdfa78a31f7feb683287573fc0","_kg_hide-input":true,"trusted":false,"collapsed":true},"cell_type":"code","source":"#The frequency of each parameter (hours)\ndef group_by(lis_p, select_p, data, Agg=''):\n    print('group by...')\n    newname = '{}'.format('_'.join(lis_p))\n    all_p = lis_p[:]\n    all_p.append(select_p)\n    if Agg=='':\n        gp = data[all_p].groupby(by=lis_p)\n        gp = gp[select_p].count().reset_index().rename(index=str, columns={select_p: newname})\n    else:\n        gp = data[all_p].groupby(by=lis_p).agg(Agg)\n        gp = gp[select_p].reset_index().rename(index=str, columns={select_p: newname})\n    print('merge...')\n    data = data.merge(gp, on=lis_p, how='left')\n    return data, newname\n#The frequency of each parameter with IP (hours)\ntrain_df, tmp1 = group_by(['ip', 'hour'], 'channel', train_df, 'count')\ntrain_df, tmp2 = group_by(['ip', 'hour', 'device'], 'channel', train_df, 'count')\ntrain_df, tmp3 = group_by(['ip', 'hour', 'app'], 'channel', train_df, 'count')                  \ntrain_df, tmp4 = group_by(['ip', 'hour', 'channel'], 'os', train_df, 'count')\ntrain_df, tmp5 = group_by(['ip', 'hour', 'os'], 'channel', train_df, 'count')\nparameter_with_IP = [tmp1,tmp2,tmp3,tmp4,tmp5]\ndel tmp1\ndel tmp2\ndel tmp3\ndel tmp4\ndel tmp5\n#train_df = train_df.drop( ['ip'], axis=1)\ngc.collect()\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"29a168c0-3457-46ae-a65f-b896afb63a95","_kg_hide-output":true,"collapsed":true,"_uuid":"23b42568be13deb73a52f3478de2ffd779387aa9","_kg_hide-input":true,"trusted":false},"cell_type":"code","source":"def calc_iv(df, feature, target, pr=False):\n    \"\"\"\n    Set pr=True to enable printing of output.\n    \n    Output: \n      * iv: float,\n      * data: pandas.DataFrame|\n    \"\"\"\n\n    lst = []\n\n    df[feature] = df[feature].fillna(\"NULL\")\n\n    for i in range(df[feature].nunique()):\n        val = list(df[feature].unique())[i]\n        lst.append([feature,                                                        # Variable\n                    val,                                                            # Value\n                    df[df[feature] == val].count()[feature],                        # All\n                    df[(df[feature] == val) & (df[target] == 0)].count()[feature],  # Good (think: Fraud == 0)\n                    df[(df[feature] == val) & (df[target] == 1)].count()[feature]]) # Bad (think: Fraud == 1)\n\n    data = pd.DataFrame(lst, columns=['Variable', 'Value', 'All', 'Good', 'Bad'])\n\n    data['Share'] = data['All'] / data['All'].sum()\n    data['Bad Rate'] = data['Bad'] / data['All']\n    data['Distribution Good'] = (data['All'] - data['Bad']) / (data['All'].sum() - data['Bad'].sum())\n    data['Distribution Bad'] = data['Bad'] / data['Bad'].sum()\n    data['WoE'] = np.log(data['Distribution Good'] / data['Distribution Bad'])\n\n    data = data.replace({'WoE': {np.inf: 0, -np.inf: 0}})\n\n    data['IV'] = data['WoE'] * (data['Distribution Good'] - data['Distribution Bad'])\n    data['abs_WoE'] = abs(data['WoE'])\n    data = data.sort_values(by=['Variable', 'Value'], ascending=[True, True])\n    data.index = range(len(data.index))\n\n    if pr:\n        print(data)\n        print('IV = ', data['IV'].sum())\n    return data\ndef loop_iv(lis_val, train_df):\n    dic_val = {}\n    for i in lis_val:\n        data = calc_iv(train_df, i, 'is_attributed')\n        dic_val[i] = data\n        print(\"Done {0}\".format(i))\n    return dic_val","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b126a3fe-07ec-4ca0-a939-809e5389d967","_kg_hide-output":true,"_uuid":"3980082678eb5039f11194731f53e27307a5b311","_kg_hide-input":true,"trusted":false,"collapsed":true},"cell_type":"code","source":"lis=list(train_df.columns)\nlis.remove('is_attributed')\nlis.remove('ip')\nlis.remove('hour')\nlis.remove('minute')\nlis.remove('second')\ndic=loop_iv(lis, train_df)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"82feba44-d7a8-4b2c-9cb5-8df42ed615d5","_kg_hide-output":true,"_uuid":"ad20e9dc04852833c20aa38329b31af07efba862","_kg_hide-input":true,"trusted":false,"collapsed":true},"cell_type":"code","source":"dic_num = {}\nfor i in lis:\n    nr_woe = dic[i]['WoE'].argmax()\n    nr_iv = dic[i]['IV'].argmax()\n    dic_num[i] = [nr_woe, nr_iv]","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"71be21c2-d9bc-44d7-a3db-8fc1c6b0b3a2","scrolled":false,"_uuid":"7cae14232a6b33107b31c457a51fd03180db8ce0","trusted":false,"collapsed":true},"cell_type":"code","source":"sum_iV={}\nfor i in lis:\n    sum_num = dic[i]['IV'].sum()\n    sum_iV[i] = sum_num\n    print(\"The sum of IV parameter in {0} : {1}\".format(i, sum_num))\nplt.figure(figsize = (16,8))\nm_colors=[]\nk_num=0\nfor num in range(len(lis)):\n    if (num//len(color))>k_num:\n        k_num+=1\n    t = num - k_num*len(color)\n    m_colors.append(color[t])\nsns.barplot(list(sum_iV.keys()), list(sum_iV.values()), alpha=0.8, palette = m_colors)\nplt.ylabel('IV value', fontsize = 12)\nplt.xlabel('Parameters', fontsize = 12)\nplt.title(\"Each parameter's IV\", fontsize = 16)\nplt.xticks(rotation='vertical')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"eb25b93f-21bf-4592-adfc-01fb2d858701","_uuid":"aef4d5f0c25f6abac84b5f9fa14c47bf717a5688","trusted":false,"_kg_hide-output":true,"_kg_hide-input":true,"collapsed":true},"cell_type":"code","source":"train_df, tmp1 = group_by(['ip','app'], 'channel', train_df, 'count')\ntrain_df, tmp2 = group_by(['ip', 'app', 'os'], 'channel', train_df, 'count')\ntrain_df, tmp3 = group_by(['ip', 'device'], 'channel', train_df, 'count')                  \ntrain_df, tmp4 = group_by(['app', 'channel'], 'os', train_df, 'count')\nparameter_IA_IAO_ID_AC = [tmp1,tmp2,tmp3,tmp4]\ndic.update(loop_iv(parameter_IA_IAO_ID_AC, train_df))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"9559a5cb-2180-4810-8637-d30bff48455b","_uuid":"24fab83ab551e75f16cf53f4fd17faffae0bf5f4","trusted":false,"_kg_hide-input":true,"collapsed":true},"cell_type":"code","source":"for i in parameter_IA_IAO_ID_AC:\n    sum_num = dic[i]['IV'].sum()\n    sum_iV[i] = sum_num\n    print(\"The sum of IV parameter in {0} : {1}\".format(i, sum_num))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"d33bb04e-8c9f-4447-a0b5-187ca1f43fc0","_kg_hide-output":true,"_uuid":"72e57b8eaed09c9368b27883ec9af99c3bcb9ada","_kg_hide-input":true,"trusted":false,"collapsed":true},"cell_type":"code","source":"#train_df = train_df.drop( parameter_with_IP, axis=1)\n#The frequency of each parameter with IP (minute)\ntrain_df, tmp1 = group_by(['ip', 'hour','minute'], 'channel', train_df, 'count')\ntrain_df, tmp2 = group_by(['ip', 'hour', 'device','minute'], 'channel', train_df, 'count')\n#train_df, tmp3 = group_by(['ip', 'hour', 'app','minute'], 'channel', train_df, 'count')                  \n#train_df, tmp4 = group_by(['ip', 'hour', 'channel','minute'], 'os', train_df, 'count')\n#train_df, tmp5 = group_by(['ip', 'hour', 'os','minute'], 'channel', train_df, 'count')\ntrain_df, tmp3 = group_by(['ip', 'hour', 'os','minute'], 'channel', train_df, 'count')\n#parameter_with_IP_minute = [tmp1,tmp2,tmp3,tmp4,tmp5]\nparameter_with_IP_minute = [tmp1,tmp2,tmp3]\ndic.update(loop_iv(parameter_with_IP_minute, train_df))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"887c39c4-1629-478d-8a7c-12735ef2178d","_uuid":"10e30b7cb4691e74d9adb8bb82b2713ad808af5c","trusted":false,"_kg_hide-input":true,"collapsed":true},"cell_type":"code","source":"for i in parameter_with_IP_minute:\n    sum_num = dic[i]['IV'].sum()\n    sum_iV[i] = sum_num\n    print(\"The sum of IV parameter in {0} : {1}\".format(i, sum_num))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"c4aa975f-7cf3-457f-a67f-737cc959ef97","_kg_hide-output":true,"_uuid":"27eae5f35364f765a7367fa9cf10e86cf5ad1def","_kg_hide-input":true,"trusted":false,"collapsed":true},"cell_type":"code","source":"##train_df = train_df.drop( parameter_with_IP_minute, axis=1)\n##The frequency of each parameter with IP (second)\n#train_df, tmp1 = group_by(['ip', 'hour','minute','second'], 'channel', train_df, 'count')\n#train_df, tmp2 = group_by(['ip', 'hour', 'device','minute','second'], 'channel', train_df, 'count')\n#train_df, tmp3 = group_by(['ip', 'hour', 'app','minute','second'], 'channel', train_df, 'count')                  \n#train_df, tmp4 = group_by(['ip', 'hour', 'channel','minute','second'], 'os', train_df, 'count')\n#train_df, tmp5 = group_by(['ip', 'hour', 'os','minute','second'], 'channel', train_df, 'count')\n#parameter_with_IP_second = [tmp1,tmp2,tmp3,tmp4,tmp5]\n#dic.update(loop_iv(parameter_with_IP_second, train_df))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"cb4423ba-e326-42e8-b411-79642bc5751c","_uuid":"93efe6ca94116f14a48661eb1c2d62a47cedfdd3","trusted":false,"collapsed":true},"cell_type":"code","source":"#for i in parameter_with_IP_second:\n#    sum_num = dic[i]['IV'].sum()\n#    sum_iV[i] = sum_num\n#    print(\"The sum of IV parameter in {0} : {1}\".format(i, sum_num))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"ab178407-de67-48dc-93c4-074f4456c468","_kg_hide-output":true,"_uuid":"281e13ee5d3af80a1d9e9ba54a1103b6cf662dae","_kg_hide-input":true,"trusted":false,"collapsed":true},"cell_type":"code","source":"#train_df = train_df.drop( parameter_with_IP_second, axis=1)\n#The frequency of each parameter with IP and channel (hours)\ntrain_df, tmp1 = group_by(['ip', 'app', 'hour', 'channel'], 'os', train_df, 'count')\n#train_df, tmp2 = group_by(['ip', 'app','minute','second', 'hour', 'channel'], 'os', train_df, 'count')\n#train_df, tmp3 = group_by(['ip', 'device', 'hour', 'channel'], 'os', train_df, 'count')\n#train_df, tmp4 = group_by(['ip', 'os', 'hour', 'channel'], 'app', train_df, 'count')\ntrain_df, tmp2 = group_by(['ip', 'device', 'hour', 'channel'], 'os', train_df, 'count')\nparameter_with_IP_channel = [tmp1,tmp2,tmp3,tmp4]\nparameter_with_IP_channel = [tmp1,tmp2]\ndic.update(loop_iv(parameter_with_IP_channel, train_df))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"1d8ac8a0-d9ce-4332-a30b-02c40e80d89e","_uuid":"4f25fc7f82edd187d315f8709bcedb1ca1ce2a1b","trusted":false,"_kg_hide-input":true,"collapsed":true},"cell_type":"code","source":"for i in parameter_with_IP_channel:\n    sum_num = dic[i]['IV'].sum()\n    sum_iV[i] = sum_num\n    print(\"The sum of IV parameter in {0} : {1}\".format(i, sum_num))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"30f1fb46-c522-43b2-8b29-53fdd77164eb","_kg_hide-output":true,"_uuid":"9f5ef339ab26482071ad4e74cc594e87ac2440ca","_kg_hide-input":true,"trusted":false,"collapsed":true},"cell_type":"code","source":"#train_df = train_df.drop( parameter_with_IP_channel, axis=1)\n#The frequency of each parameter with IP and app (hours)\ntrain_df, tmp1 = group_by(['ip', 'app', 'hour', 'device'], 'os', train_df, 'count')\ntrain_df, tmp2 = group_by(['ip', 'os', 'hour', 'app'], 'device', train_df, 'count')\n#The frequency of each parameter with IP and device (hours)\ntrain_df, tmp3 = group_by(['ip', 'os', 'hour', 'device'], 'app', train_df, 'count')\nparameter_with_IP_app = [tmp1,tmp2,tmp3]\ndic.update(loop_iv(parameter_with_IP_app, train_df))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"190a8dff-9c51-46d0-a827-acea8f9f57a3","_uuid":"bfdad4a730b1060b05b94c938e952bc2f8ffdfca","trusted":false,"_kg_hide-input":true,"collapsed":true},"cell_type":"code","source":"for i in parameter_with_IP_app:\n    sum_num = dic[i]['IV'].sum()\n    sum_iV[i] = sum_num\n    print(\"The sum of IV parameter in {0} : {1}\".format(i, sum_num))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"d0d36508-13f8-4473-bf5e-52e692b6eb35","_kg_hide-output":true,"_uuid":"910e9879edf90adf2e99c7a51e51baacb1090de5","_kg_hide-input":true,"trusted":false,"collapsed":true},"cell_type":"code","source":"#train_df = train_df.drop( parameter_with_IP_app, axis=1)\n#The frequency of each parameter with app (IP)\ntrain_df, tmp1 = group_by(['ip', 'app', 'channel'], 'os', train_df)\ntrain_df, tmp2 = group_by(['ip', 'device', 'app'], 'os', train_df)\ntrain_df, tmp3 = group_by(['ip', 'os', 'app'], 'channel', train_df)\nparameter_with_app = [tmp1,tmp2,tmp3]\ndic.update(loop_iv(parameter_with_app, train_df))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"1ef4830f-cf33-4678-9722-0f0c14221dd7","_uuid":"28e59ce14fca3ccab3a6c5bb3202bfb678135248","trusted":false,"_kg_hide-input":true,"collapsed":true},"cell_type":"code","source":"for i in parameter_with_app:\n    sum_num = dic[i]['IV'].sum()\n    sum_iV[i] = sum_num\n    print(\"The sum of IV parameter in {0} : {1}\".format(i, sum_num))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"c711257a-c6ba-49ae-98ec-13669aca5490","_kg_hide-output":true,"_uuid":"ea8df84f6b4ea41f2f75a0f5580f23a03af7dfd1","_kg_hide-input":true,"trusted":false,"collapsed":true},"cell_type":"code","source":"#train_df = train_df.drop( parameter_with_app, axis=1)\n#The frequency of each parameter with app and device(IP)\ntrain_df, tmp1 = group_by(['ip', 'app', 'device','channel'], 'os', train_df)\ntrain_df, tmp2 = group_by(['ip', 'device', 'app', 'os'], 'channel', train_df)\n#The frequency of each parameter with app and channel(IP)\n#train_df, tmp3 = group_by(['ip', 'app', 'os','channel'], 'device', train_df)\n#train_df, tmp4 = group_by(['app','channel', 'hour','minute','second' ], 'device', train_df, 'count')\ntrain_df, tmp3 = group_by(['app','channel', 'hour','minute','second' ], 'device', train_df, 'count')\n#parameter_with_app_device = [tmp1,tmp2,tmp3,tmp4]\nparameter_with_app_device = [tmp1,tmp2,tmp3]\ndic.update(loop_iv(parameter_with_app_device, train_df))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"038652df-db15-4bda-988e-70308b40248b","_uuid":"61a7574551a3d92f55730c09e15ce9eda8741079","trusted":false,"_kg_hide-input":true,"collapsed":true},"cell_type":"code","source":"for i in parameter_with_app_device:\n    sum_num = dic[i]['IV'].sum()\n    sum_iV[i] = sum_num\n    print(\"The sum of IV parameter in {0} : {1}\".format(i, sum_num))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"532bb370-1b49-4238-8a68-0de7738defd7","_kg_hide-output":true,"_uuid":"a884d5496648d7b51ba345130376328f418e151b","_kg_hide-input":true,"trusted":false,"collapsed":true},"cell_type":"code","source":"#train_df = train_df.drop( parameter_with_app_device, axis=1)\n#The frequency of each parameter with device (IP)\ntrain_df, tmp1 = group_by(['ip', 'device', 'channel'], 'os', train_df)\ntrain_df, tmp2 = group_by(['ip', 'os', 'device'], 'channel', train_df)\n#The frequency of each parameter with device and channel (IP)\n#train_df, tmp3 = group_by(['ip', 'device', 'channel', 'os'], 'app', train_df)\n##The frequency of each parameter with channel (IP)\n#train_df, tmp4 = group_by(['ip', 'os', 'channel'], 'app', train_df)\n#parameter_with_device = [tmp1,tmp2,tmp3,tmp4]\nparameter_with_device = [tmp1,tmp2]\ndic.update(loop_iv(parameter_with_device, train_df))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"fc40091f-404a-4015-9db9-cc89f8e23d8e","_uuid":"52f06bbb999d4b6e1ee1dcd76dc8e34dae57a138","trusted":false,"_kg_hide-input":true,"collapsed":true},"cell_type":"code","source":"for i in parameter_with_device:\n    sum_num = dic[i]['IV'].sum()\n    sum_iV[i] = sum_num\n    print(\"The sum of IV parameter in {0} : {1}\".format(i, sum_num))\n#train_df = train_df.drop( parameter_with_device, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"8eec042b-72d9-47af-ada8-0a2e8e546809","_uuid":"4d0ebd0c95decf31fad4fe222400dfc628ab60fa","trusted":false,"collapsed":true},"cell_type":"code","source":"v=list(sum_iV.values())\nv_m = list(sum_iV.values())\nv_n = list(sum_iV.keys())\nv.sort()\nkv = []\nfor i in range(len(v)):\n    vv = v_m.index(v[i])\n    kv.append(v_n[vv])\nprint('The Top ten IV parameters-----------------------')\nfor i in range(1, 11):\n    print('The {0} big IV paramter is {1}.....'.format(i, kv[-i]))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"24f030bb-02a9-4edb-bc2c-ffc71376e3d9","_uuid":"4b26c1f83883c0a283f39b05cf76545e457e1cca"},"cell_type":"markdown","source":"**Now, let's add the time interval parameters. **"},{"metadata":{"_cell_guid":"d8520a9b-360d-434c-94fd-3ce14beed64b","_kg_hide-output":true,"collapsed":true,"_uuid":"cddaff39667b9dd7864156ed126f685e928a9007","_kg_hide-input":true,"trusted":false},"cell_type":"code","source":"def group_by_click(lis_p, data):\n    print('group by...')\n    newname = '{}_click_time_gap'.format('_'.join(lis_p))\n    all_p = lis_p[:]\n    all_p.append('click_time')\n    data[newname] = data[all_p].groupby(by=lis_p).click_time.transform(lambda x: x.diff().shift(-1)).dt.seconds\n    data[newname] = data[newname].fillna(-1)\n    print('merge...')\n    return data, newname","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"32bd24d3-3677-49b6-81f8-a7503c34b779","_kg_hide-output":true,"collapsed":true,"_uuid":"cb05dd89da37e65bdc6a196ba7e308079ab8e019","_kg_hide-input":true,"trusted":false},"cell_type":"code","source":"df = pd.read_csv('../input/train.csv', iterator=True, chunksize=10000,nrows= 3500000, usecols=['click_time'])\ntmp_train_df = pd.concat([chunk[chunk['click_time'].str.contains(\"2017-11-06\")] for chunk in df])","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"c506a7ae-44fa-44a7-a8c9-0a8ddcb89501","_kg_hide-output":true,"_uuid":"8eef4327bbabd34a78ed365eb168d576d302059d","_kg_hide-input":true,"trusted":false,"collapsed":true},"cell_type":"code","source":"train_df['index'] = train_df.index\ntmp_train_df['index'] = tmp_train_df.index\ntrain_df = train_df.merge(tmp_train_df, on=['index'], how='left')\ntrain_df = train_df.drop( ['index'], axis=1)\ntrain_df['click_time'] = pd.to_datetime(train_df['click_time'])\ndel df\ndel tmp_train_df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"3e2bc006-d568-49f8-af0d-8a41694b3ab7","_kg_hide-output":true,"scrolled":true,"_uuid":"d25a3b314dd8019f8ed09dccbe23f5ba6387c728","_kg_hide-input":true,"trusted":false,"collapsed":true},"cell_type":"code","source":"train_df, tmp1 = group_by_click(['ip'], train_df)\ntrain_df, tmp2 = group_by_click(['ip', 'app'],train_df)\ntrain_df, tmp3 = group_by_click(['ip', 'channel'],train_df)\ntrain_df, tmp4 = group_by_click(['ip', 'device'],train_df)\nparameter_with_ip_next = [tmp1,tmp2,tmp3, tmp4]\ndic.update(loop_iv(parameter_with_ip_next, train_df))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"4afb43fc-590c-4417-bd85-a9a097e8cffc","_uuid":"983f3d0d646b48b1ea7ae9580961298793d51171","trusted":false,"_kg_hide-input":true,"collapsed":true},"cell_type":"code","source":"for i in parameter_with_ip_next:\n    sum_num = dic[i]['IV'].sum()\n    sum_iV[i] = sum_num\n    print(\"The sum of IV parameter in {0} : {1}\".format(i, sum_num))\n#train_df = train_df.drop( parameter_with_ip_next, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"d3e9faa5-4124-465c-8b13-cd7d6f4486ea","_kg_hide-output":true,"_uuid":"609ca5876aea2927f022caa827dd6de9002ec6d2","_kg_hide-input":true,"trusted":false,"collapsed":true},"cell_type":"code","source":"train_df, tmp1 = group_by_click(['ip', 'device', 'channel'],train_df)\ntrain_df, tmp2 = group_by_click(['ip', 'device', 'app'],train_df)\ntrain_df, tmp3 = group_by_click(['ip', 'channel', 'app'],train_df)\nparameter_with_ip2_next = [tmp1,tmp2,tmp3]\ndic.update(loop_iv(parameter_with_ip2_next, train_df))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"9fc91681-49c2-45ad-acc3-2f36abe6faf0","_uuid":"08f5d6f55d25e0d7c6122a73d5673d16020f784a","trusted":false,"_kg_hide-input":true,"collapsed":true},"cell_type":"code","source":"for i in parameter_with_ip2_next:\n    sum_num = dic[i]['IV'].sum()\n    sum_iV[i] = sum_num\n    print(\"The sum of IV parameter in {0} : {1}\".format(i, sum_num))\n#train_df = train_df.drop( parameter_with_ip2_next, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"66540d92-f938-4320-a4a1-2991be6608b0","_uuid":"c355b23d329cbd5eb176b8a86b4c1f6b3bb74f0b","trusted":false,"collapsed":true},"cell_type":"code","source":"v=list(sum_iV.values())\nv_m = list(sum_iV.values())\nv_n = list(sum_iV.keys())\nv.sort()\nkv = []\nfor i in range(len(v)):\n    vv = v_m.index(v[i])\n    kv.append(v_n[vv])\nprint('The Top 25 IV parameters-----------------------')\ntop_25 = {}\nfor i in range(1, 26):\n    print('The {0} big IV paramter is {1}.....'.format(i, kv[-i]))\n    top_25[kv[-i]] = sum_iV[kv[-i]]\n\nplt.figure(figsize = (22,8))\nm_colors=[]\nk_num=0\nfor num in range(len(top_25)):\n    if (num//len(color))>k_num:\n        k_num+=1\n    t = num - k_num*len(color)\n    m_colors.append(color[t])\nsns.barplot(list(top_25.keys()),list(top_25.values()), alpha=0.8, palette = m_colors)\nplt.ylabel('IV value', fontsize = 12)\nplt.xlabel('Parameters', fontsize = 12)\nplt.title(\"TOP 20 parameter's IV\", fontsize = 16)\nplt.xticks(rotation='vertical')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"680436f8-a6a2-47fc-94f3-52624bd9764f","_uuid":"34c2b6734bb559bb8d63f12adab0be69b7999ab1","collapsed":true},"cell_type":"markdown","source":"* **From the above result, we can know  the is_attributed parameter is closely related to time interval.**"},{"metadata":{"_cell_guid":"e1485d82-b4b3-4c03-a6f5-17c943f6216a","_uuid":"e1197add9117a5c7be133e44d52702acc9860acd","trusted":false,"collapsed":true},"cell_type":"code","source":"tmp = pd.DataFrame()\nsns.set(font_scale=1.2)\nfor i in top_25:\n    tmp[i] = train_df[i]\ndel train_df\ngc.collect()\ncolormap = plt.cm.viridis\nplt.figure(figsize=(26,26))\nplt.title(' The Absolute Correlation Coefficient of Features', y=1.05, size=15)\nsns.heatmap(abs(tmp.astype(float).corr()),linewidths=0.1,vmax=1.0, square=True, cmap=colormap, linecolor='white', annot=True, )\nplt.show()\nplt.savefig('feature_Correlation_Coefficient.png')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"679acae0-82b5-4cae-b9d3-5eb7fea7328d","_uuid":"bd83ade262ab28570d127892f8c4719954bea0b2","collapsed":true},"cell_type":"markdown","source":"*  **From the above result, we can choose to avoid parameters that have similar information.**"}],"metadata":{"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}