{"cells":[{"metadata":{"_uuid":"c46d53649e04773c0dc626d1ee51ae9f9b07c203"},"cell_type":"markdown","source":"# TalkingData AdTracking Fraud Detection Challenge"},{"metadata":{"_uuid":"d04b0eae371e907de819be48c900b43abd8354e2"},"cell_type":"markdown","source":"## Import libraries and load data"},{"metadata":{"trusted":true,"_uuid":"45d71f8f5603c432dd4dca951a027bf747295212"},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\n\nfrom sklearn.metrics import roc_auc_score # official evaluation score of challenge\nfrom sklearn.metrics import roc_curve\nimport xgboost as xgb\nimport lightgbm as lgbm\n\nimport warnings\nimport gc\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e640c001bbf77ee813bbfef086bfede1214d4f6d"},"cell_type":"code","source":"def load_data(which=\"sample\", skiprows=160000000):\n    dtypes = {\n        \"ip\" : \"uint64\",\n        \"app\": \"uint64\",\n        \"device\": \"uint64\",\n        \"os\": \"uint64\",\n        \"channel\": \"uint64\",\n        \"is_attributed\": \"uint64\"\n    }\n    if which == \"sample\":\n        data = pd.read_csv(\"../input/train_sample.csv\", dtype=dtypes)\n    elif which == \"whole\":\n        data = pd.read_csv(\"../input/train.csv\", names = ['ip', 'app', 'device', 'os', 'channel', 'click_time', 'attributed_time', 'is_attributed'], skiprows=skiprows, dtype=dtypes)\n    return data","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":true,"_uuid":"b3cc813335106a95404a5a5b13749900a225900c"},"cell_type":"code","source":"train_df = load_data(which=\"sample\") # which = \"sample\" or \"whole\"\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5f3c3233b746da050f92f54977a64033d9a1de20"},"cell_type":"code","source":"train_df.info()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d8a439095f9ddbf1f3c998275bf87c2e94e27637"},"cell_type":"markdown","source":"### Checking for missing values (NaN)"},{"metadata":{"trusted":true,"_uuid":"7764c7ee798ac6673f03c43c06cbe073f3672f86"},"cell_type":"code","source":"train_df.isnull().sum()/len(train_df.index)*100","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"02c85d0327e5a9f08f1e6b96c08b45fa94be42b9"},"cell_type":"code","source":"train_df = train_df.drop(\"attributed_time\", axis=1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1c25ba298dbec1ee1600d54e53c90d2d430f9b13"},"cell_type":"markdown","source":"Most of **attributed_time** values are missing, moreover, **attributed_time** is not present in the test data."},{"metadata":{"_uuid":"b7b9610268b4da30d62bc44ce3bc4f5f8057d9e0"},"cell_type":"markdown","source":"### Create Datetime Features"},{"metadata":{"trusted":true,"_uuid":"daf7c80224e095c6e814f0cff52d61a941add93a"},"cell_type":"code","source":"train_df[\"click_month\"] = pd.to_datetime(train_df[\"click_time\"]).dt.month\ntrain_df[\"click_day_of_week\"] = pd.to_datetime(train_df[\"click_time\"]).dt.dayofweek\ntrain_df[\"click_hour\"] = pd.to_datetime(train_df[\"click_time\"]).dt.hour\ntrain_df[\"click_year\"] = pd.to_datetime(train_df[\"click_time\"]).dt.year\ntrain_df = train_df.drop(\"click_time\", axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"08d80159336737c4b73c583bb732d1a60c4ca9b0"},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fa2078f94809f90e1ed5172b25011ccc4376e87e"},"cell_type":"code","source":"train_df.click_year.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1db4dd4fc9604dddc5f6bf60e79528a61fdfab4d"},"cell_type":"code","source":"train_df = train_df.drop([\"click_year\", \"click_month\"], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8b1e2f890d0ac390957213b72c645d7a1336b063"},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6e69fcb120da789ed19ea89d76d989d6414e57f3"},"cell_type":"code","source":"cols = [\"click_day_of_week\", \"click_hour\"]\ntrain_df[cols] = train_df[cols].astype(\"uint64\")\ntrain_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cd894342c66d9b86bfa6e94db7ca5604acc0169e"},"cell_type":"code","source":"train_df_int = train_df.select_dtypes(include=[\"uint64\"])\ntrain_df_int = train_df_int.apply(pd.to_numeric, downcast=\"unsigned\")\ntrain_df_int.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e7edcabefbe4d0e111cc9c8ad980ab193630f88d"},"cell_type":"code","source":"train_df = train_df.drop(train_df.dtypes[train_df.dtypes==\"uint64\"].index, axis=1)\ntrain_df = pd.concat([train_df, train_df_int], axis=1)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0de611fc8b70082bc6b64d13e04f53ed0440da54"},"cell_type":"code","source":"train_df.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"11f3f236c7bae39b403fbdab44ce8bbe8e3f895e"},"cell_type":"markdown","source":"## Univariate Analysis"},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"e8087ba3bc9597f759d805743009f97797286da0"},"cell_type":"code","source":"variable_value_counts = train_df[\"app\"].value_counts()\nvariable_value_quantile = variable_value_counts[variable_value_counts>variable_value_counts.quantile(0.8)]\nvariable_value_quantile = pd.Series(variable_value_quantile).reset_index(name=\"count\").rename(index=str, columns={\"index\": \"app\", \"count\": \"count\"})\nvariable_value_quantile","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":true,"_uuid":"f00023fae257bbc08f26311b17b470e72570f10f"},"cell_type":"code","source":"shortened_data = pd.merge(train_df, variable_value_quantile, on=\"app\", how=\"inner\").drop(\"count\", axis=1)\n# Plot app count distribution - for 80 % larger counts\nplt.figure(figsize=(18, 8))\nsns.countplot(x=\"app\", data=shortened_data)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"8c50e5247a47acd8a1f75edc7d540b9fd5a041f3"},"cell_type":"code","source":"variable_value_counts = train_df[\"device\"].value_counts()\nvariable_value_quantile = variable_value_counts[variable_value_counts>variable_value_counts.quantile(0.8)]\nvariable_value_quantile = pd.Series(variable_value_quantile).reset_index(name=\"count\").rename(index=str, columns={\"index\": \"device\", \"count\": \"count\"})\nvariable_value_quantile","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9f49fb684163112ebc6dcc93bf90ec27749b35d0"},"cell_type":"markdown","source":"The #3 app was the most clicked and 80% of the clicks are distributed between the apps in the X-axis."},{"metadata":{"scrolled":false,"trusted":true,"_uuid":"cb9d2d7e35c3bdd69b2bac18cf9674772d885fe4"},"cell_type":"code","source":"shortened_data = pd.merge(train_df, variable_value_quantile, on=\"device\", how=\"inner\").drop(\"count\", axis=1)\n# Plot device count distribution - for 80 % larger counts\nplt.figure(figsize=(18, 8))\nsns.countplot(x=\"device\", data=shortened_data)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"847021f6dc69586453290c4e1b90f5a964f14f04"},"cell_type":"markdown","source":"Despite that the value counts were restrained to the 80 % larger numbers. The number of devices that were not #1 or #2 was insignificant. Doing a quick search, the most used mobile device in 2017 was Oppo, that's problably the one indicated by #1."},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"f8013dbc0935febfc50960c755ea6e3ec10c66c1"},"cell_type":"code","source":"variable_value_counts = train_df[\"os\"].value_counts()\nvariable_value_quantile = variable_value_counts[variable_value_counts>variable_value_counts.quantile(0.8)]\nvariable_value_quantile = pd.Series(variable_value_quantile).reset_index(name=\"count\").rename(index=str, columns={\"index\": \"os\", \"count\": \"count\"})\nvariable_value_quantile","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a404df6e7bc2a5d26e0e4c0d28d04508835a0d4e"},"cell_type":"code","source":"shortened_data = pd.merge(train_df, variable_value_quantile, on=\"os\", how=\"inner\").drop(\"count\", axis=1)\n# Plot os count distribution - for 80 % larger counts\nplt.figure(figsize=(18, 8))\nsns.countplot(x=\"os\", data=shortened_data)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c7e3fc67f0a4d9dc17ffe5c7c7ec3ed052e0f161"},"cell_type":"markdown","source":"There are two most popular os in China, probably, iOS and Android."},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"dbdc6c8bebbb3f7d5e497e7513d942a3af510642"},"cell_type":"code","source":"variable_value_counts = train_df[\"channel\"].value_counts()\nvariable_value_quantile = variable_value_counts[variable_value_counts>variable_value_counts.quantile(0.5)]\nvariable_value_quantile = pd.Series(variable_value_quantile).reset_index(name=\"count\").rename(index=str, columns={\"index\": \"channel\", \"count\": \"count\"})\nvariable_value_quantile","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"80e274bc23d1816883934c32284265f8c73351a7"},"cell_type":"code","source":"shortened_data = pd.merge(train_df, variable_value_quantile, on=\"channel\", how=\"inner\").drop(\"count\", axis=1)\n# Plot os count distribution - for 80 % larger counts\nplt.figure(figsize=(18, 8))\nsns.countplot(x=\"channel\", data=shortened_data)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4d6d90612bd4e6b013d1dd4dc7359a4b61b4e119"},"cell_type":"markdown","source":"Unlike the previous categorical variables, \"channel\" counts is well distributed. This means that the clicks are well distributed between the channel ids of mobile ad publishers."},{"metadata":{"trusted":true,"_uuid":"e497d5f12f9cc257e060ba0751089a0c2c83a2e7"},"cell_type":"code","source":"train_df.channel.value_counts()[train_df.channel.value_counts() == train_df.channel.value_counts().max()]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e940f0cb6df3d1232d025f7ed279b65f4ef01e05"},"cell_type":"markdown","source":"Even that the clicks are well distributed between the channel, there is one that's exceptionally large, the channel #280."},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"a77039fafb20ad36ee048f5dd881e338c285b929"},"cell_type":"code","source":"perc_attributed = pd.DataFrame((train_df.is_attributed.value_counts()/train_df.is_attributed.value_counts().sum()*100).values, columns=[\"perc_of_occur[%]\"]).reset_index().rename(columns={\"index\": \"is_attributed\"})\nperc_attributed","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"89f46494c9ccbde347871399d152f6d1a466b19b"},"cell_type":"markdown","source":"As can be observed, the percentage of clicks that resulted in downloads were almost null. It really indicates a serious problem not only to the advertisers but for the model construction. It's hugely likely that the model becomes enbiased to class #0. \n\nDespite that the objective of this problem is to deliver the best metric score, maybe a class balancing technique will be applied further in order to make the predictions more realistic (or not)."},{"metadata":{"trusted":true,"_uuid":"291b8b4d216c047690de814b569f562642177a3f"},"cell_type":"code","source":"print(\"{:.1f}% of the IPs are unique and {:.1f}% are repetitions.\".format(len(train_df.ip.unique())/len(train_df.index)*100, 100-len(train_df.ip.unique())/len(train_df.index)*100))","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"e13fd41f9da37490ee147da05b52f795d8931da1"},"cell_type":"code","source":"ip_counts = train_df[\"ip\"].value_counts()\nsuspicious_ips = pd.DataFrame(ip_counts[ip_counts>50].reset_index(name=\"count\")).rename(columns={\"index\": \"ip\"}) # 50 clicks or more\nsuspicious_ips","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"83b1eb5b094ba326506de190e16a68c3ed4b0d64"},"cell_type":"code","source":"suspicious_ips_shortened = pd.merge(train_df, suspicious_ips, on=\"ip\", how=\"inner\").drop(\"count\", axis=1)\n# Plot IP count distribution\nplt.figure(figsize=(16, 8))\nsns.countplot(x=\"ip\", data=suspicious_ips_shortened)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1ca0da99ac1ce78bcfcff1696d45891363839ffc"},"cell_type":"markdown","source":"The \"clicks per ip\" above was limited to values above 50 clicks. This IPs are at least suspicious, for example, the IP 5348 clicked 669 times in ads, it can be considered an evidence that they are using some kind of automated process to perform the clicks (like bots), or its a group.\n\nIt's highly inlikely that an individual did that alone. \nA more profound analysis will be performed in the **bivariate analysis** section, highlighting what **channels**, **apps**, **OSes** and **devices** they are using. After that, a percentage of **clicked_and_downloaded** will be calculated.\n\n**OBS**: Considering the highly **unbalanced data** it's certain that they are entirely **click fraud** cases. "},{"metadata":{"_uuid":"6f1badb08629cd0bff7010d8e0a5171110193a07"},"cell_type":"markdown","source":"## Bivariate Analysis"},{"metadata":{"trusted":true,"_uuid":"e4c6bef2f0d47af4721d707d4244bf67a690ca7a"},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d2194bfd6992340a9e821f1a56c6fa6791996746"},"cell_type":"code","source":"suspicious_df = train_df.set_index(\"ip\").loc[suspicious_ips.ip.values]\nsuspicious_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"451377a7ea7510dcc9a0dbd56b81b200ea5c609d"},"cell_type":"code","source":"def mode(x):\n    return x.mode()\n\nsuspicious_df.groupby([\"ip\"]).apply(mode)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1fa3f871d953d218343c98db277220d76f31d79f"},"cell_type":"code","source":"ip_fraud_count = suspicious_df[suspicious_df[\"is_attributed\"]==0].groupby(\"ip\").size()\nip_fraud_perc = pd.DataFrame(ip_fraud_count/suspicious_df.groupby(\"ip\").size()*100, columns=[\"Fraud_Percentage[%]\"], dtype=\"float16\")\ndel(ip_fraud_count)\nip_fraud_perc.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e89943e6e68bf96b93351739a4653e279f2dde20"},"cell_type":"markdown","source":"As expected, almost **ALL** those suspicious IPs' clicks are **fraudulent**. The percentage that aren't **click fraud** can be considered as noise, because of its significance."},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"e355c7529d4785060ca772e9d18fb5c8ec44c830"},"cell_type":"code","source":"ip_is_attributed = train_df.groupby([\"ip\"]).is_attributed.sum()\nip_is_attributed = ip_is_attributed[ip_is_attributed > 0].sort_values(ascending=False).reset_index(name=\"is_attributed_count\")\nip_is_attributed = ip_is_attributed.iloc[:int(0.1*ip_is_attributed.shape[0])]\nip_is_attributed.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1e81ebef3bb0bed7643a65e1cb4234f58428648c"},"cell_type":"code","source":"sns.set(font_scale=1.0)\nplt.figure(figsize=(18, 6))\nsns.barplot(x=\"ip\", y=\"is_attributed_count\", data=ip_is_attributed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d947f7eb043a4393b5b37ad38ed108cfe86dcb1c"},"cell_type":"code","source":"app_is_attributed = train_df.groupby([\"app\"]).is_attributed.sum()\napp_is_attributed = app_is_attributed[app_is_attributed > 0].sort_values(ascending=False).reset_index(name=\"is_attributed_count\")\napp_is_attributed = app_is_attributed.iloc[:int(0.5*app_is_attributed.shape[0])]\napp_is_attributed.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f432af51a58e2a7906e13990d6ea4b436a1e1c64"},"cell_type":"code","source":"sns.set(font_scale=1.0)\nplt.figure(figsize=(18, 6))\nsns.barplot(x=\"app\", y=\"is_attributed_count\", data=app_is_attributed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef81998e5d8f816d0c261a3b45ce13d7ac5219f5"},"cell_type":"code","source":"os_is_attributed = train_df.groupby([\"os\"]).is_attributed.sum()\nos_is_attributed = os_is_attributed[os_is_attributed > 0].sort_values(ascending=False).reset_index(name=\"is_attributed_count\")\nos_is_attributed = os_is_attributed.iloc[:int(0.5*os_is_attributed.shape[0])]\nos_is_attributed.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"445644d59b8af4dfbf0af0dc36f947b26a765619"},"cell_type":"code","source":"sns.set(font_scale=1.0)\nplt.figure(figsize=(18, 6))\nsns.barplot(x=\"os\", y=\"is_attributed_count\", data=os_is_attributed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5b261db30f37cbeb3469f0082e1466ea5eb164ab"},"cell_type":"code","source":"device_is_attributed = train_df.groupby([\"device\"]).is_attributed.sum()\ndevice_is_attributed = device_is_attributed[device_is_attributed > 0].sort_values(ascending=False).reset_index(name=\"is_attributed_count\")\ndevice_is_attributed = device_is_attributed.iloc[:int(0.5*device_is_attributed.shape[0])]\ndevice_is_attributed.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"01ef0ea938d4b2b51ac21175008aff5bddd2dfeb"},"cell_type":"code","source":"sns.set(font_scale=1.0)\nplt.figure(figsize=(18, 6))\nsns.barplot(x=\"device\", y=\"is_attributed_count\", data=device_is_attributed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7fbe9c90dea6f1d6723085fdb2beb7a36138a449"},"cell_type":"code","source":"channel_is_attributed = train_df.groupby([\"channel\"]).is_attributed.sum()\nchannel_is_attributed = channel_is_attributed[channel_is_attributed > 0].sort_values(ascending=False).reset_index(name=\"is_attributed_count\")\nchannel_is_attributed = channel_is_attributed.iloc[:int(0.5*channel_is_attributed.shape[0])]\nchannel_is_attributed.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"51bfb0d61374dfd71d66252c29b4a8cb3ce32193"},"cell_type":"code","source":"sns.set(font_scale=1.0)\nplt.figure(figsize=(18, 6))\nsns.barplot(x=\"channel\", y=\"is_attributed_count\", data=channel_is_attributed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0446379b59e8b5a5e5e865a8428d6bc95125d429"},"cell_type":"code","source":"try:\n    \n    del channel_is_attributed\n    del device_is_attributed\n    del os_is_attributed\n    del app_is_attributed\n    del ip_is_attributed\n    \nfinally:\n    \n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8fa06266bcc24a757fa1b3aa5ff3aff9ab351c18"},"cell_type":"code","source":"train_df.groupby(\"click_day_of_week\").is_attributed.size().plot()\nplt.ylabel(\"Click Counts\")\nplt.xlabel(\"Day of week\")\nplt.xticks(ticks=[0, 1, 2, 3], labels=[\"Monday\", \"Tuesday\", \"Wednesday\", \"Thursday\"])\n_ = plt.title(\"Clicks per Weekday\", {\"fontsize\": 15})","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f1780de785234daa185b2a4d1e4e93f75fb6035"},"cell_type":"markdown","source":"The overall amount of clicks increases rapidly between Monday and Tuesday, losing strength between Tuesday and Wednesday, when it starts to decline in a ratio smaller than it grew in the first period. Probably, the downtrend will continue, decreasing slowly until the weekend."},{"metadata":{"trusted":true,"_uuid":"9c8a8108b2dea91ad3f82fbdca1cb175130a0f72"},"cell_type":"code","source":"train_df.groupby(\"click_day_of_week\").is_attributed.sum().plot()\nplt.ylabel(\"Download Counts\")\nplt.xlabel(\"Day of week\")\nplt.xticks(ticks=[0, 1, 2, 3], labels=[\"Monday\", \"Tuesday\", \"Wednesday\", \"Thursday\"])\n_ = plt.title(\"Downloads per Weekday\", {\"fontsize\": 15})","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8c367e072a294e2d7a7483a91a52f252d3aa5a3d"},"cell_type":"markdown","source":"According to the graph, the download counts increases rapidly from Monday to Tuesday, decreasing its increase ratio, but maintaining a up trend until Wednesday, when it starts to decrease until Thurday."},{"metadata":{"trusted":true,"_uuid":"54e76ba960f89054cd4f85ea0854255e77edf443"},"cell_type":"code","source":"train_df.groupby(\"click_day_of_week\").is_attributed.mean().plot()\nplt.ylabel(\"Download Ratio\")\nplt.xlabel(\"Day of week\")\nplt.xticks(ticks=[0, 1, 2, 3], labels=[\"Monday\", \"Tuesday\", \"Wednesday\", \"Thursday\"])\n_ = plt.title(\"Download Ratio per Weekday\", {\"fontsize\": 15})","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"07332c6108e9e5f269b736d79d236dc7cbc85860"},"cell_type":"markdown","source":"The **Download Ratio** follows the same patterns of **Download Counts**."},{"metadata":{"trusted":true,"_uuid":"5dd614006af91cc7ade946c615dae0b4b15b779e"},"cell_type":"code","source":"plt.figure(figsize=(10, 6))\nclick_hour_attributed = train_df.groupby(\"click_hour\").is_attributed.size()\nclick_hour_attributed[24] = click_hour_attributed[0]\nclick_hour_attributed =  click_hour_attributed.drop(0)\nclick_hour_attributed.plot()\nplt.ylabel(\"Click Counts\")\nplt.xlabel(\"Hour\")\nplt.xticks(ticks=range(1, 25), labels=range(1, 25))\n_ = plt.title(\"Clicks per Hour\", {\"fontsize\": 15})","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6c051eded10e0f71df313e88cc612687e31bc3ff"},"cell_type":"markdown","source":"The **Click Counts** decreases rapidly between 14:00 and 20:00 (where it reaches its minimum), after that it starts to increase even faster than it declined."},{"metadata":{"trusted":true,"_uuid":"d0c3bd62b981c0394e51c2113388b371f6f39b10"},"cell_type":"code","source":"plt.figure(figsize=(10, 6))\nclick_hour_attributed = train_df.groupby(\"click_hour\").is_attributed.sum()\nclick_hour_attributed[24] = click_hour_attributed[0]\nclick_hour_attributed =  click_hour_attributed.drop(0)\nclick_hour_attributed.plot()\nplt.ylabel(\"Download Counts\")\nplt.xlabel(\"Hour\")\nplt.xticks(ticks=range(1, 25), labels=range(1, 25))\n_ = plt.title(\"Downloads per Hour\", {\"fontsize\": 15})","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d1137d708a7f88c98a390ee427e0ebf7457e69f9"},"cell_type":"markdown","source":"Interestingly, the **Downloads per Hour** count has a much more no-uniform graph, with random spikes, compared to the **Clicks per Hour** graph that's much more uniform and **organized**. This random spikes reflects the interest of the user in downloading apps in certain part of the day (marketing and quality of certain apps, for instance, sparkled the interest). Where the **Clicks per Hour** graph reflects an organized approach to distribute clicks in order to make profits."},{"metadata":{"trusted":true,"_uuid":"e54cad19ac78353d9d484759afb861e642e39f16"},"cell_type":"code","source":"plt.figure(figsize=(10, 6))\nclick_hour_attributed = train_df.groupby(\"click_hour\").is_attributed.mean()\nclick_hour_attributed[24] = click_hour_attributed[0]\nclick_hour_attributed =  click_hour_attributed.drop(0)\nclick_hour_attributed.plot()\nplt.ylabel(\"Download Ratio\")\nplt.xlabel(\"Hour\")\nplt.xticks(ticks=range(1, 25), labels=range(1, 25))\n_ = plt.title(\"Downloads Ratio per Hour\", {\"fontsize\": 15})","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1cff73c819b0d993471be7f03ae15118564c5e60"},"cell_type":"markdown","source":"The reflection between the coordinated **Clicks per Hour** graph and the noisy **Downloads per Hour**."},{"metadata":{"trusted":true,"_uuid":"24a58b19c85174e61d54b4f1775d51d9321637b3"},"cell_type":"code","source":"day_week_hour_count = train_df.groupby([\"click_day_of_week\", \"click_hour\"]).is_attributed.count().reset_index(name=\"click_count\")\nday_week_hour_count[\"index\"] = day_week_hour_count[\"click_day_of_week\"].astype(str) + \"_\" + day_week_hour_count[\"click_hour\"].astype(str)\nday_week_hour_count.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e4ab3a15d11227fb1ed4fccdfb082c4c2bc0e688"},"cell_type":"code","source":"plt.figure(figsize=(50, 20))\nsns.lineplot(x=\"index\", y=\"click_count\", data=day_week_hour_count.loc[:, \"click_count\":\"index\"])\nplt.xticks(ticks=range(len(day_week_hour_count[\"index\"])), labels=day_week_hour_count[\"index\"])\nplt.tick_params(labelsize=25)\nplt.title(\"Clicks per Day of Week per Hour\")\nsns.set(font_scale=3.0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"98a94fcb7e4c830d0e8be24736756bda42152d01"},"cell_type":"markdown","source":"In this graph, some nice patterns can be spotted, depending on the week and on the hour of the day."},{"metadata":{"_uuid":"aff21143d2c8e54b1285a01364a36f8fbc3f5abb"},"cell_type":"markdown","source":"**ZOOM (30 first indexes)** "},{"metadata":{"scrolled":false,"trusted":true,"_uuid":"338098e67f7eaf8602e6e3a77b7f53c5e53f6891"},"cell_type":"code","source":"plt.figure(figsize=(50, 20))\nsns.lineplot(x=\"index\", y=\"click_count\", data=day_week_hour_count.loc[:30, \"click_count\":\"index\"])\nplt.xticks(ticks=range(len(day_week_hour_count.loc[:30, \"index\"])), labels=day_week_hour_count.loc[:30, \"index\"])\nplt.tick_params(labelsize=25)\nplt.title(\"Downloaded per Day of Week per Hour [:30]\")\nsns.set(font_scale=3.0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3ba180fb775f89affb5e0a1aaabb5c8f9bafecd5"},"cell_type":"markdown","source":"**ZOOM (From index 28 to final)** "},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"e5b25ed7f121ce7416ed5a5a7a7c549358cb5f06"},"cell_type":"code","source":"plt.figure(figsize=(50, 20))\nsns.lineplot(x=\"index\", y=\"click_count\", data=day_week_hour_count.loc[28:, \"click_count\":\"index\"])\nplt.xticks(ticks=range(len(day_week_hour_count.loc[28:, \"index\"])), labels=day_week_hour_count.loc[28:, \"index\"])\nplt.tick_params(labelsize=22)\nplt.title(\"Downloaded per Day of Week per Hour [28:]\")\nsns.set(font_scale=3.0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d6088bb00720b3c5ef51c77a2a599646c15541b9"},"cell_type":"markdown","source":"The sparkle during the downtrend click cycle always happens at noon."},{"metadata":{"scrolled":false,"trusted":true,"_uuid":"a3fb0c2e9028a1ad3573cfab5d8f913e2cf5040a"},"cell_type":"code","source":"day_week_hour_ratio = train_df.groupby([\"click_day_of_week\", \"click_hour\"]).is_attributed.mean().reset_index(name=\"is_attributed_ratio\")\nday_week_hour_ratio[\"index\"] = day_week_hour_ratio[\"click_day_of_week\"].astype(str) + \"_\" + day_week_hour_ratio[\"click_hour\"].astype(str)\nplt.figure(figsize=(50, 20))\nsns.lineplot(x=\"index\", y=\"is_attributed_ratio\", data=day_week_hour_ratio.loc[:, \"is_attributed_ratio\":\"index\"])\nplt.xticks(ticks=range(len(day_week_hour_ratio[\"index\"])), labels=day_week_hour_ratio[\"index\"])\nplt.tick_params(labelsize=25)\nplt.title(\"Downloaded Ratio per Day of Week per Hour\")\nsns.set(font_scale=1.5)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4996bb9df95d7f87627472dcaebe5b33ac47591e"},"cell_type":"markdown","source":"No relationship can be detected from this graph, it just looks like random noise/peaks. Reflecting the little correlation between the time series and user interest to download apps."},{"metadata":{"scrolled":false,"_uuid":"231f7f68ff51cef4806c81f897114a290485117e"},"cell_type":"markdown","source":"## Feature Engineering"},{"metadata":{"trusted":true,"_uuid":"61b3b0004b293957be8dbe71c0f9f9826b14876c"},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"60d7dbe25a1638e41a3e6fe23523b2077aa2adff"},"cell_type":"code","source":"new_features = [\n    {\"op\": \"mode\", \"groupby\": [\"ip\"], \"select\": \"os\", \"agg\": lambda x: x.mode() if x.mode() is int else x.mode().max()},\n    {\"op\": \"mode\", \"groupby\": [\"ip\"], \"select\": \"channel\", \"agg\": lambda x: x.mode() if x.mode() is int else x.mode().max()},\n    {\"op\": \"mode\", \"groupby\": [\"ip\"], \"select\": \"device\", \"agg\": lambda x: x.mode() if x.mode() is int else x.mode().max()},\n    \n    {\"op\": \"count\", \"groupby\": [\"ip\"], \"select\": \"app\", \"agg\": lambda x: x.count()},\n    \n    {\"op\": \"mode\", \"groupby\": [\"ip\", \"app\", \"device\"], \"select\": \"os\", \"agg\": lambda x: x.mode() if x.mode() is int else x.mode().max()},\n    {\"op\": \"mode\", \"groupby\": [\"ip\", \"app\", \"device\"], \"select\": \"channel\", \"agg\": lambda x: x.mode() if x.mode() is int else x.mode().max()},\n    {\"op\": \"mode\", \"groupby\": [\"ip\", \"app\", \"device\", \"os\"], \"select\": \"channel\", \"agg\": lambda x: x.mode() if x.mode() is int else x.mode().max()}\n]\nfor new_feature in new_features:\n    new_feature_name = str(new_feature[\"op\"]) + \"_\" + str(new_feature[\"select\"]) + \"_per_\" + '_'.join(new_feature[\"groupby\"])\n    new_feature_df = train_df.groupby(new_feature[\"groupby\"])[new_feature[\"select\"]].agg(new_feature[\"agg\"]).reset_index(name=new_feature_name)\n    train_df = pd.merge(train_df, new_feature_df, how=\"inner\", on=new_feature[\"groupby\"])\n                                                          ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"def867f52613ad196d235b9417649f7c3950d513"},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"564994732bd3431d5ee4be87ec15670951ba4b84"},"cell_type":"code","source":"test_df = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"52dea0143e99f198cdc50b59662045f68283ffd7"},"cell_type":"code","source":"ip_occur_train = len(test_df.ip.value_counts()[train_df.ip.unique()].index)*100/len(test_df.ip.value_counts().index)\nprint(round(ip_occur_train, 2), \"% of the test set IPs have appeared in the train set\", round(100 - ip_occur_train, 2), \"% are new occurences.\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6327281e665d8e7071fe6b38a48e16784dad19e7"},"cell_type":"markdown","source":"For the reason above, features built on top of the IP feature and the target variable would not be an effective predictive feature. Dropping `click_day_of_week` column in train dataframe."},{"metadata":{"trusted":true,"_uuid":"cba9e9357ec65f08952eebe516e66bbf4d613829"},"cell_type":"code","source":"train_df = train_df.drop(\"click_day_of_week\", axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c6b2559ea24f749b1e2c6a65fac1819a36278045"},"cell_type":"code","source":"def perc_of_train_fea_cat_in_test_data(test_df, features):\n    for feature in features:\n        test_df_unique = test_df[feature].unique()\n        train_df_unique = train_df[feature].unique()\n        train_df_unique_len = len(train_df_unique)\n        count = 0\n        for unique_feature in test_df_unique:\n            if unique_feature in train_df_unique:\n                count += 1\n        perc = round(count / train_df_unique_len * 100, 2)\n        print(perc, \"% of feature named: \" + feature.upper() + \"'s categories of the training set have appeared in the test set.\")\n        \ndef perc_of_new_fea_in_test(test_df, features):\n    for feature in features:\n        test_df_unique = test_df[feature].unique()\n        test_df_unique_len = len(test_df_unique)\n        train_df_unique = train_df[feature].unique()\n        count = 0\n        for unique_feature in test_df_unique:\n            if unique_feature not in train_df_unique:\n                count += 1\n        perc = round((count / test_df_unique_len) * 100, 2)\n        print(perc, \"% of categories in feature: \" + feature.upper() + \" are new occurences in the test set.\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"42c699bf114cfccc52dc4c246fc931f5a746b9bc"},"cell_type":"code","source":"perc_of_train_fea_cat_in_test_data(test_df, test_df.columns[(test_df.columns!=\"click_id\") & (test_df.columns!=\"click_time\")])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"710dababa70b7cb85a7cf8d139164b0c24028781"},"cell_type":"code","source":"perc_of_new_fea_in_test(test_df, test_df.columns[(test_df.columns!=\"click_id\") & (test_df.columns!=\"click_time\")])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"41bd0498ce4b60bad29db582a4ad08c358f86d27"},"cell_type":"markdown","source":"Notice that the values presented above are not valid for the sample took from the training data. After the model is tested on the training sample data, the statistics above should be repeated for the whole training set."},{"metadata":{"_uuid":"30d45a19d1d302706591133ea0e86970af1d39fb"},"cell_type":"markdown","source":"## Model Training (Sample data)"},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"e1a9fe6b8de3e4f8001fc4b31571fcfdb02bda3f"},"cell_type":"code","source":"# train_df[\"attributed_time\"] = pd.to_datetime(train_df[\"attributed_time\"])\n# train_df.attributed_time[~train_df.attributed_time.isnull()]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"becfecb17b6d5de8617bbd9925e44707e6fc75b6"},"cell_type":"code","source":"# train_df[\"attributed_time_day\"] = train_df[\"attributed_time\"].dt.day\n# train_df[\"attributed_time_hour\"] = train_df[\"attributed_time\"].dt.hour\n# train_df[\"attributed_time_weekday\"] = train_df[\"attributed_time\"].dt.dayofweek\n# train_df = train_df.drop(\"attributed_time\", axis=1)\n# train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"35eec40fe3cab29a0cd90361248c99f707d51836"},"cell_type":"code","source":"train_df.is_attributed.mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"111a93e7f7d1a3daac600ee44743ad01fa769ae8"},"cell_type":"code","source":"train_df = train_df.drop(\"ip\", axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"51a5d8760ffc5a20161e3984eba6478aceb41886"},"cell_type":"code","source":"import xgboost as xgb\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.metrics import make_scorer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4492c2c8db149337a63b42bc55be6bf5b8b4a433"},"cell_type":"code","source":"KFold = StratifiedKFold(n_splits=int(train_df.shape[0]/10000), shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7589917171ad3d9b9d13ac49beec3f256f0bc022"},"cell_type":"code","source":"scale_pos_weight = round(train_df.is_attributed.value_counts()[0]/train_df.is_attributed.value_counts()[1], 2)\nscale_pos_weight","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"776ef26fae04ef9a39c89762a48e1d194be2050a"},"cell_type":"code","source":"param_grid = {\"max_depth\": [2, 4, 5, 10],\n             \"learning_rate\": [0.0001, 0.001, 0.01],\n             \"n_estimators\": [10, 100, 200],\n             }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f4b871ba85dc80b19a252bf3e2f0489dc7d4e4e0"},"cell_type":"code","source":"bst = xgb.XGBModel(objective=\"binary:logistic\", booster=\"dart\",\n                  scale_pos_weight=scale_pos_weight, n_jobs=-1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1ba9e9e1a480f9db5b2b49ee6c1a9753e6454207"},"cell_type":"code","source":"grid_search = GridSearchCV(estimator=bst,\n                        param_grid=param_grid,\n                        scoring=make_scorer(roc_auc_score),\n                        cv=KFold,\n                        verbose=1,\n                        return_train_score=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d6cbbad2a00d5c9e60c9ae18414394c0ee8cf86d","scrolled":true},"cell_type":"code","source":"grid_search.fit(X=train_df.drop(\"is_attributed\", axis=1), y=train_df[\"is_attributed\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a0317c1000cba256ed253ad2c64b84d26c645071"},"cell_type":"code","source":"xgb_df = pd.DataFrame(grid_search.cv_results_)\nxgb_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"60e752421894669386e3bf36bbcd21efb9168ca0"},"cell_type":"code","source":"print(\"The best auc score is:\", grid_search.best_score_)\nprint(\"The best params are:\", grid_search.best_params_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c63685c195a3f3d81293cb4a4085592dcb55f28d"},"cell_type":"code","source":"plt.plot(list(range(1, 37)), xgb_df[\"mean_train_score\"], label=\"Train Score\")\nplt.plot(list(range(1, 37)), xgb_df[\"mean_test_score\"], label=\"Test Score\")\nplt.grid()\nplt.xlabel(\"Param Index\")\nplt.ylabel(\"AUC Score\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"812fc2e1e63b4c36ac57d8ecebb21125add4f829"},"cell_type":"code","source":"param_grid = {\"max_depth\": [2, 4, 5, 10],\n             \"learning_rate\": [0.0001, 0.001, 0.01],\n             \"n_estimators\": [10, 100, 200],\n             }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c9da350bf29421989c3130822f60c3c335d49891"},"cell_type":"code","source":"lg = lgbm.LGBMClassifier(objective=\"binary\", scale_pos_weight=scale_pos_weight, n_jobs=-1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a5ddd2c2fd94c4a06998bf471e6010a4d7f21cf5"},"cell_type":"code","source":"grid_search_lg = GridSearchCV(estimator=lg,\n                        param_grid=param_grid,\n                        scoring=make_scorer(roc_auc_score),\n                        cv=KFold,\n                        verbose=1,\n                        return_train_score=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"beb085986ea4770196fe492a728dab30f468f017"},"cell_type":"code","source":"grid_search_lg.fit(X=train_df.drop(\"is_attributed\", axis=1), y=train_df[\"is_attributed\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e2632ea4b54643cc5e8fb2239611cdb184d6d179"},"cell_type":"code","source":"lg_df = pd.DataFrame(grid_search_lg.cv_results_)\nlg_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"acfcd66ade2ebf7ce84d03f3568c66df73d0b46d"},"cell_type":"code","source":"print(\"The best auc score is:\", grid_search_lg.best_score_)\nprint(\"The best params are:\", grid_search_lg.best_params_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"10e55f12bd9ca94f769f67bcd709ab46e8cdca77"},"cell_type":"code","source":"plt.plot(list(range(1, 37)), lg_df[\"mean_train_score\"], label=\"Train Score\")\nplt.plot(list(range(1, 37)), lg_df[\"mean_test_score\"], label=\"Test Score\")\nplt.grid()\nplt.xlabel(\"Param Index\")\nplt.ylabel(\"AUC Score\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fb5c1f5007fd3a6cb7348037f617a079a1272f86"},"cell_type":"markdown","source":"### Final Model (Sample Data)"},{"metadata":{"trusted":true,"_uuid":"dc1cf340d5a628a407ceef1f5e299ab2c70e6cd2"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_test, y_train, y_test = train_test_split(train_df.drop(\"is_attributed\", axis=1), train_df[\"is_attributed\"], test_size=.3, shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7dedc5cfc54d31ade56262ea4bbe00eadc72aa84"},"cell_type":"code","source":"print(\"Ratio of 'is_attributed (1)' in train data:\", y_train.mean())\nprint(\"Ratio of 'is_attributed (1)' in test data:\", y_test.mean())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"21ae96d0955b797704edbff81f9ed630f7c64bec"},"cell_type":"markdown","source":"#### xgBoost"},{"metadata":{"trusted":true,"_uuid":"c0a03873abfc15edffb03b3d679f308858dcdc2c"},"cell_type":"code","source":"xgb_final = xgb.XGBClassifier(objective=\"binary:logistic\", booster=\"dart\",\n                  scale_pos_weight=scale_pos_weight, n_jobs=-1, **grid_search.best_params_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d9976e894a011ad9dc79857b535175ae4ae39ccd"},"cell_type":"code","source":"xgb_final.fit(X_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ff8670abd6b0ea2d6a1ef4428bb626583aa2a759"},"cell_type":"code","source":"xgb_predict = xgb_final.predict(X_test)\nxgb_proba = xgb_final.predict_proba(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"04226725af6e8bc994427ac543ee8931c5992779"},"cell_type":"code","source":"xgb_predict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9251cb25c9481c8804661cfba3f33b825175d27c"},"cell_type":"code","source":"xgb_proba","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c8b30f28b5d6275219dc78744ad31c691a0f02ca"},"cell_type":"code","source":"roc_auc_score(y_test, xgb_proba[:, 1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1f67dbab1a9b9a85165ec91b816c8b36b130a46d"},"cell_type":"code","source":"count = 0\nfor i, j in zip(y_test.values, xgb_predict):\n    if i == j:\n        count += 1\nprint(\"Accuracy:\", count/xgb_predict.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ba2a801a76f7238cc626a522910770afe2bf0723"},"cell_type":"markdown","source":"The XGBoost model achieved a `roc_auc_score` of approximately **90.7%** and Accuracy of **94.5%** (at threshold = **50%**)."},{"metadata":{"trusted":true,"_uuid":"eeb785e08cd33dda6965d91977e53ba4f09fe803"},"cell_type":"code","source":"fpr_xgb, tpr_xgb, thresholds_xgb = roc_curve(y_test, xgb_proba[:, 1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0611bc358f7cb7efadac919c0e9b7078b43dcab5"},"cell_type":"code","source":"plt.plot(fpr_xgb, tpr_xgb)\nplt.grid()\nplt.title(\"XGBoost ROC Curve\")\nplt.xlabel(\"False Positive Ratio (FPR)\")\nplt.ylabel(\"True Positive Ratio (TPR)\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ec9428c482291b7edca59b71719af69ae8570015"},"cell_type":"code","source":"ROC_df_xgb = pd.DataFrame(data=np.concatenate([thresholds_xgb.reshape(-1, 1), tpr_xgb.reshape(-1, 1), fpr_xgb.reshape(-1, 1)], axis=1), columns=[\"Threshold\", \"True Positive Ratio (TPR)\", \"False Positive Ratio (FPR)\"])\nROC_df_xgb.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c049f0f432d2ba498c5fbd67ac9f3f003c5d0dc9"},"cell_type":"markdown","source":"#### lightGBM"},{"metadata":{"trusted":true,"_uuid":"5cf2864b6cdcade1d398f4e41fcc5f1dc46da67e"},"cell_type":"code","source":"lg_final = lgbm.LGBMClassifier(objective=\"binary\", scale_pos_weight=scale_pos_weight, n_jobs=-1, **grid_search_lg.best_params_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2728c42696d60b6ce87d3a99659f245bfaa61b4b","scrolled":true},"cell_type":"code","source":"lg_final.fit(X_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b5892842a77f4e622810142618fcb3963ab867b2"},"cell_type":"code","source":"lg_predict = lg_final.predict(X_test)\nlg_proba = lg_final.predict_proba(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5688665607c9e51ea7ed544b73f6955a99e073ff"},"cell_type":"code","source":"lg_predict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c4e72b8f8ae71c86197596f86b410d011ed0ed51"},"cell_type":"code","source":"lg_proba","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2ca24ffaeca03df95a655f2865835e83bc0d9ac9"},"cell_type":"code","source":"roc_auc_score(y_test, lg_proba[:, 1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"df67e36ca650d281c76a997bd802f40fd2b40e4d"},"cell_type":"code","source":"count = 0\nfor i, j in zip(y_test.values, lg_predict):\n    if i == j:\n        count += 1\nprint(\"Accuracy:\", count/lg_predict.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5c38ea6c1401dc2d07ba3b05fd896862c5d8b8da"},"cell_type":"markdown","source":"The LightGBM model achieved a `roc_auc_score` of approximately **92.9%** and Accuracy of **95.3%** (at threshold = **50%**)."},{"metadata":{"trusted":true,"_uuid":"c251726db6ad6b450c2c63fb90b521c49f1069c6"},"cell_type":"code","source":"fpr_lg, tpr_lg, thresholds_lg = roc_curve(y_test, xgb_proba[:, 1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8b32df192d239e177f5f236fde0f9deec673849c"},"cell_type":"code","source":"plt.plot(fpr_lg, tpr_lg)\nplt.grid()\nplt.title(\"LightGBM ROC Curve\")\nplt.xlabel(\"False Positive Ratio (FPR)\")\nplt.ylabel(\"True Positive Ratio (TPR)\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef37b3d27d1ce12eb94c64fb5e8703974201d2ad"},"cell_type":"code","source":"thresholds_lg","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9e2a73073fda7192965422daeff505a3b6230900"},"cell_type":"code","source":"ROC_df_lg = pd.DataFrame(data=np.concatenate([thresholds_lg.reshape(-1, 1), tpr_lg.reshape(-1, 1), fpr_lg.reshape(-1, 1)], axis=1), columns=[\"Threshold\", \"True Positive Ratio (TPR)\", \"False Positive Ratio (FPR)\"])\nROC_df_lg","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f3f284674d35f4f8b8ca1f51d121340120825a8"},"cell_type":"code","source":"try:\n    \n    del train_df\n    del X_train\n    del X_test\n    del y_train\n    del y_test\n    del test_df # Delete it because of RAM shortage when working on the whole data; It's gonna be loaded later again after the final model is trained and tested on the whole data.\n    del ROC_df_xgb\n    del ROC_df_lg\n    \nexcept:\n    pass\n    \nfinally:\n    _ = gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"45f9bc99c7f849c01ec96d45ad72fa002e94c4c8"},"cell_type":"markdown","source":"## Loading whole dataset"},{"metadata":{"trusted":true,"_uuid":"3905d77694b431411d028c467e6f1cdbbb992a5a"},"cell_type":"code","source":"train_whole =  load_data(which=\"whole\")\ntrain_whole = train_whole.drop(\"attributed_time\", axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"50717da7f626467c2ed0bd35a3297909757a8e3e"},"cell_type":"code","source":"train_whole.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b82429a4eb9e216c5ef8faae24976f494fa68410"},"cell_type":"code","source":"train_whole.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"12976e5fee2a1f2f1a9e22c56e92ca1d90fbfd03"},"cell_type":"code","source":"int_columns = [\"ip\", \"app\", \"device\", \"os\", \"channel\", \"is_attributed\"]\ntrain_whole[int_columns] = train_whole[int_columns].apply(pd.to_numeric, downcast=\"unsigned\")\ntrain_whole.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a3f00c45f21806b51350886520496c0b94ff6d3e"},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"caa51de41642a7a34ce381e521d64dedf997c03d"},"cell_type":"code","source":"import sys\nvar, obj = None, None\ntotal_size = 0\nfor var, obj in locals().items():\n    print(str(var) + \" : \" + str(sys.getsizeof(obj)))\n    total_size += sys.getsizeof(obj)\nprint(\"Total memory usage:\", total_size)\ndel total_size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d1f129406fdb580bbe1f6f6552072eb452c47793"},"cell_type":"code","source":"try:\n    del train_df_int\n    del variable_value_counts\n    del variable_value_quantile\n    del shortened_data\n    del ip_counts\n    del suspicious_ips\n    del suspicious_ips_shortened\n    del suspicious_df\n    del click_hour_attributed\n    del day_week_hour_count\n    del day_week_hour_ratio\n    del new_feature_df\n    del xgb_df\n    del lg_df\n    del xgb_predict\n    del lg_predict\n    del StratifiedKFold    \nfinally:\n    _ = gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b679e75ed9811b93b5bb33e49a23ccb97e552483"},"cell_type":"code","source":"import sys\nvar, obj = None, None\ntotal_size = 0\nfor var, obj in locals().items():\n    print(str(var) + \" : \" + str(sys.getsizeof(obj)))\n    total_size += sys.getsizeof(obj)\nprint(\"Total memory usage:\", total_size)\ndel total_size","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dc5b7a69f0e5107b20a6a2d3b4aee6768d7601c5"},"cell_type":"markdown","source":"### Data Processing"},{"metadata":{"trusted":true,"_uuid":"a215030ce4a61f6b2a94f28315ae9f1f86c84fc2"},"cell_type":"code","source":"train_whole[\"click_month\"] = pd.to_datetime(train_whole[\"click_time\"]).dt.month\ntrain_whole[\"click_day_of_week\"] = pd.to_datetime(train_whole[\"click_time\"]).dt.dayofweek\ntrain_whole[\"click_hour\"] = pd.to_datetime(train_whole[\"click_time\"]).dt.hour\ntrain_whole[\"click_year\"] = pd.to_datetime(train_whole[\"click_time\"]).dt.year\ntrain_whole = train_whole.drop(\"click_time\", axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e6fdf98a19afac453ae391d0a78b33a418811035"},"cell_type":"code","source":"train_whole.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bcb0bc45b9dddcf2b4af5759b8427fd86725b43d"},"cell_type":"code","source":"train_whole = train_whole.drop([\"click_year\", \"click_month\"], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"73705f309db3bfc66f6432ca0588fe2c5980ae8d"},"cell_type":"code","source":"int_columns = [\"click_day_of_week\", \"click_hour\"]\ntrain_whole[int_columns] = train_whole[int_columns].apply(pd.to_numeric, downcast=\"unsigned\")\ntrain_whole.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1e79ce07a8f630e18f6477c36b0f9a66784739f5"},"cell_type":"code","source":"for new_feature in new_features:\n    new_feature_name = str(new_feature[\"op\"]) + \"_\" + str(new_feature[\"select\"]) + \"_per_\" + '_'.join(new_feature[\"groupby\"])\n    new_feature_df = train_whole.groupby(new_feature[\"groupby\"])[new_feature[\"select\"]].agg(new_feature[\"agg\"]).reset_index(name=new_feature_name)\n    train_whole = pd.merge(train_whole, new_feature_df, how=\"inner\", on=new_feature[\"groupby\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b534dc67d4c62bddb568ac7545782516d4a4b3a2"},"cell_type":"code","source":"train_whole.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d536b96e7dc6ccaaeb57318ca47a1f37e2bf20ca"},"cell_type":"code","source":"# float_columns = [\"attributed_time_day\", \"attributed_time_hour\", \"attributed_time_weekday\"]\n# train_whole[float_columns] = train_whole[float_columns].apply(pd.to_numeric, downcast=\"float\")\nint_columns = [\"count_app_per_ip\"]\ntrain_whole[int_columns] = train_whole[int_columns].apply(pd.to_numeric, downcast=\"unsigned\")\ntrain_whole.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6025c5ff058dfb5c941008de4309195cecb9d8f9"},"cell_type":"code","source":"train_whole = train_whole.drop(\"ip\", axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"d9085c9e6e69348d0bcf170a8a029c01dfc52009"},"cell_type":"code","source":"vars_ = dir()\nvar_list = []\nfor var in vars_:\n    if not var.startswith(\"_\"):\n        var_list.append(var)\n        \nvar_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"10f4a9e016e1f0474989adc60478c730738a4d7e"},"cell_type":"code","source":"var, obj = None, None\ntotal_size = 0\nfor var, obj in locals().items():\n    print(str(var) + \" : \" + str(sys.getsizeof(obj)) + \" Bytes\")\n    total_size += sys.getsizeof(obj)\nprint(\"Total memory usage:\", total_size/1000000000, \"GB\")\ndel total_size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f1e85accbd64a2845a30d77886157631e4c12c0"},"cell_type":"code","source":"try:\n    del new_feature_df\nfinally:\n    _ = gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"048e89226308f6a5a74f180e8550590df29ae24b"},"cell_type":"code","source":"var, obj = None, None\ntotal_size = 0\nfor var, obj in locals().items():\n    print(str(var) + \" : \" + str(sys.getsizeof(obj)) + \" Bytes\")\n    total_size += sys.getsizeof(obj)\nprint(\"Total memory usage:\", total_size/1000000000, \"GB\")\ndel total_size","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"76dc46220ee80ae420fbe9193372b7673f8de652"},"cell_type":"markdown","source":"* ## Final Model (Whole - lightGBM)"},{"metadata":{"trusted":true,"_uuid":"b2314ad520853b0935061ebef64cb194367bef1b"},"cell_type":"code","source":"# Not enough memory, maybe some more can be freed in order to perform this operation and test the accuracy of the model trained on almost all the data\n# X_train, X_test, y_train, y_test = train_test_split(train_whole.drop(\"is_attributed\", axis=1), train_whole[\"is_attributed\"], test_size=.3, shuffle=True)\n# try:\n#     del train_whole\n# except:\n#     pass","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"438410350a41e927d62e9f01e2efd4a6d12ce99e"},"cell_type":"code","source":"# print(\"Ratio of 'is_attributed' in y_train:\", y_train.mean())\n# print(\"Ratio of 'is_attributed' in y_test:\", y_test.mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"597d272ce427b40ffa9487da6d01ccd6c0c7a159"},"cell_type":"code","source":"# lg_final = lgbm.LGBMClassifier(objective=\"binary\", is_unbalance=True, n_jobs=-1, **grid_search_lg.best_params_).fit(X_train, y_train)\nlg_final = lgbm.LGBMClassifier(objective=\"binary\", scale_pos_weight=scale_pos_weight, n_jobs=-1, **grid_search_lg.best_params_).fit(train_whole.drop(\"is_attributed\", axis=1), train_whole[\"is_attributed\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d5ff71019ab2abdcc85e6d699f22b0912845edf3"},"cell_type":"code","source":"try:\n    del train_whole\n#     del X_train\n#     del X_test\n#     del y_train\n#     del y_test\nfinally:\n    _ = gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b5a6835ee45c78dbc8cd3643cfd0f075c58fb369"},"cell_type":"markdown","source":"## Prepare the Test Data"},{"metadata":{"trusted":true,"_uuid":"94b5d1b6ac9929e87739c4e3f91b543ca49ebf07"},"cell_type":"code","source":"test_df = pd.read_csv(\"../input/test.csv\")\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f278a985ea8de1b8083051aa51db626c339fb968"},"cell_type":"code","source":"test_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b1e48ae8165f9e5e631989fd3702b83810caf16e"},"cell_type":"code","source":"test_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"58e63faa5b18cef4c9ab5a94f06d24514f2d0ff7"},"cell_type":"code","source":"test_df[\"click_month\"] = pd.to_datetime(test_df[\"click_time\"]).dt.month\ntest_df[\"click_day_of_week\"] = pd.to_datetime(test_df[\"click_time\"]).dt.dayofweek\ntest_df[\"click_hour\"] = pd.to_datetime(test_df[\"click_time\"]).dt.hour\ntest_df[\"click_year\"] = pd.to_datetime(test_df[\"click_time\"]).dt.year\ntest_df = test_df.drop(\"click_time\", axis=1)\ntest_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"94c544a84d57f98fad7c996e93a7dd36156faef4"},"cell_type":"code","source":"test_df = test_df.drop([\"click_year\", \"click_month\"], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d1134c35fc168e865a3072c6695ec497e5b4cef4"},"cell_type":"code","source":"test_df = test_df.astype(\"uint64\")\ntest_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b6868d96aa8f2a6b064ae76e99742905e0815134"},"cell_type":"code","source":"test_df_int = test_df.select_dtypes(include=[\"uint64\"])\ntest_df_int = test_df_int.apply(pd.to_numeric, downcast=\"unsigned\")\ntest_df = test_df.drop(test_df.dtypes[test_df.dtypes==\"uint64\"].index, axis=1)\ntest_df = pd.concat([test_df, test_df_int], axis=1)\ntest_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9f5d466ecbce5a9167ef01cb2dafaec037cf4e31"},"cell_type":"code","source":"for new_feature in new_features:\n    new_feature_name = str(new_feature[\"op\"]) + \"_\" + str(new_feature[\"select\"]) + \"_per_\" + '_'.join(new_feature[\"groupby\"])\n    new_feature_df = test_df.groupby(new_feature[\"groupby\"])[new_feature[\"select\"]].agg(new_feature[\"agg\"]).reset_index(name=new_feature_name)\n    test_df = pd.merge(test_df, new_feature_df, how=\"inner\", on=new_feature[\"groupby\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f215be94c68a7f81be400cacf96b7f037f188b72"},"cell_type":"code","source":"try:\n    del new_feature_df\n    del test_df_int\nfinally:\n    _ = gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"faa9bd104952dfc85d65e1b8d1b959b342ea3cee"},"cell_type":"code","source":"test_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3024b2e708a006eee3d893e92e02d205cdc86885"},"cell_type":"code","source":"int_columns = [\"count_app_per_ip\"]\ntest_df[int_columns] = test_df[int_columns].apply(pd.to_numeric, downcast=\"unsigned\")\ntest_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b40ad3c03d98dd111bf7e523a34e48c1b9328c15"},"cell_type":"code","source":"test_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3f224488aaca6ab85be41c20789f0fe5320107ac"},"cell_type":"markdown","source":"## Predictions for the Test Data"},{"metadata":{"trusted":true,"_uuid":"f97352ffb71b6d984ec88399de7320e00ef080fc"},"cell_type":"code","source":"click_id = test_df.click_id\nX_test = test_df.drop([\"click_id\", \"ip\"], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"270b530856ca903acd3c7a4d201cc41087597aa0"},"cell_type":"code","source":"lg_predict = lg_final.predict_proba(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"156bfddcb784ecf144adcb9919e23a97d05199a3"},"cell_type":"code","source":"lg_predict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"16f6e967e638b097719034c64f5f236a8f759a7e"},"cell_type":"code","source":"try:\n    del test_df\n    del X_test\nexcept:\n    pass\nfinally:\n    _ = gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4d5b655f8f40d7f5634ef723b10846efac6ef10f"},"cell_type":"code","source":"results = pd.concat([click_id, pd.Series(lg_predict[:, 1], name=\"is_attributed\")], axis=1)\nresults.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aa50508c90ebc30a0ebb68eec49d309704dfd138"},"cell_type":"code","source":"results = results.sort_values(by=\"click_id\", axis=0).reset_index().drop(\"index\", axis=1)\nresults.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7bc9e468e227ba2e8f3910efb14ad4fb15316068"},"cell_type":"code","source":"results.to_csv(\"submission_file.csv\", sep=',', index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6664ce0391df9d3cdd071d9406cd1ae10fb2976e"},"cell_type":"markdown","source":""}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}