{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Importing Libraries","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import gc\nimport time\n\n# Data Manipulation Packages\nimport numpy as np\nimport pandas as pd\nimport datetime\n\n# Data Visualisation Packages\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Modelling Packages\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.cluster import KMeans\nimport lightgbm as lgb\n\n# Packages for K-means clustering\nimport plotly as py\nimport plotly.graph_objs as go\nfrom sklearn.cluster import KMeans","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loading Data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/talkingdata-adtracking-fraud-detection/train.csv' ,nrows=2000000, parse_dates=['click_time','attributed_time'] )\ntrain.head(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preliminary Analysis","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features = ['ip','app','device','os', 'channel']\nfor i in features:\n    print(i + \" : \" + str(train[i].nunique()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Splitting into Training and Validation Data\ntrain_x , val_x = train_test_split(train , random_state = 0 , test_size = 0.2)\ntrain_x.shape\nval_x.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Checking for unique values in train data\nfeatures = ['ip','app','device','os', 'channel']\nfor i in features:\n    print(i + \" : \" + str(train_x[i].nunique()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Checking for unique values in validation data\nfeatures = ['ip','app','device','os', 'channel']\nfor i in features:\n    print(i + \" : \" + str(val_x[i].nunique()))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Feature Generation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Creating features from click_time\ntrain_x['cl_year'] = train['click_time'].dt.year.astype('int64')\ntrain_x['cl_month'] = train['click_time'].dt.month.astype('int64')\ntrain_x['cl_day'] = train['click_time'].dt.day.astype('int64')\ntrain_x['cl_hour'] = train['click_time'].dt.hour.astype('int64')\ntrain_x['cl_minute'] = train['click_time'].dt.minute.astype('int64')\ntrain_x['cl_second'] = train['click_time'].dt.second.astype('int64')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_x.head()\ntrain_x.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Dropping columns with zero variance\ndropcl = ['cl_year','cl_month','cl_day']\ntrain_x.drop( dropcl , axis = 1, inplace = True)\ntrain_x.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Dropping attributed_time column\ntrain_x.drop(['attributed_time'], axis = 1, inplace = True)\ntrain_x.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_x.sort_index(inplace = True)\ntrain_x.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_x[(train_x['ip']== 45745) & (train_x['app']==3) ].channel.value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Feature 1 : Adding count of ip,app,device,os","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_x['ip_chan_count'] = train_x.groupby(['ip','channel'])['is_attributed'].transform('count')\ntrain_x['app_chan_count'] = train_x.groupby(['app','channel'])['is_attributed'].transform('count')\ntrain_x['device_chan_count'] = train_x.groupby(['device','channel'])['is_attributed'].transform('count')\ntrain_x['os_chan_count'] = train_x.groupby(['os','channel'])['is_attributed'].transform('count')\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Feature 2: Adding Count of Combinations of ip,app,device,os,channel","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_x['ip_app_chan_count'] = train_x.groupby(['ip','app','channel'])['is_attributed'].transform('count')\ntrain_x['ip_device_chan_count'] = train_x.groupby(['ip','device','channel'])['is_attributed'].transform('count')\ntrain_x['ip_os_chan_count'] = train_x.groupby(['ip','os','channel'])['is_attributed'].transform('count')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_x['app_device_count'] = train_x.groupby(['app','device','channel'])['is_attributed'].transform('count')\ntrain_x['app_os_count'] = train_x.groupby(['app','os','channel'])['is_attributed'].transform('count')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_x.rename(columns = {'app_device_count':'app_device_chan_count','app_os_count':'app_os_chan_count'}, inplace = True) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_x['device_os_chan_count'] = train_x.groupby(['device','os','channel'])['is_attributed'].transform('count')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_x.drop(['cl_hour'], axis = 1 , inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_y = train_x[['is_attributed']].copy()\ntrain_x.drop(['is_attributed','click_time'], axis = 1, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# K-means Clustering","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# K-means Clustering between ip_chan_count and cl_minute\nX1 = train_x[['ip_chan_count' , 'cl_minute']].iloc[: , :].values\ninertia = []\nfor n in range(1 , 11):\n    algorithm = (KMeans(n_clusters = n ,init='k-means++', n_init = 10 ,max_iter=300, \n                        tol=0.0001,  random_state= 111  , algorithm='elkan') )\n    algorithm.fit(X1)\n    inertia.append(algorithm.inertia_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"algorithm = (KMeans(n_clusters = 4 ,init='k-means++', n_init = 10 ,max_iter=300, \n                        tol=0.0001,  random_state= 111  , algorithm='elkan') )\nalgorithm.fit(X1)\nlabels1 = algorithm.labels_\ncentroids1 = algorithm.cluster_centers_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"h = 0.02\nx_min, x_max = X1[:, 0].min() - 1, X1[:, 0].max() + 1\ny_min, y_max = X1[:, 1].min() - 1, X1[:, 1].max() + 1\nxx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))\nZ = algorithm.predict(np.c_[xx.ravel(), yy.ravel()]) \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plotting the final clustered Data\nplt.figure(1 , figsize = (15 , 7) )\nplt.clf()\nZ = Z.reshape(xx.shape)\nplt.imshow(Z , interpolation='nearest', \n           extent=(xx.min(), xx.max(), yy.min(), yy.max()),\n           cmap = plt.cm.Pastel2, aspect = 'auto', origin='lower')\n\nplt.scatter( x = 'ip_chan_count' ,y = 'cl_minute' , data = train_x , c = labels1 , \n            s = 200 )\nplt.scatter(x = centroids1[: , 0] , y =  centroids1[: , 1] , s = 300 , c = 'red' , alpha = 0.5)\nplt.ylabel('cl_minute') , plt.xlabel('ip_chan_count')\nplt.show()\n\n# Adding the encoded labels column to the dataframe\ntrain_x['K-Means_encoding'] = labels1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Applying all transformations to Validation data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"val_x['cl_minute'] = val_x['click_time'].dt.minute.astype('int64')\nval_x['cl_second'] = val_x['click_time'].dt.second.astype('int64')\n\nval_x.drop(['attributed_time'], axis = 1, inplace = True)\nval_x.sort_index(inplace = True)\n\nval_x['ip_chan_count'] = val_x.groupby(['ip','channel'])['is_attributed'].transform('count')\nval_x['app_chan_count'] = val_x.groupby(['app','channel'])['is_attributed'].transform('count')\nval_x['device_chan_count'] = val_x.groupby(['device','channel'])['is_attributed'].transform('count')\nval_x['os_chan_count'] = val_x.groupby(['os','channel'])['is_attributed'].transform('count')\n\nval_x['ip_app_chan_count'] = val_x.groupby(['ip','app','channel'])['is_attributed'].transform('count')\nval_x['ip_device_chan_count'] = val_x.groupby(['ip','device','channel'])['is_attributed'].transform('count')\nval_x['ip_os_chan_count'] = val_x.groupby(['ip','os','channel'])['is_attributed'].transform('count')\n\nval_x['app_device_count'] = val_x.groupby(['app','device','channel'])['is_attributed'].transform('count')\nval_x['app_os_count'] = val_x.groupby(['app','os','channel'])['is_attributed'].transform('count')\nval_x.rename(columns = {'app_device_count':'app_device_chan_count','app_os_count':'app_os_chan_count'}, inplace = True) \n\nval_x['device_os_chan_count'] = val_x.groupby(['device','os','channel'])['is_attributed'].transform('count')\n\nval_y = val_x[['is_attributed']].copy()\nval_x.drop(['is_attributed','click_time'], axis = 1, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model fiting using LightGBM","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Defining a Function that will return the trained model\n\ndef lgb_modelfit_nocv(params, train_x, val_x, predictors ,train_y, val_y, objective='binary', metrics='auc',\n                 feval=None, early_stopping_rounds=20, num_boost_round=3000, verbose_eval=10, categorical_features=None):\n    lgb_params = {\n        'boosting_type': 'gbdt',\n        'objective': objective,\n        'metric':metrics,\n        'learning_rate': 0.01,\n        #'is_unbalance': 'true',  #because training data is unbalance (replaced with scale_pos_weight)\n        'num_leaves': 31,  # we should let it be smaller than 2^(max_depth)\n        'max_depth': -1,  # -1 means no limit\n        'min_child_samples': 20,  # Minimum number of data need in a child(min_data_in_leaf)\n        'max_bin': 255,  # Number of bucketed bin for feature values\n        'subsample': 0.6,  # Subsample ratio of the training instance.\n        'subsample_freq': 0,  # frequence of subsample, <=0 means no enable\n        'colsample_bytree': 0.3,  # Subsample ratio of columns when constructing each tree.\n        'min_child_weight': 5,  # Minimum sum of instance weight(hessian) needed in a child(leaf)\n        'subsample_for_bin': 200000,  # Number of samples for constructing bin\n        'min_split_gain': 0,  # lambda_l1, lambda_l2 and min_gain_to_split to regularization\n        'reg_alpha': 0,  # L1 regularization term on weights\n        'reg_lambda': 0,  # L2 regularization term on weights\n        'nthread': 4,\n        'verbose': 0,\n        'metric':metrics\n    }\n\n    lgb_params.update(params)\n\n    print(\"preparing validation datasets\")\n    \n    xgtrain = lgb.Dataset(train_x[predictors].values, label=train_y['is_attributed'].values,\n                          feature_name=predictors,\n                          categorical_feature=categorical_features\n                          )\n    xgvalid = lgb.Dataset(val_x[predictors].values, label=val_y['is_attributed'].values,\n                          feature_name=predictors,\n                          categorical_feature=categorical_features\n                          )\n\n    evals_results = {}\n\n    bst1 = lgb.train(lgb_params, \n                     xgtrain, \n                     valid_sets=[xgtrain, xgvalid], \n                     valid_names=['train','valid'], \n                     evals_result=evals_results, \n                     num_boost_round=num_boost_round,\n                     early_stopping_rounds=early_stopping_rounds,\n                     verbose_eval=10, \n                     feval=feval)\n\n    n_estimators = bst1.best_iteration\n    print(\"\\nModel Report\")\n    print(\"n_estimators : \", n_estimators)\n    print(metrics+\":\", evals_results['valid'][metrics][n_estimators-1])\n\n    return bst1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Using grid search with cross-validation.\n## This will be run only once and the optimal parameters will be used for training.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# def algorithm_pipeline(X_train_data, X_test_data, y_train_data, y_test_data, \n#                        model, param_grid, cv=10, scoring_fit='neg_mean_squared_error',\n#                        do_probabilities = False):\n#     gs = GridSearchCV(\n#         estimator=model,\n#         param_grid=param_grid, \n#         cv=cv, \n#         n_jobs=-1, \n#         scoring=scoring_fit,\n#         verbose=2\n#     )\n#     fitted_model = gs.fit(X_train_data, y_train_data)\n    \n#     if do_probabilities:\n#       pred = fitted_model.predict_proba(X_test_data)\n#     else:\n#       pred = fitted_model.predict(X_test_data)\n    \n#     return fitted_model, pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model = lgb.LGBMClassifier()\n# param_grid = {\n#     'num_leaves': [7,15,31]  \n#     'max_depth': [3,4,5]  \n#     'min_child_samples': [50,100,150]  \n#     'max_bin': [100,150,200]  \n#     'subsample': [0.7,0.8,0.9]  \n#     'colsample_bytree': [0.7,0.8,0.9] \n#     \n# }\n\n# model, pred = algorithm_pipeline(train_x, val_x, train_y, val_y, model, \n#                                  param_grid, cv=5, scoring_fit='accuracy')\n\n# print(model.best_score_)\n# print(model.best_params_)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Calling the model function","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Calling the function that returns the trained model\npredictors = ['ip','app','device','os', 'channel', 'cl_minute', 'cl_second', \n              'ip_chan_count','app_chan_count','device_chan_count','os_chan_count',\n              'ip_app_chan_count','ip_device_chan_count','ip_os_chan_count','app_device_chan_count','app_os_chan_count'\n             , 'device_os_chan_count']\ncategorical = ['ip','app', 'device', 'os', 'channel', 'cl_minute', 'cl_second']\n\nparams = {\n    'learning_rate': 0.15,\n    #'is_unbalance': 'true', # replaced with scale_pos_weight argument\n    'num_leaves': 7,  # 2^max_depth - 1\n    'max_depth': 3,  # -1 means no limit\n    'min_child_samples': 100,  # Minimum number of data need in a child(min_data_in_leaf)\n    'max_bin': 100,  # Number of bucketed bin for feature values\n    'subsample': 0.7,  # Subsample ratio of the training instance.\n    'subsample_freq': 1,  # frequence of subsample, <=0 means no enable\n    'colsample_bytree': 0.9,  # Subsample ratio of columns when constructing each tree.\n    'min_child_weight': 0,  # Minimum sum of instance weight(hessian) needed in a child(leaf)\n    'scale_pos_weight':99 # because training data is extremely unbalanced \n}\n\nmodel = lgb_modelfit_nocv(params, \n                        train_x, \n                        val_x, \n                        predictors, \n                        train_y,\n                        val_y,\n                        objective='binary', \n                        metrics='auc',\n                        early_stopping_rounds=30, \n                        verbose_eval=True, \n                        num_boost_round=500, \n                        categorical_features=categorical)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submitting final model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Reading the Test Data\ntest = pd.read_csv('../input/talkingdata-adtracking-fraud-detection/test.csv' ,parse_dates=['click_time'] )\ntest.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#test.set_index('click_id')\ntest.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Feature Engineering of Test data\ntest['cl_minute'] = test['click_time'].dt.minute.astype('int64')\ntest['cl_second'] = test['click_time'].dt.second.astype('int64')\n\n\ntest['ip_chan_count'] = test.groupby(['ip','channel'])['click_time'].transform('count')\ntest['app_chan_count'] = test.groupby(['app','channel'])['click_time'].transform('count')\ntest['device_chan_count'] = test.groupby(['device','channel'])['click_time'].transform('count')\ntest['os_chan_count'] = test.groupby(['os','channel'])['click_time'].transform('count')\n\ntest['ip_app_chan_count'] = test.groupby(['ip','app','channel'])['click_time'].transform('count')\ntest['ip_device_chan_count'] = test.groupby(['ip','device','channel'])['click_time'].transform('count')\ntest['ip_os_chan_count'] = test.groupby(['ip','os','channel'])['click_time'].transform('count')\n\ntest['app_device_count'] = test.groupby(['app','device','channel'])['click_time'].transform('count')\ntest['app_os_count'] = test.groupby(['app','os','channel'])['click_time'].transform('count')\ntest.rename(columns = {'app_device_count':'app_device_chan_count','app_os_count':'app_os_chan_count'}, inplace = True) \n\ntest['device_os_chan_count'] = test.groupby(['device','os','channel'])['click_time'].transform('count')\n\ntest.drop(['click_time'], axis = 1, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()\ntest.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()\n\n# Creating the submission file\ntest_dummy = pd.read_csv('../input/talkingdata-adtracking-fraud-detection/test.csv' ,parse_dates=['click_time'] )\nsub = pd.DataFrame()\nsub['click_id'] = test_dummy['click_id'].astype('int')\ndel test_dummy\n\nprint(\"Predicting...\")\nsub['is_attributed'] = model.predict(test[predictors])\nprint(\"writing...\")\nsub.to_csv('submission1_lgb.csv',index=False)\nprint(\"done...\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"# Method 2\n# # Pipelining all categorical features\n# categorical = ['ip','app', 'device', 'os', 'channel', 'cl_minute', 'cl_second']\n\n# # Preparing the LightGBM Data Containers\n# lgb_train_data = lgb.Dataset(train_x, label= train_y, categorical_feature=categorical)\n# lgb_val_data = lgb.Dataset(val_x, label= val_y)\n\n# #Parameters of the model\n# params = {\n#     'learning_rate': 0.15,\n#     #'is_unbalance': 'true', # replaced with scale_pos_weight argument\n#     'num_leaves': 7,  # 2^max_depth - 1\n#     'max_depth': 3,  # -1 means no limit\n#     'min_child_samples': 100,  # Minimum number of data need in a child(min_data_in_leaf)\n#     'max_bin': 100,  # Number of bucketed bin for feature values\n#     'subsample': 0.7,  # Subsample ratio of the training instance.\n#     'subsample_freq': 1,  # frequence of subsample, <=0 means no enable\n#     'colsample_bytree': 0.9,  # Subsample ratio of columns when constructing each tree.\n#     'min_child_weight': 0,  # Minimum sum of instance weight(hessian) needed in a child(leaf)\n#     'scale_pos_weight':99 # because training data is extremely unbalanced \n# }\n\n# # Training the model\n# model = lgb.train(params,\n#                        lgb_train_data,\n#                        valid_sets=lgb_val_data,\n#                        num_boost_round=5000,\n#                        early_stopping_rounds=100)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Testing purposes","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Test code to view table\n#train_x.head()\ntrain_y.head()\n#val_x.head()\n#val_y.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train_x.shape\n#train_y.shape\n#val_x.shape\nval_y.shape","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}