{"cells":[{"metadata":{"_uuid":"d2df72fec0a41bb169adc1d46e1ef89adefc35f4"},"cell_type":"markdown","source":"#### Special thanks to other kernels authors : \n\n- https://www.kaggle.com/yuliagm/how-to-work-with-big-datasets-on-16g-ram-dask\n- https://www.kaggle.com/kailex/talkingdata-eda-and-class-imbalance\n- https://www.kaggle.com/nanomathias/feature-engineering-importance-testing\n- https://www.kaggle.com/pranav84/lightgbm-fixing-unbalanced-data-auc-0-9787\n-  https://www.kaggle.com/rteja1113/lightgbm-with-count-features\n- https://www.kaggle.com/kanncaa1/roc-curve-with-k-fold-cv\n- https://www.kaggle.com/valkling/mercari-rnn-2ridge-models-with-notes-0-42755/versions#base=2202774&new=2519287\n- https://www.kaggle.com/wenjiebai/if-you-run-on-entire-dataset-lb-0-9798\n\n"},{"metadata":{"trusted":true,"_uuid":"66b42c4df135002f4bcb9eaf0c6b6be1c0731545"},"cell_type":"code","source":"##author : Ahmet Turkmen \nimport pandas as pd \nimport numpy as np \nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport plotly.plotly as py\nimport plotly.graph_objs as go\nimport plotly \nimport xgboost as xgb \nimport lightgbm as lgb \nimport gc as memory_free\nfrom skopt import BayesSearchCV \nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.tree import DecisionTreeClassifier\nfrom matplotlib.colors import ListedColormap\nfrom scipy import interp\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier\nfrom xgboost import XGBClassifier \nfrom sklearn.metrics import roc_curve\nfrom sklearn.svm import SVC\nimport matplotlib.patches as patches\nfrom sklearn.neural_network import MLPClassifier\nfrom sklearn.gaussian_process import GaussianProcessClassifier\nfrom sklearn.gaussian_process.kernels import RBF\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score \nfrom sklearn.metrics import roc_curve\nfrom sklearn.metrics import roc_auc_score,auc\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"07efb420d974e3cfd080ff09c9d71608c1df45b8"},"cell_type":"code","source":"def plot_2d_space(X, y, label='Classes'):   \n    colors = ['#1F77B4', '#FF7F0E']\n    markers = ['o', 's']\n    for l, c, m in zip(np.unique(y), colors, markers):\n        plt.scatter(\n            X[y==l, 0],\n            X[y==l, 1],\n            c=c, label=l, marker=m\n        )\n    plt.title(label)\n    plt.legend(loc='upper right')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"55eb2437dcf8f85d27a7b14807e70ade7c0d69f6"},"cell_type":"code","source":"def print_status(optimum_result):\n    models = pd.DataFrame(bayes_cv_hyper_tuning.cv_results_)\n    best_parameters = pd.Series(bayes_cv_hyper_tuning.best_params_)\n    print ('Model {}\\n Best ROC-AUC: {}\\n best parameters: {}\\n'.format(len(models),np.round(bayes_cv_hyper_tuning.best_score_,4),bayes_cv_hyper_tuning.best_params_))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bcd183b8d3be3dee17bce56232d84dd9c64165d8"},"cell_type":"code","source":"## Preparing data in order to minimize memory-usage\n## Datapreprocessing\ndtypes = {\n        'ip'            : 'uint32',\n        'app'           : 'uint16',\n        'device'        : 'uint16',\n        'os'            : 'uint16',\n        'channel'       : 'uint16',\n        'is_attributed' : 'uint8',\n        'click_id'      : 'uint32'}\n\ndf  = pd.read_csv('../input/train_sample.csv',dtype=dtypes)\n\n\n## train dataset is used because since it is a competition and we want to make class project by evaluating performance \n## of different classification algoritms on data by making some manipulation on data, we used train set by splitting it \n## %30 > test , %70 train. \n## 'is_attributed'  our label. \n\n## check null values in colunms \nprint(df.isnull().sum())\nprint(df.shape[0])\nprint('total number of null values in attributed_time feature is {} '.format(df.isnull().sum()['attributed_time']))\n## it might be good idea to remove attributed_time coloumn all, it might create noisy on data. \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7adebb41f53f4893bd0b5cb7fcc75efb528e58cf"},"cell_type":"code","source":"print('Extracting new features...')\ndf['hour'] = pd.to_datetime(df.click_time).dt.hour.astype('uint8')\ndf['day'] = pd.to_datetime(df.click_time).dt.day.astype('uint8')\n\nmemory_free.collect()\n\nprint('grouping by ip-day-hour combination...')\ngp = df[['ip','day','hour','channel']].groupby(by=['ip','day','hour'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'ip_tcount'})\ndf = df.merge(gp, on=['ip','day','hour'], how='left')\ndel gp\nmemory_free.collect()\n\nprint('grouping by ip-app combination...')\ngp = df[['ip', 'app', 'channel']].groupby(by=['ip', 'app'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'ip_app_count'})\ndf = df.merge(gp, on=['ip','app'], how='left')\ndel gp\nmemory_free.collect()\n\n\nprint('grouping by ip-app-os combination...')\ngp = df[['ip','app', 'os', 'channel']].groupby(by=['ip', 'app', 'os'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'ip_app_os_count'})\ndf = df.merge(gp, on=['ip','app', 'os'], how='left')\ndel gp\nmemory_free.collect()\n\n\n# Adding features with var and mean hour (inspired from nuhsikander's script)\nprint('grouping by : ip_day_chl_var_hour')\ngp = df[['ip','day','hour','channel']].groupby(by=['ip','day','channel'])[['hour']].var().reset_index().rename(index=str, columns={'hour': 'ip_tchan_count'})\ndf = df.merge(gp, on=['ip','day','channel'], how='left')\ndel gp\nmemory_free.collect()\n\nprint('grouping by : ip_app_os_var_hour')\ngp = df[['ip','app', 'os', 'hour']].groupby(by=['ip', 'app', 'os'])[['hour']].var().reset_index().rename(index=str, columns={'hour': 'ip_app_os_var'})\ndf = df.merge(gp, on=['ip','app', 'os'], how='left')\ndel gp\nmemory_free.collect()\n\nprint('grouping by : ip_app_channel_var_day')\ngp = df[['ip','app', 'channel', 'day']].groupby(by=['ip', 'app', 'channel'])[['day']].var().reset_index().rename(index=str, columns={'day': 'ip_app_channel_var_day'})\ndf = df.merge(gp, on=['ip','app', 'channel'], how='left')\ndel gp\nmemory_free.collect()\n\nprint('grouping by : ip_app_chl_mean_hour')\ngp = df[['ip','app', 'channel','hour']].groupby(by=['ip', 'app', 'channel'])[['hour']].mean().reset_index().rename(index=str, columns={'hour': 'ip_app_channel_mean_hour'})\nprint(\"merging...\")\ntrain_df = df.merge(gp, on=['ip','app', 'channel'], how='left')\ndel gp\nmemory_free.collect()\n\nprint(\"vars and data type: \")\n\ndf=df.fillna(0)\ndf['ip_tcount'] = df['ip_tcount'].astype('uint16')\ndf['ip_app_count'] = df['ip_app_count'].astype('uint16')\ndf['ip_app_os_count'] = df['ip_app_os_count'].astype('uint16')\ndf['ip_tchan_count']=df['ip_tchan_count'].astype('uint32')\ndf['ip_app_os_var']=df['ip_app_os_var'].astype('uint32')\ndf['ip_app_channel_var_day']=df['ip_app_channel_var_day'].astype('uint32')\ndf.info()\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0fbd0904e73c9a0b8f3fd6ab34375cdde1191de7"},"cell_type":"raw","source":"df.to_csv('new_df.csv',sep=',',index=False)"},{"metadata":{"trusted":true,"_uuid":"4974c1492594a3dd93d0541215ec6e8747abae8e"},"cell_type":"code","source":"## check number of label, is it balanced or unbalanced data. \nlabel_dist=df.is_attributed.value_counts()\nprint('Proportion:', round(label_dist[1] / label_dist[0], 5), ': 1')\nprint(df.is_attributed.value_counts())\nlabel_dist.plot(kind='bar', title='Count (target)');","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8d39026da677484bb1d364cefa1141ad2e0c8265"},"cell_type":"markdown","source":"The dataset is unbalanced data to be able to make classification properly, some preprocessing techniques should be applied. Simple metrics such as accuracy give us wrong information."},{"metadata":{"_uuid":"2062ef3513d747685489e0044ad2071b549f376f"},"cell_type":"markdown","source":"#### Normalized Gini Coefficient"},{"metadata":{"trusted":true,"_uuid":"73e6770cf80ec910cf5a8930ed590798d11e7dbf"},"cell_type":"code","source":"def gini(actual, pred):\n    assert (len(actual) == len(pred))\n    all = np.asarray(np.c_[actual, pred, np.arange(len(actual))], dtype=np.float)\n    all = all[np.lexsort((all[:, 2], -1 * all[:, 1]))]\n    totalLosses = all[:, 0].sum()\n    giniSum = all[:, 0].cumsum().sum() / totalLosses\n\n    giniSum -= (len(actual) + 1) / 2.\n    return giniSum / len(actual)\n\ndef gini_normalized(actual, pred):\n    return gini(actual, pred) / gini(actual, actual)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"82e6f2fba4f5e60b6938fdc0b3414a2e7c53ca1a"},"cell_type":"markdown","source":"##### Deprecated\n\ndef plot_confusion_matrix(y_test,y_pred):\n    from sklearn.metrics import confusion_matrix\n    confusion_matrix = confusion_matrix(y_true=y_test, y_pred=y_pred)\n    print('Confusion matrix:\\n', confusion_matrix)\n    labels = ['Not Fraudulent', 'Fraudulent']\n    fig = plt.figure()\n    ax = fig.add_subplot(111)\n    cax = ax.matshow(confusion_matrix, cmap=plt.cm.Blues)\n    fig.colorbar(cax)\n    ax.set_xticklabels([''] + labels)\n    ax.set_yticklabels([''] + labels)\n    plt.xlabel('Predicted')\n    plt.ylabel('Expected')\n    plt.show()"},{"metadata":{"trusted":true,"_uuid":"0c8c03715dbbe177a236e2b1d32a0d91c0102ee1"},"cell_type":"code","source":"bayes_cv_hyper_tuning = BayesSearchCV(\n    estimator  = xgb.XGBClassifier(\n        n_jobs=1,\n        objective='binary:logistic',\n        eval_metric='auc',\n        silent=1,\n        tree_method = 'approx'\n    ),\n    search_spaces={\n        'learning_rate':(0.01,1.0,'log-uniform'),\n        'min_child_weight':(0,10),\n        'max_depth':(0,50),\n        'max_delta_step':(0,20),\n        'subsample':(0.01,1.0,'uniform'),\n        'n_estimators':(50,100),\n        'scale_pos_weight':(1e-6,500,'log-uniform')\n    },\n    scoring = 'roc_auc',\n    cv = StratifiedKFold(n_splits = 5, shuffle=True, random_state=42),\n    n_jobs = 3,\n    n_iter=10,\n    verbose=0,\n    refit=True,\n    random_state=42\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e54505f7e67841edfb1abe5f38bf370ebb48c7bc"},"cell_type":"code","source":"def plot_confusion_matrix(cm, classes,\n                          normalize=False,\n                          title='Confusion matrix',\n                          cmap=plt.cm.Blues):\n    import itertools\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n        print(\"Normalized confusion matrix\")\n    else:\n        print('Confusion matrix, without normalization')\n\n    print(cm)\n\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    fmt = '.2f' if normalize else 'd'\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, format(cm[i, j], fmt),\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n    plt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ab636e27aa52122610e64108892754b6a72bc9f6"},"cell_type":"markdown","source":"### Applying StratifiedKFold to Classification Algorithms and Plotting ROC Curve"},{"metadata":{"trusted":true,"_uuid":"61f8012f64ff2a41b578023b23b5e8d507fff0e0"},"cell_type":"code","source":"def plot_roc_graph(df,classifier,class_name):\n    from sklearn.metrics import classification_report \n    scaler = StandardScaler()\n    cv = StratifiedKFold(n_splits = 10, shuffle=True, random_state=42)\n    x = df.loc[:, df.columns != 'is_attributed']\n    y = df.loc[:,'is_attributed']\n    x = scaler.fit_transform(x)\n    dtype = [('ip','uint32'), ('app','uint16'), ('device','uint16'),('os','uint16'),('channel','uint16')]\n    index = ['Row'+str(i) for i in range(1, len(x)+1)]\n    x = pd.DataFrame(x, index=index)\n    # plot arrows\n    fig1 = plt.figure(figsize=[12,12])\n    ax1 = fig1.add_subplot(111,aspect = 'equal')\n    ax1.add_patch(patches.Arrow(0.45,0.5,-0.25,0.25,width=0.3,color='green',alpha = 0.5))\n    ax1.add_patch(patches.Arrow(0.5,0.45,0.25,-0.25,width=0.3,color='red',alpha = 0.5))\n\n    tprs = []\n    aucs = []\n    mean_fpr = np.linspace(0,1,100)\n    i = 1\n    for train,test in cv.split(x,y):\n        prediction = classifier.fit(x.iloc[train],y.iloc[train]).predict_proba(x.iloc[test])\n        fpr, tpr, t = roc_curve(y[test], prediction[:, 1])\n        tprs.append(interp(mean_fpr, fpr, tpr))\n        roc_auc = auc(fpr, tpr)\n        aucs.append(roc_auc)\n        plt.plot(fpr, tpr, lw=2, alpha=0.3, label='ROC fold %d (AUC = %0.2f)' % (i, roc_auc))\n        i= i+1\n     \n    plt.plot([0,1],[0,1],linestyle = '--',lw = 2,color = 'black')\n    mean_tpr = np.mean(tprs, axis=0)\n    mean_auc = auc(mean_fpr, mean_tpr)\n    plt.plot(mean_fpr, mean_tpr, color='blue',\n             label=r'Mean ROC (AUC = %0.2f )' % (mean_auc),lw=2, alpha=1)\n\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.title('ROC - '+class_name)\n    plt.legend(loc=\"lower right\")\n    plt.text(0.32,0.7,'More accurate area',fontsize = 12)\n    plt.text(0.63,0.4,'Less accurate area',fontsize = 12)\n    plt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bd20ea38d85ee038f96030e0d082d5bc6946418e"},"cell_type":"markdown","source":"- If the train_sample.csv dataset divided into two, %30 > test, %70> train, then it gives following result, however the following accuracy is NOT valid. Since there is so small percentage of not_fraudulant data, it gives high accuracy, however when confusion matrix and ROC are investigated, then we can easyliy observe that accuracy is lower than assumed one. "},{"metadata":{"trusted":true,"_uuid":"27c3f130879e42ef289c1387f1b189e9aa458145"},"cell_type":"code","source":"\ndf=df.drop(['click_time','attributed_time'],axis=1)\nfeatures=df.drop(['is_attributed'],axis=1).columns \n# scaler = StandardScaler()\nX_org=df[features]\ny_org=df['is_attributed']\n# X_org = scaler.fit_transform(X_org)\nX_train,X_test, y_train,y_test=train_test_split(X_org,y_org,test_size=0.3,random_state=1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9c9f1cce9dbd3e087f37d047e69aae077316ae30"},"cell_type":"code","source":"model = XGBClassifier()\nmodel.fit(X_train,y_train)\ny_pred=model.predict(X_test)\nprobs=model.predict_proba(X_test)\naccu = accuracy_score(y_test,y_pred)\nprint('Accuracy : %.2f%%' % (accu *100.0))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8a569354159bd80f0a60967f09a871938ea0e2a0"},"cell_type":"markdown","source":"The accuracy is 99.77 % which is NOT realistic. In unbalanced data Normalized Gini Coefficient  might be good metric to evaluate which is used in imbalanced datasets. \n\n- It can be proven that this accuracy is NOT correct, by looking at classification report of model."},{"metadata":{"trusted":true,"_uuid":"b1eea75b1c442853abe2266daea1bc496593420f"},"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\nconf_mat=confusion_matrix(y_test,y_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"59600a7928575544661c2091ded868035191ad7a"},"cell_type":"code","source":"print(y_test.value_counts())\nlabels = ['Not Fraudulent', 'Fraudulent']\n# Plot non-normalized confusion matrix\nplt.figure()\nplot_confusion_matrix(conf_mat,labels, title='Confusion matrix, without normalization')\n# Plot normalized confusion matrix\nplt.figure()\nplot_confusion_matrix(conf_mat, classes=labels, normalize=True,\n                      title='Normalized confusion matrix')\n\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"262946345c9b24551f737da72971e8e7a071ed6e"},"cell_type":"markdown","source":"When confusion matrix and number of instance (0,1  is_attributed) investigated, False Negative rate is very high which means that 67 records out of 73 classified as Not_Fraudulent, even those are Fraudulent. \n"},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"823d7dd5cc58dc70e1e6744f1be15b46c8a16c4d"},"cell_type":"code","source":"print(df.is_attributed.value_counts())\ndf.is_attributed.value_counts().plot(kind='pie',title='Distribution of data')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6dff8190472528b0b50e74ed3a85da3b314bbdd2"},"cell_type":"code","source":"features=df.drop(['is_attributed'],axis=1).columns \nX_org=df[features]\ny_org=df['is_attributed']\nX_train,X_test, y_train,y_test=train_test_split(X_org,y_org,test_size=0.3,random_state=1)\ndecision_tree_classifer = DecisionTreeClassifier(criterion='gini')\n\ndecision_tree_classifer.fit(X_train,y_train)\ny_pred=decision_tree_classifer.predict(X_test)\nprobs=decision_tree_classifer.predict_proba(X_test)\naccu = accuracy_score(y_test,y_pred)\nprint(y_test.value_counts())\nlabels = ['Not Fraudulent', 'Fraudulent']\n\ncnf_matrix = confusion_matrix(y_test, y_pred)\nnp.set_printoptions(precision=2)\n\n# Plot non-normalized confusion matrix\nplt.figure()\nplot_confusion_matrix(cnf_matrix, classes=labels,\n                      title='Confusion matrix, without normalization')\n\n# Plot normalized confusion matrix\nplt.figure()\nplot_confusion_matrix(cnf_matrix, classes=labels, normalize=True,\n                      title='Normalized confusion matrix')\n\nplt.show()\n\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"00b182043bd1d1de1ee31ddc6bc65b8a5b098c65"},"cell_type":"code","source":"plot_roc_graph(df,decision_tree_classifer,'DecisionTreeClassifier')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a032a156eccd01e7ff8d0daf57f44ca3f39a267e"},"cell_type":"code","source":"from sklearn.naive_bayes import GaussianNB\nnaive_bayes_gaussian =  GaussianNB()\nplot_roc_graph(df,naive_bayes_gaussian,'GaussianNB')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0d0ae93a60594774e9473a8b6a69fb8bd044a8b3"},"cell_type":"code","source":"random_forest = RandomForestClassifier(max_depth=5, n_estimators=10, max_features=1)\nplot_roc_graph(df,random_forest,'RandomForest')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"925202e0ba870e487a26867db1acc2efd691e9e5"},"cell_type":"markdown","source":"##### BayesSearchCV (Findinf optimum parameters and imply cross-validation)"},{"metadata":{"trusted":true,"_uuid":"60d9229b8ef4f5632558ad81d65af948d9b4fcba"},"cell_type":"code","source":"X = df.loc[:, df.columns != 'is_attributed']\ny = df.loc[:,'is_attributed']\nbayes_cv_hyper_tuning.fit(X.values,y.values, callback=print_status)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e74ad5b77ae4ba6ff6ecf1c63d0e370bfe35dfba"},"cell_type":"markdown","source":"### Resampling Techniques\n\n- In this dataset, we should implement some under-sampling, over-sampling and other resampling techniques in order to get valid better results. \n\n\n![Resampling Techniuqes](https://github.com/ahmetturkmen/PythonBeginner/blob/master/resampling.png?raw=true![image.png](attachment:image.png)\n\n\n- Both of them has advantages and drawbacks, in undersampling, majority of data reduced number of minority data in order get balance between them, in similiar logic, in oversampling minority of data duplicated to number of majority data in dataset. So, in undersampling, data is lost, in oversampling we have overfitting problem. \n\n\n"},{"metadata":{"_uuid":"5c25a5cf385766a2cc8ea2b750d4c9173392794a"},"cell_type":"markdown","source":"#### Undersampling"},{"metadata":{"trusted":true,"_uuid":"fa612b4fe229704d3e36e2d7ce1a75f43bae0d0f"},"cell_type":"code","source":"not_fraudulent = df[df['is_attributed']==0]\nfraudulent  = df[df['is_attributed']==1]\nnot_fraudulent_under = not_fraudulent.sample(label_dist[1])\ndf_undered=pd.concat([not_fraudulent_under,fraudulent],axis=0)\nprint('Randomly under-sampled:\\n{}'.format(df_undered.is_attributed.value_counts()))\ndf_undered.is_attributed.value_counts().plot(kind='pie',title='Dist. of resampled data')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e9953e0934fcddcdbf9596e42d5af6554275c462"},"cell_type":"code","source":"df_undered=df_undered.reset_index()\ndf_undered=df_undered.drop(['index'],axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"599be450a79581dc4c65e6e3d0d8638878f0c1f7"},"cell_type":"code","source":"plot_roc_graph(df_undered,random_forest,'RandomForest > UnderSampled')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f8b9b91398cc457ee8f00d23b4a455af0a70cf72"},"cell_type":"code","source":"df_undered.columns","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f31bbe4b9a01147609e6a2a6a2bcfbb603e29f38"},"cell_type":"markdown","source":"#### Oversampling"},{"metadata":{"trusted":true,"_uuid":"81c4d21bc2df4eb0e6ef0bed082057aee5201b53"},"cell_type":"code","source":"\nfraudulent_over = fraudulent.sample(label_dist[0],replace=True)\ndf_over=pd.concat([not_fraudulent,fraudulent_over],axis=0)\nprint('Randomly under-sampled:\\n{}'.format(df_over.is_attributed.value_counts()))\ndf_over.is_attributed.value_counts().plot(kind='bar',title='Dist. of resampled data')\ndf_over = df_over.reset_index()\ndf_over = df_over.drop(['index'],axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"048472be25f1c835c0e8c5e60abec5563fc13fcf"},"cell_type":"code","source":"plot_roc_graph(df_over,naive_bayes_gaussian,'Naive Bayes - Oversampling')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"87b56aec7e780abe6157ee30c7a90449a4044ee2"},"cell_type":"code","source":"X_under=df_undered[features]\ny_under=df_undered['is_attributed']\nbayes_cv_hyper_tuning.fit(X_under.values,y_under.values, callback=print_status)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d12799077c3bcf89d3d883c09e1cd6b8763c02dc"},"cell_type":"code","source":"from sklearn.decomposition import PCA\npca = PCA(n_components=2)\nX_org = pca.fit_transform(X_org)\nplot_2d_space(X_org, y_org, 'Imbalanced dataset (2 PCA components)')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f4c65f836b21ab2b66299832f60584c039ce158f"},"cell_type":"code","source":"from imblearn.under_sampling import RandomUnderSampler\n\nrus = RandomUnderSampler(return_indices=True)\nX_rus, y_rus, id_rus = rus.fit_sample(X_org, y_org)\n\nprint('Removed indexes:', id_rus)\n\nplot_2d_space(X_rus, y_rus, 'Random under-sampling')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7ea2af36f1300d2b24f28681996016adc4d9ee80"},"cell_type":"markdown","source":"#### Over Sampling: SMOTE"},{"metadata":{"trusted":true,"_uuid":"de131dc8785baf8b74f54ac535e7e788587fe1bf"},"cell_type":"code","source":"from imblearn.over_sampling import SMOTE\nsmote = SMOTE(ratio='minority')\nX_sm, y_sm = smote.fit_sample(X_org, y_org)\nplot_2d_space(X_sm, y_sm, 'SMOTE over-sampling')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1aeb8a70d1b5583c5c995e0e8548d59dba90cb4d"},"cell_type":"code","source":"df_train_all = pd.read_csv('../input/train.csv',nrows=37000000,dtype=dtypes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"70ca156790b35a49a00c9f653412977fcee7994d"},"cell_type":"code","source":"df_train_all.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"35d9ade71a430128d4054f6d39db7038437e8316"},"cell_type":"code","source":"fraudulant_addition=df_train_all[df_train_all['is_attributed']==1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"56717affd428308ea501014e5fc3d5819acde4f2"},"cell_type":"code","source":"fraudulant_addition=fraudulant_addition.drop(['click_time','attributed_time'],axis=1)\ndf_=pd.concat([fraudulant_addition,df])","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"89268f321ac65941808677b064a4e802fc5b6b43"},"cell_type":"code","source":"df_.is_attributed.value_counts().plot(kind='bar',title='regenerated dataset')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9d995ade4bf1ba2bef281d83ef22477ab768fe25"},"cell_type":"code","source":"df_=df_.reset_index()\ndf_=df_.drop(['index'],axis=1)\n","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"fc27ab40ddb60a1baf44f6de5db61fcfe905c292"},"cell_type":"code","source":"X_regenerated = df_[features]\ny_regenerated = df_['is_attributed']\nX_train,X_test, y_train,y_test=train_test_split(X_regenerated,y_regenerated,test_size=0.3,random_state=1)\nbayes_cv_hyper_tuning.fit(X_train.values,y_train.values, callback=print_status)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d276e1f7fe1196b208ce1be713bdd29a7a6a58c"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"widgets":{"application/vnd.jupyter.widget-state+json":{"state":{},"version_major":2,"version_minor":0}}},"nbformat":4,"nbformat_minor":1}