{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"id":"iSSh1R2LPkgh","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np","execution_count":null,"outputs":[]},{"metadata":{"id":"b7sEKOLiP97z","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"df=pd.read_csv(\"../input/talkingdata-adtracking-fraud-detection/train_sample.csv\")","execution_count":null,"outputs":[]},{"metadata":{"id":"p2Hb6KEQQHYA","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"del df['attributed_time']","execution_count":null,"outputs":[]},{"metadata":{"id":"uqkUCJYMQbT3","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"# Convert click_time into time,month,year and day\nvalue='click_time'\ndf[value]=pd.to_datetime(df[value])\ndf['year']=df[value].dt.year\ndf['month']=df[value].dt.month\ndf['day']=df[value].dt.day\ndf['time']=df[value].dt.hour\ndf['day_week']=df[value].dt.weekday_name","execution_count":null,"outputs":[]},{"metadata":{"id":"Ls4HujFxQgco","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"del df['click_time']","execution_count":null,"outputs":[]},{"metadata":{"id":"-KAFqm4RQ9K2","colab_type":"code","outputId":"b87fb5a5-c0f0-434c-e2f8-1245edaebf74","colab":{"base_uri":"https://localhost:8080/","height":204},"trusted":true},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"id":"W99_GIRzYzF7","colab_type":"code","outputId":"75ceecab-ab9e-41c6-fbf7-2e4be198cf5b","colab":{"base_uri":"https://localhost:8080/","height":204},"trusted":true},"cell_type":"code","source":"df.tail()","execution_count":null,"outputs":[]},{"metadata":{"id":"qlfs-5ZNY3gy","colab_type":"code","outputId":"d4d3b009-57a6-44f4-f499-736440afec8f","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":true},"cell_type":"code","source":"df.is_attributed.unique()","execution_count":null,"outputs":[]},{"metadata":{"id":"5AVWTBYQZSYr","colab_type":"code","outputId":"bab12351-6c7b-41e1-f565-4666a146547a","colab":{"base_uri":"https://localhost:8080/","height":68},"trusted":true},"cell_type":"code","source":"df.is_attributed.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"id":"BOKs7Rd_Z_Ph","colab_type":"code","outputId":"a62ef00f-f85d-408f-dc9b-b9ffdc72be11","colab":{"base_uri":"https://localhost:8080/","height":1000},"trusted":true},"cell_type":"code","source":"from sklearn.utils import shuffle\ndf = shuffle(df)\ndf","execution_count":null,"outputs":[]},{"metadata":{"id":"8sWCm_HBRdhC","colab_type":"code","outputId":"a31f05ba-ba29-43e4-e18c-a2a03daabd7c","colab":{"base_uri":"https://localhost:8080/","height":153},"trusted":true},"cell_type":"code","source":"#Select the variables to be one-hot encoded\none_hot_features = ['day_week']\n# Convert categorical variables into dummy/indicator variables (i.e. one-hot encoding).\none_hot_encoded = pd.get_dummies(df[one_hot_features],drop_first=True)\none_hot_encoded.info(verbose=True, memory_usage=True, null_counts=True)","execution_count":null,"outputs":[]},{"metadata":{"id":"ljBJt70eRkoY","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"# Replacing categorical columns with dummies\nfdf = df.drop(one_hot_features,axis=1)\ntrain = pd.concat([df, one_hot_encoded] ,axis=1)","execution_count":null,"outputs":[]},{"metadata":{"id":"k-BJvFNodMBH","colab_type":"code","outputId":"26f53f13-19e8-4fbe-cce0-d8dd41501502","colab":{"base_uri":"https://localhost:8080/","height":340},"trusted":true},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{"id":"kjb-xEgrf-gp","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"ones=train['is_attributed']==1\nzeros=train['is_attributed']==0","execution_count":null,"outputs":[]},{"metadata":{"id":"q3vEeU2Jgh2_","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"ones=train[ones]\nzeros=train[zeros]\n","execution_count":null,"outputs":[]},{"metadata":{"id":"jlKBLo9oxhM_","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"import random\n#n=random.randint(227,99773)\n#print(\"Randaom Number is:\",n)\nsample_zeros=zeros.sample(n=2043)","execution_count":null,"outputs":[]},{"metadata":{"id":"nWYbKon4gsWU","colab_type":"code","outputId":"0ae3ce3a-71d1-4cac-e782-0dc6249a6914","colab":{"base_uri":"https://localhost:8080/","height":224},"trusted":true},"cell_type":"code","source":"ones.head()\n","execution_count":null,"outputs":[]},{"metadata":{"id":"_XLY4Y-xhEgh","colab_type":"code","outputId":"ff75d7f8-b679-432a-bbc4-ec1f572ef5a7","colab":{"base_uri":"https://localhost:8080/","height":224},"trusted":true},"cell_type":"code","source":"zeros.head()","execution_count":null,"outputs":[]},{"metadata":{"id":"B9kQ2YD3xwhr","colab_type":"code","outputId":"86694e00-bbfa-4d19-fc62-5770e94da201","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":true},"cell_type":"code","source":"ones.shape","execution_count":null,"outputs":[]},{"metadata":{"id":"BzvBqEwFx0Gj","colab_type":"code","outputId":"044be9d6-7072-4438-dcb9-bead7f5b3d32","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":true},"cell_type":"code","source":"sample_zeros.shape","execution_count":null,"outputs":[]},{"metadata":{"id":"WeX4ft1Qx3Pk","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"train_dataset=  pd.concat([ones, sample_zeros] ,axis=0)","execution_count":null,"outputs":[]},{"metadata":{"id":"lTlXpHgcyWdv","colab_type":"code","outputId":"9b369f36-08e7-475f-840a-b2d8eb7f1a59","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":true},"cell_type":"code","source":"train_dataset.shape","execution_count":null,"outputs":[]},{"metadata":{"id":"eJClgxOK1mXz","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"train_dataset['click_id'] = range(1, len(train_dataset)+1)","execution_count":null,"outputs":[]},{"metadata":{"id":"J6Gj-fk94JTF","colab_type":"code","outputId":"6b08dd59-e4ad-41e9-9c71-30392ca30a21","colab":{"base_uri":"https://localhost:8080/","height":224},"trusted":true},"cell_type":"code","source":"train_dataset.head()","execution_count":null,"outputs":[]},{"metadata":{"id":"ijlLcPrHSMuq","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"#Standardize rows into uniform scale\n\nX = train_dataset.drop(['is_attributed','click_id'],axis=1)\ny = train_dataset['is_attributed']","execution_count":null,"outputs":[]},{"metadata":{"id":"B1ewl6U6Smh_","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"del X['day_week']","execution_count":null,"outputs":[]},{"metadata":{"id":"ez1yJk8hz_Yn","colab_type":"code","outputId":"6e08ca81-1bdc-4509-e07e-739dbaca3b1e","colab":{"base_uri":"https://localhost:8080/","height":68},"trusted":true},"cell_type":"code","source":"y.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"id":"F9xEYgpnROJX","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"# Importing Models\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nfrom sklearn import svm, tree, linear_model, neighbors, naive_bayes, ensemble, discriminant_analysis, gaussian_process\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis\nfrom sklearn.ensemble import AdaBoostClassifier, RandomForestClassifier, GradientBoostingClassifier\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.svm import SVC, LinearSVC\nfrom xgboost import XGBClassifier\nfrom lightgbm import LGBMClassifier\n\n# Importing other tools\nfrom sklearn import model_selection\nfrom sklearn.metrics import confusion_matrix, classification_report, make_scorer\nfrom sklearn.metrics import accuracy_score, recall_score, precision_recall_curve\nfrom sklearn.model_selection import StratifiedKFold, cross_validate\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.calibration import CalibratedClassifierCV","execution_count":null,"outputs":[]},{"metadata":{"id":"V8aX9qi2RUsJ","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"# Defining random seed\nseed=42\n\n# Creating Models\n\nlr = LogisticRegression()\nlda = LinearDiscriminantAnalysis()\nsvc = SVC(random_state=seed, probability=True)\ndtree = DecisionTreeClassifier(random_state=seed)\nrf = RandomForestClassifier(10, random_state=seed)\ngdb = GradientBoostingClassifier(random_state=seed)\nadb = AdaBoostClassifier(random_state=seed)\nxgb = XGBClassifier(random_state=seed)\nknn = KNeighborsClassifier()\nlgbm = LGBMClassifier(random_state=seed)\n\nfirst_models = [ lr,\n                lda,\n                svc,\n                dtree,\n                rf,\n                gdb,\n                adb,\n                xgb, \n                knn,\n                lgbm]\nfirst_model_names = ['Logistic Regression',\n                     'LDA',\n                     'SVM',\n                     'Decision Tree', \n                     'Random Forest',\n                     'GradientBoosting',\n                     'AdaBoost',\n                     'XGB', \n                     'K-Neighbors',\n                     'Light GBM'] \n\n# Defining other steps\nn_folds = 5\nskf = model_selection.ShuffleSplit(n_splits = n_folds, test_size = .3, train_size = .7, random_state = seed ) \nstd_sca = StandardScaler()","execution_count":null,"outputs":[]},{"metadata":{"id":"1_Joe-DFR8RZ","colab_type":"code","outputId":"c2f393c1-d4db-4d75-fc95-94521eee226f","colab":{"base_uri":"https://localhost:8080/","height":187},"trusted":true},"cell_type":"code","source":"MLA_columns = ['MLA Name', 'MLA Parameters','MLA Train Accuracy Mean', 'MLA Test Accuracy Mean', 'MLA Time']\nMLA_compare = pd.DataFrame(columns = MLA_columns)\n\n#create table to compare MLA predictions\nMLA_predict = train_dataset[['click_id']]\n\ntrain_size = X.shape[0]\nn_models = len(first_models)\noof_pred = np.zeros((train_size, n_models))\nscores = []\nrow_index=0\n\nfor n, model in enumerate(first_models):\n    print('-'*25,first_model_names[n],'-'*int(45-len(first_model_names[n])))\n    model_pipeline = Pipeline(steps=[('Scaler', std_sca),\n                                     ('Estimator', model)])\n    MLA_name = model.__class__.__name__\n    MLA_compare.loc[row_index, 'MLA Name'] = MLA_name\n    MLA_compare.loc[row_index, 'MLA Parameters'] = str(model.get_params())\n    \n    cv_results = model_selection.cross_validate(model, X, y, cv  = skf, return_train_score=True,scoring=None)\n\n    MLA_compare.loc[row_index, 'MLA Time'] = cv_results['fit_time'].mean()\n    MLA_compare.loc[row_index, 'MLA Train Accuracy Mean'] = cv_results['train_score'].mean()\n    MLA_compare.loc[row_index, 'MLA Test Accuracy Mean'] = cv_results['test_score'].mean()  \n    \n    model_pipeline.fit(X, y)\n    MLA_predict[MLA_name] = model_pipeline.predict(X)\n    row_index+=1\n        ","execution_count":null,"outputs":[]},{"metadata":{"id":"Y6wPnvFD1CNq","colab_type":"code","outputId":"9315434c-7ecb-49f7-c314-be7e1c8b6da1","colab":{"base_uri":"https://localhost:8080/","height":546},"trusted":true},"cell_type":"code","source":"#print and sort table: https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.sort_values.html\nMLA_compare['Differenec'] = (MLA_compare['MLA Test Accuracy Mean'] - MLA_compare['MLA Train Accuracy Mean'])*100\nMLA_compare.sort_values(by = ['Differenec'], ascending = False, inplace = True)\nMLA_compare","execution_count":null,"outputs":[]},{"metadata":{"id":"OuN70wl8u19Z","colab_type":"code","outputId":"b170bab1-dbe5-4b6c-f837-3bc68e8324cf","colab":{"base_uri":"https://localhost:8080/","height":438},"trusted":true},"cell_type":"code","source":"feature_names = X.columns\nfeat_imp_df = pd.DataFrame(columns=first_model_names, index=feature_names)\n\n# Dropping the Models that don't have feature importances for this analysis\nfeat_imp_df.drop(['SVM','K-Neighbors'], axis=1, inplace=True)\n\n\n# I'm using absolute values for logistic Regression and LDA because we only care about the magnitude of the coefficient, not its direction \nfeat_imp_df['Logistic Regression'] = np.abs(lr.coef_.ravel())\nfeat_imp_df['LDA'] = np.abs(lda.coef_.flatten())\nfeat_imp_df['Decision Tree'] = dtree.feature_importances_\nfeat_imp_df['Random Forest'] = rf.feature_importances_\nfeat_imp_df['GradientBoosting'] = gdb.feature_importances_\nfeat_imp_df['AdaBoost'] = adb.feature_importances_\nfeat_imp_df['XGB'] = xgb.feature_importances_\nfeat_imp_df['Light GBM'] = lgbm.feature_importances_\nfeat_imp_df","execution_count":null,"outputs":[]},{"metadata":{"id":"GiIrShLE0TUm","colab_type":"code","outputId":"b6a2a86c-7bf3-4896-8421-5513abf69d3c","colab":{"base_uri":"https://localhost:8080/","height":495},"trusted":true},"cell_type":"code","source":"# http://www.menucool.com/rgba-color-picker\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\n%matplotlib inline\n\nfrom sklearn.preprocessing import MinMaxScaler\nmms = MinMaxScaler()\n\nscaled_fi = pd.DataFrame(data=mms.fit_transform(feat_imp_df),\n                         columns=feat_imp_df.columns,\n                         index=feat_imp_df.index)\nscaled_fi['Overall'] = scaled_fi.sum(axis=1)\n\nordered_ranking = scaled_fi.sort_values('Overall', ascending=False)\nfig, ax = plt.subplots(figsize=(10,7), dpi=80)\nsns.barplot(data=ordered_ranking, y=ordered_ranking.index, x='Overall', palette='magma')\nfor index,data in enumerate(tuple(ordered_ranking.Overall)):\n    plt.text(y=index , x=data , s=f\"{data}\" , fontdict=dict(fontsize=15), color=\"#CC0000\")\nax.spines['right'].set_visible(False)\nax.spines['top'].set_visible(False)\nax.spines['bottom'].set_visible(False)\nax.xaxis.set_visible(False)\nax.grid(False)\nax.set_title('Feature Importances for all Models');","execution_count":null,"outputs":[]},{"metadata":{"id":"makPd_rq05E6","colab_type":"code","outputId":"d23ab979-101d-49df-c7cf-431b06f09252","colab":{"base_uri":"https://localhost:8080/","height":187},"trusted":true},"cell_type":"code","source":"list(scaled_fi[:10].index)","execution_count":null,"outputs":[]},{"metadata":{"id":"oZjAhROt57LR","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":" X =  X[list(scaled_fi[:10].index)].head()","execution_count":null,"outputs":[]},{"metadata":{"id":"zGlYDDUi6DNY","colab_type":"code","outputId":"cd1ac4c5-8d00-4719-9bc5-0bfcfe5ef304","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":true},"cell_type":"code","source":"ordered_ranking.index[:-3:-1]","execution_count":null,"outputs":[]},{"metadata":{"id":"onTRqeDA6SWN","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"train_v2 = train_dataset.drop(ordered_ranking.index[:-3:-1], axis=1)\n\nX_v1 = train_v2.drop(['is_attributed'],axis=1)\ny_v1 = train_v2['is_attributed']\n","execution_count":null,"outputs":[]},{"metadata":{"id":"8y3a2Vkb7Se7","colab_type":"code","outputId":"b10c6f5c-3c79-4661-da01-a6800684f57b","colab":{"base_uri":"https://localhost:8080/","height":224},"trusted":true},"cell_type":"code","source":"X_v1.head()","execution_count":null,"outputs":[]},{"metadata":{"id":"h3peaCzj7Xn_","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"del X_v1['day_week']\ndel X_v1['click_id']","execution_count":null,"outputs":[]},{"metadata":{"id":"3WhpeHfU6aR5","colab_type":"code","outputId":"20dc00b8-3286-4ccb-b4c0-0cee4a029b1b","colab":{"base_uri":"https://localhost:8080/","height":187},"trusted":true},"cell_type":"code","source":"  MLA_columns = ['MLA Name', 'MLA Parameters','MLA Train Accuracy Mean', 'MLA Test Accuracy Mean', 'MLA Time']\nMLA_compare = pd.DataFrame(columns = MLA_columns)\n\n#create table to compare MLA predictions\nMLA_predict = train_dataset[['click_id']]\n\ntrain_size = X_v1.shape[0]\nn_models = len(first_models)\noof_pred = np.zeros((train_size, n_models))\nscores = []\nrow_index=0\n\nfor n, model in enumerate(first_models):\n    print('-'*25,first_model_names[n],'-'*int(45-len(first_model_names[n])))\n    model_pipeline = Pipeline(steps=[('Scaler', std_sca),\n                                     ('Estimator', model)])\n    MLA_name = model.__class__.__name__\n    MLA_compare.loc[row_index, 'MLA Name'] = MLA_name\n    MLA_compare.loc[row_index, 'MLA Parameters'] = str(model.get_params())\n    \n    #score model with cross validation: http://scikit-learn.org/stable/modules/generated/sklearn.model_selection.cross_validate.html#sklearn.model_selection.cross_validate\n    cv_results = model_selection.cross_validate(model, X_v1, y_v1, cv  = skf, return_train_score=True)\n\n    MLA_compare.loc[row_index, 'MLA Time'] = cv_results['fit_time'].mean()\n    MLA_compare.loc[row_index, 'MLA Train Accuracy Mean'] = cv_results['train_score'].mean()\n    MLA_compare.loc[row_index, 'MLA Test Accuracy Mean'] = cv_results['test_score'].mean()  \n    \n    model_pipeline.fit(X_v1, y_v1)\n    MLA_predict[MLA_name] = model_pipeline.predict(X_v1)\n    \n    #model_pipeline.fit(X, y)\n    #val_pred = model_pipeline.predict(x_val)\n    #oof_pred[X, n] = model_pipeline.predict_proba(X)[:,1]\n    row_index+=1\n        ","execution_count":null,"outputs":[]},{"metadata":{"id":"wooEREo77JIO","colab_type":"code","outputId":"ae41c447-e188-4e13-974e-e28369477888","colab":{"base_uri":"https://localhost:8080/","height":546},"trusted":true},"cell_type":"code","source":"#print and sort table: https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.sort_values.html\nMLA_compare['Differenec'] = (MLA_compare['MLA Test Accuracy Mean'] - MLA_compare['MLA Train Accuracy Mean'])*100\nMLA_compare.sort_values(by = ['Differenec'], ascending = False, inplace = True)\nMLA_compare","execution_count":null,"outputs":[]},{"metadata":{"id":"EXN2SeLZ7rLq","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"first_models = [rf,lgbm,knn]\nfirst_model_names = ['rf','lgbm', 'knn'] ","execution_count":null,"outputs":[]},{"metadata":{"id":"EY1bRw2z8AcH","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"LGBM_param_grid = {'lgbm__learning_rate': [0.1],\n    'lgbm__n_estimators': [100],\n    'lgbm__num_leaves': [31], # large num_leaves helps improve accuracy but might lead to over-fitting\n    'lgbm__boosting_type' : ['gbdt', 'dart'], # for better accuracy -> try dart\n    \n                  }\n\nRF_param_grid = {\n                \n               'rf__n_estimators': [100,200], #default=10\n            'rf__criterion': ['gini', 'entropy'], #default=”gini”\n            'rf__max_depth': [2,4,8] ,#default=None\n            'rf__oob_score': ['True']\n}\n\nGDB_param_grid = {\n    \"gdb__loss\":[\"deviance\"],\n    \"gdb__learning_rate\": [0.01,0.1],\n    \n    \"gdb__max_depth\":[3,5],\n   \n    \"gdb__criterion\": [\"friedman_mse\",  \"mae\"],\n    \n    \"gdb__n_estimators\":[10]\n    \n}\nXGB_param_grid = {\n    'xgb__min_child_weight': [1, 5],\n        'xgb__gamma': [0.5, 1],\n        'xgb__max_depth': [3, 4, 5]\n}\n\nKNN_param_grid = {\n    'knn__n_neighbors':[5,6,7],\n          'knn__leaf_size':[1,2,3,5],\n          \n}","execution_count":null,"outputs":[]},{"metadata":{"id":"yU5OFM3V8EEz","colab_type":"code","outputId":"631fcb01-5b80-487f-8d36-919c3f96a93b","colab":{"base_uri":"https://localhost:8080/","height":516},"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import GridSearchCV\ncolumns = ['Name', 'Parameters', 'Train Accuracy Mean', 'Test Accuracy']\n#models = [LGBMClassifier(),RandomForestClassifier()]\nparams_grid = [RF_param_grid,LGBM_param_grid,KNN_param_grid]\n\nafter_model_compare = pd.DataFrame(columns = columns)\n\nrow_index = 0\nfor n,alg in enumerate(first_models):\n    print('-'*25,first_model_names[n],'-'*int(29-len(first_model_names[n])))\n    print(alg)\n    model_pipeline = Pipeline(steps=[('Scaler', std_sca),\n                                     (first_model_names[n], first_models[n])])\n    \n    gs_alg = GridSearchCV(model_pipeline, param_grid = params_grid[0], cv = skf, scoring = 'accuracy', n_jobs=-1,return_train_score=True)\n    params_grid.pop(0)\n\n    #set name and parameters\n    model_name = alg.__class__.__name__\n    after_model_compare.loc[row_index, 'Name'] = model_name\n    \n    gs_alg.fit(X_v1, y_v1)\n   \n    after_model_compare.loc[row_index, 'Parameters'] = str(gs_alg.best_params_)\n \n    \n    after_model_compare.loc[row_index, 'Train Accuracy Mean'] = gs_alg.cv_results_['mean_train_score'][gs_alg.best_index_]\n    after_model_compare.loc[row_index, 'Test Accuracy'] = gs_alg.cv_results_['mean_test_score'][gs_alg.best_index_]\n    \n    row_index+=1\n    print(row_index, alg.__class__.__name__, 'trained...')\n\nafter_model_compare","execution_count":null,"outputs":[]},{"metadata":{"id":"UKEDiuuB8XeF","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"after_model_compare['Difference'] = (after_model_compare['Test Accuracy']-after_model_compare['Train Accuracy Mean'])*100","execution_count":null,"outputs":[]},{"metadata":{"id":"QhwofvKf8sO8","colab_type":"code","outputId":"e7369a5a-afa9-42ab-c2ce-52b4e603661b","colab":{"base_uri":"https://localhost:8080/","height":142},"trusted":true},"cell_type":"code","source":"after_model_compare","execution_count":null,"outputs":[]},{"metadata":{"id":"FA-wR_Xo_OOl","colab_type":"code","outputId":"f65239e0-8a17-4185-d901-74a66c825696","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":true},"cell_type":"code","source":"X_v1.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_v1.head()","execution_count":null,"outputs":[]},{"metadata":{"id":"Pocaazll8t91","colab_type":"code","outputId":"91d7b3b2-aee4-4d03-d386-c5d7e19d7996","colab":{"base_uri":"https://localhost:8080/","height":153},"trusted":true},"cell_type":"code","source":"vote_est = [\n    #Ensemble Methods: http://scikit-learn.org/stable/modules/ensemble.html\n    \n    ('lda', LinearDiscriminantAnalysis()),\n    \n    ('ada', AdaBoostClassifier(random_state=seed)),\n    \n    ('lr', LogisticRegression()),\n    \n    #Nearest Neighbor: http://scikit-learn.org/stable/modules/neighbors.html\n    ('knn', neighbors.KNeighborsClassifier()),\n    \n    #xgboost: http://xgboost.readthedocs.io/en/latest/model.html\n   ('xgb', XGBClassifier())\n  \n\n]\n\n\n#Hard Vote or majority rules\nvote_hard = ensemble.VotingClassifier(estimators = vote_est , voting = 'hard')\nvote_hard_cv = model_selection.cross_validate(vote_hard, X_v1, y_v1, cv  = skf,return_train_score=True)\nvote_hard.fit(X_v1, y_v1)\n#print(\"Hard Voting Training w/bin score mean: {:.2f}\". format(vote_hard_cv['train_score'].mean()*100)) \nprint(\"Hard Voting Train w/bin score mean: {:.2f}\". format(vote_hard_cv['train_score'].mean()*100))\nprint(\"Hard Voting Test w/bin score mean: {:.2f}\". format(vote_hard_cv['test_score'].mean()*100))\nprint(\"Hard Voting Test w/bin score 3*std: +/- {:.2f}\". format(vote_hard_cv['test_score'].std()*100*5))\nprint('-'*10)\n\n\n#Soft Vote or weighted probabilities\nvote_soft = ensemble.VotingClassifier(estimators = vote_est , voting = 'soft')\nvote_soft_cv = model_selection.cross_validate(vote_soft, X_v1, y_v1, cv  = skf,return_train_score=True)\nvote_soft.fit(X_v1, y_v1)\n\n#print(\"Soft Voting Training w/bin score mean: {:.2f}\". format(vote_soft_cv['train_score'].mean()*100)) \nprint(\"Soft Voting Train w/bin score mean: {:.2f}\". format(vote_soft_cv['train_score'].mean()*100))\nprint(\"Soft Voting Test w/bin score mean: {:.2f}\". format(vote_soft_cv['test_score'].mean()*100))\nprint(\"Soft Voting Test w/bin score 3*std: +/- {:.2f}\". format(vote_soft_cv['test_score'].std()*100*5))\nprint('-'*10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test=pd.read_csv(\"../input/talkingdata-adtracking-fraud-detection/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Convert click_time into time,month,year and day\nvalue='click_time'\ndf_test[value]=pd.to_datetime(df_test[value])\ndf_test['year']=df_test[value].dt.year\ndf_test['month']=df_test[value].dt.month\ndf_test['day']=df_test[value].dt.day\ndf_test['time']=df_test[value].dt.hour\ndf_test['day_week']=df_test[value].dt.weekday_name","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del df_test['click_time']\ndel df_test['click_id']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del df_test['day_week']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data={'day_week_Tuesday': np.zeros(df_test.shape[0], dtype='int'),\n     'day_week_Thursday': np.zeros(df_test.shape[0], dtype='int'),\n     'day_week_Wednesday':np.zeros(df_test.shape[0], dtype='int')}\nday = pd.DataFrame(data, columns = ['day_week_Thursday','day_week_Tuesday','day_week_Wednesday'])\nday","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.concat([df_test, day] ,axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del test['year']\ndel test['month']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission=pd.read_csv(\"../input/talkingdata-adtracking-fraud-detection/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission['is_attributed']=vote_soft.predict(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv(\"submission.csv\",index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"colab":{"name":"Untitled0.ipynb","version":"0.3.2","provenance":[],"collapsed_sections":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"nbformat":4,"nbformat_minor":1}