{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","collapsed":true,"scrolled":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport gc\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\n\npath = \"../input/\"","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"5933be7d-bfa8-4bb5-892d-fb4b2f054970","_uuid":"66cce7440b2c67c16009a7851287383b159ac519"},"cell_type":"markdown","source":"----\n## Exploring the Data\n\nExplore the sample training data , which are 100,000 randomly-selected rows of training data."},{"metadata":{"_cell_guid":"9a8d04ce-a447-4b9b-a402-e852c58c0c2c","_uuid":"384da0b52636c8938db575cddddaf5d4dfd57b31","trusted":false,"collapsed":true},"cell_type":"code","source":"sample_train = pd.read_csv(path + \"train_sample.csv\")\ndisplay(sample_train.head(n=5))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b8c4f428-5ea7-4530-997f-525b5e7d4be2","_uuid":"db3bd878590d78ce5c8c6c0860eb0158d100e8b7","trusted":false,"collapsed":true},"cell_type":"code","source":"#https://www.kaggle.com/anokas/talkingdata-adtracking-eda\npal = sns.color_palette()\nplt.figure(figsize=(15, 8))\ncols = ['ip', 'app', 'device', 'os', 'channel']\nuniques = sample_train[cols].nunique()\nsns.set(font_scale=1.2)\nax = sns.barplot(cols, uniques, palette=pal, log=True)\nax.set(xlabel='Feature', ylabel='log(unique count)', title='Number of unique values per feature')\nfor p, uniq in zip(ax.patches, uniques):\n    height = p.get_height()\n    ax.text(p.get_x()+p.get_width()/2.,\n            height + 10,\n            uniq,\n            ha=\"center\") ","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"16a39dd9-4a20-4c14-8dd7-10a80228fe1b","_uuid":"846d25fb32113405b197f72ef213fcbdde978970","trusted":false,"collapsed":true},"cell_type":"code","source":"#https://www.kaggle.com/anokas/talkingdata-adtracking-eda\ndef plot_distribution(dt):\n    plt.figure(figsize=(8, 8))\n    sns.set(font_scale=1.2)\n    mean = (dt.is_attributed.values == 1).mean()\n    n_is_attributed = len(dt[dt.is_attributed==1])\n    n_not_attributed = len(dt[dt.is_attributed==0])\n    greater_percent = float(n_is_attributed)/float(n_not_attributed)*100\n    ax = sns.barplot(['Downloaded (1)', 'Not downloaded (0)'], [mean, 1-mean], palette=pal)\n    ax.set(xlabel='Target Value', ylabel='Probability', title='Target value distribution')\n    for p, uniq in zip(ax.patches, [mean, 1-mean]):\n        height = p.get_height()\n        ax.text(p.get_x()+p.get_width()/2.,\n                height+0.01,\n                '{}%'.format(round(uniq * 100, 2)),\n                ha=\"center\") \nplot_distribution(sample_train)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"e8747c6f-d198-4c12-9774-3bc7a59b8ef3","_kg_hide-input":true,"_kg_hide-output":true,"_uuid":"454bf703741c75ca8fca888bae2133145ccf839e","trusted":false,"collapsed":true},"cell_type":"code","source":"def print_data_distribution(dt):\n    n_records = dt.shape[0]\n    n_is_attributed = len(dt[dt.is_attributed==1])\n    n_not_attributed = len(dt[dt.is_attributed==0])\n    greater_percent = float(n_is_attributed)/float(n_not_attributed)*100\n    # Print the results\n    print (\"Total number of records: {}\".format(n_records))\n    print (\"Clicks downloading app: {}\".format(n_is_attributed))\n    print (\"Clicking not downloading app: {}\".format(n_not_attributed))\n    print (\"Percentage of clicks downloading app: {:.2f}%\".format(greater_percent))\n    \nprint_data_distribution(sample_train)\ndel sample_train;gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"777a186a-a269-469b-b152-ae7abe36227d","collapsed":true,"_uuid":"4996ef8b648daa8c76ba1611adeb7858b095107b"},"cell_type":"markdown","source":"----\n## Data processing"},{"metadata":{"_cell_guid":"dcd824db-196a-4f2f-b693-fdb7abca15c1","_uuid":"70b9003a26c15ebd9a18905a61a9bca14025cfd0"},"cell_type":"markdown","source":"> ### Upsampling the positive target value data to improve the imblance in the training data"},{"metadata":{"_cell_guid":"50148b46-a6b5-4a86-b81a-75fb4d5be0b7","_kg_hide-output":false,"_uuid":"6fce40a6c8559a63caa432020ebd2a0b53629472","trusted":false,"collapsed":true},"cell_type":"code","source":"import dask.dataframe as ddf\n\ndtypes = {'ip'            : 'uint32',\n          'app'           : 'uint16',\n          'device'        : 'uint16',\n          'os'            : 'uint16',\n          'channel'       : 'uint16',\n          'is_attributed' : 'uint8',\n          }\n\ndask_df = ddf.read_csv('../input/train.csv',dtype=dtypes)\ndf_neg = dask_df[(dask_df['is_attributed'] == 0)].compute()\ndf_neg = df_neg.sample(n=20000)\ndf_pos = dask_df[(dask_df['is_attributed'] == 1)].compute()\ndf_pos = df_pos.sample(n=5000)\ntrain = pd.concat([df_pos,df_neg]).sample(frac=1).reset_index()\ndel dask_df, df_pos, df_neg; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"f0de89a0-2b60-479c-971c-c62c38ba9c71","_uuid":"8adc3fac264b1d53466493d2176ffbcf5236eb85","trusted":false,"collapsed":true},"cell_type":"code","source":"print_data_distribution(train)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b6503e87-462f-4aff-a7c2-e4d781b2e5a6","_uuid":"22bc0160c6b58b9c6e432f33d48cfffa36a0410e","trusted":false,"collapsed":true},"cell_type":"code","source":"plot_distribution(train)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"f4ac6f64-580c-4736-9ad8-3baac7fd717b","collapsed":true,"_uuid":"b5bdbe82e94b5a7805d7e5b6dbcc8adcdd5ba34d"},"cell_type":"markdown","source":"### Process click time\n\nSplit **click_time** into  4 segments  : day, hour, minute,second\n\nSince the data that TalkingData provided  is a dataset covering approximately 200 million clicks over 4 days,  YEAR, MONTH, DAY is meaningless for model to learn. \n"},{"metadata":{"_cell_guid":"9d867e0f-9883-4983-acf9-0e039d0a65f6","collapsed":true,"_uuid":"a8379c5f920baf37e4fc0001fffbb5b81e8028b4","trusted":false},"cell_type":"code","source":"def process_time(dt):\n    dt['sec'] = pd.to_datetime(dt.click_time).dt.second.astype('uint8')\n    dt['min'] = pd.to_datetime(dt.click_time).dt.minute.astype('uint8')\n    dt['hour'] = pd.to_datetime(dt.click_time).dt.hour.astype('uint8')\n    #dt['day'] = pd.to_datetime(dt.click_time).dt.day.astype('uint8')\n    #dt['month'] = pd.to_datetime(dt.click_time).dt.month.astype('uint8')\n    #dt['wday'] = pd.to_datetime(dt.click_time).dt.dayofweek.astype('uint8')\n    dt.drop(['click_time'], axis=1, inplace=True)\nprocess_time(train)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"5b76a2c1-50a9-4196-9dcd-f6527cb3197f","_uuid":"df6f9fbcd0d76caac1b7894519c3e897f6c6ed2d"},"cell_type":"markdown","source":"### drop ip and attributed_time\nAlthough ip can be seen as categorical data, the amount of ip unique values is really too large to process with one-hot encoding.  Therefore, we drop this feature so as not to affect the model peformance.  Meanwhile, we also attributed_time feature since is_attributed can totally work as taget label value."},{"metadata":{"_cell_guid":"77372aff-220b-48db-b06b-7cc8d741db26","_uuid":"8608ec8ddb192cf7343591827594d46df0086d8c","trusted":false,"collapsed":true},"cell_type":"code","source":"attributed = train['is_attributed']\ntrain.drop(['index','ip','attributed_time','is_attributed'], axis=1, inplace=True)\ndisplay(train.head(n=1))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"18f3ade2-6eca-4999-9008-8a8cb016216f","_uuid":"bc9c181fe29035bacac373e4457caff939600acf"},"cell_type":"markdown","source":"### one-hot encoding\n\nSince some standard one-hot encoding functionssuch as get_dummies of pandas library are very memory consuming, we have to create our own one-hod encoding function as below."},{"metadata":{"_cell_guid":"a38a8de2-657b-46a5-bd1a-3923cd415d28","_uuid":"d2edb7f7fa669c70232b48b7d04e239136d51f77","trusted":false,"collapsed":true},"cell_type":"code","source":"feature_len_dict = {'app':800, 'device':4230, 'os':900, 'channel':500, 'hour':24, 'min':60, 'sec':60 }\nfeatures_final = pd.DataFrame(np.zeros((len(train),sum(feature_len_dict.values()))))\n\nfeatures_columns = []\nfor k,v in feature_len_dict.items():\n    for i in range(v):\n        features_columns.append(k+'_'+str(i))\nfeatures_final.columns = features_columns\n\n\ndef update_features(final,dt):\n    for name, values in dt.iteritems():\n        for i in range(len(values)):\n            col = '{name}_{value}'.format(name=name, value=values.values[i])            \n            final[col][i] = 1\n\n# customerized one-hot encoding\nupdate_features(features_final,train)\n\ndel train;gc.collect()\nencoded = features_final.shape[1]\nprint(\"{} total features after one-hot encoding.\".format(encoded))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"e6d939fc-2218-4d64-b855-2cdf9fc54076","_uuid":"9b3e56fea9c5f26ddb38e668698fca05992788a1"},"cell_type":"markdown","source":"### shuffle and split data"},{"metadata":{"_cell_guid":"176a95fa-f5bc-4607-a2f8-2cd1feab238e","_uuid":"86c1dde5fe42a7fecd1121bedcb90f61215014a2","trusted":false,"collapsed":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_test, y_train, y_test = train_test_split(features_final, \n                                                    attributed, \n                                                    test_size = 0.2, \n                                                    random_state = 0)\ndel features_final;gc.collect()\nprint(\"Training set has {} samples.\".format(X_train.shape[0]))\nprint(\"Testing set has {} samples.\".format(X_test.shape[0]))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"5487be8a-ec9a-4df4-bb62-cd49f7afd389","collapsed":true,"_uuid":"834c73eb5d06cb168bfc7bb8e810dee0d3be42c6"},"cell_type":"markdown","source":"### Benchmark - Naive predicator"},{"metadata":{"_cell_guid":"3ab59398-549c-4967-b0c1-6bd63dfbe164","_uuid":"0f6ae23b187e7fe18118e6502d7c90cddc431906","trusted":false,"collapsed":true},"cell_type":"code","source":"# https://github.com/udacity/machine-learning/tree/master/projects/finding_donors\n   \nTP = np.sum(attributed) \nFP = attributed.count() - TP # Specific to the naive case\nTN = 0 # No predicted negatives in the naive case\nFN = 0 # No predicted negatives in the naive case\n\n# Calculate accuracy, precision and recall\naccuracy = float(TP) / float(attributed.count())\nprecision = float(TP) / float(TP + FP)\nrecall = float(TP) / float(TP + FN)\n\n# Calculate F-score using the formula above for beta = 0.5 and correct values for precision and recall.\nfscore = (1 + 0.5 * 0.5) * ((precision * recall) / (0.5 * 0.5 * precision + recall))\n\n# Print the results \nprint(\"Naive Predictor: [Accuracy score: {:.4f}, F-score: {:.4f}]\".format(accuracy, fscore))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"65bd4de6-df32-47ae-91e7-23ed5eac9476","_uuid":"fd09b25196115436292b7534cef6af8ba40ace81"},"cell_type":"markdown","source":"----\n## Model evaluation\n"},{"metadata":{"_cell_guid":"f24807c1-7351-40ee-b9ef-893921fae2ac","collapsed":true,"_uuid":"792346ca71527f301824b731a12086535a182ee3","trusted":false},"cell_type":"code","source":"from sklearn.naive_bayes import GaussianNB\nfrom sklearn import tree\nfrom sklearn.linear_model import LogisticRegression\nfrom time import time\nfrom sklearn.metrics import fbeta_score,accuracy_score\nfrom sklearn.model_selection import KFold\n\n# Split and shuffle data with K-Fold for cross validation\nkf = KFold(n_splits=3, random_state=None, shuffle=True)\n\n# Create models' instances\nclf_A = tree.DecisionTreeClassifier(criterion='entropy', splitter='best', min_samples_split=2, min_samples_leaf=1)\nclf_B = GaussianNB(priors=None)\nclf_C = LogisticRegression(penalty='l2',  C=1.0, multi_class='ovr')\n\n# results dictionary\nresults = {}\n    \nfor clf in [clf_A, clf_B, clf_C]:\n    clf_name = clf.__class__.__name__\n    results[clf_name] = {}\n    i = 0\n    for train_index, test_index in kf.split(X_train):\n        results[clf_name][i] = {}\n\n        # Fit the learner to the training data \n        start = time();\n        learner = clf.fit(X_train.iloc[train_index], y_train.iloc[train_index]);\n        results[clf_name][i]['train_time'] = time()  - start\n\n        # get predictions on test set and training data\n        start = time();\n        predictions_train = learner.predict(X_train.iloc[train_index]);\n        predictions_test = learner.predict(X_train.iloc[test_index]);\n        results[clf_name][i]['pred_time'] = time() - start\n\n        # get accuracy and f-beta score on training data\n        results[clf_name][i]['acc_train' ] = accuracy_score(y_train.iloc[train_index], predictions_train)\n        results[clf_name][i]['f_train'] = fbeta_score(y_train.iloc[train_index], predictions_train,beta=0.5)\n\n        # get accuracy and f-beta score on test data\n        results[clf_name][i]['acc_test'] = accuracy_score(y_train.iloc[test_index], predictions_test)\n        results[clf_name][i]['f_test'] = fbeta_score(y_train.iloc[test_index], predictions_test,beta=0.5)\n        i = i + 1\n        ","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"457f476d-b324-4ba7-b995-85619b4f826c","_uuid":"1f1c81006420b7450db6a45052970035cc06e485","trusted":false,"collapsed":true},"cell_type":"code","source":"print('{}\\t{}\\t{}(Sec)\\t{}(Sec)\\t{}\\t{}\\t{}\\t{}\\t'.format('model','round','train_time','pred_time','acc_train', 'f_train','acc_test','f_test'))\nfor k in results.keys():\n    for f in results[k]:\n        v = results[k][f]\n        print('{}\\t{}\\t{:.2f}\\t{:.2f}\\t{:.4f}\\t{:.4f}\\t{:.4f}\\t{:.4f}\\t'.format(k,f+1,v['train_time'],v['pred_time'],v['acc_train'],v['f_train'],v['acc_test'],v['f_test']))\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"e8e206ec-1b23-4e59-98e3-931657b97981","collapsed":true,"_uuid":"9a62b4541700c89b161e7b965c855510f6f64893","trusted":false},"cell_type":"code","source":"# %matplotlib inline\n# for  metric in metrics.keys():    \n#     plt.figure(figsize=(8, 8))\n#     sns.set(font_scale=1.2)\n#     ax = sns.barplot([k for k in results.keys()], [results[learner][metric] for learner in results.keys()], palette=pal)\n#     ax.set( ylabel=metrics[metric], title=metric)\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"3fa87980-7c10-42f2-9462-45001b2125c5","collapsed":true,"_uuid":"2714817a14d9cb212c15c34170200750d4d470e0","trusted":false},"cell_type":"code","source":"# from sklearn.metrics import confusion_matrix\n# %matplotlib inline\n\n# for clf in predictors:\n#     # Compute confusion matrix for a model\n#     model = clf\n#     cm = confusion_matrix(y_test.values, model.predict(X_test))\n#     plt.figure(figsize=(3, 3))\n#     # view with a heatmap\n#     sns.heatmap(cm, annot=True, cmap='Blues', xticklabels=['no', 'yes'], yticklabels=['no', 'yes'])\n#     plt.ylabel('True label')\n#     plt.xlabel('Predicted label')\n#     plt.title('Confusion matrix for : {}'.format(model.__class__.__name__));","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"f2fb4c9c-c0c6-47a6-804b-fcf79d02a99b","_uuid":"c3e4ac99be37e89e96dd604a7717c23ba019146b"},"cell_type":"markdown","source":"----\n## Model tuning"},{"metadata":{"_cell_guid":"b764d1ac-3ea8-4c90-81dc-e3309b660d20","_uuid":"cd69e142b79e1de066a211e5a80a75a01c99ab34","trusted":false,"collapsed":true},"cell_type":"code","source":"#https://github.com/udacity/machine-learning/tree/master/projects/finding_donors\n# Import 'GridSearchCV', 'make_scorer', and any other necessary libraries\nfrom sklearn.metrics import make_scorer\nfrom sklearn.grid_search import GridSearchCV\nfrom sklearn.cross_validation import ShuffleSplit\n\n# choose cross-validation iterator\ncv = ShuffleSplit(X_train.shape[0], n_iter=10, test_size=0.2, random_state=0)\n\n# Initialize the classifier\nclf = LogisticRegression(penalty='l2',  C=1.0, multi_class='ovr')\n\n# Create the parameters list you wish to tune, using a dictionary if needed.\nparameters = {'penalty': ['l1','l2'],'C':np.logspace(-3,3,7)}\n\n# Make an fbeta_score scoring object using make_scorer()\nscorer = make_scorer(fbeta_score,beta=0.5)\n\n# Perform grid search on the classifier using 'scorer' as the scoring method using GridSearchCV()\ngrid_obj = GridSearchCV(estimator=clf, cv=cv,param_grid=parameters,scoring=scorer)\n\n# Fit the grid search object to the training data and find the optimal parameters using fit()\ngrid_fit = grid_obj.fit(X_train, y_train)\n\n# Get the estimator\nbest_clf = grid_fit.best_estimator_\n\n# Make predictions using the unoptimized and model\npredictions = (clf.fit(X_train, y_train)).predict(X_test)\nbest_predictions = best_clf.predict(X_test)\n\n# Report the before-and-afterscores\nprint (\"Unoptimized model\\n------\")\nprint (\"Accuracy score on testing data: {:.4f}\".format(accuracy_score(y_test, predictions)))\nprint (\"F-score on testing data: {:.4f}\".format(fbeta_score(y_test, predictions, beta = 0.5)))\nprint (\"\\nOptimized Model\\n------\")\nprint (\"Final accuracy score on the testing data: {:.4f}\".format(accuracy_score(y_test, best_predictions)))\nprint (\"Final F-score on the testing data: {:.4f}\".format(fbeta_score(y_test, best_predictions, beta = 0.5)))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"6e397f54-8cf3-45b4-9efc-220f2bc8e492","_uuid":"733d8b36854674bac99a43163bbb1a26362be5ad"},"cell_type":"markdown","source":"----\n## Feature importance analysis"},{"metadata":{"_cell_guid":"d72ce339-c124-4ce0-956b-121288d0305c","collapsed":true,"_uuid":"acefce680e15728f6f3fe59fc8c69deae26c3716","trusted":false},"cell_type":"code","source":"# Train the supervised model on the training set using .fit(X_train, y_train)\nmodel = tree.DecisionTreeClassifier(random_state=42)\nmodel.fit(X_train, y_train)\n\n# Extract the feature importances using .feature_importances_ \nimportances = model.feature_importances_ \n\n# Define the number of feature for importance analysis\nFEATURE_NUMBER = 10\n\n# Display the five most important features\nindices = np.argsort(importances)[::-1]\ncolumns = X_train.columns.values[indices[:FEATURE_NUMBER]]\nvalues = importances[indices][:FEATURE_NUMBER]","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"8f464d62-d5bb-4d61-9175-a6e6d1dad785","_uuid":"d726eef807e4376d29f5053dc2abb409a6305dd2","trusted":false,"collapsed":true},"cell_type":"code","source":"sns.set(style=\"whitegrid\")\n\n# Initialize the matplotlib figure\nf, ax = plt.subplots(figsize=(12, 6))\n\n# Plot the total crashes\nsns.set_color_codes(\"pastel\")\nsns.barplot(columns, np.cumsum(values), label=\"Culcumative feature weight\", color=\"b\")\n\n# Plot the crashes where alcohol was involved\nsns.set_color_codes(\"muted\")\nsns.barplot(columns, values, label=\"Feature weight\", color=\"b\")\n\n# Add a legend and informative axis label\nax.legend(ncol=2, loc=\"upper left\", frameon=True)\nax.set( ylabel=\"Weight\",\n       xlabel=\"Normalized Weights for First Ten Most Predictive Features\")\nsns.despine( right=False)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"0fd5072a-608c-4c81-829d-619578c505f7","_uuid":"4d69ec176587f068199309a08868341c8a4d4bce","trusted":false,"collapsed":true},"cell_type":"code","source":"#https://github.com/udacity/machine-learning/tree/master/projects/finding_donors\n# Import functionality for cloning a model\nfrom sklearn.base import clone\n\n# Reduce the feature space\nX_train_reduced = X_train[X_train.columns.values[(np.argsort(importances)[::-1])[:FEATURE_NUMBER]]]\nX_test_reduced = X_test[X_test.columns.values[(np.argsort(importances)[::-1])[:FEATURE_NUMBER]]]\n\n# Train on the \"best\" model found from grid search earlier\nclf = (clone(best_clf)).fit(X_train_reduced, y_train)\n\n# Make new predictions\nreduced_predictions = clf.predict(X_test_reduced)\n\n# Report scores from the final model using both versions of data\nprint (\"Final Model trained on full data\\n------\")\nprint (\"Accuracy on testing data: {:.4f}\".format(accuracy_score(y_test, best_predictions)))\nprint (\"F-score on testing data: {:.4f}\".format(fbeta_score(y_test, best_predictions, beta = 0.5)))\nprint (\"\\nFinal Model trained on reduced data\\n------\")\nprint (\"Accuracy on testing data: {:.4f}\".format(accuracy_score(y_test, reduced_predictions)))\nprint (\"F-score on testing data: {:.4f}\".format(fbeta_score(y_test, reduced_predictions, beta = 0.5)))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"mimetype":"text/x-python","nbconvert_exporter":"python","name":"python","file_extension":".py","version":"3.6.4","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3"}},"nbformat":4,"nbformat_minor":1}