{"cells":[{"metadata":{"_uuid":"13489fbea8dfa4c359520b8c5c99e82453efcf99"},"cell_type":"markdown","source":"### Notebook setup"},{"metadata":{"trusted":false,"_uuid":"7416ca9bbfa0a612b33cb4fcb2bb2a0046218040"},"cell_type":"code","source":"%matplotlib inline\n%autosave 30\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport multiprocessing as mp\nfrom tqdm import tqdm_notebook as tqdm\n\npd.set_option('display.max_rows', 10)\nnp.random.seed(42)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8910f2a6d7b12a4a4828da12331f2320d6b98dd0"},"cell_type":"markdown","source":"### Datasets\n\n| Dataset             | Comment                                           | Size (clicks/size)  |\n|---------------------|---------------------------------------------------|---------------------|\n| train.csv           | the training set                                  | 57.537.506 / 2.5 GB |\n| train_sample.csv    | sample of training set                            | 100.000 / 3.9 MB |\n| test.csv            | the test set                                      | 18.790.469 / 824 MB |\n| test_supplement.csv | larger test set, subset used for Kaggle evaluation | 57.537.505 / 2.5GB  |\n\n\nEvery click record has following associated fields:\n- click_time: timestamp of click (UTC)\n- ip: ip address of click\n- device: device type of user mobile phone\n- os: os version id of user mobile phone\n- channel: channel id of mobile ad publisher\n- app: app id for marketing\n\nTraining data contains two additional fields:\n- is_attributed: was app downloaded, to be predicted\n- attributed_time: time of app download\n\nFor the given click data, the goal is to output the probability that the user will download the app.\n"},{"metadata":{"_uuid":"e632c0c6514a2351501edc638688d4a93a111603"},"cell_type":"markdown","source":"Read train and test sets for local modeling and evaluation."},{"metadata":{"trusted":false,"_uuid":"7be42d764c86feeeb9e731a501ae1187658b7454"},"cell_type":"code","source":"def read_dataset(path, nrows=None):\n    \"\"\"Reads dataset from CSV file, with optional number of rows\"\"\"\n    \n    df = pd.read_csv(path, nrows=nrows)\n    df['click_time'] = pd.to_datetime(df['click_time'])\n    df = df.set_index('click_time')\n    df.sort_index(inplace=True)\n    \n    return df\n\n# Training data set\ndataset_train = read_dataset('../input/train_sample.csv')\n\n# Benchmark test set, same as used in benchmark model: https://rpubs.com/el16/410747\ndataset_test = read_dataset('../input/train.csv', nrows=1000000)\n\n# Larger test set, used for Kaggle Leaderboard evaluation\ndataset_test_all = read_dataset('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3828a8e548a376e6e03532246153c1276fcfba2d"},"cell_type":"markdown","source":"### Data exploration"},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"ab214068e31f7bcf34f7c340a52dc855a7b2da66"},"cell_type":"code","source":"dataset_train.describe()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"5a55b25c1a5c8dd011e9a32ec968c84aadc4c222"},"cell_type":"code","source":"ser = dataset_train.nunique()\npd.DataFrame({'attribute':ser.index, 'unique values':ser.values})","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"180b8a6fde23c3fe50a81dbae5e4bee257f8ce2d"},"cell_type":"code","source":"dataset_train.hist(bins=20, figsize=(10,10));","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"da8d3946f6e83c893d97905e4fb5a8b7ed0d78a9"},"cell_type":"markdown","source":"Distribution of `is_attributed` feature shows heavily unbalanced dataset towards no download cases."},{"metadata":{"trusted":false,"_uuid":"3a506d651a32d3ecd5ff06bf580c09044aa3c8db"},"cell_type":"code","source":"corr = dataset_train.corr()\ncorr.style.background_gradient()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"a6736bb392873d3c19987417f46cc9b0bf48cb0d"},"cell_type":"code","source":"dataset_train[(dataset_train.is_attributed == 1)].hist(bins=20, figsize=(10,10));","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"72252591f9a7faea5913d71f25e253c0a9ddcfdd"},"cell_type":"code","source":"top_apps = dataset_train.groupby(['ip'])['is_attributed'].agg(\n    {\"is_attributed\": sum}).sort_values(\n    \"is_attributed\", ascending=False).head(10).reset_index()\ntop_apps","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"60cc693d5b3f7766c3c8c8e8497b4fadf18756b2"},"cell_type":"code","source":"top_apps = dataset_train.groupby(['app','channel'])['is_attributed'].agg(\n    {\"is_attributed\": sum}).sort_values(\n    \"is_attributed\", ascending=False).head(10).reset_index()\ntop_apps","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"c2ca4523ce93ad2b4d2c66bd66257fdf79156091"},"cell_type":"code","source":"dataset_train.apply(lambda x: 1 if x['is_attributed'] == 1 else 0, axis=1).plot(figsize=(20, 5));","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"86393e0e668ce274e1654bf6073b7f2c3c740d5d"},"cell_type":"code","source":"dataset_train.apply(lambda x: 1 if x['app'] == 19 else 0, axis=1).plot(figsize=(20, 5));","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"630622795e763889a8f84deb3bb663d6c25ddec8"},"cell_type":"code","source":"dataset_train.apply(lambda x: 1 if x['channel'] == 213 and x['app'] == 19 else 0, axis=1).plot(figsize=(20, 5));","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5c4f513d44ce24de40e07e2243d250fff1616f36"},"cell_type":"markdown","source":"### Benchmark models\n\nFor benchmark we will consider two models, one outputing constant value and second using random value from uniform distribution. Both will be evaluated on test set and the one with higher score will be used as baseline."},{"metadata":{"trusted":false,"_uuid":"9e145d5972249ceed2259b4766ab03d1eb664b0d"},"cell_type":"code","source":"dataset_test.head()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"a764a0524adb1e0d1201c6d920e16411f75f5998"},"cell_type":"code","source":"benchmark_const = dataset_test_all[['click_id']].copy()\nbenchmark_const['is_attributed'] = 0\ndisplay(benchmark_const)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"6575c2b22b62b7fabb7ea38271e1c5f3c59f007c"},"cell_type":"code","source":"benchmark_rand = dataset_test_all[['click_id']].copy()\nbenchmark_rand['is_attributed'] = np.random.uniform(size=len(benchmark_rand))\ndisplay(benchmark_rand)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"192380e4ab0c736f9f0f00ec6f02f5cc6e8b74aa"},"cell_type":"markdown","source":"Constant model that predicts no app download scores 0.5 on Kaggle evaluator. Random model that uses uniform distribution scores 0.4995 as private score (82% of test data) and 0.5003 as public score (18% of the test data). One approach to this problem is done by [Elior Tal](https://rpubs.com/el16/410747) and his solution scores 0.825 on a subset of the test set. In his work he compares performance of random forest and boosting algorithms. We will use this as a benchmark model as it is well documented and scores better than our initial baseline models. Additionally we will use existing Kaggle solutions on the [leaderboard](https://www.kaggle.com/c/talkingdata-adtracking-fraud-detection/leaderboard ) as a supporting benchmark."},{"metadata":{"_uuid":"759794d8c8bc52d3975d4e216a502aca9a369cab"},"cell_type":"markdown","source":"### Data preprocessing\n\n* Extract hour of the day\n* For each attribute extract duration since last click with that attribute\n* Remove attributes: attributed_time, click_time, is_attributed"},{"metadata":{"trusted":false,"_uuid":"e2fd973d808e1fed6c86379766de8cb8a26aedd7"},"cell_type":"code","source":"from sklearn import preprocessing\n\ndef prepare_column(job):\n    \"\"\"Generate new features from target column\"\"\"\n    \n    df = job['df']\n    col = job['feat']\n    \n    print ('Preparing column: ', col)\n    def calculate_last(row, attr, attr_map):\n        attr_val = row[attr]\n        if attr_val in attr_map:\n            st = attr_map.get(attr_val)\n            et = row['click_time']\n            val = min((et - st).total_seconds(), 86400)\n        else:\n            val = 86400\n        attr_map[attr_val] = row['click_time']\n        return val\n\n    new_col = 'last_'+col\n    df[new_col] = df.apply(calculate_last, axis=1, attr=col, attr_map={})\n    x = df[[new_col]].values.astype(float)\n    min_max_scaler = preprocessing.MinMaxScaler()\n    x_scaled = min_max_scaler.fit_transform(x)\n    df[new_col] = x_scaled\n    return df[new_col]\n\ndef prepare_dataset(df):\n    \"\"\"Preprocess raw input dataset to enhanced dataset with additional features\"\"\"\n    \n    df['hour'] = df.index.hour\n    df['click_time'] = df.index\n    pool = mp.Pool(maxtasksperchild=1000)\n\n    jobs = [{'df':df,'feat':feat} for feat in ['ip', 'app', 'os', 'channel', 'device']]\n    with tqdm(total=len(jobs), desc=\"Preparing features\") as pbar:\n        for feat in pool.imap(prepare_column, jobs):\n            df = pd.concat([df, feat], axis=1)\n            pbar.update()\n            \n    pool.close()\n    pool.join()\n\n    return df.drop(labels=['is_attributed', 'attributed_time', 'click_time'], axis=1, errors='ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"43d5f2f07befa7e2412c38d373cfb8f102bf63b3"},"cell_type":"code","source":"print('Preparing train dataset')\ntrain_y = dataset_train['is_attributed']\n%time train_X = prepare_dataset(dataset_train)\n\nprint('Preparing test dataset')\ntest_y = dataset_test['is_attributed']\n%time test_X = prepare_dataset(dataset_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"3260cb1125d42dee5c2fb75f5cc6ff499c381ef2"},"cell_type":"code","source":"train_X['last_app'].plot(figsize=(20, 5));","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"fb2f52b7da5427fd5a692ea7dc9eb562df550020"},"cell_type":"code","source":"train_X.tail()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e6645d27f886d8d3259f74ccb67157ba0bca40fe"},"cell_type":"markdown","source":"Additional features to be considered adding:\n- for each attribute number of clicks in previous N minutes with same attribute\nvalue"},{"metadata":{"_uuid":"31951bf7071f8f62b4e67604487adcca618a4853"},"cell_type":"markdown","source":"### Model training"},{"metadata":{"trusted":false,"_uuid":"02135c8f3cbcd274c402579ad6b2317ab3a2c7f4"},"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.metrics import roc_auc_score, classification_report, confusion_matrix\nfrom sklearn import tree\nfrom sklearn.ensemble import GradientBoostingClassifier\nfrom sklearn.ensemble import RandomForestClassifier\n\ndef evaluate_model(clf, tstX=test_X, tsty=test_y, verbose=True):\n    \"\"\"Evaluate model using AUC score on given test data\"\"\"\n    \n    predicted_proba_y = clf.predict_proba(tstX)[:,1]\n    predicted_y = clf.predict(tstX)\n    clf_name = str(clf).split('(')[0]\n    if verbose:\n        display(clf)\n        print(clf_name + ' confusion matrix:')\n        display(pd.DataFrame(confusion_matrix(tsty, predicted_y)))\n    return [{'clf': clf_name,\n            'auc': roc_auc_score(tsty, predicted_proba_y)}]\n\n# Initial model evaluation using fixed random state\nresults = []\nfor clf in [LogisticRegression(random_state=42), \n            GaussianNB(), \n            tree.DecisionTreeClassifier(random_state=42),\n            GradientBoostingClassifier(random_state=42),\n            RandomForestClassifier(random_state=42)]:\n    clf.fit(train_X, train_y)\n    results += evaluate_model(clf)\n    \nresults = pd.DataFrame(results).sort_values(by=['auc'])\n\ndisplay(results)\nresults.plot.bar(x='clf');","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"b7a86f7751ce1a1ade318aef8039c16f9d4ca430"},"cell_type":"code","source":"%%time\n\n# https://www.analyticsvidhya.com/blog/2018/05/improve-model-performance-cross-validation-in-python-r/\nfrom sklearn.model_selection import GridSearchCV, StratifiedKFold\nfrom sklearn.metrics import make_scorer\nfrom sklearn.externals import joblib\n\ndef optimize_model(estimator, params, cv):\n    scorer = make_scorer(roc_auc_score)\n    gs = GridSearchCV(estimator=estimator, param_grid=params, \n                      scoring=scorer, cv=cv, verbose=3,\n                      n_jobs=-1)\n    return gs.fit(train_X, train_y)\n\ncv = StratifiedKFold(n_splits=5, shuffle=True)\nmodel = optimize_model(GradientBoostingClassifier(random_state=42), {\n        'loss' : ['deviance', 'exponential'],\n        'learning_rate': [0.1, 0.2, 0.3],\n        'n_estimators': [50, 100, 200],\n    }, cv=cv)\n\njoblib.dump(model, 'final_model.joblib') \ndisplay(pd.DataFrame(evaluate_model(model)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"0b52b674d4ae86458db1dafd18f8c637f82c2939"},"cell_type":"code","source":"model.estimator","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c2195b893cbba6e8a77cb4ec7dc36f2d96e17ef5"},"cell_type":"markdown","source":"### Sensitivity analysis"},{"metadata":{"trusted":false,"_uuid":"4d4c5e8a64e686345b312172c90585742dd33197"},"cell_type":"code","source":"dataset_test_sens = dataset_test.copy()\n\ntime_delta = pd.Series([ pd.Timedelta(minutes=np.random.randint(-120, 121)) for i in range(len(test_X)) ])\ndataset_test_sens.index = dataset_test_sens.index + time_delta\ndataset_test_sens['click_time'] = dataset_test_sens.index\ndataset_test_sens.sort_index(inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"963089ec0e79a3f1ef2b4d3a7686d99ee4669e54"},"cell_type":"code","source":"display(dataset_test_sens.head())\ndisplay(dataset_test.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"009be9efebbd8cbd3856507e3514d09ecf9b8895"},"cell_type":"code","source":"print('Preparing modified test dataset')\ntest_sens_y = dataset_test_sens['is_attributed']\n%time test_sens_X = prepare_dataset(dataset_test_sens)\n\ntest_sens_X.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"82d417ea0eb73f66af4638b3d4cf6efd696e66a9"},"cell_type":"code","source":"from sklearn.externals import joblib\n\nmodel = joblib.load('final_model.joblib')\n\ndisplay(pd.DataFrame(evaluate_model(model, test_sens_X, test_sens_y)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"57aa37257c191cdbe8c5e32c810cf97b34ce62c6"},"cell_type":"markdown","source":"### Confidence intervals"},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"6d466e3af9ec3a08591e62256fe32f2a6a7d4da7"},"cell_type":"code","source":"from sklearn.externals import joblib\nfrom sklearn.utils import resample\n\nn_bootstraps = 1000\nbootstraps = []\nmodel = joblib.load('final_model.joblib')\n\ndef bootstrap_score(i):\n    sample_X, sample_y = resample(test_X, test_y)\n    res = evaluate_model(model, sample_X, sample_y, False)\n    return res[0]['auc']\n\nwith tqdm(total=n_bootstraps, desc=\"Preparing bootstraps\") as pbar:\n    pool = mp.Pool(maxtasksperchild=1000)\n\n    for bootstrap in pool.imap(bootstrap_score, range(n_bootstraps)):\n        bootstraps.append(bootstrap)\n        pbar.update()\n    pool.close()\n    pool.join()\n        \npd.DataFrame(bootstraps).hist();\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"ef0cac74f586df55b968f976665ec481f8e01deb"},"cell_type":"code","source":"alpha = 0.95\np = ((1.0 - alpha) / 2.0) * 100\nlower = max(0.0, np.percentile(bootstraps, p))\np = (alpha + ((1.0 - alpha) / 2.0)) * 100\nupper = min(1.0, np.percentile(bootstraps, p))\nprint('%.1f confidence interval %.2f%% and %.2f%%' % (alpha*100, lower*100, upper*100))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.0"}},"nbformat":4,"nbformat_minor":1}