{"cells":[{"metadata":{"_uuid":"079b88b0daa642afb776a4e0504b03f206416e3c"},"cell_type":"markdown","source":"## Updated the kernel with:\n* Feature normalization in training and test (improved accuracy 15%)\n* Slightly bigger autoencoder network\n* Added t-SNE visualization"},{"metadata":{"_uuid":"5409098486d0927856e59da322acee465bb85662","trusted":true},"cell_type":"code","source":"# Load libraries\n\nimport numpy as np \nimport pandas as pd \nimport datetime\nimport os\nimport time\n\nimport gc  # garbage collection\n\nimport pickle\nfrom scipy import stats\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nfrom pylab import rcParams\n%matplotlib inline\nsns.set(style='whitegrid', palette='muted', font_scale=1.5)\nrcParams['figure.figsize'] = 12, 5\n\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import train_test_split\nimport tensorflow as tf\nfrom keras.models import Model, load_model\nfrom keras.layers import Input, Dense\nfrom keras.callbacks import ModelCheckpoint, TensorBoard\nfrom keras import regularizers\n\nnp.random.seed(5)","execution_count":1,"outputs":[]},{"metadata":{"_uuid":"0176c7a3569b96670502082d922e88ba3f51f7b2"},"cell_type":"markdown","source":"# EDA\nRedoing little bit, see the below great kernel for details:  \nhttps://www.kaggle.com/yuliagm/talkingdata-eda-plus-time-patterns"},{"metadata":{"_uuid":"0fbb39a13296c9a83236cbf7565377ae65522915","trusted":true},"cell_type":"code","source":"# Load only 500k rows from train data, test data will be loaded later\n\ntrain = pd.read_csv('../input/train.csv', nrows =500000, parse_dates=['click_time'])","execution_count":3,"outputs":[]},{"metadata":{"_uuid":"0a83531eb303b4a5170db62adbc50ad53cfbf5e0","trusted":true},"cell_type":"code","source":"# Quick check\ntrain.head()","execution_count":4,"outputs":[]},{"metadata":{"_uuid":"eacfa7ed406e420c1d5748c5abc431f34a6a1973"},"cell_type":"markdown","source":"## What are the given features in the dataset?"},{"metadata":{"_uuid":"2321b5fbe3b57d2500ddf55699122751861fa159","trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10, 6))\ncols = ['ip', 'app', 'device', 'os', 'channel']\nuniques = [len(train[col].unique()) for col in cols]\nsns.set(font_scale=1.2)\nax = sns.barplot(cols, uniques, log=True)\nax.set(xlabel='Feature', ylabel='log(unique count)', title='Number of unique values per feature')\nfor p, uniq in zip(ax.patches, uniques):\n    height = p.get_height()\n    ax.text(p.get_x()+p.get_width()/2.,\n            height + 10,\n            uniq,\n            ha=\"center\") ","execution_count":5,"outputs":[]},{"metadata":{"_uuid":"3d984113f1f00c3e509cffbe3b2c1d52d7edf08a"},"cell_type":"markdown","source":"## Attributed class has high imbalance!"},{"metadata":{"_uuid":"5aba709892648400c00a7f888adfe3b0f50db659","trusted":true},"cell_type":"code","source":"mean = (train.is_attributed.values == 1).mean()\nax = sns.barplot(['App Downloaded (1)', 'Not Downloaded (0)'], [mean, 1-mean])\nax.set(ylabel='Proportion', title='App Downloaded vs Not Downloaded')\nfor p, uniq in zip(ax.patches, [mean, 1-mean]):\n    height = p.get_height()\n    ax.text(p.get_x()+p.get_width()/2.,\n            height+0.01,\n            '{}%'.format(round(uniq * 100, 2)),\n            ha=\"center\")","execution_count":6,"outputs":[]},{"metadata":{"_uuid":"2c03d219e6bdeb39ecb1039bc3fee7f4dad5b5d9"},"cell_type":"markdown","source":"Data is highly skewed. < 0.2% of clicks actually downloaded apps.  \n99.8% accuracy may completely miss to capture what click actually got converted to download an app. \n\n## Feature Engineering: let us develop more features from the given ones\n\nI followed part of below kernel, did not take all the features:  \nhttps://www.kaggle.com/nanomathias/feature-engineering-importance-testing"},{"metadata":{"_uuid":"f8bb4c305fef5c13bf7f2bf71af15f63013b3703","collapsed":true,"trusted":true},"cell_type":"code","source":"# extract day, minute, hour, second from the click_time\ntrain['day'] = train['click_time'].dt.day.astype('uint8')\ntrain['hour'] = train['click_time'].dt.hour.astype('uint8')\ntrain['minute'] = train['click_time'].dt.minute.astype('uint8')\ntrain['second'] = train['click_time'].dt.second.astype('uint8')","execution_count":7,"outputs":[]},{"metadata":{"_uuid":"a6d75f66c9950eda786a46845710c0ae454fa6d6","collapsed":true,"trusted":true},"cell_type":"code","source":"# Groupby Aggregation\nGROUPBY_AGGREGATIONS = [\n    \n    # V1 - GroupBy Features #\n    #########################    \n    # Variance in day, for ip-app-channel\n    {'groupby': ['ip','app','channel'], 'select': 'day', 'agg': 'var'},\n    # Variance in hour, for ip-app-os\n    {'groupby': ['ip','app','os'], 'select': 'hour', 'agg': 'var'},\n    # Variance in hour, for ip-day-channel\n    {'groupby': ['ip','day','channel'], 'select': 'hour', 'agg': 'var'},\n    # Count, for ip-day-hour\n    {'groupby': ['ip','day','hour'], 'select': 'channel', 'agg': 'count'},\n    # Count, for ip-app\n    {'groupby': ['ip', 'app'], 'select': 'channel', 'agg': 'count'},        \n    # Count, for ip-app-os\n    {'groupby': ['ip', 'app', 'os'], 'select': 'channel', 'agg': 'count'},\n    # Count, for ip-app-day-hour\n    {'groupby': ['ip','app','day','hour'], 'select': 'channel', 'agg': 'count'},\n    # Mean hour, for ip-app-channel\n    {'groupby': ['ip','app','channel'], 'select': 'hour', 'agg': 'mean'}, \n    \n    # V2 - GroupBy Features #\n    #########################\n    # Average clicks on app by distinct users; is it an app they return to?\n    {'groupby': ['app'], \n     'select': 'ip', \n     'agg': lambda x: float(len(x)) / len(x.unique()), \n     'agg_name': 'AvgViewPerDistinct'\n    },\n    # How popular is the app or channel?\n    {'groupby': ['app'], 'select': 'channel', 'agg': 'count'},\n    {'groupby': ['channel'], 'select': 'app', 'agg': 'count'},\n    \n    # V3 - GroupBy Features                                              #\n    # https://www.kaggle.com/bk0000/non-blending-lightgbm-model-lb-0-977 #\n    ###################################################################### \n    {'groupby': ['ip'], 'select': 'channel', 'agg': 'nunique'}, \n    {'groupby': ['ip'], 'select': 'app', 'agg': 'nunique'}, \n    {'groupby': ['ip','day'], 'select': 'hour', 'agg': 'nunique'}, \n    {'groupby': ['ip','app'], 'select': 'os', 'agg': 'nunique'}, \n    {'groupby': ['ip'], 'select': 'device', 'agg': 'nunique'}, \n    {'groupby': ['app'], 'select': 'channel', 'agg': 'nunique'}, \n    {'groupby': ['ip', 'device', 'os'], 'select': 'app', 'agg': 'nunique'}, \n    {'groupby': ['ip','device','os'], 'select': 'app', 'agg': 'cumcount'}, \n    {'groupby': ['ip'], 'select': 'app', 'agg': 'cumcount'}, \n    {'groupby': ['ip'], 'select': 'os', 'agg': 'cumcount'}, \n    {'groupby': ['ip','day','channel'], 'select': 'hour', 'agg': 'var'}    \n]\n\n# Apply all the groupby transformations\nfor spec in GROUPBY_AGGREGATIONS:\n    \n    # Name of the aggregation we're applying\n    agg_name = spec['agg_name'] if 'agg_name' in spec else spec['agg']\n    \n    # Name of new feature\n    new_feature = '{}_{}_{}'.format('_'.join(spec['groupby']), agg_name, spec['select'])\n    \n    # Info\n    #print(\"Grouping by {}, and aggregating {} with {}\".format(\n    #    spec['groupby'], spec['select'], agg_name\n    #))\n    \n    # Unique list of features to select\n    all_features = list(set(spec['groupby'] + [spec['select']]))\n    \n    # Perform the groupby\n    gp = train[all_features]. \\\n        groupby(spec['groupby'])[spec['select']]. \\\n        agg(spec['agg']). \\\n        reset_index(). \\\n        rename(index=str, columns={spec['select']: new_feature})\n        \n    # Merge back to X_total\n    if 'cumcount' == spec['agg']:\n        train[new_feature] = gp[0].values\n    else:\n        train = train.merge(gp, on=spec['groupby'], how='left')\n        \n     # Clear memory\n    del gp\n    gc.collect()","execution_count":8,"outputs":[]},{"metadata":{"_uuid":"04d5f426abae28814b67700074c89b67665db255","collapsed":true,"trusted":true},"cell_type":"code","source":"# Clicks on app ad before & after\n\nHISTORY_CLICKS = {\n    'identical_clicks': ['ip', 'app', 'device', 'os', 'channel'],\n    'app_clicks': ['ip', 'app']\n}\n\n# Go through different group-by combinations\nfor fname, fset in HISTORY_CLICKS.items():\n    \n    # Clicks in the past\n    train['prev_'+fname] = train. \\\n        groupby(fset). \\\n        cumcount(). \\\n        rename('prev_'+fname)\n        \n    # Clicks in the future\n    train['future_'+fname] = train.iloc[::-1]. \\\n        groupby(fset). \\\n        cumcount(). \\\n        rename('future_'+fname).iloc[::-1]","execution_count":9,"outputs":[]},{"metadata":{"_uuid":"fe1db317778bc651f277d6f4b7a04d435ab841e5","collapsed":true,"trusted":true},"cell_type":"code","source":"#train.info()","execution_count":10,"outputs":[]},{"metadata":{"_uuid":"a69fd9dd21c7134fc947889c299ed9f38a49bbeb","collapsed":true,"trusted":true},"cell_type":"code","source":"train = train.drop(['click_time', 'attributed_time'], axis=1)","execution_count":11,"outputs":[]},{"metadata":{"_uuid":"8e45c26356c4af0704c5a919782c8b3a4d88f369","collapsed":true,"trusted":true},"cell_type":"code","source":"train=train.replace(np.nan, 0)","execution_count":12,"outputs":[]},{"metadata":{"_uuid":"03a2abb8d30dc6927033d9c2d81bc3c24859e115","trusted":true},"cell_type":"code","source":"train.isnull().values.any()","execution_count":13,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bb28a85037c55e913f5165add7f12f8cb2bda744"},"cell_type":"code","source":"traincolnames = list(train.columns.values)","execution_count":14,"outputs":[]},{"metadata":{"_uuid":"b1ba15569593a34f6ac15167754cab31e7f67e51"},"cell_type":"markdown","source":"### Normalize the features for model input"},{"metadata":{"trusted":true,"_uuid":"2b7976d483f71f5d60cce0c09dc2ea5ff2b33680"},"cell_type":"code","source":"for v in traincolnames:\n    if v!= 'is_attributed':\n        train[v] = StandardScaler().fit_transform(train[v].values.reshape(-1, 1))","execution_count":15,"outputs":[]},{"metadata":{"_uuid":"6c9e27c48c75d57c74d1f952535e7b40c4c0c9f4","trusted":true},"cell_type":"code","source":"train.shape","execution_count":16,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b8d872db820de178e7c9ec4c1f6e2f523984e56d"},"cell_type":"code","source":"train.describe()","execution_count":17,"outputs":[]},{"metadata":{"_uuid":"eedd5ab5657d41b447aaad722928d67b5113c6f5"},"cell_type":"markdown","source":"## Model building with Autoencoder Neural network using Keras\n\nI shall apply Autoencoder neural network with Keras for this highly skewed dataset.  \nCheck my previous Kernel [here](https://www.kaggle.com/mnpathak1/fraud-detection-analysis-with-nn) for similar analysis and references on Fraud transaction analysis.   "},{"metadata":{"_uuid":"0802e6fbea67e07945bc328c8c80d29fd24daa67","collapsed":true,"trusted":true},"cell_type":"code","source":"Converted = train[train.is_attributed == 1]\nDidNotConvert = train[train.is_attributed == 0]","execution_count":18,"outputs":[]},{"metadata":{"_uuid":"194e5a58d8a8842110383e82b757edf88311b1fc","trusted":true},"cell_type":"code","source":"Converted.shape","execution_count":19,"outputs":[]},{"metadata":{"_uuid":"31d579f5d779947a02b6842d9bd9cee827d0f031","trusted":true},"cell_type":"code","source":"DidNotConvert.shape","execution_count":20,"outputs":[]},{"metadata":{"_uuid":"fb0818fe034ff70e4bfd9642bad135f6abcbf8c9","collapsed":true,"trusted":true},"cell_type":"code","source":"X_train, X_test = train_test_split(train, test_size=0.2, random_state=5)  # split the train data for training model\nX_train = X_train[X_train.is_attributed == 0]    # train on 0 class\nX_train = X_train.drop(['is_attributed'], axis=1)   \ny_test = X_test['is_attributed']\nX_test = X_test.drop(['is_attributed'], axis=1)  \nX_train = X_train.values\nX_test = X_test.values","execution_count":21,"outputs":[]},{"metadata":{"_uuid":"0dcf3d76b1507b946fb0c5fe0684310258869f9c","trusted":true},"cell_type":"code","source":"X_train.shape, X_test.shape","execution_count":22,"outputs":[]},{"metadata":{"_uuid":"88b6b11021ddede51dda193c4dacd9adedba7932"},"cell_type":"markdown","source":"I am using a 9 layer network icluding the input and output layers.  Number of neurons in each layers are 35, 20, 10, 5, 4, 5, 10, 20, 35. Input and output layers have same number of neurons."},{"metadata":{"_uuid":"b511808929760e4d2d31a10e06ec134e58f32441","collapsed":true,"trusted":true},"cell_type":"code","source":"input_dim = X_train.shape[1]\nencoding_dim = 20\n\ninput_layer = Input(shape=(input_dim, ))\nencoder = Dense(encoding_dim, activation=\"relu\", activity_regularizer=regularizers.l1(10e-5))(input_layer)\nencoder = Dense(int(encoding_dim / 2), activation=\"relu\")(encoder)\nencoder = Dense(int(encoding_dim / 4), activation=\"relu\")(encoder)\nencoder = Dense(int((encoding_dim / 4)-1), activation=\"relu\")(encoder)\ndecoder = Dense(int(encoding_dim / 4), activation='relu')(encoder)\ndecoder = Dense(int(encoding_dim / 2), activation='relu')(decoder)\ndecoder = Dense(int(encoding_dim ), activation='relu')(decoder)\ndecoder = Dense(input_dim, activation='relu')(decoder)\nautoencoder = Model(inputs=input_layer, outputs=decoder)","execution_count":23,"outputs":[]},{"metadata":{"_uuid":"eb02a7c4ec2727ffc6746d5c9ba88ba62a03dafd","collapsed":true,"trusted":true},"cell_type":"code","source":"epoch = 10    # large number of iterations help neural network accuracy\nbatch_size = 25  # small batch size around 30 is typically good","execution_count":24,"outputs":[]},{"metadata":{"_uuid":"ab383339ed6bb2dbe9d566befd7c751e5ec16138","collapsed":true,"trusted":true},"cell_type":"code","source":"autoencoder.compile(optimizer='adam', loss='mean_squared_error', metrics=['accuracy'])","execution_count":25,"outputs":[]},{"metadata":{"_uuid":"02be529b35d5c8103674b510d702e2f53e136b89","collapsed":true,"trusted":true},"cell_type":"code","source":"# If needed to use the model later, save it locally\n#checkpointer = ModelCheckpoint(filepath=\"model3.h5\", verbose=0, save_best_only=True)\n#tensorboard = TensorBoard(log_dir='./logs3', histogram_freq=0,   write_graph=True, write_images=True)","execution_count":26,"outputs":[]},{"metadata":{"_uuid":"f0c568b6f1d3220eda9d7150f7068244f2499e29","trusted":true},"cell_type":"code","source":"history = autoencoder.fit(X_train, X_train,\n                    epochs=epoch,\n                    batch_size=batch_size,\n                    shuffle=True,\n                    validation_data=(X_test, X_test),\n                    verbose=1,\n                    #callbacks=[checkpointer, tensorboard]\n                    ).history","execution_count":27,"outputs":[]},{"metadata":{"_uuid":"688e5955dfb438ae40139b16568015084989328c"},"cell_type":"markdown","source":"## Model evaluation\n\n### Loss vs. epoch"},{"metadata":{"_uuid":"8fd53410e73c0fa269cef77cf16264b1b1d55204","trusted":true},"cell_type":"code","source":"plt.plot(history['loss'])\nplt.plot(history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper right');","execution_count":28,"outputs":[]},{"metadata":{"_uuid":"bc43d76d7b993d497d20244efe97f8588733721b"},"cell_type":"markdown","source":"### Accuracy vs. epoch"},{"metadata":{"_uuid":"f851428b6052bc0ab7512a5913f6df69c400932e","trusted":true},"cell_type":"code","source":"plt.plot(history['acc'])\nplt.plot(history['val_acc'])\nplt.title('Model Accuracy')\nplt.ylabel('Accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper right');","execution_count":29,"outputs":[]},{"metadata":{"_uuid":"9f0993c5995d0a0798dea628a5c5b05838feb905"},"cell_type":"markdown","source":"## Model error and characteristics\n\nPredict on test data and calculate mse."},{"metadata":{"_uuid":"615e98047a933d35be67e38fe65f72900dd53c1a","scrolled":true,"trusted":true},"cell_type":"code","source":"predictions = autoencoder.predict(X_test)\nmse = np.mean(np.power(X_test - predictions, 2), axis=1)\nerror_df = pd.DataFrame({'reconstruction_error': mse, 'Converted': y_test})\nerror_df.describe()","execution_count":30,"outputs":[]},{"metadata":{"_uuid":"5ddade30f09131a0be1c784a5aad7f1c5e48b231","trusted":true},"cell_type":"code","source":"predictions.shape","execution_count":31,"outputs":[]},{"metadata":{"_uuid":"ee9d90cdbf3fe78435b2e6ce47158132321497a4"},"cell_type":"markdown","source":"Distribution of error for non-converted (significant majority of the population)."},{"metadata":{"_uuid":"a39b63a775ce9348f92c3da6306909bcfab8b1be","trusted":true},"cell_type":"code","source":"fig = plt.figure()\nax = fig.add_subplot(111)\nDidNotConvert_error_df = error_df[(error_df['Converted']== 0) & (error_df['reconstruction_error'] )]\n_ = ax.hist(DidNotConvert_error_df.reconstruction_error.values, bins=20)","execution_count":32,"outputs":[]},{"metadata":{"_uuid":"6d7968787acc5c31d958f2224631141e9e2f0ade"},"cell_type":"markdown","source":"Distribution of error for all that converted."},{"metadata":{"_uuid":"a70bac5d72464aeabe9f53b9bb8fb1f6ec6e5e3e","trusted":true},"cell_type":"code","source":"fig = plt.figure()\nax = fig.add_subplot(111)\nConverted_error_df = error_df[error_df['Converted'] == 1]\n_ = ax.hist(Converted_error_df.reconstruction_error.values, bins=20)","execution_count":33,"outputs":[]},{"metadata":{"_uuid":"eac80a11c4f77f6f99a4bb0066e571a135087a2b"},"cell_type":"markdown","source":"### ROC\n\nSince the dataset is highly skewed (prediction class < 0.2%), ROC is actually not significant for this problem."},{"metadata":{"_uuid":"e7193a1f9f0a92927a72fa18586562cecc142c7a","collapsed":true,"trusted":true},"cell_type":"code","source":"from sklearn.metrics import (confusion_matrix, precision_recall_curve, auc,\n                             roc_curve, recall_score, classification_report, f1_score,\n                             precision_recall_fscore_support)","execution_count":34,"outputs":[]},{"metadata":{"_uuid":"8cfaad94704dbafbf186e04b3b2f89570447f98f","trusted":true},"cell_type":"code","source":"fpr, tpr, thresholds = roc_curve(error_df.Converted, error_df.reconstruction_error)\nroc_auc = auc(fpr, tpr)\n\nplt.title('Receiver Operating Characteristic')\nplt.plot(fpr, tpr, label='AUC = %0.4f'% roc_auc)\nplt.legend(loc='lower right')\nplt.plot([0,1],[0,1],'r--')\nplt.xlim([-0.001, 1])\nplt.ylim([0, 1.001])\nplt.ylabel('True Positive Rate')\nplt.xlabel('False Positive Rate')\nplt.show();","execution_count":35,"outputs":[]},{"metadata":{"_uuid":"f7329703b454cb4a568b6d9f6da7721fa1f04cd8"},"cell_type":"markdown","source":"### Recall vs. precision"},{"metadata":{"_uuid":"77e87ed7c8c2f24778a0b4e37871785de2c51def","trusted":true},"cell_type":"code","source":"precision, recall, th = precision_recall_curve(error_df.Converted, error_df.reconstruction_error)\nplt.plot(recall, precision, 'b', label='Precision-Recall curve')\nplt.title('Recall vs Precision')\nplt.xlabel('Recall')\nplt.ylabel('Precision')\nplt.show()","execution_count":36,"outputs":[]},{"metadata":{"_uuid":"99fd60ed27da9700c9af4c86d1707c90b6717895"},"cell_type":"markdown","source":"### Precision at different threshold"},{"metadata":{"_uuid":"a894a8d1ec43c912ed98afc799475cc1fb7517bb","trusted":true},"cell_type":"code","source":"plt.plot(th, precision[1:], 'b', label='Threshold-Precision curve')\nplt.title('Precision for different threshold values')\nplt.xlabel('Threshold')\nplt.ylabel('Precision')\nplt.show()","execution_count":37,"outputs":[]},{"metadata":{"_uuid":"ab8141a543fd16d897f8c065dbb89197b8e562c4"},"cell_type":"markdown","source":"### Recall at different threshold"},{"metadata":{"_uuid":"9f6d2f67d55b8d5e2d0d1bceca579dddab30d9fd","trusted":true},"cell_type":"code","source":"plt.plot(th, recall[1:], 'b', label='Threshold-Recall curve')\nplt.title('Recall for different threshold values')\nplt.xlabel('Reconstruction error')\nplt.ylabel('Recall')\nplt.show()","execution_count":38,"outputs":[]},{"metadata":{"_uuid":"1ae5a447c68f15f4a925b8c4c1508b1a1aa48c24"},"cell_type":"markdown","source":"### Error scatterplot showing a threshold"},{"metadata":{"_uuid":"c811458604776777345b5625074f01968ebdc5ea","collapsed":true,"trusted":true},"cell_type":"code","source":"threshold = 0.5","execution_count":39,"outputs":[]},{"metadata":{"_uuid":"b41d378d087b984738e3db11547607edfbed0a21","trusted":true},"cell_type":"code","source":"groups = error_df.groupby('Converted')\nfig, ax = plt.subplots()\n\nfor name, group in groups:\n    ax.plot(group.index, group.reconstruction_error, marker='o', ms=3.5, linestyle='',\n            label= \"Converted\" if name == 1 else \"Did not convert\")\nax.hlines(threshold, ax.get_xlim()[0], ax.get_xlim()[1], colors=\"r\", zorder=100, label='Threshold')\nax.legend()\nplt.title(\"Reconstruction error for different classes\")\nplt.ylabel(\"Reconstruction error\")\nplt.xlabel(\"Data point index\")\nplt.show();","execution_count":40,"outputs":[]},{"metadata":{"_uuid":"a2f5df81bbff4350d6adc2004ccc15590066c57d"},"cell_type":"markdown","source":"### COnfusion matrix for aparticular threshold"},{"metadata":{"_uuid":"9735ab3221a53f4392177b9051925b3effe2a645","trusted":true},"cell_type":"code","source":"LABELS = [\"Did not convert\", \"Converted\"]\ny_pred = [1 if e > threshold else 0 for e in error_df.reconstruction_error.values]\nconf_matrix = confusion_matrix(error_df.Converted, y_pred)\nplt.figure(figsize=(8, 6))\nsns.heatmap(conf_matrix, xticklabels=LABELS, yticklabels=LABELS, annot=True, fmt=\"d\");\nplt.title(\"Confusion matrix\")\nplt.ylabel('Actual class')\nplt.xlabel('Predicted class')\nplt.show()","execution_count":41,"outputs":[]},{"metadata":{"_uuid":"edb629a564ba1a775c76bfb81b10bbea07c2852e"},"cell_type":"markdown","source":"Model captures part of click conversion but also labels significant false click conversion. Therefore need to improve the model:\n* Need more training with more number of epochs, more data and CPU/GPU space (I had 340% CPU usage during this trial). I trained the model on 500k data, did not dare to use >1GB training data on my local machine or here. \n* Better feature selection. I did not add all the features from the previous analysis provided due to memory issue, did not select or remove any particular features.\n* Need to explore different Autoencoder network.\n* Compare with other ML outputs such as logistic regression, RF classifier, GBM.\n\nNow I am going to prepare the test dataset to apply this model.\n\n## Applying features to test data\n\nDid not do it together with train dataset and taking only 500k rows of test dataset due to memory / timeout issues. "},{"metadata":{"_uuid":"2e65110cae3c348431c27aa85678eeef7898539e","trusted":true},"cell_type":"code","source":"test = pd.read_csv('../input/test.csv', nrows=500000, parse_dates=['click_time'])  # I am not using the whole test set\ntest.head()","execution_count":43,"outputs":[]},{"metadata":{"_uuid":"edca7d41ee11d560f9ef8adcbd2604753f650611","trusted":true},"cell_type":"code","source":"# Check feature counts on test dataset\n\nplt.figure(figsize=(10, 6))\ncols = ['ip', 'app', 'device', 'os', 'channel']\nuniques = [len(test[col].unique()) for col in cols]\nsns.set(font_scale=1.2)\nax = sns.barplot(cols, uniques, log=True)\nax.set(xlabel='Feature', ylabel='log(unique count)', title='Number of unique values per feature')\nfor p, uniq in zip(ax.patches, uniques):\n    height = p.get_height()\n    ax.text(p.get_x()+p.get_width()/2.,\n            height + 10,\n            uniq,\n           ha=\"center\") ","execution_count":44,"outputs":[]},{"metadata":{"_uuid":"4d974d289b52845e037607b503dd6b0188baa64c"},"cell_type":"markdown","source":"### Get same features on test datset as in the model or train dataset "},{"metadata":{"_uuid":"56638154af7688901c11b22d50568f5303864c94","collapsed":true,"trusted":true},"cell_type":"code","source":"# extract day, minute, hour, second from the click_time\ntest['day'] = test['click_time'].dt.day.astype('uint8')\ntest['hour'] = test['click_time'].dt.hour.astype('uint8')\ntest['minute'] = test['click_time'].dt.minute.astype('uint8')\ntest['second'] = test['click_time'].dt.second.astype('uint8')","execution_count":45,"outputs":[]},{"metadata":{"_uuid":"6453c08c98c9fa9a6f9e51d9d223c61a47236cd4","collapsed":true,"trusted":true},"cell_type":"code","source":"# Groupby Aggregation\nGROUPBY_AGGREGATIONS = [\n    \n    # V1 - GroupBy Features #\n    #########################    \n    # Variance in day, for ip-app-channel\n    {'groupby': ['ip','app','channel'], 'select': 'day', 'agg': 'var'},\n    # Variance in hour, for ip-app-os\n    {'groupby': ['ip','app','os'], 'select': 'hour', 'agg': 'var'},\n    # Variance in hour, for ip-day-channel\n    {'groupby': ['ip','day','channel'], 'select': 'hour', 'agg': 'var'},\n    # Count, for ip-day-hour\n    {'groupby': ['ip','day','hour'], 'select': 'channel', 'agg': 'count'},\n    # Count, for ip-app\n    {'groupby': ['ip', 'app'], 'select': 'channel', 'agg': 'count'},        \n    # Count, for ip-app-os\n    {'groupby': ['ip', 'app', 'os'], 'select': 'channel', 'agg': 'count'},\n    # Count, for ip-app-day-hour\n    {'groupby': ['ip','app','day','hour'], 'select': 'channel', 'agg': 'count'},\n    # Mean hour, for ip-app-channel\n    {'groupby': ['ip','app','channel'], 'select': 'hour', 'agg': 'mean'}, \n    \n    # V2 - GroupBy Features #\n    #########################\n    # Average clicks on app by distinct users; is it an app they return to?\n    {'groupby': ['app'], \n     'select': 'ip', \n     'agg': lambda x: float(len(x)) / len(x.unique()), \n     'agg_name': 'AvgViewPerDistinct'\n    },\n    # How popular is the app or channel?\n    {'groupby': ['app'], 'select': 'channel', 'agg': 'count'},\n    {'groupby': ['channel'], 'select': 'app', 'agg': 'count'},\n    \n    # V3 - GroupBy Features                                              #\n    # https://www.kaggle.com/bk0000/non-blending-lightgbm-model-lb-0-977 #\n    ###################################################################### \n    {'groupby': ['ip'], 'select': 'channel', 'agg': 'nunique'}, \n    {'groupby': ['ip'], 'select': 'app', 'agg': 'nunique'}, \n    {'groupby': ['ip','day'], 'select': 'hour', 'agg': 'nunique'}, \n    {'groupby': ['ip','app'], 'select': 'os', 'agg': 'nunique'}, \n    {'groupby': ['ip'], 'select': 'device', 'agg': 'nunique'}, \n    {'groupby': ['app'], 'select': 'channel', 'agg': 'nunique'}, \n    {'groupby': ['ip', 'device', 'os'], 'select': 'app', 'agg': 'nunique'}, \n    {'groupby': ['ip','device','os'], 'select': 'app', 'agg': 'cumcount'}, \n    {'groupby': ['ip'], 'select': 'app', 'agg': 'cumcount'}, \n    {'groupby': ['ip'], 'select': 'os', 'agg': 'cumcount'}, \n    {'groupby': ['ip','day','channel'], 'select': 'hour', 'agg': 'var'}    \n]\n\n# Apply all the groupby transformations\nfor spec in GROUPBY_AGGREGATIONS:\n    \n    # Name of the aggregation we're applying\n    agg_name = spec['agg_name'] if 'agg_name' in spec else spec['agg']\n    \n    # Name of new feature\n    new_feature = '{}_{}_{}'.format('_'.join(spec['groupby']), agg_name, spec['select'])\n    \n    # Info\n    #print(\"Grouping by {}, and aggregating {} with {}\".format(\n    #    spec['groupby'], spec['select'], agg_name\n    #))\n    \n    # Unique list of features to select\n    all_features = list(set(spec['groupby'] + [spec['select']]))\n    \n    # Perform the groupby\n    gp = test[all_features]. \\\n        groupby(spec['groupby'])[spec['select']]. \\\n        agg(spec['agg']). \\\n        reset_index(). \\\n        rename(index=str, columns={spec['select']: new_feature})\n        \n    # Merge back to X_total\n    if 'cumcount' == spec['agg']:\n        test[new_feature] = gp[0].values\n    else:\n        test = test.merge(gp, on=spec['groupby'], how='left')\n        \n     # Clear memory\n    del gp\n    gc.collect()","execution_count":46,"outputs":[]},{"metadata":{"_uuid":"c74d399d22b0fd0f927bdb0f20b0730df6f89671","collapsed":true,"trusted":true},"cell_type":"code","source":"# Clicks on app ad before & after\n\nHISTORY_CLICKS = {\n    'identical_clicks': ['ip', 'app', 'device', 'os', 'channel'],\n    'app_clicks': ['ip', 'app']\n}\n\n# Go through different group-by combinations\nfor fname, fset in HISTORY_CLICKS.items():\n    \n    # Clicks in the past\n    test['prev_'+fname] = test. \\\n        groupby(fset). \\\n        cumcount(). \\\n        rename('prev_'+fname)\n        \n    # Clicks in the future\n    test['future_'+fname] = test.iloc[::-1]. \\\n        groupby(fset). \\\n        cumcount(). \\\n        rename('future_'+fname).iloc[::-1]","execution_count":47,"outputs":[]},{"metadata":{"_uuid":"65c864611a2b58ed202021123d8e047a17887422","collapsed":true,"trusted":true},"cell_type":"code","source":"test = test.drop(['click_time'], axis=1)","execution_count":48,"outputs":[]},{"metadata":{"_uuid":"5080e21b2ac5573a0fdd9be4225b0fa1c478f271","collapsed":true,"trusted":true},"cell_type":"code","source":"test=test.replace(np.nan, 0)","execution_count":49,"outputs":[]},{"metadata":{"_uuid":"28e63c95eea5e1bc9afd73fa5992a28b5148747e","trusted":true},"cell_type":"code","source":"test.isnull().values.any()","execution_count":50,"outputs":[]},{"metadata":{"_uuid":"a49c9d6b07c2787037a7cb2a64d3fafa0d7315f8","trusted":true},"cell_type":"code","source":"test.shape","execution_count":51,"outputs":[]},{"metadata":{"_uuid":"d941a5a60cc1fc60fbb8e85009ad3d187b2725c6"},"cell_type":"markdown","source":"### Normalize features in test data"},{"metadata":{"trusted":true,"_uuid":"180f7cd666141067fa4ba221cae2801c4b9597e1"},"cell_type":"code","source":"for v in list(test.columns.values):\n    if v!= 'click_id':\n        test[v] = StandardScaler().fit_transform(test[v].values.reshape(-1, 1))","execution_count":52,"outputs":[]},{"metadata":{"_uuid":"8df4e65b3ffd0ff189b1da7bbb7a1d7b13b7f7f2","trusted":true},"cell_type":"code","source":"test.head()","execution_count":53,"outputs":[]},{"metadata":{"_uuid":"bec5e2b5fb779836e37972220dbde43b894c2062"},"cell_type":"markdown","source":"## Predict based on the mmodel with threshold on test data and submission\n\nI am going to appply the model on the test data and based on the threshold above, I am going to assign if the click was converted to download."},{"metadata":{"_uuid":"28619620d574c9b1f1cd6c6f4bff017251e8e1ed","collapsed":true,"trusted":true},"cell_type":"code","source":"click_id = test['click_id']","execution_count":54,"outputs":[]},{"metadata":{"_uuid":"6314210fb26290e83ae60dcc643089d89c5ac458","collapsed":true,"trusted":true},"cell_type":"code","source":"test = test.drop(['click_id'], axis=1)","execution_count":55,"outputs":[]},{"metadata":{"_uuid":"47b8ba831524b97fa69e71badcf82df275daf8d1","collapsed":true,"trusted":true},"cell_type":"code","source":"predictions_test = autoencoder.predict(test)","execution_count":56,"outputs":[]},{"metadata":{"_uuid":"8cf5d01a25c0df7f6acfb3c73387e42335aa0bd9","trusted":true},"cell_type":"code","source":"predictions_test.shape","execution_count":57,"outputs":[]},{"metadata":{"_uuid":"c7a8bffe648e7f0c18486998add10c91eee9e4bf","trusted":true},"cell_type":"code","source":"mse_test = np.mean(np.power(test - predictions_test, 2), axis=1)\ntest_error_df = pd.DataFrame({'reconstruction_error': mse_test})\ntest_error_df.describe()","execution_count":58,"outputs":[]},{"metadata":{"_uuid":"1476d1acb05243be0b6f2289eae904d78e588113","collapsed":true,"trusted":true},"cell_type":"code","source":"y_test = [1 if e > threshold else 0 for e in test_error_df.reconstruction_error.values]","execution_count":59,"outputs":[]},{"metadata":{"_uuid":"fe6ed0be5980f436d24be8bc304c67722a638914","trusted":true},"cell_type":"code","source":"click_idData = pd.DataFrame(click_id)\ny_testData = pd.DataFrame(y_test)\ny_testData.columns = ['is_attributed']\nresult = pd.concat([click_idData, y_testData], axis=1, join_axes=[click_idData.index])\nresult.head()","execution_count":60,"outputs":[]},{"metadata":{"_uuid":"0dbdd1e007d9735312d0bdd51bc2f71582a4ec3c","collapsed":true,"trusted":true},"cell_type":"code","source":"#result.to_csv(\"TalkingData_Submission_v3.csv\",index=False)","execution_count":61,"outputs":[]},{"metadata":{"_uuid":"486897211830137be00c7db9c61681b6f246a1e8"},"cell_type":"markdown","source":"## Visualizing data with t-SNE analysis\n\nI am taking all is_attributed=1 and sample from is_attributed=0 in the train data to get the t-SNE visualization."},{"metadata":{"collapsed":true,"trusted":true,"_uuid":"5ddf7290b19033ef4cd7d4f1b861e1e6753e1e1a"},"cell_type":"code","source":"from sklearn.manifold import TSNE","execution_count":62,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ae1c4b02b6c11dce785a721ecf42633308fd9605"},"cell_type":"code","source":"# Set the dataset for t-SNE plot\n\ndf2 = train[train.is_attributed == 1]\ndf2 = pd.concat([df2, train[train.is_attributed == 0].sample(n = 5000)], axis = 0)","execution_count":63,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c68cd1a664f4312983da54235fcebe5cdc947a11"},"cell_type":"code","source":"#Scale features to improve the training ability of TSNE.\nstandard_scaler = StandardScaler()\ndf2_std = standard_scaler.fit_transform(df2)","execution_count":64,"outputs":[]},{"metadata":{"collapsed":true,"trusted":true,"_uuid":"d981221b9cf88fedea8affdb45939c379202d309"},"cell_type":"code","source":"#Set y equal to the target values i.e. is_attributed column and all rows\ny = df2.iloc[:,5].values","execution_count":65,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"215f2ba3c2cffd6026ccfd19f2626236c72d2bfa"},"cell_type":"code","source":"tsne = TSNE(n_components=2, random_state=0)\nx_test_2d = tsne.fit_transform(df2_std)","execution_count":66,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1b58e64a89a273ea4d1b117290796d1b484e1240"},"cell_type":"code","source":"#Build the scatter plot with the two types of transactions.\ncolor_map = {0:'red', 1:'blue'}\nplt.figure()\nfor idx, cl in enumerate(np.unique(y)):\n    plt.scatter(x = x_test_2d[y==cl,0], \n                y = x_test_2d[y==cl,1], \n                c = color_map[idx], \n                label = cl)\nplt.xlabel('X in t-SNE')\nplt.ylabel('Y in t-SNE')\nplt.legend(loc='upper left')\nplt.title('t-SNE visualization of a sample of train data')\nplt.show()","execution_count":67,"outputs":[]},{"metadata":{"_uuid":"e27714c572caf5ccd315b4d8fbbbdb8123f65701"},"cell_type":"markdown","source":"We see that converted (is_attributed=1) and rest of non-converted clicks are separated in this t-SNE plot. Feature engineeting and normalization seem to have worked."}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.5.5"}},"nbformat":4,"nbformat_minor":1}