{"cells":[{"metadata":{"_uuid":"49e9bb2b773f22126b9e9cd62692ffd0c5c3c878"},"cell_type":"markdown","source":"**An RNN Approach to TalkingData Fraud Detection Dataset\n**\n\nThis kernel is inspired by the work of the following folks. I have added the GRU, RNN layers to it.\n\n**Alexander Kireev**\nhttps://www.kaggle.com/alexanderkireev/deep-learning-support-9663\n\n**Andy Harless:**\nhttps://www.kaggle.com/aharless/variation-on-alexander-kireev-s-dl\n\n**Noobhound:**\nhttps://www.kaggle.com/knowledgegrappler/a-simple-nn-solution-with-keras-0-48611-pl\n\n**Yyll008:**\nhttps://www.kaggle.com/yyll008/gru-25-12-12-with-keras-512-64-relu-sgdr-lb0-432\n"},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"a405903db38229d5f944ce01223d84407afb3908"},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom datetime import datetime ","execution_count":1,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"a09133c9d63df5164f6fa1f30e4c0b94487f49d6"},"cell_type":"code","source":"init_time = datetime.now()","execution_count":2,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"c49c731ee2de9c8e54044633cbb1b05e39d2248d"},"cell_type":"code","source":"# CONSTANTS\n\nIS_DEV = True\nTREAT_OUTLIERS = False\n\nif(IS_DEV == True):\n    TRAIN_FILE = '../input/train.csv'\n    TEST_FILE = '../input/test.csv'\n    COMPRESSION = None\n    SKIP_ROWS = range(1, 170000000)\nelse:\n    TRAIN_FILE = '../input/train.csv.zip'\n    TEST_FILE = '../input/test.csv.zip'\n    COMPRESSION = 'zip'\n    \nTRAIN_SAMPLE_FILE = '../input/' + 'train_sample.csv'\n    \nLEARNING_RATE_INIT = 0.001\nLEARNING_RATE_END = 0.0001\nBATCH_SIZE = 20000\nEPOCHS = 2\n\nEMBEDDING_N = 50\nDENSE_N = 1024\n\nSPATIAL_DROPOUT_1D = 0.2\nDROPOUT_1 = 0.2\nDROPOUT_2 = 0.2","execution_count":3,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"7ce8ebfcaf9b5b447b1ab4d8d984cb01571a59a0"},"cell_type":"code","source":"dtypes = {\n    'ip': 'uint32',\n    'app': 'uint16',\n    'device': 'uint16',\n    'os': 'uint16',\n    'channel': 'uint16',\n    'is_attributed': 'uint8',\n    'click_id': 'uint32'\n}","execution_count":4,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"a6bb7e534e625f25cb0bbfd26a7e65bead16a90c"},"cell_type":"code","source":"# Ensure file path is correct.\ntrain_sample = pd.read_csv(    \n    TRAIN_SAMPLE_FILE, \n    dtype=dtypes,\n    usecols=['ip', 'app', 'device', 'os', 'channel', 'click_time', 'is_attributed'],\n    compression = None,\n    header=0,\n    #skiprows=range(1, 181886954),  # 2016-01-01)\n    engine='c'\n)\ntrain_sample.head()","execution_count":5,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"b2a236b771070f439118a0f37969d607da84be76"},"cell_type":"code","source":"# Load Data\nprint('Loading data...')\nstart_time = datetime.now()\ntrain = pd.read_csv(    \n    TRAIN_FILE, \n    dtype=dtypes,\n    usecols=['ip', 'app', 'device', 'os', 'channel', 'click_time', 'is_attributed'],\n    compression = COMPRESSION,\n    header=0,\n    skiprows=SKIP_ROWS,  # 2016-01-01)\n    engine='c'\n)\n\ntest = pd.read_csv(    \n    TEST_FILE, \n    dtype=dtypes,\n    usecols=['ip', 'app', 'device', 'os', 'channel', 'click_time', 'click_id'],\n    compression = COMPRESSION,\n    header=0,\n    #skiprows=range(1, 17886954),  # 2016-01-01)\n    engine='c'\n)\n\nprint('End of Loading Data: {time_taken}'.format(time_taken=datetime.now() - start_time))","execution_count":6,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"71a1164aa577edd02cc6d8dfb4b3c574699441ce"},"cell_type":"code","source":"train.head()","execution_count":7,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"e0c5a883b2109b83e1c4480c62f72e4e2e84cc3a"},"cell_type":"code","source":"if(False):\n\n    import seaborn as sns\n    import matplotlib.pyplot as plt\n\n    from mlxtend.preprocessing import minmax_scaling\n    from scipy import stats\n\n\n    positive_is_attributed = train.is_attributed.loc[train.is_attributed > 0]\n\n    # Normalize\n    normalized_is_attributed = stats.boxcox(positive_is_attributed)[0]\n    fig, ax = plt.subplots(1, 3)\n    sns.distplot(train.is_attributed, ax=ax[0])\n    ax[0].set_title('Original Data')\n    sns.distplot(normalized_is_attributed[0], ax=ax[1])\n    ax[1].set_title('Normalized Data')","execution_count":8,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"807fac2ca153e2537e8d485000efe3fc30aee850"},"cell_type":"code","source":"# Remove Outliers\nprint(len(train))\n\nclass Outliers():\n\n    data = None\n    def __init__(self, data):\n        \"\"\"\n        INPUT\n        data - Dataframe having date column in MM-DD-YYYY format\n        \"\"\"\n        self.data = data\n\n    def treat_outliers_by_iqr(self, colName, what_sd=1.5):\n        mean_value = np.mean(self.data[colName].values)\n        sd_value = np.std(self.data[colName].values)\n\n        max_value = mean_value + what_sd * sd_value\n        min_value = mean_value - what_sd * sd_value\n\n        new_data = self.data[(self.data[colName] >= min_value) & (self.data[colName] <= max_value)]\n        outliers = self.data[(self.data[colName] > max_value) | (self.data[colName] < min_value)]\n\n        print('%Outliers: {outlier_percentage:<8} Mean: {mean_value:<8} SD: {sd_value:<8} IQR Min: {iqr_min:<8} IQR Max: {iqr_max:<8} Min: {mini:<8} Max: {maxi:<8}  Outliers: {outliers:<8} #App: {app_count:<8} #Obs: {size}'.format(\n            outlier_percentage=np.around((len(outliers)/len(self.data))*100, decimals=2), \n            mean_value=np.around(mean_value, decimals=2), sd_value=np.around(sd_value, decimals=2), \n            iqr_min=np.around(min_value, decimals=2),\n            iqr_max=np.around(max_value, decimals=2),\n            mini=np.around(min(self.data[colName].values), decimals=2), \n            maxi=np.around(max(self.data[colName].values), decimals=2),\n            outliers=len(outliers),\n            app_count=len(new_data.app.unique()), \n            size=len(new_data)\n        ))\n\n        return outliers, new_data\n    \nif(TREAT_OUTLIERS == True):   \n    print('Remove Outliers...')\n    start_time = datetime.now()\n\n    outs = Outliers(train)\n    outliers, train = outs.treat_outliers_by_iqr('is_attributed')\n\n    print('End of Outlier Removal: {time_taken}'.format(time_taken=datetime.now() - start_time))","execution_count":9,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"b9c0ebcf27312247fddb12a6f96e70c2f0f6c0f6"},"cell_type":"code","source":"train_size = len(train)\ntrain = train.append(test)\ndel test\n\nimport gc\ngc.collect()","execution_count":10,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"f481e39da58a020287fa9cbf4efb4d95430f579a"},"cell_type":"code","source":"# Datetime Object\nprint('Making Hour, Day and Weekday Columns...')\nstart_time = datetime.now()\n\ndate_time = pd.to_datetime(train.click_time)\n\ntrain['hour'] = date_time.dt.hour.astype('uint8')\ntrain['day'] = date_time.dt.day.astype('uint8')\ntrain['wday'] = date_time.dt.dayofweek.astype('uint8')\n\nprint('End of Making HDWd: {time_taken}'.format(time_taken=datetime.now() - start_time))","execution_count":11,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"0f0a4392609541a058405aaaa7b9b65d07d04476"},"cell_type":"code","source":"print('Making number of channels by IP, Day, HOUR...')\ngroup_ip_dhc = train[['ip', 'day', 'hour', 'channel']].groupby(by=['ip', 'day', 'hour'])\ngroup_ip_dhc = group_ip_dhc[['channel']].count().reset_index().rename(index=str, columns={'channel': 'qty'})\ntrain = train.merge(group_ip_dhc, on=['ip', 'day', 'hour'], how='left')\n\ndel group_ip_dhc\ngc.collect\n\ntrain.head()\n\nprint('End of Making channel count by IDH: {time_taken}'.format(time_taken=datetime.now() - start_time))","execution_count":12,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"a405cf14e81e7d8c05720da38923d0dc10db7a7d"},"cell_type":"code","source":"# Group by IP, APP, OS Combination\nprint('Making number of channels by IP, APP...')\nstart_time = datetime.now()\n\ngroup_ip_ac = train[['ip', 'app', 'channel']].groupby(by=['ip', 'app'])\ngroup_ip_ac = group_ip_ac[['channel']].count().reset_index().rename(index=str, columns={'channel': 'ip_app_count'})\ntrain = train.merge(group_ip_ac, on=['ip', 'app'], how='left')\n\ndel group_ip_ac\ngc.collect()\n\ntrain.head()\n\nprint('End of Making channel count by IA: {time_taken}'.format(time_taken=datetime.now() - start_time))","execution_count":13,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"12717e06099489bffde43b4c85855db6fbc70431"},"cell_type":"code","source":"print('Making number of channels by IP, APP, OS...')\nstart_time = datetime.now()\n\ngroup_ip_aoc = train[['ip', 'app', 'os', 'channel']].groupby(by=['ip', 'app', 'os'])\ngroup_ip_aoc = group_ip_aoc[['channel']].count().reset_index().rename(index=str, columns={'channel': 'ip_app_os_count'})\ntrain = train.merge(group_ip_aoc, on=['ip', 'app', 'os'], how='left')\n\ndel group_ip_aoc\ngc.collect()\n\nprint('End of  Making channel count by IAO: {time_taken}'.format(time_taken=datetime.now() - start_time))","execution_count":14,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"a7e9018dd806239d413117a5c2ef4a11301dff64"},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder","execution_count":15,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"6fb2a5b9425113f4c296ea11d4315218754d267d"},"cell_type":"code","source":"print('Label encoding and fit transformation...')\n\nstart_time = datetime.now()\ntrain[['app', 'device', 'os', 'channel', 'hour', 'day', 'wday']].apply(LabelEncoder().fit_transform)\n\nprint('End of  Making channel count by IAO: {time_taken}'.format(time_taken=datetime.now() - start_time))\n# train.head()","execution_count":16,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"41a39e68b7ac9102b19ca0b38ee71fa4aabc7907"},"cell_type":"code","source":"print('Splitting train and test data...')\nstart_time = datetime.now()\n\ntest = train[train_size:]\nprint(len(test))\ntrain = train[:train_size]\n\ny_train = train['is_attributed'].values\nprint(len(train))\ntrain.drop(['click_id', 'click_time', 'ip', 'is_attributed'], 1, inplace=True)\n\nprint('End of Splitting Train and Test Dataset: {time_taken}'.format(time_taken=datetime.now() - start_time))","execution_count":17,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"89d116aecebffaa4b70d487e6f8c834247ecbc20"},"cell_type":"code","source":"# Neural Network\nfrom keras.layers import Input, Embedding, Dense, Flatten, Dropout, concatenate\nfrom keras.layers import BatchNormalization, SpatialDropout1D, GRU\nfrom keras.callbacks import Callback\nfrom keras.models import Model\nfrom keras.optimizers import Adam","execution_count":18,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"c88f14ee41f1b32846785a92bb3c4f53a8a58506"},"cell_type":"code","source":"max_app = np.max([train['app'].max(), test['app'].max()]) + 1\nmax_channel = np.max([train['channel'].max(), test['channel'].max()]) + 1\nmax_device = np.max([train['device'].max(), test['device'].max()]) + 1\nmax_os = np.max([train['os'].max(), test['os'].max()]) + 1\nmax_hour = np.max([train['hour'].max(), test['hour'].max()]) + 1\nmax_day = np.max([train['day'].max(), test['day'].max()]) + 1\nmax_wday = np.max([train['wday'].max(), test['wday'].max()]) + 1\nmax_qty = np.max([train['qty'].max(), test['qty'].max()]) + 1\nmax_c1 = np.max([train['ip_app_count'].max(), test['ip_app_count'].max()]) + 1\nmax_c2 = np.max([train['ip_app_os_count'].max(), test['ip_app_os_count'].max()]) + 1\n","execution_count":19,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"d8b103d63b68aa21839258a6a5934e5f1d679c67"},"cell_type":"code","source":"print('Preparing dataset for training...')\nstart_time = datetime.now()\n\ndef get_keras_data(dataset):\n    X = {\n        'app': np.array(dataset.app),\n        'channel': np.array(dataset.channel),\n        'device': np.array(dataset.device),\n        'os': np.array(dataset.os),\n        'hour': np.array(dataset.hour),\n        'day': np.array(dataset.day),\n        'wday': np.array(dataset.wday),\n        'qty': np.array(dataset.qty),\n        'c1': np.array(dataset.ip_app_count),\n        'c2': np.array(dataset.ip_app_os_count)\n    }\n    return X\n\ntrain = get_keras_data(train)\n\nin_app = Input(shape=[1], name='app')\nemb_app = Embedding(max_app, EMBEDDING_N)(in_app)\n\nin_channel = Input(shape=[1], name='channel')\nemb_channel = Embedding(max_channel, EMBEDDING_N)(in_channel)\n\nin_device = Input(shape=[1], name='device')\nemb_device = Embedding(max_device, EMBEDDING_N)(in_device)\n\nin_os = Input(shape=[1], name='os')\nemb_os = Embedding(max_os, EMBEDDING_N)(in_os)\n\nin_hour = Input(shape=[1], name='hour')\nemb_hour = Embedding(max_hour, EMBEDDING_N)(in_hour)\n\nin_day = Input(shape=[1], name='day')\nemb_day = Embedding(max_day, EMBEDDING_N)(in_day)\n\nin_wday = Input(shape=[1], name='wday')\nemb_wday = Embedding(max_wday, EMBEDDING_N)(in_wday)\n\nin_qty = Input(shape=[1], name='qty')\nemb_qty = Embedding(max_qty, EMBEDDING_N)(in_qty)\n\nin_c1 = Input(shape=[1], name='c1')\nemb_c1 = Embedding(max_c1, EMBEDDING_N)(in_c1)\n\nin_c2 = Input(shape=[1], name='c2')\nemb_c2 = Embedding(max_c2, EMBEDDING_N)(in_c2)\n\n\nprint('End of Dataset Preparation: {time_taken}'.format(time_taken=datetime.now() - start_time))","execution_count":20,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"14fb33c266fe31ae6b4609aef375657b0c80df23"},"cell_type":"code","source":"print('Create RNN Layers...')\nstart_time = datetime.now()\n\nrnn_layer_1 = GRU(16)(emb_app)\nrnn_layer_2 = GRU(8)(emb_channel)\nrnn_layer_3 = GRU(8)(emb_device)\nrnn_layer_4 = GRU(8)(emb_os)\n\nfe = concatenate([\n    Flatten()(emb_app), \n    Flatten()(emb_channel), \n    Flatten()(emb_device), \n    Flatten()(emb_os), \n    Flatten()(emb_hour), \n    Flatten()(emb_day), \n    Flatten()(emb_wday), \n    Flatten()(emb_qty), \n    Flatten()(emb_c1), \n    Flatten()(emb_c2),\n    rnn_layer_1,\n    rnn_layer_2,\n    rnn_layer_3,\n    rnn_layer_4\n])\nprint('End of RNN Layer Creation: {time_taken}'.format(time_taken=datetime.now() - start_time))","execution_count":21,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"1f9b323fe80d7af70aec9219b120d381bf8a0602"},"cell_type":"code","source":"print('Building TF architecture...')\nstart_time = datetime.now()\n\n#s_dout = SpatialDropout1D(SPATIAL_DROPOUT_1D)(fe)\n#x = Flatten()(s_dout)\nx = Dropout(DROPOUT_1)(Dense(DENSE_N, activation='relu')(fe))\nx = Dropout(DROPOUT_1)(Dense(512, activation='relu')(x))\nx = Dropout(DROPOUT_2)(Dense(256, activation='relu')(fe))\nx = Dropout(DROPOUT_2)(Dense(128, activation='relu')(x))\noutp = Dense(1,activation='sigmoid')(x)\nmodel = Model(inputs=[in_app, in_channel, in_device, in_os,\n    in_hour, in_day, in_wday, in_qty, in_c1, in_c2], outputs=outp)\n\nprint('End of Building TF Architecture: {time_taken}'.format(time_taken=datetime.now() - start_time))","execution_count":22,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"e6c2860c4016752452c305715ba0d59d753328f6"},"cell_type":"code","source":"exp_decay = lambda init, fin, steps: (init/fin)**(1/(steps -1)) - 1","execution_count":23,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"a19438efd0471dbf9a4e25f12af1a1f0d402dd7b"},"cell_type":"code","source":"steps = int(len(train) / BATCH_SIZE) * EPOCHS\nlr_init, lr_fin = LEARNING_RATE_INIT, LEARNING_RATE_END\nlr_decay = exp_decay(lr_init, lr_fin, steps)","execution_count":24,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"f2c58ff6548bff155661ece47e25eae1acce8a6a"},"cell_type":"code","source":"print('Creating the model...')\nstart_time = datetime.now()\n\noptimizer_adam = Adam(lr=0.001, decay=lr_decay)\nmodel.compile(loss='binary_crossentropy', optimizer=optimizer_adam, metrics=['accuracy'])\nmodel.summary()\n\nprint('End of Creating the Model: {time_taken}'.format(time_taken=datetime.now() - start_time))","execution_count":25,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"7bbf9a727e97d4dc472d2f65a1062dcec5ecbf7c"},"cell_type":"code","source":"print('Training...')\nstart_time = datetime.now()\n\nclass_weight = {0:0.01, 1:0.99}\nmodel.fit(train, y_train, batch_size=BATCH_SIZE, epochs=EPOCHS, class_weight=class_weight, shuffle=True, verbose=1)\ndel train, y_train\ngc.collect()\n\n\nprint('End of Training: {time_taken}'.format(time_taken=datetime.now() - start_time))","execution_count":18,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"047f01979fc32ba16685b9fa0bcb99a5097effc2"},"cell_type":"code","source":"print('Preparing test dataset...')\nstart_time = datetime.now()\n\nsubmit = pd.DataFrame()\nsubmit['click_id'] = test['click_id'].astype('int')\ntest.drop(['click_id', 'click_time', 'ip', 'is_attributed'], 1, inplace=True)\ntest = get_keras_data(test)\n\nprint('End of Test Dataset: {time_taken}'.format(time_taken=datetime.now() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"d5418d803e11b4360f579084b9db5256d8f28253"},"cell_type":"code","source":"print('Preparing predicted output...')\nstart_time = datetime.now()\n\nsubmit['is_attributed'] = model.predict(test, batch_size=BATCH_SIZE, verbose=2)\ndel test\ngc.collect()\nsubmit.to_csv('output.csv', index=False)\n\nprint('End of Predicted Output: {time_taken}'.format(time_taken=datetime.now() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"5d51dee55834df7570a3803bea4d82ba54dfb393"},"cell_type":"code","source":"print('The END: {time_taken}'.format(time_taken=datetime.now() - init_time))","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"a0920a4d1767b257c68047818f3e344f23170d15"},"cell_type":"code","source":"if(IS_DEV == False):\n    import h5py\n    model.save_weights('../output/weights.h5')\n    model.save('../output/model.h5')","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false,"_uuid":"3e09a6bd9e2fa07daef3094208b5947b092244fa"},"cell_type":"code","source":"\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}