{"cells":[{"metadata":{"_uuid":"f98c71b9c12134633477169bf8de487bcc98d5cb","_cell_guid":"f6c832c3-e3b1-455d-bb74-5ff26e17d819"},"cell_type":"markdown","source":"Even though Deep Learning doesn't seem to be doing that well in this competition, I believe it is still an interesting approach and a great learning experience.\n\nIn this notebook I continue and improve on the work by Mr. Kireev from his great DL script [here](https://www.kaggle.com/alexanderkireev/deep-learning-support-9663). All credit goes to him, I have simply tuned hyperparameters and messed around with the layers of the network (+ some validation).\n\nFirst we will read the data, starting after row 131886954."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","collapsed":true,"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":false},"cell_type":"code","source":"import gc\nimport time\nimport numpy as np\nimport pandas as pd\nfrom sklearn.cross_validation import train_test_split\nimport warnings\nwarnings.filterwarnings('ignore')\nimport os\nos.environ['OMP_NUM_THREADS'] = '4'\n\npath = '../input/'\ndtypes = {\n        'ip'            : 'uint32',\n        'app'           : 'uint16',\n        'device'        : 'uint16',\n        'os'            : 'uint16',\n        'channel'       : 'uint16',\n        'is_attributed' : 'uint8',\n        'click_id'      : 'uint32'\n        }\n\nprint('loading')\n\ntrain_columns = ['ip', 'app', 'device', 'os', 'channel', 'click_time', 'is_attributed']\ntest_columns = ['click_id', 'ip', 'app', 'device', 'os', 'channel', 'click_time']\ntrain_df = pd.read_csv(path+\"train.csv\", dtype=dtypes, skiprows = range(1, 131886954), usecols=train_columns)\ntest_df = pd.read_csv(path+\"test.csv\", dtype=dtypes, usecols=test_columns)\n\nprint('finished loading')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c44d5e1927b0a98391e7a8c70149899d76029e1b","_cell_guid":"90ad58c7-0eca-462c-be9c-5015b373b2eb"},"cell_type":"markdown","source":"The preprocessing is the same as the [original kernel](https://www.kaggle.com/alexanderkireev/deep-learning-support-9663). I highly suggest you add features like 'next_click', found in the most high-scoring kernels of the competition."},{"metadata":{"_uuid":"07bcf27d2da01c7a30a5c53eecbfe20dc4bd0925","collapsed":true,"_cell_guid":"c1ce582d-c8b0-4740-8dc0-f4300b63d04d","trusted":false},"cell_type":"code","source":"def prep_data(d):\n    print('hour, day, wday....')\n    d['hour'] = pd.to_datetime(d.click_time).dt.hour.astype('uint8')\n    d['day'] = pd.to_datetime(d.click_time).dt.day.astype('uint8')\n    d['wday']  = pd.to_datetime(d.click_time).dt.dayofweek.astype('uint8')\n    print('grouping by ip-day-hour combination....')\n    gp = d[['ip', 'day', 'hour', 'channel']].groupby(by=['ip', 'day', 'hour'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'qty'})\n    d = d.merge(gp, on=['ip', 'day', 'hour'], how='left')\n    del gp; gc.collect()\n    print('group by ip-app combination....')\n    gp = d[['ip', 'app', 'channel']].groupby(by=['ip', 'app'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'ip_app_count'})\n    d = d.merge(gp, on=['ip', 'app'], how='left')\n    del gp; gc.collect()\n    print('group by ip-app-os combination....')\n    gp = d[['ip', 'app', 'os', 'channel']].groupby(by=['ip', 'app', 'os'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'ip_app_os_count'})\n    d = d.merge(gp, on=['ip', 'app', 'os'], how='left')\n    del gp; gc.collect()\n    print(\"vars and d type....\")\n    d['qty'] = d['qty'].astype('uint16')\n    d['ip_app_count'] = d['ip_app_count'].astype('uint16')\n    d['ip_app_os_count'] = d['ip_app_os_count'].astype('uint16')\n    print(\"label encoding....\")\n    from sklearn.preprocessing import LabelEncoder\n    d[['app', 'device', 'os', 'channel', 'hour', 'day', 'wday']].apply(LabelEncoder().fit_transform)\n    print('dropping')\n    d.drop(['click_time', 'ip'], 1, inplace=True)\n    \n    return d","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"272772bf7040b4c4f12942bbc0b824c6a6862a7d","collapsed":true,"_cell_guid":"63aae1c3-920f-4fa6-a481-e0d60ee1c21c","trusted":false},"cell_type":"code","source":"train_df = prep_data(train_df)\ntest_df = prep_data(test_df)\nprint(\"finished\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"99e0a55d2b29714f0d95c14dba6d779a840b449b","_cell_guid":"b872e056-9933-4a31-827b-ce7f9d7d44c6"},"cell_type":"markdown","source":"We will keep 10% of the training dataset for validation."},{"metadata":{"_uuid":"4f10e5e381b477ca179a52a54d32b7cc9f0b19ec","collapsed":true,"_cell_guid":"25c8b89a-9974-4384-8ffe-5b5c3daadf8c","trusted":false},"cell_type":"code","source":"RANDOM_SEED = 1\nimport random\nrandom.seed(RANDOM_SEED)\nnp.random.seed(RANDOM_SEED)\n\nX_train, X_test = train_test_split(train_df, test_size=0.1, random_state=RANDOM_SEED)\n\ny_train = X_train['is_attributed']\nX_train = X_train.drop(['is_attributed'], axis=1)\ny_test = X_test['is_attributed']\nX_test = X_test.drop(['is_attributed'], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2c39e0026dfea87ef77f013a7e986d20f15b5e4e","_cell_guid":"845063d6-4dc3-4074-8fb4-eabc24775f6e"},"cell_type":"markdown","source":"## Building Our Model"},{"metadata":{"_uuid":"dd687254fa92eb3aa164bb1cae8c98f737d25314","collapsed":true,"_cell_guid":"8348140a-05a6-4adf-b72a-bcaaed4012f5","trusted":false},"cell_type":"code","source":"from keras.layers import Input, Embedding, Dense, Flatten, Dropout, concatenate\nfrom keras.layers import BatchNormalization, SpatialDropout1D, Conv1D\nfrom keras.callbacks import Callback\nfrom keras.models import Model\nfrom keras.optimizers import Adam","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"813eca588b6fb58312513855742a52ae0a68b74d","collapsed":true,"_cell_guid":"94c22103-510a-4da6-bce9-94d532c10bcf","trusted":false},"cell_type":"code","source":"def get_keras_data(dataset):\n    X = {\n        'app': np.array(dataset.app),\n        'ch': np.array(dataset.channel),\n        'dev': np.array(dataset.device),\n        'os': np.array(dataset.os),\n        'h': np.array(dataset.hour),\n        'd': np.array(dataset.day),\n        'wd': np.array(dataset.wday),\n        'qty': np.array(dataset.qty),\n        'c1': np.array(dataset.ip_app_count),\n        'c2': np.array(dataset.ip_app_os_count)\n    }\n    return X\n\ntrain = get_keras_data(X_train)\ntest = get_keras_data(X_test)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4deb2b4ba4553e582f44a31ee18c689427368c36","collapsed":true,"_cell_guid":"8f2db467-84c6-495a-9ae5-d0b73f06e106","trusted":false},"cell_type":"code","source":"max_app = np.max([train_df['app'].max(), test_df['app'].max()])+1\nmax_ch = np.max([train_df['channel'].max(), test_df['channel'].max()])+1\nmax_dev = np.max([train_df['device'].max(), test_df['device'].max()])+1\nmax_os = np.max([train_df['os'].max(), test_df['os'].max()])+1\nmax_h = np.max([train_df['hour'].max(), test_df['hour'].max()])+1\nmax_d = np.max([train_df['day'].max(), test_df['day'].max()])+1\nmax_wd = np.max([train_df['wday'].max(), test_df['wday'].max()])+1\nmax_qty = np.max([train_df['qty'].max(), test_df['qty'].max()])+1\nmax_c1 = np.max([train_df['ip_app_count'].max(), test_df['ip_app_count'].max()])+1\nmax_c2 = np.max([train_df['ip_app_os_count'].max(), test_df['ip_app_os_count'].max()])+1","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"def5b2fc34cae09852c26f1f694a6e22bdb33e4c","collapsed":true,"_cell_guid":"4a6d042c-be87-418f-b617-b87f38a90aa5","trusted":false},"cell_type":"code","source":"emb_n = 50\n\nin_app = Input(shape=[1], name = 'app')\nemb_app = Embedding(max_app, emb_n)(in_app)\nin_ch = Input(shape=[1], name = 'ch')\nemb_ch = Embedding(max_ch, emb_n)(in_ch)\nin_dev = Input(shape=[1], name = 'dev')\nemb_dev = Embedding(max_dev, emb_n)(in_dev)\nin_os = Input(shape=[1], name = 'os')\nemb_os = Embedding(max_os, emb_n)(in_os)\nin_h = Input(shape=[1], name = 'h')\nemb_h = Embedding(max_h, emb_n)(in_h) \nin_d = Input(shape=[1], name = 'd')\nemb_d = Embedding(max_d, emb_n)(in_d) \nin_wd = Input(shape=[1], name = 'wd')\nemb_wd = Embedding(max_wd, emb_n)(in_wd) \nin_qty = Input(shape=[1], name = 'qty')\nemb_qty = Embedding(max_qty, emb_n)(in_qty) \nin_c1 = Input(shape=[1], name = 'c1')\nemb_c1 = Embedding(max_c1, emb_n)(in_c1) \nin_c2 = Input(shape=[1], name = 'c2')\nemb_c2 = Embedding(max_c2, emb_n)(in_c2) \nfe = concatenate([(emb_app), (emb_ch), (emb_dev), (emb_os), (emb_h), \n                 (emb_d), (emb_wd), (emb_qty), (emb_c1), (emb_c2)])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1027f33e1aaa320c3f72cf5cb2f2e4cb0a1bc255","_cell_guid":"c29dff1a-79c2-40bc-b425-6d6e036ea9b1"},"cell_type":"markdown","source":"Below we will build the main core of the network. If you want to mess around, I suggest you start by adjusting the `Dropout` parameters. I have noticed that up to 0.5 it works really well."},{"metadata":{"_uuid":"5809695337539ae5103be1e53a92f1a15cd123e8","collapsed":true,"_cell_guid":"f86e3fc1-d6c3-43dc-9859-a20eefe8d75e","trusted":false},"cell_type":"code","source":"s_dout = SpatialDropout1D(0.25)(fe)\n\nfl1 = Flatten()(s_dout)\nconv1 = Conv1D(70, kernel_size=2, strides=1, padding='same')(s_dout)\nfl2 = Flatten()(conv1)\nconv2 = Conv1D(70, kernel_size=4, strides=1, padding='same')(s_dout)\nfl3 = Flatten()(conv2)\n\nconcat = concatenate([(fl1), (fl2), (fl3)])\n\nx = Dropout(0.25)(Dense(500, activation='relu')(concat))\nx = Dropout(0.35)(Dense(750, activation='relu')(x))\nx = Dropout(0.45)(Dense(1000, activation='relu')(x))\noutp = Dense(1, activation='sigmoid')(x)\n\nmodel = Model(inputs=[in_app,in_ch,in_dev,in_os,in_h,in_d,in_wd,in_qty,in_c1,in_c2], outputs=outp)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cae8e7df9d43cb90b6873e8babd274deb96a9065","_cell_guid":"ff7ee343-f95d-4192-88ca-d9df90915800"},"cell_type":"markdown","source":"Time to train our network!\n\nPossible improvements include increasing `batch_size` and adjusting the `lr_decay` values."},{"metadata":{"_uuid":"b0b087386ae755bd94162b8711bb0a46e2ea263f","collapsed":true,"_cell_guid":"7ada4048-e658-4e39-8bec-9e811f89eafc","trusted":false},"cell_type":"code","source":"batch_size = 50000\nepochs = 1\nexp_decay = lambda init, fin, steps: (init/fin)**(1/(steps-1)) - 1\nsteps = int(len(train_df) / batch_size) * epochs\nlr_init, lr_fin = 0.0025, 0.0001\nlr_decay = exp_decay(lr_init, lr_fin, steps)\noptimizer_adam = Adam(lr=0.0025, decay=lr_decay)\nmodel.compile(loss='binary_crossentropy', optimizer=optimizer_adam, metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8b5ed06eb8d44ac31bab54954ce6e7336b26f06e","collapsed":true,"_cell_guid":"2b637a71-7f81-40f5-b464-23ff1be1c9d9","trusted":false},"cell_type":"code","source":"class_weight = {0:.01,1:.99} # magic\nmodel.fit(train, y_train, batch_size=batch_size, epochs=epochs, class_weight=class_weight, shuffle=True, validation_data=(test, y_test), verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c70234f84a09238047d87340d00aa071a489bc08","collapsed":true,"_cell_guid":"b87cb312-67d2-473d-8481-79a88d18bb1d","trusted":false},"cell_type":"code","source":"predictions = model.predict(test, batch_size=batch_size, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"40e405b3f6d5672983fa03bc3b1a33637d078ff7","_cell_guid":"b92df68b-28d4-49a8-a86d-a3f59f62fee6"},"cell_type":"markdown","source":"## Validation\n\nHere we will perform some basic validation on 10% of the collected training data set. We will calculate the ROC-AUC score, from this script [here](https://www.kaggle.com/antmarakis/calculating-and-plotting-auc-score)."},{"metadata":{"_uuid":"783c77b1b49a56ed56365f984f19e213c7d6a11d","collapsed":true,"_cell_guid":"4f62ec99-9f6a-4920-94c8-b0b4493e0d51","trusted":false},"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom sklearn.metrics import (confusion_matrix, precision_recall_curve, auc,\n                             roc_curve, recall_score, classification_report, f1_score,\n                             precision_recall_fscore_support)\n\nfpr, tpr, thresholds = roc_curve(y_test, predictions)\nroc_auc = auc(fpr, tpr)\n\nplt.title('Receiver Operating Characteristic')\nplt.plot(fpr, tpr, label='AUC = %0.4f'% roc_auc)\nplt.legend(loc='lower right')\nplt.plot([0,1],[0,1],'r--')\nplt.xlim([-0.001, 1])\nplt.ylim([0, 1.001])\nplt.ylabel('True Positive Rate')\nplt.xlabel('False Positive Rate')\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4e9290b1c37fd37e94f2b280ca12fdfcbadb5ac6","_cell_guid":"b743923e-6618-4bb7-9baf-3ab6f8e0850e"},"cell_type":"markdown","source":"## Submission\n\nFinally, creating the submission file:"},{"metadata":{"_uuid":"f409d871bc4238f01c173d1eccc5c9e3c7ec951b","collapsed":true,"_cell_guid":"fb5dc002-eaff-4bd0-8442-c81cd14e7d2a","trusted":false},"cell_type":"code","source":"ids = test_df['click_id']\ntest_df.drop('click_id', axis=1, inplace=True)\nt = get_keras_data(test_df)\nsub_preds = model.predict(t, batch_size=batch_size, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"987913094f3ee0313facde2cc2aeb514995781e6","collapsed":true,"_cell_guid":"724a4189-d014-425b-b038-c5607cc308da","trusted":false},"cell_type":"code","source":"sub = pd.DataFrame()\nsub['click_id'] = ids\nsub['is_attributed'] = sub_preds\nsub.to_csv('dl_sub.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}