{"cells":[{"cell_type":"code","execution_count":2,"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","collapsed":true,"scrolled":true},"outputs":[],"source":["import gc\n","import numpy as np\n","import pandas as pd\n","from scipy.sparse import csr_matrix\n","from sklearn.naive_bayes import BernoulliNB\n","from sklearn.metrics import accuracy_score, roc_auc_score, f1_score, recall_score, precision_score"]},{"cell_type":"markdown","metadata":{"_cell_guid":"922f67c5-2530-4303-8c6d-9b576d893393","_uuid":"f08288f6c99ccf7219521689dfa577d3a530d510"},"source":["## A. Training\n","*Using the first 40 million rows (for now)*"]},{"cell_type":"markdown","metadata":{"_cell_guid":"c0492bad-ec65-44df-9f3d-c2e3cb4095fa","_uuid":"cf00daf558452aa6d6aa606ec994149b0f273f66"},"source":["### A1. Feature engineering\n","*Probability of each class to have a specific app, device, os or channel - only top 20 probabilities are used because of memory constraints*"]},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"4e9fa8f7-c6d9-499e-b443-cb7cdddb3c96","_uuid":"6ed49b6170b16348654e8bc4927322433c353513","collapsed":true},"outputs":[],"source":["def prepare(data, return_probs=False):\n","    LIMIT = 20\n","    \n","    cl = data[data['is_attributed'] == 1]\n","    vc_app_pos = cl['app'].value_counts(True)[:LIMIT].to_dict()\n","    vc_dev_pos = cl['device'].value_counts(True)[:LIMIT].to_dict()\n","    vc_os_pos = cl['os'].value_counts(True)[:LIMIT].to_dict()\n","    vc_chnl_pos = cl['channel'].value_counts(True)[:LIMIT].to_dict()\n","\n","    cl = data[data['is_attributed'] == 0]\n","    vc_app_neg = cl['app'].value_counts(True)[:LIMIT].to_dict()\n","    vc_dev_neg = cl['device'].value_counts(True)[:LIMIT].to_dict()\n","    vc_os_neg = cl['os'].value_counts(True)[:LIMIT].to_dict()\n","    vc_chnl_neg = cl['channel'].value_counts(True)[:LIMIT].to_dict()\n","\n","    del cl\n","    gc.collect()\n","\n","    name = 'pos'\n","    data['is_' + name + '_app_'] = data['app'].apply(lambda x: vc_app_pos.get(x, 0))\n","    data['is_' + name + '_dev_'] = data['device'].apply(lambda x: vc_dev_pos.get(x, 0))\n","    data['is_' + name + '_os_'] = data['os'].apply(lambda x: vc_os_pos.get(x, 0))\n","    data['is_' + name + '_chnl_'] = data['channel'].apply(lambda x: vc_chnl_pos.get(x, 0))\n","    gc.collect()\n","\n","    name = 'neg'\n","    data['is_' + name + '_app_'] = data['app'].apply(lambda x: vc_app_neg.get(x, 0))\n","    data['is_' + name + '_dev_'] = data['device'].apply(lambda x: vc_dev_neg.get(x, 0))\n","    data['is_' + name + '_os_'] = data['os'].apply(lambda x: vc_os_neg.get(x, 0))\n","    data['is_' + name + '_chnl_'] = data['channel'].apply(lambda x: vc_chnl_neg.get(x, 0))\n","    gc.collect()\n","\n","    data.drop(['app', 'device', 'os', 'channel'], axis=1, inplace=True)\n","    gc.collect()\n","\n","    X = csr_matrix(data.drop('is_attributed', axis=1).values, dtype='float64')\n","    y = data['is_attributed'].values.flatten()\n","    \n","    if return_probs:\n","        return X, y, vc_app_pos, vc_dev_pos, vc_os_pos, vc_chnl_pos, vc_app_neg, vc_dev_neg, vc_os_neg, vc_chnl_neg\n","    else:\n","        return X, y"]},{"cell_type":"markdown","metadata":{"_cell_guid":"b0cb4e95-0aaa-481e-a997-fc21a500da29","_uuid":"afca788eaaba67d5a98f336fe56f57203c964520"},"source":["*Load first 40 million rows*"]},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true},"outputs":[],"source":["data = pd.read_csv('../input/train.csv', \n","                   usecols=[1,2,3,4,7], \n","                   engine='c', \n","                   encoding='ascii', \n","                   na_filter=False, \n","                   dtype=np.uint16,\n","                   nrows=40000000,\n","                   skiprows=1,)\n","data.columns = ['app', 'device', 'os', 'channel', 'is_attributed']\n","gc.collect()"]},{"cell_type":"markdown","metadata":{"_cell_guid":"e493e5c4-07d0-467e-8ff9-c7974dc008fe","_uuid":"b907d1ca070b3a39a5e4496b21556e538381411a"},"source":["*Apply feature engineering*"]},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"884588bf-9a84-4af7-bdc9-29a70c8c438b","_uuid":"b72952421f7ba068c9cbe3603cb534d258ea97c2","collapsed":true},"outputs":[],"source":["X, y = prepare(data)\n","del data\n","gc.collect()"]},{"cell_type":"markdown","metadata":{"_cell_guid":"10ed57e8-2d93-474e-abe4-d0df6feeb383","_uuid":"ba4b04902a7598631c768af2d5efac797073fced"},"source":["### A2. Predictive modelling\n","*Bernoulli Naive Bayes with prior set to the expected value*"]},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"aa3cedc6-f395-4d38-be1b-eabad54b1412","_uuid":"096e430d91b3961590edbf5067dabef169ab9e76","collapsed":true,"scrolled":true},"outputs":[],"source":["y_mean = y.mean()\n","prior = [1. - y_mean, y_mean]\n","m = BernoulliNB(class_prior=prior).partial_fit(X, y, classes=[0,1])\n","gc.collect()"]},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"aa5dc5fe-b77d-469b-a904-5b21ff72c229","_uuid":"ed2bc6f49b8e9c016fa921e08e1e2b04b6c31bb9","collapsed":true},"outputs":[],"source":["y_pred = m.predict_proba(X)[:,1]\n","del X\n","gc.collect()\n","y_pred_round = np.round(y_pred).astype('uint8')"]},{"cell_type":"markdown","metadata":{"_cell_guid":"7e495c3f-760f-416c-b862-32c6fe71656b","_uuid":"2a8f7f95bae9a92ef417b080971384db42f5eb87"},"source":["### A3. Training scores"]},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"10a072fa-1441-4ec4-a873-4d544116287c","_uuid":"633f5ba577f5e30e0ed140cb1613c2bcc2542656","collapsed":true},"outputs":[],"source":["metrics = [\n","    accuracy_score(y, y_pred_round),\n","    roc_auc_score(y, y_pred),\n","    f1_score(y, y_pred_round),\n","    recall_score(y, y_pred_round),\n","    precision_score(y, y_pred_round),\n","]\n","metric_names = ['accuracy', 'roc_auc', 'f1', 'recall', 'precision']\n","del y, y_pred, y_pred_round\n","gc.collect()\n","pd.Series(dict(zip(metric_names, metrics)))"]},{"cell_type":"markdown","metadata":{"_cell_guid":"21c917b8-f797-429a-9bb4-b6b8f5571ff3","_uuid":"03ce64b5636f241244ef5b3a8339dfb9f67afc49"},"source":["## B. Cross-validation\n","*Using the next 40 million rows of the training set*"]},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"e637ed05-cbfe-4d2c-9451-323c160bc723","_uuid":"7bb7e2c9dde8ada3f783d1cf0db14604ab603f42","collapsed":true},"outputs":[],"source":["data = pd.read_csv('../input/train.csv', \n","                   usecols=[1,2,3,4,7], \n","                   engine='c', \n","                   encoding='ascii', \n","                   na_filter=False, \n","                   dtype=np.uint16,\n","                   skiprows=40000001,\n","                   nrows=40000000,\n","                   header=None)\n","data.columns = ['app', 'device', 'os', 'channel', 'is_attributed']\n","gc.collect()"]},{"cell_type":"markdown","metadata":{"_cell_guid":"86e9316e-ee95-41e1-af72-d92b477d183c","_uuid":"06062d6c48a0fe3734e074280ba8eb981a0cd139"},"source":["### B1. Apply same feature engineering on validation set"]},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"f6fe24bd-848d-4ac7-b633-226b94ede808","_uuid":"33a2f1d8aeac00b83911abb762078c9cfb02bb3d","collapsed":true},"outputs":[],"source":["X, y = prepare(data)\n","del data\n","gc.collect()"]},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"dbf444a7-e7db-4875-8efa-073cacc97e99","_uuid":"3ae903e1c5cc325a79b01644df26ccb84770d2c1","collapsed":true},"outputs":[],"source":["y_pred = m.predict_proba(X)[:,1]\n","y_pred_round = np.round(y_pred).astype('uint8')"]},{"cell_type":"markdown","metadata":{"_cell_guid":"606ad9d3-7513-4579-890f-c5805835dfc8","_uuid":"1e3c3809b92190c77e16012fcf5a837ab72ca93b"},"source":["### B2. Cross-Validation Scores"]},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"1709e143-9d26-476b-b22c-b1a7882f9af4","_uuid":"3c9b05a7f08842e6b55a58609b26ac72fdc60147","collapsed":true},"outputs":[],"source":["metrics = [\n","    accuracy_score(y, y_pred_round),\n","    roc_auc_score(y, y_pred),\n","    f1_score(y, y_pred_round),\n","    recall_score(y, y_pred_round),\n","    precision_score(y, y_pred_round),\n","]\n","del y_pred, y_pred_round\n","gc.collect()\n","metric_names = ['accuracy', 'roc_auc', 'f1', 'recall', 'precision']\n","pd.Series(dict(zip(metric_names, metrics)))"]},{"cell_type":"markdown","metadata":{"_cell_guid":"da94a854-47cc-4f65-8f60-7d415daf39c8","_uuid":"4c7909c483e4bb055511b1a4d20686d494258685"},"source":["## C. Prediction on test set\n","*Now, we fit the model on the entire dataset*"]},{"cell_type":"markdown","metadata":{"_cell_guid":"0de0672a-f0f6-49d2-95cf-58705db2da59","_uuid":"535a3843ecc4500e57afdd98f9bcc8d9f647bb11"},"source":["### C1. Fit the remaining data to the existing model to improve performance further"]},{"cell_type":"markdown","metadata":{"_cell_guid":"57c205fe-b8e6-4e80-b6b5-c2d14e953849","_uuid":"2be89d651fcc581338d646cbefb13bd466bd4054"},"source":["*Second set of 40M rows (validation set)*"]},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"988e8c03-7a02-420b-849f-8f7ac74a21c7","_uuid":"0ac33828c4cbd541beb7eac43ac9dbb025a81fc5","collapsed":true},"outputs":[],"source":["m.partial_fit(X, y)\n","del X\n","del y\n","gc.collect()"]},{"cell_type":"markdown","metadata":{"_cell_guid":"db32ed03-af49-409b-a297-4398794c7112","_uuid":"6759de1b746732d09c71ed94f13c83c578ab310a"},"source":["*Third set of 40M rows*"]},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"bdc0fa20-c90e-49b0-80f8-a695645bbbb8","_uuid":"ac8d66a686b0cf85d8c152c7afef9149d8a66311","collapsed":true},"outputs":[],"source":["data = pd.read_csv('../input/train.csv', \n","                   usecols=[1,2,3,4,7], \n","                   engine='c', \n","                   encoding='ascii', \n","                   na_filter=False, \n","                   dtype=np.uint16,\n","                   skiprows=80000001,\n","                   nrows=40000000,\n","                   header=None)\n","data.columns = ['app', 'device', 'os', 'channel', 'is_attributed']\n","gc.collect()\n","\n","X, y = prepare(data)\n","del data\n","m.partial_fit(X, y)\n","del X, y\n","gc.collect()"]},{"cell_type":"markdown","metadata":{"_cell_guid":"111b7bac-fd6b-4cc2-b4f8-d7ffe7bec5f5","_uuid":"b8643f6cf5b4572bae2c33ba135b92d8267e4dce"},"source":["*Remaining set of rows (final)*"]},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"3c0ba780-b87b-4ac0-9df0-754d7742466d","_uuid":"504292e33ab1e9d070a955a9ae0b36afc965baf6","collapsed":true},"outputs":[],"source":["data = pd.read_csv('../input/train.csv', \n","                   usecols=[1,2,3,4,7], \n","                   engine='c', \n","                   encoding='ascii', \n","                   na_filter=False, \n","                   dtype=np.uint16,\n","                   skiprows=120000001,\n","                   header=None)\n","data.columns = ['app', 'device', 'os', 'channel', 'is_attributed']\n","gc.collect()\n","\n","X, y, vc_app_pos, vc_dev_pos, vc_os_pos, vc_chnl_pos, vc_app_neg, vc_dev_neg, vc_os_neg, vc_chnl_neg = prepare(data, return_probs=True)\n","del data\n","m.partial_fit(X, y)\n","del X, y\n","gc.collect()"]},{"cell_type":"markdown","metadata":{"_cell_guid":"799d0488-d836-4a2b-b38c-dc3e4d0cab97","_uuid":"278c94eea96754588501a99312c002d5c7ad7234"},"source":["### C2. Apply same feature engineering on test set (using probabilities from final set)"]},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"c4f7cd8b-b2a0-436a-909d-d5f0b0833081","_uuid":"d958e12d21da4ae95509fb2fa9f9096943878278","collapsed":true},"outputs":[],"source":["data = pd.read_csv('../input/test.csv', \n","                   usecols=['click_id', 'app', 'device', 'os', 'channel'],\n","                   engine='c', \n","                   encoding='ascii', \n","                   na_filter=False, \n","                   dtype=np.uint32)"]},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"ebaffb64-d5f5-4a55-9518-130dd1340dcc","_uuid":"f2ad641d8eddd05d13a34cdd53374e0ce6be7b86","collapsed":true},"outputs":[],"source":["ids = data['click_id'].values.flatten().tolist()\n","data.drop('click_id', axis=1, inplace=True)"]},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"3ded9685-957e-4e41-8617-9b1aa2f36651","_uuid":"17d3d32a5ba37bda9fcb8a3098c1c61ba3bc6a24","collapsed":true},"outputs":[],"source":["name = 'pos'\n","data['is_' + name + '_app_'] = data['app'].apply(lambda x: vc_app_pos.get(x, 0))\n","data['is_' + name + '_dev_'] = data['device'].apply(lambda x: vc_dev_pos.get(x, 0))\n","data['is_' + name + '_os_'] = data['os'].apply(lambda x: vc_os_pos.get(x, 0))\n","data['is_' + name + '_chnl_'] = data['channel'].apply(lambda x: vc_chnl_pos.get(x, 0))\n","gc.collect()\n","\n","name = 'neg'\n","data['is_' + name + '_app_'] = data['app'].apply(lambda x: vc_app_neg.get(x, 0))\n","data['is_' + name + '_dev_'] = data['device'].apply(lambda x: vc_dev_neg.get(x, 0))\n","data['is_' + name + '_os_'] = data['os'].apply(lambda x: vc_os_neg.get(x, 0))\n","data['is_' + name + '_chnl_'] = data['channel'].apply(lambda x: vc_chnl_neg.get(x, 0))\n","gc.collect()\n","\n","data.drop(['app', 'device', 'os', 'channel'], axis=1, inplace=True)\n","gc.collect()\n","\n","X = csr_matrix(data.values, dtype='float64')\n","\n","del data\n","gc.collect()"]},{"cell_type":"markdown","metadata":{"_cell_guid":"532fd70c-3e2d-4ef3-9eeb-d222cc4b7241","_uuid":"2e3165275b94b8b9afd485ccd23bc929fa61b225"},"source":["### C3. Save results\n","*Re-calibrate the probabilities by rank and feed to sigmoid function for smoothing*"]},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"f7600ad0-eaa3-4174-b2b5-4dd6ae2ad783","_uuid":"6a0468c7bb18eed3b513653ef1ad782d2de6e366","collapsed":true},"outputs":[],"source":["ranked = pd.Series(m.predict_proba(X)[:,1]).rank(method='min').astype('float64')\n","ranked /= len(ranked)\n","ranked -= 0.5 # center at zero\n","ranked *= 12 # stretch out to the interval (-6, 6)\n","ranked = 1 / (1 + np.exp(-ranked)) # apply sigmoid\n","pd.DataFrame({\n","    'click_id': ids,\n","    'is_attributed': ranked\n","}).to_csv('result.csv', index=False, float_format='%.4f')"]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.4"}},"nbformat":4,"nbformat_minor":1}