{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":false},"cell_type":"markdown","source":"## Pre-processing"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom datetime import datetime\n\nimport warnings\nwarnings.filterwarnings('ignore')\n%matplotlib inline\n\nfrom sklearn.utils import resample\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn import metrics\nfrom sklearn import model_selection\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.ensemble import RandomForestClassifier\n!pip install lightgbm\nimport lightgbm as lgb","execution_count":1,"outputs":[]},{"metadata":{"_uuid":"2b05261662561b0a0a42d280ee8b115f537a613e"},"cell_type":"markdown","source":"## Load in Data"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"df655f1d53bde1014b8b40a010eaacd273503dbb"},"cell_type":"code","source":"DATA_PATH = r\"../input\"\n\ndef load_data(data_path=DATA_PATH):\n    # PATHS TO FILE\n    train_path = os.path.join(data_path, \"train.csv\")\n    test_path = os.path.join(data_path, \"test.csv\")\n    ssize = 50000000\n    return pd.read_csv(train_path,nrows=ssize), pd.read_csv(test_path)\n\ntrain, test = load_data()","execution_count":2,"outputs":[]},{"metadata":{"_uuid":"4e481c1095f5f494b49510009f4d1e5049222dcc"},"cell_type":"markdown","source":"## Data Exploration"},{"metadata":{"trusted":true,"_uuid":"5897fd8dcefbc00f10046265bc95ff797b2bdbbe"},"cell_type":"code","source":"# Training sample\nprint(train.shape)\ntrain.head()","execution_count":3,"outputs":[]},{"metadata":{"_uuid":"f70ea6abc6d2ad16bf173d18275664b57ac79586"},"cell_type":"markdown","source":"We notice that all the missing values in `attributed_time` are for observations that did not convert into a download (`is_attributed`=0)."},{"metadata":{"trusted":true,"_uuid":"7983d5014b05db3182cc18acfeb8ff2a6b7867e5"},"cell_type":"code","source":"# Plot the proportion of clicks that converted into a download or not\nplt.figure(figsize=(6,6))\nmean = (train.is_attributed.values == 1).mean()\nax = sns.barplot(['Converted (1)', 'Not Converted (0)'], [mean, 1-mean])\nax.set(ylabel='Proportion', title='Proportion of clicks converted into app downloads')\nfor p, uniq in zip(ax.patches, [mean, 1-mean]):\n    height = p.get_height()\n    ax.text(p.get_x()+p.get_width()/2.,\n            height+0.01,\n            '{}%'.format(round(uniq * 100, 2)),\n            ha=\"center\")","execution_count":4,"outputs":[]},{"metadata":{"_uuid":"66ebd9ec750e74df931880c95607c6d07ef84b90"},"cell_type":"markdown","source":"## Undersampling"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"184244c172d867671982e4591dadad5f2af79c15"},"cell_type":"code","source":"# Separate the 2 classes\ntrain_0 = train[train['is_attributed'] == 0]\ntrain_1 = train[train['is_attributed'] == 1]","execution_count":5,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"000df66e9b91578a410ec0760da2dbd1dac15b09"},"cell_type":"code","source":"print(len(train_1))\nprint(train_0.shape)\nprint(train.shape)\ntrain['is_attributed'].value_counts()","execution_count":6,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"a1be3944f306e10d4be3f0fb152045c1744100fe"},"cell_type":"code","source":"# Undersample class 0 (without replacement)\ntrain0_undersampled = resample(train_0, replace=False, n_samples=len(train_1), random_state=142)","execution_count":7,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cb1d3959fb271d87395556b07ed69645a811963a"},"cell_type":"code","source":"# Combine minority class with downsampled majority class\ntrain_us = pd.concat([train0_undersampled, train_1])\n \n# Display new class counts\ntrain_us.is_attributed.value_counts()","execution_count":8,"outputs":[]},{"metadata":{"_uuid":"326712591cc08b7f21d4c1bd05fd0dc3d8a50925"},"cell_type":"markdown","source":"## Feature Engineering"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"214f3e37cc1f7e4d8e0748982e5909185d52b6c3"},"cell_type":"code","source":"# Extract features from click_time\ndef ppClicktime(df):\n    df['click_time'] = pd.to_datetime(df['click_time'])\n    df['wday'] = df['click_time'].dt.dayofweek\n    df['hour'] = df['click_time'].dt.hour\n    return df\ntrain_pp = ppClicktime(train)\ntest_pp = ppClicktime(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"81a09f6820fda3477c50b75af25275edf46f5657"},"cell_type":"code","source":"# Drop click_time\ntrain_pp.drop('click_time', axis = 1, inplace = True)\ntest_pp.drop('click_time', axis = 1, inplace = True)\nprint(len(test_pp))\ntest_pp.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"785ec5f337536220d37964e7ae4d72398b6a8654"},"cell_type":"code","source":"# Write to csv\ntrain_pp.to_csv(\"train_pp.csv\",index=None)","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}