{"cells":[{"metadata":{"_uuid":"161e21affa8ec4636358009ed81673c7c0d5ee2e","_cell_guid":"c4907bdc-cf9b-41be-b0d7-bbb8f5e92203"},"cell_type":"markdown","source":"# Implementação de Random Forrest\n## Equipe \"Econometristas\" - Alessandro Rivello e Raul Guarini"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","collapsed":true,"trusted":false},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport time\nimport gc\n# Input data files are available in the \"../input/\" directory\ndtypes = {\n    'ip'            : 'uint32',\n    'app'           : 'uint16',\n    'device'        : 'uint16',\n    'os'            : 'uint16',\n    'channel'       : 'uint16',\n    'is_attributed' : 'uint8',\n    'click_id'      : 'uint32'\n}\n\npath = '../input/'\n\n# Funcoes para manejar os dados de click_time\ndef diff_timestamp(df, drop_na = False):\n    df['click_time'] = pd.to_datetime(df['click_time'])\n    df['diff'] = df.groupby('ip').click_time.diff().dt.total_seconds()\n    # Se drop_na = True, a funcao joga fora informacao\n    if drop_na == True:\n        nulls = df['diff'].isnull()\n        df.drop(nulls[nulls == True].index, inplace = True)\n    return df\n\ndef handleClickHour_raul(df):\n    df['click_hour']= pd.to_datetime(df['click_time']).dt.hour.astype('uint8')\n    #df['click_minute'] = pd.to_datetime(df['click_time']).dt.minute.astype('uint8')\n    #df['click_second'] = pd.to_datetime(df['click_time']).dt.second.astype('uint8')\n    df = df.drop(['click_time'], axis=1)   \n    return df","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ba4d728548f5004fd1aaea79135d07bfbc28675c"},"cell_type":"markdown","source":"Nesta implementação, utilizamos as últimas 35 milhões de linhas do dataset. O que motivou essa escolha foi tentar trabalhar com dados mais próximos dos dados de teste na questão temporal."},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false,"collapsed":true},"cell_type":"code","source":"# Importando dados de teste\ntrain_columns = ['ip', 'app', 'device', 'os', 'channel', 'click_time', 'is_attributed']\nstart_time = time.time()\ndf_train_30m = pd.read_csv(path + 'train.csv', dtype=dtypes, skiprows=range(1,163333333), \n                           nrows=35000000, usecols=train_columns)\nprint('Loaded df_train_30m with {:f} seconds'.format(time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1ab1d76d881b8a2fbdaccd65b61fd159cfea9ce9","_cell_guid":"74d67712-8f50-4765-9108-c05e5a9c8f28","trusted":false,"collapsed":true},"cell_type":"code","source":"# Importando dados de treinamento\nstart_time = time.time()\ndf_test = pd.read_csv(path + 'test.csv', dtype=dtypes)\nprint('Loaded df_test in {:.2f} seconds'.format(round(time.time() - start_time, 3)))\n\ntrain_record_index = df_train_30m.shape[0]\n\n# Ajeitando o click_hour \ndf_train_30m = diff_timestamp(df_train_30m)\ndf_train_30m = handleClickHour_raul(df_train_30m)\ndf_test = diff_timestamp(df_test)\ndf_test = handleClickHour_raul(df_test)\ngc.collect();\nprint('ClickTime data correctly handled.')\n\n# DataFrame de submissão\ndf_submit = pd.DataFrame()\ndf_submit['click_id'] = df_test['click_id']\n\n# Extraindo o target de treinamento\nLearning_Y = df_train_30m['is_attributed']\nprint('Training target correctly extracted.')\n\n# Dropando informação redundante\ndf_test = df_test.drop(['click_id'], axis=1)\ndf_train_30m = df_train_30m.drop(['is_attributed'], axis=1)\ngc.collect();\n\n# Juntando os dois datasets para podermos criar features nos dois ao mesmo tempo\ndf_merge = pd.concat([df_train_30m, df_test])\n\n# Liberando espaço na memória\ndel df_train_30m, df_test\ngc.collect();\nprint('Data was correctly concatenated')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"41e9e7e68751b5755418359a10639334e8907b89"},"cell_type":"markdown","source":"Na célula acima criamos um dataframe com dados de teste e treinamento para a crianção de features utilizando informação de ambos datasets. Abaixo, criamos uma feature que é a contagem de vezes que um mesmo ip apareceu."},{"metadata":{"_uuid":"729bb9508e43ce8b0bc18baf15bdd25f9524e1f9","_cell_guid":"d191b6b0-a51e-459c-b090-52fd444f4bf2","trusted":false,"collapsed":true},"cell_type":"code","source":"# Criando feature que conta quantos cliques aquele IP deu\nstart_time = time.time()\ndf_ip_count = df_merge['ip'].value_counts().reset_index(name = 'ip_count')\ndf_ip_count.columns = ['ip', 'ip_count']\nprint('Loaded df_ip_count with {:.2f} seconds'.format(time.time() - start_time))\nprint('Starting to merge with main dataset...')\ndf_merge = df_merge.merge(df_ip_count, on='ip', how='left', sort=False)\ndf_merge['ip_count'] = df_merge['ip_count'].astype('uint16')\nprint('Merging operation completed.')\ndel df_ip_count\ndf_merge = df_merge.drop(['ip'], axis=1)\ngc.collect();","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fb84f78661de3cac14ab83e0a741f643840a902f","_cell_guid":"96b3869d-bf5f-4fd2-a001-d369727e4933","trusted":false,"collapsed":true},"cell_type":"code","source":"df_merge.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ae1ea6ee9b9ce9bf2d8b9466ccfb8f0ea1fc8678","_cell_guid":"5024e5ad-a06e-49c8-973d-5c86cfcef2bf"},"cell_type":"markdown","source":"### Preparativos para Machine Learning\nImplementamos uma maneira de avaliar o modelo com ROC score e criamos dados de cross-validation."},{"metadata":{"_uuid":"d2ec93a30040a651fd1d866b347fa9c5cde36d0d","_cell_guid":"67143cc8-fba7-467b-8997-d1cb5c1125c0","collapsed":true,"trusted":false},"cell_type":"code","source":"# Usando a métrica do ROC Score\nfrom sklearn.metrics import roc_auc_score\n#from sklearn.metrics import accuracy_score\n#from sklearn.metrics import average_precision_score\nfrom sklearn.metrics import f1_score\n\ndef clf_eval(y_true, y_pred):\n    print('Classification Report')\n   # print('F-1 Score: {}'.format(f1_score(y_true, y_pred)))\n    print('ROC Score: {}'.format(roc_auc_score(y_true, y_pred)))\n    return roc_auc_score(y_true, y_pred)\n\n# Recuperando os dados de treino e teste\ndf_train = df_merge[:train_record_index]\ndf_test = df_merge[train_record_index:]\ndel df_merge\ngc.collect();","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dcb8daad3238e66c94abdb475884df79f1499140","_cell_guid":"63b14b73-4893-4016-ac93-b643ed7fc26d","trusted":false,"collapsed":true},"cell_type":"code","source":"# Checando que fração dos dois datasets tem clicks com NaN na coluna diff (que seriam primeiros cliques!)\nprint('Fração de NaN em df_train: {:.4f} %'.format(100*len(df_train[df_train['diff'].isnull()])/len(df_train)))\nprint('Fração de NaN em df_test: {:.4f} %'.format(100*len(df_test[df_test['diff'].isnull()])/len(df_test)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fb34b5584a5404b704d2c132b3993992db220d1e","_cell_guid":"b6e7d9f9-ad6a-495f-b873-df7fe435d0ea"},"cell_type":"markdown","source":"Há poucas observações com valores NaN na coluna diff. Precisamos preenchê-los. Faremos isso colocando zeros no lugar. Afinal, por definição, não há click anterior a eles."},{"metadata":{"_uuid":"8cc6c1118310f99d3bff8dab80d706813f3bb87f","_cell_guid":"2f5dfabe-71e7-496a-b9c3-5df9abd45a17","collapsed":true,"trusted":false},"cell_type":"code","source":"df_train['diff'].fillna(0, inplace = True)\ndf_test['diff'].fillna(0, inplace = True)\ngc.collect();\n\n# Otimizando a memória\ndf_train['diff'] = df_train['diff'].astype('uint8')\ndf_test['diff'] = df_test['diff'].astype('uint8')\nLearning_Y = Learning_Y.astype('uint8')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1db1b791eca65c2b93361337f0252bbb5ddfc203","_cell_guid":"e2a22a71-79c1-4284-aade-34e97be1134d","trusted":false,"collapsed":true},"cell_type":"code","source":"df_test.info()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7fda4e94cd279893eb36bb83b91d928206c740dc","_cell_guid":"8b929131-b21a-46f5-981a-427737fd3873","trusted":false,"collapsed":true},"cell_type":"code","source":"# Criando conjunto de cross-validation\nfrom sklearn import model_selection\nX_train, X_cv, Y_train, Y_cv = model_selection.train_test_split(df_train, Learning_Y, train_size = 0.8)\ngc.collect();\nprint('Data splitting into training and cross validation is done.')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"111cb502e8719449bdeac87dfdf390156dc47f19","_cell_guid":"38d305c6-daeb-4985-8603-c6f773610329"},"cell_type":"markdown","source":"### Implementando Random Forest\nChecamos que um número muito grande de árvores não alterava tanto o resultado. Os parâmetros abaixo são os melhores que encontramos em nossos testes."},{"metadata":{"_uuid":"60d7756318095688ca297dd82c4be7dd120770ce","_cell_guid":"4b43f77f-6372-481a-96f8-fcd6017871fa","trusted":false,"collapsed":true},"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nprint('Starting to fit Random Forest Model... The machine is learning...')\nstart_time = time.time()\nrf = RandomForestClassifier(n_estimators=13, max_depth=13, random_state=13, verbose=2, n_jobs = 4)\n\ncols = ['app', 'os', 'channel', 'device', 'click_hour', 'diff', 'ip_count']\nrf.fit(X_train[cols], Y_train)\nprint('The machine has learned.')\nprint('RandomForest has fitted X_train with {:.2f} seconds'.format(time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"231ecfafd79aae0ef0a16b31564d58f3077329af","_cell_guid":"6795a072-7e92-468e-a38d-467a32bd080d","trusted":false,"collapsed":true},"cell_type":"code","source":"print('Starting cross-validation prediction phase...')\nstart_time = time.time()\npredictions = rf.predict_proba(X_cv[cols])[:,1]\nprint('Prediction done. Elapsed time: {:.2f} seconds'.format(time.time() - start_time))\n\n# Avaliando\nclf_eval(Y_cv, predictions)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9ef696de3fe71cce0862a1addf16b4ed5e5f4036","_cell_guid":"47912f51-70f3-4917-83f1-895f14735f90"},"cell_type":"markdown","source":"A seguir, testamos se alguma técnica de  **undersampling** poderis nos ajudar."},{"metadata":{"_uuid":"e6f41e213fa4d40d863afdb42ec264b36c015e93","_cell_guid":"8d482b29-04c6-41c6-ba8b-726780680547","collapsed":true,"trusted":false},"cell_type":"code","source":"# Implementando UnderSampler\nfrom imblearn.under_sampling import RandomUnderSampler\nrus = RandomUnderSampler(return_indices=True)\nX_resampled, Y_resampled, idx_resampled = rus.fit_sample(X = X_train, y = Y_train)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4217f2d5e4dfb2288e82ea829f741db94ca5b489","_cell_guid":"52ccfd23-bfdd-4e54-8353-104d72de67b9","collapsed":true,"trusted":false},"cell_type":"code","source":"# Transformando em Dataframes\nX_resampled = pd.DataFrame(X_resampled, columns=df_train.columns)\nY_resampled = pd.DataFrame(Y_resampled, columns=['is_attributed'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2ae18d1ba534cd4b35e53e24a90736d8e00418e4","_cell_guid":"f1abf128-b8c2-4cb2-a4ea-7541638cf400","trusted":false,"collapsed":true},"cell_type":"code","source":"# Implementando o Random Forest com resampling\nrf_resampled = RandomForestClassifier(n_estimators=13, \n                                      max_depth=13, \n                                      random_state=13, \n                                      verbose=2, \n                                      n_jobs = 5)\nrf_resampled.fit(X_resampled, Y_resampled)\nprint('The machine has learned.')\nprint('RandomForest has fitted X_train with {:.2f} seconds'.format(time.time() - start_time))\nprint('Starting cross-validation prediction phase...')\npredictions = rf.predict_proba(X_cv)[:,1]\nprint('Prediction done. Elapsed time: {:.2f} seconds'.format(time.time() - start_time))\n\n# Avaliando\nclf_eval(Y_cv, predictions)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"265a029be2d83694823daaa8361ef1d249e7a7a1","_cell_guid":"7be2457b-ad03-4b03-86fe-da7a10fc3e05"},"cell_type":"markdown","source":"O resultado foi pior do que com os dados anteriores.\n### Preparando submissão"},{"metadata":{"_uuid":"fbae21db8c5b33e321de732912ed233810647a4a","_cell_guid":"a263354f-644c-4145-b1f2-43adcf81c2cc","trusted":false,"collapsed":true},"cell_type":"code","source":"# Predicao\nprint('Starting prediction phase...')\nstart_time = time.time()\npredictions = rf.predict_proba(df_test[cols])\nprint('Prediction done. Elapsed time: {:.2f} seconds'.format(time.time() - start_time))\n\n# Creating the submission dataset\ndf_submit['is_attributed'] = predictions[:,1]\nprint('Submission dataset created.')\n\n# Preparing submssion\ndf_submit.to_csv('timed_rf_raul.csv', index=False)\nprint('Submission dataset saved correctly.')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}