{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!/opt/conda/bin/python3.7 -m pip install --upgrade pip -q\n!pip install --upgrade xgboost","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Functions copied from: https://www.kaggle.com/obougacha/ingv-xgboost-baseline/comments?select=Train.csv"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from xgboost import XGBRegressor, plot_tree\nimport tensorflow as tf\nimport numpy as np \nimport pandas as pd\nimport os\nimport random\nfrom sklearn.model_selection import KFold, train_test_split\nfrom sklearn.metrics import mean_absolute_error\nimport matplotlib.pyplot as plt\nimport xgboost as xgb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ls /kaggle/input/volcano-pca","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission = pd.read_csv('/kaggle/input/predict-volcanic-eruptions-ingv-oe/sample_submission.csv') \n\n# y = pd.read_csv('/kaggle/input/predict-volcanic-eruptions-ingv-oe/train.csv')['time_to_eruption']\ny = np.load('/kaggle/input/volcano-pca/y_aug2.npy')\n\n# X = pd.read_csv('/kaggle/input/volcanobench/train_p.csv').to_numpy()\n# X_test = pd.read_csv('/kaggle/input/volcanobench/test_p.csv').to_numpy()\n\nX = pd.read_csv('/kaggle/input/volcano-pca/reduced_X_norm.csv')[:y.shape[0]].to_numpy()\nX_test = np.load('/kaggle/input/volcano-pca/reduced_X_test_norm.csv.npy')\n\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.3, random_state=16)\n\n# dtrain = xgb.DMatrix(X_train, label=y_train)\n# dval = xgb.DMatrix(X_val, label=y_val)\n# dtest = xgb.DMatrix(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def submit_prediction(pred, write=True, file_name=\"\", Return=True):\n    \"\"\"\n        Submits prediction and modifies the negative values.\n        \n        input:\n            pred: the predicted values\n            write: boolean value indicating if the predictions should be\n                written to a csv file.\n            file_name: name of the csv file to write\n            Return: if the edited predictions should be returned\n        \n        return:\n            returns the edited predictions if asked for (Return==1)\n    \"\"\"\n    # Replace the negative values with the mean of the data\n    pred = np.where(pred < 0, np.mean(pred), pred)\n    \n    sample_submission['time_to_eruption'] = pred  # Formatting\n    if write: sample_submission.to_csv(f'{file_name}.csv', index=False)  # Write the file to a csv file\n    \n    if Return: return pred\n    \n\ndef kfold_validation(X, y, X_test,n_fold=5, seeds=[0, 1, 2, 3, 4, 5, 6]):\n    \"\"\"\n        Runs a repeated KFold on a given model and data\n        \n        input:\n            model: model to traing the data\n            X, y, X_test: datasets needed for training and prediction\n            n_rep, n_fold: parameters of Repeated KFold\n        \n        return:\n            model: trained model\n            preds: predictions at each fold\n            avg_preds: average of all predictions\n    \"\"\"\n    index = 0  # Keep track of the loss and val_loss (history object)\n    prediction = np.zeros((X_test.shape[0])) # For every single prediction\n    preds = np.empty((len(seeds) * n_fold, X_test.shape[0])) # Saving all the predictions\n    \n#     param = {\n#         'booster': 'gbtree',\n#         'max_depth': 20,\n#         'gamma': 1e4,\n#         'min_child_weight': 5,  \n#         'tree_method': 'gpu_hist',\n#         'objective':'reg:squarederror',\n#         'n_jobs':-1,\n#         'reg_lambda':1e-3,\n#         'eta':0.05,  \n#         'eval_metric': 'mae',\n#         'verbosity': 1,\n#         'predictor': 'gpu_predictor'\n#     }\n    \n    param = {\n        'booster': 'dart',\n        'sample_type': 'weighted',\n        'rate_drop': 0.6,\n        'one_drop': 1,\n        'max_depth': 20,\n        'gamma': 1e4,\n        'min_child_weight': 5,  \n        'tree_method': 'gpu_hist',\n        'objective':'reg:squarederror',\n        'n_jobs':-1,\n        'reg_lambda':1e-3,\n        'eta':0.3,  \n        'eval_metric': 'mae',\n        'verbosity': 1,\n        'predictor': 'gpu_predictor'\n    }\n    \n    dtest = xgb.DMatrix(X_test)\n    \n    for seed in seeds:\n        kf = KFold(n_splits=n_fold, random_state=seed,shuffle=True)\n        LOAD_MODEL = False\n    \n        for train_indices, val_indices in kf.split(X, y):\n            # Data divided into Train and Validation splits\n            X_train, X_val = X[train_indices, :], X[val_indices,: ]\n            y_train, y_val = y[train_indices], y[val_indices]\n            \n            dtrain = xgb.DMatrix(X_train, label=y_train)\n            dval = xgb.DMatrix(X_val, label=y_val)\n\n            print(f'{seed}{index + 1}th fold, Validation Indices: ')\n            \n            if not LOAD_MODEL:\n                bst = xgb.train(param, \n                    dtrain, \n                    num_boost_round=1000, \n                    evals=[(dtrain, 'train'), (dval, 'eval')],\n                    verbose_eval =True,\n                    early_stopping_rounds=50\n                   )\n                    \n                LOAD_MODEL = True\n            else:\n                bst = xgb.train(param, \n                    dtrain, \n                    num_boost_round=1000, \n                    evals=[(dtrain, 'train'), (dval, 'eval')],\n                    verbose_eval =True,\n                    early_stopping_rounds=50,\n                    xgb_model=f'/kaggle/working/xg{seed}'\n                   )\n                \n            bst.save_model(f'xg{seed}')\n    \n\n            #------------------ Predictions -------------------\n            model_prediction = bst.predict(dtest)\n\n            model_prediction = submit_prediction(model_prediction, \n                                                 write=False, \n                                                 file_name=f\"sub{index}\", \n                                                 Return=True)\n\n            # Saving the predictions for each fold\n            preds[index] = model_prediction\n            index += 1\n\n            # Starting different fold or end of folding\n            print('#----------------#----------------#----------------#----------------#----------------#')\n        \n    # Averaging the predictions\n    p = pd.DataFrame(preds)\n    p = p.sum() / (n_fold * len(seeds))\n        \n    avg_pred = submit_prediction(p, \n                      write=True, \n                      file_name=f\"S_avg\", \n                      Return=True)\n    \n    return preds, avg_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds, avg_pred = kfold_validation(X, y, X_test, 5, seeds=[11])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.DataFrame(preds).to_csv('preds_xg10.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"avg_pred[[1,2,3]] * 100","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}