{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.026471,"end_time":"2022-07-05T17:51:33.077210","exception":false,"start_time":"2022-07-05T17:51:33.050739","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-05T18:08:32.049655Z","iopub.execute_input":"2022-07-05T18:08:32.050396Z","iopub.status.idle":"2022-07-05T18:08:32.057474Z","shell.execute_reply.started":"2022-07-05T18:08:32.050358Z","shell.execute_reply":"2022-07-05T18:08:32.056115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Importing required libraries","metadata":{"papermill":{"duration":0.006561,"end_time":"2022-07-05T17:51:33.090606","exception":false,"start_time":"2022-07-05T17:51:33.084045","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import pandas as pd\nfrom xgboost import XGBClassifier\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.model_selection import cross_val_score","metadata":{"papermill":{"duration":1.503749,"end_time":"2022-07-05T17:51:34.600970","exception":false,"start_time":"2022-07-05T17:51:33.097221","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-05T18:08:32.895142Z","iopub.execute_input":"2022-07-05T18:08:32.895572Z","iopub.status.idle":"2022-07-05T18:08:33.653987Z","shell.execute_reply.started":"2022-07-05T18:08:32.895537Z","shell.execute_reply":"2022-07-05T18:08:33.653004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Approach\n1. Load data\n2. Select useful features\n3. Extract numerical and categorical columns\n4. Make pipeline\n5. Do Hyperparameter tuning with cross-validation\n6. Check MAE\n7. Find best parameters\n8. Train model\n9. Predicting `survived` on `test_data`\n10. Submit predictions","metadata":{"papermill":{"duration":0.006225,"end_time":"2022-07-05T17:51:34.613924","exception":false,"start_time":"2022-07-05T17:51:34.607699","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## 1. Loading data","metadata":{"papermill":{"duration":0.006384,"end_time":"2022-07-05T17:51:34.626878","exception":false,"start_time":"2022-07-05T17:51:34.620494","status":"completed"},"tags":[]}},{"cell_type":"code","source":"titanic_data = pd.read_csv(\"../input/titanic/train.csv\", index_col=\"PassengerId\")\ntest_data = pd.read_csv(\"../input/titanic/test.csv\", index_col=\"PassengerId\")\n\ntitanic_data.head()","metadata":{"papermill":{"duration":0.068181,"end_time":"2022-07-05T17:51:34.701520","exception":false,"start_time":"2022-07-05T17:51:34.633339","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-05T18:37:12.906169Z","iopub.execute_input":"2022-07-05T18:37:12.906858Z","iopub.status.idle":"2022-07-05T18:37:12.934151Z","shell.execute_reply.started":"2022-07-05T18:37:12.906821Z","shell.execute_reply":"2022-07-05T18:37:12.933234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Selecting useful features","metadata":{"papermill":{"duration":0.006588,"end_time":"2022-07-05T17:51:34.715192","exception":false,"start_time":"2022-07-05T17:51:34.708604","status":"completed"},"tags":[]}},{"cell_type":"code","source":"outliers = titanic_data[(titanic_data[\"Survived\"]==1) & (titanic_data[\"Fare\"] > 200)].index\ntitanic_data.drop(labels=outliers, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T18:37:13.294076Z","iopub.execute_input":"2022-07-05T18:37:13.294451Z","iopub.status.idle":"2022-07-05T18:37:13.301996Z","shell.execute_reply.started":"2022-07-05T18:37:13.294417Z","shell.execute_reply":"2022-07-05T18:37:13.300836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"useful_features = [\"Pclass\", \"Sex\", \"Age\", \"SibSp\", \"Parch\", \"Ticket\", \"Fare\", \"Embarked\"]\nX = titanic_data[useful_features]\nY = titanic_data[\"Survived\"]\ntest_data = test_data[useful_features]","metadata":{"papermill":{"duration":0.021967,"end_time":"2022-07-05T17:51:34.744201","exception":false,"start_time":"2022-07-05T17:51:34.722234","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-05T18:45:54.710550Z","iopub.execute_input":"2022-07-05T18:45:54.710936Z","iopub.status.idle":"2022-07-05T18:45:54.718851Z","shell.execute_reply.started":"2022-07-05T18:45:54.710903Z","shell.execute_reply":"2022-07-05T18:45:54.717501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. Extracting numerical and categorical columns","metadata":{"papermill":{"duration":0.006634,"end_time":"2022-07-05T17:51:34.758022","exception":false,"start_time":"2022-07-05T17:51:34.751388","status":"completed"},"tags":[]}},{"cell_type":"code","source":"num_cols = X.select_dtypes(exclude=\"object\").columns\ncat_cols = X.select_dtypes(\"object\").columns\nnum_cols, cat_cols","metadata":{"papermill":{"duration":0.022206,"end_time":"2022-07-05T17:51:34.787181","exception":false,"start_time":"2022-07-05T17:51:34.764975","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-05T18:45:55.155749Z","iopub.execute_input":"2022-07-05T18:45:55.156829Z","iopub.status.idle":"2022-07-05T18:45:55.168025Z","shell.execute_reply.started":"2022-07-05T18:45:55.156771Z","shell.execute_reply":"2022-07-05T18:45:55.167021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4. Making pipeline","metadata":{"papermill":{"duration":0.006696,"end_time":"2022-07-05T17:51:34.801093","exception":false,"start_time":"2022-07-05T17:51:34.794397","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class CreatePipeline:\n    \"\"\"Create Pipeline\n    methods:\n        pipeline: Create Final Pipeline\n        \n        create_model: Create the provided model\n        \n        numerical_transformer: Transform numerical cols\n        \n        categorical_transformer: Transform categorical cols \\\n        OneHotEncoding / OrdinalEncoding\n        \n        data_preprocessor: Preprocess the data using ColumnTransformer     \n        \"\"\"\n    \n    def pipeline(self, *, preprocessor, model, verbose=False):\n        \"\"\"Creates pipeline\n        params:\n            preprocessor\n            model\n        \"\"\"\n        steps = [(\"preprocessor\", preprocessor),\n                 (\"model\", model)]\n        return Pipeline(steps=steps, verbose=verbose)\n    \n    \n    def numerical_transformer(self, *, strategy=\"mean\", **params):\n        \"\"\"Transform numerical columns using `SimpleImputer`.\n        params:\n            strategy: \"mean\" | \"median\" | \"most_frequent\" | \"constant\"\n            **params: extra keyword args for SimpleImputer\"\"\"\n        \n        transformer = SimpleImputer(strategy=strategy, **params)\n        return transformer\n\n    \n    def categorical_transformer(self, *, \n                                imp_strategy=\"most_frequent\", \n                                encoder_type=\"Ordinal\", \n                                imp_params={}, encoder_params={}):\n        \"\"\"Transform categorical columns by making Pipeline\n        `SimpleImputer` | `OneHotEncoder` | `OrdinalEncoder`.\n        args:\n            imp_strategy: strategy for imputer values can be\n                \"most_frequent\" | \"constant\"\n            encoder_type: encoder type,\n                \"Ordinal\" | \"OneHot\"\n        kwargs:\n            imp_params: keyword args for `SimpleImputer`.\n            encoder_params: keyword args for encoder.`\n        \"\"\"\n        if not encoder_type in (\"Ordinal\", \"OneHot\"):\n            raise ValueError(f\"Inappropriate value for encoder_type passed: {encoder_type}\\\n            Takes one of 'Ordinal' | 'OneHot'.\")\n        \n        encoder = OrdinalEncoder if encoder_type==\"Ordinal\" else OneHotEncoder\n        transformer = Pipeline(steps=[\n            (\"imputer\", SimpleImputer(strategy=imp_strategy, **imp_params)),\n            (encoder_type, encoder(**encoder_params))\n        ])\n        return transformer\n    \n    \n    def data_preprocessor(self, *, transformers):\n        \"\"\"Preprocess the data using `ColumnTransformer`.\n        Pass extact list of transformers\n        to be passed in `ColumnTransformer`.\n        each tuple consist of: (transformer_name,\n                                transformer,\n                                list_of_columns).\"\"\"\n        preprocessor = ColumnTransformer(transformers=transformers)\n        return preprocessor\n    \n    \n    def create_model(self, *, model, random_state=0, n_estimators=1000, **kwargs):\n        \"\"\"Creates the model.\n        **kwargs: keyword args for model.\"\"\"\n        my_model = model(random_state=random_state, n_estimators=n_estimators, **kwargs)\n        return my_model","metadata":{"papermill":{"duration":0.025381,"end_time":"2022-07-05T17:51:34.833443","exception":false,"start_time":"2022-07-05T17:51:34.808062","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-05T18:45:56.317480Z","iopub.execute_input":"2022-07-05T18:45:56.318316Z","iopub.status.idle":"2022-07-05T18:45:56.331657Z","shell.execute_reply.started":"2022-07-05T18:45:56.318273Z","shell.execute_reply":"2022-07-05T18:45:56.330460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cp = CreatePipeline()\nnum_transformer = cp.numerical_transformer(strategy=\"median\")\ncat_transformer = cp.categorical_transformer(encoder_type=\"OneHot\", encoder_params={\"handle_unknown\": \"ignore\"})\npreprocessor = cp.data_preprocessor(\n                    transformers=[(\"num\", num_transformer, num_cols),\n                                  (\"cat\", cat_transformer, cat_cols)\n                                 ])","metadata":{"papermill":{"duration":0.016687,"end_time":"2022-07-05T17:51:34.857272","exception":false,"start_time":"2022-07-05T17:51:34.840585","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-05T18:45:56.634145Z","iopub.execute_input":"2022-07-05T18:45:56.634572Z","iopub.status.idle":"2022-07-05T18:45:56.640722Z","shell.execute_reply.started":"2022-07-05T18:45:56.634537Z","shell.execute_reply":"2022-07-05T18:45:56.639861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5. Doing hyperparameter tuning with Cross-validation using `XGBClassifier`","metadata":{"papermill":{"duration":0.006627,"end_time":"2022-07-05T17:51:34.871079","exception":false,"start_time":"2022-07-05T17:51:34.864452","status":"completed"},"tags":[]}},{"cell_type":"code","source":"n_estimators = [350, 500, 750]\nmax_depths = [5, 10, 20]\nlearning_rate = [0.05, 0.1]\nmaes = {}\ni = 0\nfor n in n_estimators:\n    for md in max_depths:\n        for rate in learning_rate:\n            i += 1\n            model = cp.create_model(model=XGBClassifier, n_estimators=n, max_depth=md, learning_rate=rate)\n            pipeline = cp.pipeline(preprocessor=preprocessor, model=model)\n            scores = -1 * cross_val_score(pipeline, X, Y, cv=10, verbose=True,\n                                    scoring=\"neg_mean_absolute_error\")\n            mae = scores.mean()\n            maes[i] = [n, md, rate, mae]","metadata":{"papermill":{"duration":493.997081,"end_time":"2022-07-05T17:59:48.875047","exception":false,"start_time":"2022-07-05T17:51:34.877966","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-05T18:45:57.379394Z","iopub.execute_input":"2022-07-05T18:45:57.379774Z","iopub.status.idle":"2022-07-05T18:53:18.790754Z","shell.execute_reply.started":"2022-07-05T18:45:57.379743Z","shell.execute_reply":"2022-07-05T18:53:18.789898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 7. Checking MAE","metadata":{"papermill":{"duration":0.008234,"end_time":"2022-07-05T17:59:48.892538","exception":false,"start_time":"2022-07-05T17:59:48.884304","status":"completed"},"tags":[]}},{"cell_type":"code","source":"for i in maes:\n    n, md, rate, mae = maes[i]\n    print(f\"{i}.\\tN_estimators: {n}\\tmax_depth: {md}\\tlearning_rate: {rate}\\tMAE: {mae}\")","metadata":{"papermill":{"duration":0.018241,"end_time":"2022-07-05T17:59:48.919064","exception":false,"start_time":"2022-07-05T17:59:48.900823","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-05T18:53:18.794906Z","iopub.execute_input":"2022-07-05T18:53:18.795432Z","iopub.status.idle":"2022-07-05T18:53:18.804205Z","shell.execute_reply.started":"2022-07-05T18:53:18.795399Z","shell.execute_reply":"2022-07-05T18:53:18.803083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 8. Best parameters\n#### `n_estimators: 350`\n#### `max_depth: 5`\n#### `learning_rate: 0.05`\n#### `MAE: 0.17108150470219435`","metadata":{"papermill":{"duration":0.008239,"end_time":"2022-07-05T17:59:48.936127","exception":false,"start_time":"2022-07-05T17:59:48.927888","status":"completed"},"tags":[]}},{"cell_type":"code","source":"best_n_estimators = 350\nbest_max_depth = 5\nbest_rate = 0.05","metadata":{"papermill":{"duration":0.016868,"end_time":"2022-07-05T17:59:48.961485","exception":false,"start_time":"2022-07-05T17:59:48.944617","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-05T19:00:01.395394Z","iopub.execute_input":"2022-07-05T19:00:01.395807Z","iopub.status.idle":"2022-07-05T19:00:01.401758Z","shell.execute_reply.started":"2022-07-05T19:00:01.395770Z","shell.execute_reply":"2022-07-05T19:00:01.400620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 9. Training `XGBClassifier` model with best parameters","metadata":{"papermill":{"duration":0.008583,"end_time":"2022-07-05T17:59:48.978533","exception":false,"start_time":"2022-07-05T17:59:48.969950","status":"completed"},"tags":[]}},{"cell_type":"code","source":"model = cp.create_model(model=XGBClassifier, n_estimators=best_n_estimators, learning_rate=best_rate, max_depth=best_max_depth)\npipeline = cp.pipeline(preprocessor=preprocessor, model=model)\npipeline.fit(X, Y)","metadata":{"papermill":{"duration":1.555016,"end_time":"2022-07-05T17:59:50.542027","exception":false,"start_time":"2022-07-05T17:59:48.987011","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-05T19:00:02.268474Z","iopub.execute_input":"2022-07-05T19:00:02.268853Z","iopub.status.idle":"2022-07-05T19:00:03.836183Z","shell.execute_reply.started":"2022-07-05T19:00:02.268822Z","shell.execute_reply":"2022-07-05T19:00:03.834978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 10. Predicting `Survived` on `test_data`","metadata":{"papermill":{"duration":0.009139,"end_time":"2022-07-05T17:59:50.560258","exception":false,"start_time":"2022-07-05T17:59:50.551119","status":"completed"},"tags":[]}},{"cell_type":"code","source":"test_preds = pipeline.predict(test_data)\ntest_preds","metadata":{"papermill":{"duration":0.039927,"end_time":"2022-07-05T17:59:50.608926","exception":false,"start_time":"2022-07-05T17:59:50.568999","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-05T19:00:04.019177Z","iopub.execute_input":"2022-07-05T19:00:04.020407Z","iopub.status.idle":"2022-07-05T19:00:04.054093Z","shell.execute_reply.started":"2022-07-05T19:00:04.020352Z","shell.execute_reply":"2022-07-05T19:00:04.053232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 11. Submitting predictions","metadata":{"papermill":{"duration":0.008487,"end_time":"2022-07-05T17:59:50.627616","exception":false,"start_time":"2022-07-05T17:59:50.619129","status":"completed"},"tags":[]}},{"cell_type":"code","source":"output = pd.DataFrame({\"PassengerId\": test_data.index, \"Survived\": test_preds})\noutput.to_csv(\"./submission_3.csv\", index=False)","metadata":{"papermill":{"duration":0.023314,"end_time":"2022-07-05T17:59:50.659817","exception":false,"start_time":"2022-07-05T17:59:50.636503","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-05T19:00:07.019015Z","iopub.execute_input":"2022-07-05T19:00:07.019961Z","iopub.status.idle":"2022-07-05T19:00:07.029605Z","shell.execute_reply.started":"2022-07-05T19:00:07.019919Z","shell.execute_reply":"2022-07-05T19:00:07.028578Z"},"trusted":true},"execution_count":null,"outputs":[]}]}