{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-07T17:38:48.879149Z","iopub.execute_input":"2022-07-07T17:38:48.879612Z","iopub.status.idle":"2022-07-07T17:38:48.911162Z","shell.execute_reply.started":"2022-07-07T17:38:48.879521Z","shell.execute_reply":"2022-07-07T17:38:48.909957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Approach\n1. Import libraries\n2. Load data\n2. Impute outliers with median (columns)\n3. Get best Mutual Information columns\n4. Create new columns with them\n5. Select useful features\n6. Extract numerical and categorical columns\n7. Make Pipeline\n8. Do hyperparameter tuning with cross-validation (`XGBClassifier`)\n9. Check MAE score\n10. Select best parameters\n11. Train model\n12. Predict `Survived` on `test_data`\n13. Submit predictions","metadata":{}},{"cell_type":"markdown","source":"## 1. Importing required libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline\nfrom xgboost import XGBClassifier\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.feature_selection import mutual_info_classif","metadata":{"execution":{"iopub.status.busy":"2022-07-07T17:41:31.165138Z","iopub.execute_input":"2022-07-07T17:41:31.165518Z","iopub.status.idle":"2022-07-07T17:41:31.175159Z","shell.execute_reply.started":"2022-07-07T17:41:31.165489Z","shell.execute_reply":"2022-07-07T17:41:31.174132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Load data","metadata":{}},{"cell_type":"code","source":"data = pd.read_csv(\"../input/titanic/train.csv\", index_col=\"PassengerId\")\ntest_data = pd.read_csv(\"../input/titanic/test.csv\", index_col=\"PassengerId\")\n\ndata.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:14:09.300856Z","iopub.execute_input":"2022-07-07T19:14:09.301286Z","iopub.status.idle":"2022-07-07T19:14:09.328600Z","shell.execute_reply.started":"2022-07-07T19:14:09.301252Z","shell.execute_reply":"2022-07-07T19:14:09.327834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. Imputing outliers with their column's median value","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(12, 6))\nsns.scatterplot(x=data[\"Fare\"], y=data.index, hue=\"Survived\", data=data)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:14:09.846855Z","iopub.execute_input":"2022-07-07T19:14:09.847856Z","iopub.status.idle":"2022-07-07T19:14:10.069577Z","shell.execute_reply.started":"2022-07-07T19:14:09.847808Z","shell.execute_reply":"2022-07-07T19:14:10.068458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"outliers = data[data[\"Fare\"]>200].index\noutliers","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:14:10.177373Z","iopub.execute_input":"2022-07-07T19:14:10.177736Z","iopub.status.idle":"2022-07-07T19:14:10.185835Z","shell.execute_reply.started":"2022-07-07T19:14:10.177707Z","shell.execute_reply":"2022-07-07T19:14:10.184938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.loc[outliers, \"Fare\"] = data[\"Fare\"].median()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:14:10.404653Z","iopub.execute_input":"2022-07-07T19:14:10.405546Z","iopub.status.idle":"2022-07-07T19:14:10.412301Z","shell.execute_reply.started":"2022-07-07T19:14:10.405508Z","shell.execute_reply":"2022-07-07T19:14:10.411091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12, 6))\nsns.scatterplot(x=data[\"Fare\"], y=data.index, hue=\"Survived\", data=data)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:14:10.625919Z","iopub.execute_input":"2022-07-07T19:14:10.626783Z","iopub.status.idle":"2022-07-07T19:14:10.923737Z","shell.execute_reply.started":"2022-07-07T19:14:10.626733Z","shell.execute_reply":"2022-07-07T19:14:10.922483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4. Getting best Mutual Information columns","metadata":{}},{"cell_type":"code","source":"def impute_data(data, num_strategy=\"median\", cat_strategy=\"most_frequent\"):\n    X = data.copy()\n    index = X.index\n    num_cols = X.select_dtypes(exclude=\"object\").columns\n    cat_cols = X.select_dtypes(\"object\").columns    \n    \n    num_imputer = SimpleImputer(strategy=num_strategy)\n    cat_imputer = SimpleImputer(strategy=cat_strategy)\n    \n    X[num_cols] = pd.DataFrame(num_imputer.fit_transform(X[num_cols]), index=index, columns=num_cols)\n    X[cat_cols] = pd.DataFrame(cat_imputer.fit_transform(X[cat_cols]), index=index, columns=cat_cols)\n    return X","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:14:19.226857Z","iopub.execute_input":"2022-07-07T19:14:19.227889Z","iopub.status.idle":"2022-07-07T19:14:19.236937Z","shell.execute_reply.started":"2022-07-07T19:14:19.227828Z","shell.execute_reply":"2022-07-07T19:14:19.235601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_mi_score(X, y):\n    X = impute_data(X)\n    \n    # Converting values of discrete features to numerical values\n    for col in X.select_dtypes([\"object\"]):\n        X[col] = X[col].factorize()[0]\n    \n    discrete_features = X.dtypes == int\n    mi_scores = mutual_info_classif(X, y, discrete_features=discrete_features, random_state=0)\n    mi_scores = pd.Series(mi_scores, name=\"MI Scores\", index=X.columns).sort_values(ascending=False)\n    return mi_scores","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:14:19.554835Z","iopub.execute_input":"2022-07-07T19:14:19.555504Z","iopub.status.idle":"2022-07-07T19:14:19.562802Z","shell.execute_reply.started":"2022-07-07T19:14:19.555466Z","shell.execute_reply":"2022-07-07T19:14:19.561728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = data.copy() \nY = X.pop(\"Survived\")\nmi_scores = get_mi_score(X, Y)\nmi_scores","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:14:19.763078Z","iopub.execute_input":"2022-07-07T19:14:19.763936Z","iopub.status.idle":"2022-07-07T19:14:19.823106Z","shell.execute_reply.started":"2022-07-07T19:14:19.763892Z","shell.execute_reply":"2022-07-07T19:14:19.821881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data[mi_scores.index]","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:14:19.970876Z","iopub.execute_input":"2022-07-07T19:14:19.971415Z","iopub.status.idle":"2022-07-07T19:14:19.995838Z","shell.execute_reply.started":"2022-07-07T19:14:19.971373Z","shell.execute_reply":"2022-07-07T19:14:19.994778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data[\"Ticket\"].isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:14:20.181431Z","iopub.execute_input":"2022-07-07T19:14:20.182354Z","iopub.status.idle":"2022-07-07T19:14:20.190621Z","shell.execute_reply.started":"2022-07-07T19:14:20.182316Z","shell.execute_reply":"2022-07-07T19:14:20.189686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data[\"Ticket\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:14:20.381795Z","iopub.execute_input":"2022-07-07T19:14:20.382925Z","iopub.status.idle":"2022-07-07T19:14:20.391451Z","shell.execute_reply.started":"2022-07-07T19:14:20.382886Z","shell.execute_reply":"2022-07-07T19:14:20.390216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5. Creating new columns","metadata":{}},{"cell_type":"markdown","source":"#### Creating 2 new columns `Tikcet_code`, `Ticket_number` from `Ticket` column","metadata":{}},{"cell_type":"code","source":"def split_tickets(tickets):\n    raw_tickets = [i.split(\" \") for i in tickets]\n    for ticket in raw_tickets:\n        if len(ticket) == 3:\n            if ticket[1][0].isdigit():\n                ticket[1:] = [ticket[1] + ticket[2]]\n            else:    # if 2nd element is word \n                ticket[:2] = [ticket[0] + ticket[1]]\n\n        if len(ticket) != 2:\n            ticket[:-1] = [\"\"]\n    return pd.DataFrame(raw_tickets, columns=[\"Ticket_code\", \"Ticket_number\"], index=tickets.index)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:14:20.565700Z","iopub.execute_input":"2022-07-07T19:14:20.566763Z","iopub.status.idle":"2022-07-07T19:14:20.573445Z","shell.execute_reply.started":"2022-07-07T19:14:20.566721Z","shell.execute_reply":"2022-07-07T19:14:20.572339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_tickets_df = split_tickets(data[\"Ticket\"])","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:14:20.808306Z","iopub.execute_input":"2022-07-07T19:14:20.809207Z","iopub.status.idle":"2022-07-07T19:14:20.815603Z","shell.execute_reply.started":"2022-07-07T19:14:20.809164Z","shell.execute_reply":"2022-07-07T19:14:20.814574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = X.join(new_tickets_df)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:14:21.036081Z","iopub.execute_input":"2022-07-07T19:14:21.037089Z","iopub.status.idle":"2022-07-07T19:14:21.042703Z","shell.execute_reply.started":"2022-07-07T19:14:21.037045Z","shell.execute_reply":"2022-07-07T19:14:21.041628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:14:21.200398Z","iopub.execute_input":"2022-07-07T19:14:21.200994Z","iopub.status.idle":"2022-07-07T19:14:21.221595Z","shell.execute_reply.started":"2022-07-07T19:14:21.200949Z","shell.execute_reply":"2022-07-07T19:14:21.220855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Creating `Total_members` by counting total from `SibSp` and `Parch`","metadata":{}},{"cell_type":"code","source":"X[\"Total_members\"] = X[[\"SibSp\", \"Parch\"]].sum(axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:14:21.362836Z","iopub.execute_input":"2022-07-07T19:14:21.363435Z","iopub.status.idle":"2022-07-07T19:14:21.370636Z","shell.execute_reply.started":"2022-07-07T19:14:21.363402Z","shell.execute_reply":"2022-07-07T19:14:21.369585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:14:29.256762Z","iopub.execute_input":"2022-07-07T19:14:29.257590Z","iopub.status.idle":"2022-07-07T19:14:29.281733Z","shell.execute_reply.started":"2022-07-07T19:14:29.257548Z","shell.execute_reply":"2022-07-07T19:14:29.280608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mi_scores","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:14:29.583915Z","iopub.execute_input":"2022-07-07T19:14:29.584686Z","iopub.status.idle":"2022-07-07T19:14:29.591381Z","shell.execute_reply.started":"2022-07-07T19:14:29.584646Z","shell.execute_reply":"2022-07-07T19:14:29.590536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X[\"Pclass\"].isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:14:29.831619Z","iopub.execute_input":"2022-07-07T19:14:29.832629Z","iopub.status.idle":"2022-07-07T19:14:29.839123Z","shell.execute_reply.started":"2022-07-07T19:14:29.832592Z","shell.execute_reply":"2022-07-07T19:14:29.838135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Creating `PclassRatio` from count of each `Pclass`","metadata":{}},{"cell_type":"code","source":"def create_pclass_ratios(data):\n    raw_pclassratio = pd.DataFrame((data.groupby(\"Pclass\")[\"Pclass\"].agg(\"count\") / data.Pclass.count()) * 100)\n    raw_pclassratio.rename(columns={\"Pclass\": \"PclassRatio\"}, inplace=True)\n    \n    pclass_ratios = []\n    for i in data[\"Pclass\"]:\n        pclass_ratios.append(raw_pclassratio.loc[i, \"PclassRatio\"])\n    return pclass_ratios","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:14:30.366766Z","iopub.execute_input":"2022-07-07T19:14:30.367493Z","iopub.status.idle":"2022-07-07T19:14:30.375413Z","shell.execute_reply.started":"2022-07-07T19:14:30.367457Z","shell.execute_reply":"2022-07-07T19:14:30.374284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X[\"PclassRatio\"] = create_pclass_ratios(X)\nX","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:14:30.584620Z","iopub.execute_input":"2022-07-07T19:14:30.585550Z","iopub.status.idle":"2022-07-07T19:14:30.620643Z","shell.execute_reply.started":"2022-07-07T19:14:30.585504Z","shell.execute_reply":"2022-07-07T19:14:30.619911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Creating `PclassAvgFare` column based on `Fare` as per `Pclass`","metadata":{}},{"cell_type":"code","source":"def create_avg_pclass_fares(data):\n    average_fares_pclass = pd.DataFrame(data.groupby(\"Pclass\")[\"Fare\"].mean())\n    average_fares_pclass\n\n    fares = []\n    for i in data[\"Pclass\"]:\n        fares.append(average_fares_pclass.loc[i, \"Fare\"])\n    return fares","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:14:31.114335Z","iopub.execute_input":"2022-07-07T19:14:31.114848Z","iopub.status.idle":"2022-07-07T19:14:31.120300Z","shell.execute_reply.started":"2022-07-07T19:14:31.114815Z","shell.execute_reply":"2022-07-07T19:14:31.119400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X[\"PclassAvgFare\"] = create_avg_pclass_fares(X)\nX","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:14:31.429289Z","iopub.execute_input":"2022-07-07T19:14:31.430005Z","iopub.status.idle":"2022-07-07T19:14:31.466434Z","shell.execute_reply.started":"2022-07-07T19:14:31.429955Z","shell.execute_reply":"2022-07-07T19:14:31.465508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### checking MI score again on new columns","metadata":{}},{"cell_type":"code","source":"mi_scores = get_mi_score(X, Y)\nmi_scores","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:14:32.165291Z","iopub.execute_input":"2022-07-07T19:14:32.166255Z","iopub.status.idle":"2022-07-07T19:14:32.242146Z","shell.execute_reply.started":"2022-07-07T19:14:32.166216Z","shell.execute_reply":"2022-07-07T19:14:32.241065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Creating same features for our `test_data`","metadata":{}},{"cell_type":"code","source":"test_data = test_data.join(split_tickets(test_data[\"Ticket\"]))","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:14:32.820593Z","iopub.execute_input":"2022-07-07T19:14:32.821250Z","iopub.status.idle":"2022-07-07T19:14:32.828858Z","shell.execute_reply.started":"2022-07-07T19:14:32.821214Z","shell.execute_reply":"2022-07-07T19:14:32.828137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data[\"PclassRatio\"] = create_pclass_ratios(test_data)\ntest_data[\"PclassAvgFare\"] = create_avg_pclass_fares(test_data)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:14:33.120152Z","iopub.execute_input":"2022-07-07T19:14:33.120540Z","iopub.status.idle":"2022-07-07T19:14:33.138834Z","shell.execute_reply.started":"2022-07-07T19:14:33.120509Z","shell.execute_reply":"2022-07-07T19:14:33.137214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data[\"Total_members\"] = test_data[[\"SibSp\", \"Parch\"]].sum(axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:14:33.384642Z","iopub.execute_input":"2022-07-07T19:14:33.385111Z","iopub.status.idle":"2022-07-07T19:14:33.394366Z","shell.execute_reply.started":"2022-07-07T19:14:33.385072Z","shell.execute_reply":"2022-07-07T19:14:33.393238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:14:33.691793Z","iopub.execute_input":"2022-07-07T19:14:33.692211Z","iopub.status.idle":"2022-07-07T19:14:33.713828Z","shell.execute_reply.started":"2022-07-07T19:14:33.692178Z","shell.execute_reply":"2022-07-07T19:14:33.713010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 6. Selecting useful features","metadata":{}},{"cell_type":"code","source":"useful_features = mi_scores[mi_scores > 0.017].index\nX = X[useful_features]\ntest_data = test_data[useful_features]\nuseful_features","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:14:34.344049Z","iopub.execute_input":"2022-07-07T19:14:34.345218Z","iopub.status.idle":"2022-07-07T19:14:34.354976Z","shell.execute_reply.started":"2022-07-07T19:14:34.345175Z","shell.execute_reply":"2022-07-07T19:14:34.353854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 7. Extracting numerical and categorical columns","metadata":{}},{"cell_type":"code","source":"num_cols = X.select_dtypes(exclude=\"object\").columns\ncat_cols = X.select_dtypes(\"object\").columns\nnum_cols, cat_cols","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:14:35.738218Z","iopub.execute_input":"2022-07-07T19:14:35.738825Z","iopub.status.idle":"2022-07-07T19:14:35.749871Z","shell.execute_reply.started":"2022-07-07T19:14:35.738782Z","shell.execute_reply":"2022-07-07T19:14:35.748660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 8. Making Pipeline","metadata":{}},{"cell_type":"code","source":"class CreatePipeline:\n    \"\"\"Create Pipeline\n    methods:\n        pipeline: Create Final Pipeline\n        \n        create_model: Create the provided model\n        \n        numerical_transformer: Transform numerical cols\n        \n        categorical_transformer: Transform categorical cols \\\n        OneHotEncoding / OrdinalEncoding\n        \n        data_preprocessor: Preprocess the data using ColumnTransformer     \n        \"\"\"\n    \n    def pipeline(self, *, preprocessor, model, verbose=False):\n        \"\"\"Creates pipeline\n        params:\n            preprocessor\n            model\n        \"\"\"\n        steps = [(\"preprocessor\", preprocessor),\n                 (\"model\", model)]\n        return Pipeline(steps=steps, verbose=verbose)\n    \n    \n    def numerical_transformer(self, *, strategy=\"mean\", **params):\n        \"\"\"Transform numerical columns using `SimpleImputer`.\n        params:\n            strategy: \"mean\" | \"median\" | \"most_frequent\" | \"constant\"\n            **params: extra keyword args for SimpleImputer\"\"\"\n        \n        transformer = SimpleImputer(strategy=strategy, **params)\n        return transformer\n\n    \n    def categorical_transformer(self, *, \n                                imp_strategy=\"most_frequent\", \n                                encoder_type=\"Ordinal\", \n                                imp_params={}, encoder_params={}):\n        \"\"\"Transform categorical columns by making Pipeline\n        `SimpleImputer` | `OneHotEncoder` | `OrdinalEncoder`.\n        args:\n            imp_strategy: strategy for imputer values can be\n                \"most_frequent\" | \"constant\"\n            encoder_type: encoder type,\n                \"Ordinal\" | \"OneHot\"\n        kwargs:\n            imp_params: keyword args for `SimpleImputer`.\n            encoder_params: keyword args for encoder.`\n        \"\"\"\n        if not encoder_type in (\"Ordinal\", \"OneHot\"):\n            raise ValueError(f\"Inappropriate value for encoder_type passed: {encoder_type}\\\n            Takes one of 'Ordinal' | 'OneHot'.\")\n        \n        encoder = OrdinalEncoder if encoder_type==\"Ordinal\" else OneHotEncoder\n        transformer = Pipeline(steps=[\n            (\"imputer\", SimpleImputer(strategy=imp_strategy, **imp_params)),\n            (encoder_type, encoder(**encoder_params))\n        ])\n        return transformer\n    \n    \n    def data_preprocessor(self, *, transformers):\n        \"\"\"Preprocess the data using `ColumnTransformer`.\n        Pass extact list of transformers\n        to be passed in `ColumnTransformer`.\n        each tuple consist of: (transformer_name,\n                                transformer,\n                                list_of_columns).\"\"\"\n        preprocessor = ColumnTransformer(transformers=transformers)\n        return preprocessor\n    \n    \n    def create_model(self, *, model, random_state=0, n_estimators=1000, **kwargs):\n        \"\"\"Creates the model.\n        **kwargs: keyword args for model.\"\"\"\n        my_model = model(random_state=random_state, n_estimators=n_estimators, **kwargs)\n        return my_model","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:14:36.504246Z","iopub.execute_input":"2022-07-07T19:14:36.504641Z","iopub.status.idle":"2022-07-07T19:14:36.515530Z","shell.execute_reply.started":"2022-07-07T19:14:36.504607Z","shell.execute_reply":"2022-07-07T19:14:36.514714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cp = CreatePipeline()\nnum_transformer = cp.numerical_transformer(strategy=\"median\")\ncat_transformer = cp.categorical_transformer(encoder_type=\"OneHot\", encoder_params={\"handle_unknown\": \"ignore\"})\npreprocessor = cp.data_preprocessor(\n                    transformers=[(\"num\", num_transformer, num_cols),\n                                  (\"cat\", cat_transformer, cat_cols)\n                                 ])","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:14:36.862443Z","iopub.execute_input":"2022-07-07T19:14:36.862911Z","iopub.status.idle":"2022-07-07T19:14:36.868936Z","shell.execute_reply.started":"2022-07-07T19:14:36.862877Z","shell.execute_reply":"2022-07-07T19:14:36.867862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 9. Doing hyperparameter tuning with cross-validation (`XGBClassifier`)","metadata":{}},{"cell_type":"code","source":"n_estimators = [500, 750, 1000]\nmax_depths = [5, 10]\nlearning_rate = [0.05, 0.1]\nmaes = {}\ni = 0\nfor n in n_estimators:\n    for md in max_depths:\n        for rate in learning_rate:\n            i += 1\n            model = cp.create_model(model=XGBClassifier, n_estimators=n, max_depth=md, learning_rate=rate)\n            pipeline = cp.pipeline(preprocessor=preprocessor, model=model)\n            scores = -1 * cross_val_score(pipeline, X, Y, cv=10, verbose=True,\n                                    scoring=\"neg_mean_absolute_error\")\n            mae = scores.mean()\n            maes[i] = [n, md, rate, mae]","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:14:37.550939Z","iopub.execute_input":"2022-07-07T19:14:37.552034Z","iopub.status.idle":"2022-07-07T19:22:12.016577Z","shell.execute_reply.started":"2022-07-07T19:14:37.551996Z","shell.execute_reply":"2022-07-07T19:22:12.015415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 10. Checking MAE scores","metadata":{}},{"cell_type":"code","source":"for i in maes:\n    n, md, rate, mae = maes[i]\n    print(f\"{i}.\\tN_estimators: {n}\\tmax_depth: {md}\\tlearning_rate: {rate}\\tMAE: {mae}\")","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:22:12.018387Z","iopub.execute_input":"2022-07-07T19:22:12.018692Z","iopub.status.idle":"2022-07-07T19:22:12.025478Z","shell.execute_reply.started":"2022-07-07T19:22:12.018665Z","shell.execute_reply":"2022-07-07T19:22:12.024448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min(maes, key=lambda x: maes[x][3])    # but selecting 5th","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:22:12.026998Z","iopub.execute_input":"2022-07-07T19:22:12.028597Z","iopub.status.idle":"2022-07-07T19:22:12.044388Z","shell.execute_reply.started":"2022-07-07T19:22:12.028509Z","shell.execute_reply":"2022-07-07T19:22:12.043450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 11. Selecting best parameters\n#### `n_estimators: 750`\n#### `learning_rate: 0.05`\n#### `max_depth: 5`\n#### `MAE: 0.1929962546816479`","metadata":{}},{"cell_type":"code","source":"best_n_estimators = 750\nbest_max_depth = 5\nbest_rate = 0.05","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:24:19.225915Z","iopub.execute_input":"2022-07-07T19:24:19.226323Z","iopub.status.idle":"2022-07-07T19:24:19.231025Z","shell.execute_reply.started":"2022-07-07T19:24:19.226293Z","shell.execute_reply":"2022-07-07T19:24:19.230180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 12. Training model on best parameters","metadata":{}},{"cell_type":"code","source":"model = cp.create_model(model=XGBClassifier, n_estimators=best_n_estimators, learning_rate=best_rate, max_depth=best_max_depth)\npipeline = cp.pipeline(preprocessor=preprocessor, model=model)\npipeline.fit(X, Y)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:24:21.579343Z","iopub.execute_input":"2022-07-07T19:24:21.579956Z","iopub.status.idle":"2022-07-07T19:24:25.006244Z","shell.execute_reply.started":"2022-07-07T19:24:21.579915Z","shell.execute_reply":"2022-07-07T19:24:25.005088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 13. Predicting `Survived` on `test_data`","metadata":{}},{"cell_type":"code","source":"test_preds = pipeline.predict(test_data)\ntest_preds","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:24:29.552746Z","iopub.execute_input":"2022-07-07T19:24:29.553141Z","iopub.status.idle":"2022-07-07T19:24:29.588329Z","shell.execute_reply.started":"2022-07-07T19:24:29.553109Z","shell.execute_reply":"2022-07-07T19:24:29.587158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 14. Submitting predictions","metadata":{}},{"cell_type":"code","source":"output = pd.DataFrame({\"PassengerId\": test_data.index, \"Survived\": test_preds})\noutput.to_csv(\"./submission_3.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:24:33.662732Z","iopub.execute_input":"2022-07-07T19:24:33.663762Z","iopub.status.idle":"2022-07-07T19:24:33.675094Z","shell.execute_reply.started":"2022-07-07T19:24:33.663717Z","shell.execute_reply":"2022-07-07T19:24:33.674045Z"},"trusted":true},"execution_count":null,"outputs":[]}]}