{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.032345,"end_time":"2022-07-07T19:24:59.678645","exception":false,"start_time":"2022-07-07T19:24:59.646300","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:40:50.434508Z","iopub.execute_input":"2022-07-07T19:40:50.435133Z","iopub.status.idle":"2022-07-07T19:40:50.479163Z","shell.execute_reply.started":"2022-07-07T19:40:50.435004Z","shell.execute_reply":"2022-07-07T19:40:50.478157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Approach\n1. Import libraries\n2. Load data\n2. Impute outliers with median (columns)\n3. Get best Mutual Information columns\n4. Create new columns with them\n5. Select useful features\n6. Extract numerical and categorical columns\n7. Make Pipeline\n8. Do hyperparameter tuning with cross-validation (`XGBClassifier`)\n9. Check MAE score\n10. Select best parameters\n11. Train model\n12. Predict `Survived` on `test_data`\n13. Submit predictions","metadata":{"papermill":{"duration":0.01698,"end_time":"2022-07-07T19:24:59.713058","exception":false,"start_time":"2022-07-07T19:24:59.696078","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## 1. Importing required libraries","metadata":{"papermill":{"duration":0.016522,"end_time":"2022-07-07T19:24:59.746333","exception":false,"start_time":"2022-07-07T19:24:59.729811","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline\nfrom xgboost import XGBClassifier\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.feature_selection import mutual_info_classif","metadata":{"papermill":{"duration":1.586995,"end_time":"2022-07-07T19:25:01.349896","exception":false,"start_time":"2022-07-07T19:24:59.762901","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:40:50.480988Z","iopub.execute_input":"2022-07-07T19:40:50.481740Z","iopub.status.idle":"2022-07-07T19:40:51.753392Z","shell.execute_reply.started":"2022-07-07T19:40:50.481694Z","shell.execute_reply":"2022-07-07T19:40:51.751791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Load data","metadata":{"papermill":{"duration":0.009991,"end_time":"2022-07-07T19:25:01.370992","exception":false,"start_time":"2022-07-07T19:25:01.361001","status":"completed"},"tags":[]}},{"cell_type":"code","source":"data = pd.read_csv(\"../input/titanic/train.csv\", index_col=\"PassengerId\")\ntest_data = pd.read_csv(\"../input/titanic/test.csv\", index_col=\"PassengerId\")\n\ndata.head()","metadata":{"papermill":{"duration":0.069913,"end_time":"2022-07-07T19:25:01.451376","exception":false,"start_time":"2022-07-07T19:25:01.381463","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:40:51.755110Z","iopub.execute_input":"2022-07-07T19:40:51.755562Z","iopub.status.idle":"2022-07-07T19:40:51.823455Z","shell.execute_reply.started":"2022-07-07T19:40:51.755522Z","shell.execute_reply":"2022-07-07T19:40:51.822440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. Imputing outliers with their column's median value","metadata":{"papermill":{"duration":0.017415,"end_time":"2022-07-07T19:25:01.486444","exception":false,"start_time":"2022-07-07T19:25:01.469029","status":"completed"},"tags":[]}},{"cell_type":"code","source":"plt.figure(figsize=(12, 6))\nsns.scatterplot(x=data[\"Fare\"], y=data.index, hue=\"Survived\", data=data)","metadata":{"papermill":{"duration":0.323256,"end_time":"2022-07-07T19:25:01.827380","exception":false,"start_time":"2022-07-07T19:25:01.504124","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:40:51.825671Z","iopub.execute_input":"2022-07-07T19:40:51.826911Z","iopub.status.idle":"2022-07-07T19:40:52.206079Z","shell.execute_reply.started":"2022-07-07T19:40:51.826869Z","shell.execute_reply":"2022-07-07T19:40:52.204644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"outliers = data[data[\"Fare\"]>200].index\noutliers","metadata":{"papermill":{"duration":0.027278,"end_time":"2022-07-07T19:25:01.873079","exception":false,"start_time":"2022-07-07T19:25:01.845801","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:40:52.208096Z","iopub.execute_input":"2022-07-07T19:40:52.208743Z","iopub.status.idle":"2022-07-07T19:40:52.219863Z","shell.execute_reply.started":"2022-07-07T19:40:52.208696Z","shell.execute_reply":"2022-07-07T19:40:52.218693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.loc[outliers, \"Fare\"] = data[\"Fare\"].median()","metadata":{"papermill":{"duration":0.030628,"end_time":"2022-07-07T19:25:01.922744","exception":false,"start_time":"2022-07-07T19:25:01.892116","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:40:52.221085Z","iopub.execute_input":"2022-07-07T19:40:52.221714Z","iopub.status.idle":"2022-07-07T19:40:52.232142Z","shell.execute_reply.started":"2022-07-07T19:40:52.221655Z","shell.execute_reply":"2022-07-07T19:40:52.230820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12, 6))\nsns.scatterplot(x=data[\"Fare\"], y=data.index, hue=\"Survived\", data=data)","metadata":{"papermill":{"duration":0.276052,"end_time":"2022-07-07T19:25:02.217579","exception":false,"start_time":"2022-07-07T19:25:01.941527","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:40:52.234004Z","iopub.execute_input":"2022-07-07T19:40:52.234837Z","iopub.status.idle":"2022-07-07T19:40:52.536389Z","shell.execute_reply.started":"2022-07-07T19:40:52.234782Z","shell.execute_reply":"2022-07-07T19:40:52.535169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4. Getting best Mutual Information columns","metadata":{"papermill":{"duration":0.019798,"end_time":"2022-07-07T19:25:02.257549","exception":false,"start_time":"2022-07-07T19:25:02.237751","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def impute_data(data, num_strategy=\"median\", cat_strategy=\"most_frequent\"):\n    X = data.copy()\n    index = X.index\n    num_cols = X.select_dtypes(exclude=\"object\").columns\n    cat_cols = X.select_dtypes(\"object\").columns    \n    \n    num_imputer = SimpleImputer(strategy=num_strategy)\n    cat_imputer = SimpleImputer(strategy=cat_strategy)\n    \n    X[num_cols] = pd.DataFrame(num_imputer.fit_transform(X[num_cols]), index=index, columns=num_cols)\n    X[cat_cols] = pd.DataFrame(cat_imputer.fit_transform(X[cat_cols]), index=index, columns=cat_cols)\n    return X","metadata":{"papermill":{"duration":0.028718,"end_time":"2022-07-07T19:25:02.306045","exception":false,"start_time":"2022-07-07T19:25:02.277327","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:40:52.537915Z","iopub.execute_input":"2022-07-07T19:40:52.538862Z","iopub.status.idle":"2022-07-07T19:40:52.548258Z","shell.execute_reply.started":"2022-07-07T19:40:52.538821Z","shell.execute_reply":"2022-07-07T19:40:52.546769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_mi_score(X, y):\n    X = impute_data(X)\n    \n    # Converting values of discrete features to numerical values\n    for col in X.select_dtypes([\"object\"]):\n        X[col] = X[col].factorize()[0]\n    \n    discrete_features = X.dtypes == int\n    mi_scores = mutual_info_classif(X, y, discrete_features=discrete_features, random_state=0)\n    mi_scores = pd.Series(mi_scores, name=\"MI Scores\", index=X.columns).sort_values(ascending=False)\n    return mi_scores","metadata":{"papermill":{"duration":0.027833,"end_time":"2022-07-07T19:25:02.354247","exception":false,"start_time":"2022-07-07T19:25:02.326414","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:40:52.549854Z","iopub.execute_input":"2022-07-07T19:40:52.550598Z","iopub.status.idle":"2022-07-07T19:40:52.561923Z","shell.execute_reply.started":"2022-07-07T19:40:52.550554Z","shell.execute_reply":"2022-07-07T19:40:52.560486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = data.copy() \nY = X.pop(\"Survived\")\nmi_scores = get_mi_score(X, Y)\nmi_scores","metadata":{"papermill":{"duration":0.084508,"end_time":"2022-07-07T19:25:02.451512","exception":false,"start_time":"2022-07-07T19:25:02.367004","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:40:52.566681Z","iopub.execute_input":"2022-07-07T19:40:52.567539Z","iopub.status.idle":"2022-07-07T19:40:52.645998Z","shell.execute_reply.started":"2022-07-07T19:40:52.567417Z","shell.execute_reply":"2022-07-07T19:40:52.644580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data[mi_scores.index]","metadata":{"papermill":{"duration":0.043559,"end_time":"2022-07-07T19:25:02.515412","exception":false,"start_time":"2022-07-07T19:25:02.471853","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:40:52.647746Z","iopub.execute_input":"2022-07-07T19:40:52.648477Z","iopub.status.idle":"2022-07-07T19:40:52.675039Z","shell.execute_reply.started":"2022-07-07T19:40:52.648410Z","shell.execute_reply":"2022-07-07T19:40:52.674112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data[\"Ticket\"].isna().sum()","metadata":{"papermill":{"duration":0.030019,"end_time":"2022-07-07T19:25:02.566706","exception":false,"start_time":"2022-07-07T19:25:02.536687","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:40:52.676361Z","iopub.execute_input":"2022-07-07T19:40:52.676992Z","iopub.status.idle":"2022-07-07T19:40:52.688323Z","shell.execute_reply.started":"2022-07-07T19:40:52.676942Z","shell.execute_reply":"2022-07-07T19:40:52.687010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data[\"Ticket\"].value_counts()","metadata":{"papermill":{"duration":0.032521,"end_time":"2022-07-07T19:25:02.619811","exception":false,"start_time":"2022-07-07T19:25:02.587290","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:40:52.690178Z","iopub.execute_input":"2022-07-07T19:40:52.690744Z","iopub.status.idle":"2022-07-07T19:40:52.707094Z","shell.execute_reply.started":"2022-07-07T19:40:52.690693Z","shell.execute_reply":"2022-07-07T19:40:52.706076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5. Creating new columns","metadata":{"papermill":{"duration":0.012836,"end_time":"2022-07-07T19:25:02.646185","exception":false,"start_time":"2022-07-07T19:25:02.633349","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"#### Creating 2 new columns `Tikcet_code`, `Ticket_number` from `Ticket` column","metadata":{"papermill":{"duration":0.012794,"end_time":"2022-07-07T19:25:02.672094","exception":false,"start_time":"2022-07-07T19:25:02.659300","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def split_tickets(tickets):\n    raw_tickets = [i.split(\" \") for i in tickets]\n    for ticket in raw_tickets:\n        if len(ticket) == 3:\n            if ticket[1][0].isdigit():\n                ticket[1:] = [ticket[1] + ticket[2]]\n            else:    # if 2nd element is word \n                ticket[:2] = [ticket[0] + ticket[1]]\n\n        if len(ticket) != 2:\n            ticket[:-1] = [\"\"]\n    return pd.DataFrame(raw_tickets, columns=[\"Ticket_code\", \"Ticket_number\"], index=tickets.index)","metadata":{"papermill":{"duration":0.023204,"end_time":"2022-07-07T19:25:02.708149","exception":false,"start_time":"2022-07-07T19:25:02.684945","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:40:52.709169Z","iopub.execute_input":"2022-07-07T19:40:52.709972Z","iopub.status.idle":"2022-07-07T19:40:52.718588Z","shell.execute_reply.started":"2022-07-07T19:40:52.709919Z","shell.execute_reply":"2022-07-07T19:40:52.717506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_tickets_df = split_tickets(data[\"Ticket\"])","metadata":{"papermill":{"duration":0.02168,"end_time":"2022-07-07T19:25:02.743158","exception":false,"start_time":"2022-07-07T19:25:02.721478","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:40:52.720160Z","iopub.execute_input":"2022-07-07T19:40:52.721006Z","iopub.status.idle":"2022-07-07T19:40:52.734083Z","shell.execute_reply.started":"2022-07-07T19:40:52.720969Z","shell.execute_reply":"2022-07-07T19:40:52.732911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = X.join(new_tickets_df)","metadata":{"papermill":{"duration":0.02362,"end_time":"2022-07-07T19:25:02.780196","exception":false,"start_time":"2022-07-07T19:25:02.756576","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:40:52.735698Z","iopub.execute_input":"2022-07-07T19:40:52.736037Z","iopub.status.idle":"2022-07-07T19:40:52.755302Z","shell.execute_reply.started":"2022-07-07T19:40:52.736005Z","shell.execute_reply":"2022-07-07T19:40:52.753886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X","metadata":{"papermill":{"duration":0.039255,"end_time":"2022-07-07T19:25:02.832922","exception":false,"start_time":"2022-07-07T19:25:02.793667","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:40:52.757326Z","iopub.execute_input":"2022-07-07T19:40:52.757827Z","iopub.status.idle":"2022-07-07T19:40:52.786331Z","shell.execute_reply.started":"2022-07-07T19:40:52.757791Z","shell.execute_reply":"2022-07-07T19:40:52.785153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Creating `Total_members` by counting total from `SibSp` and `Parch`","metadata":{"papermill":{"duration":0.013486,"end_time":"2022-07-07T19:25:02.860106","exception":false,"start_time":"2022-07-07T19:25:02.846620","status":"completed"},"tags":[]}},{"cell_type":"code","source":"X[\"Total_members\"] = X[[\"SibSp\", \"Parch\"]].sum(axis=1)","metadata":{"papermill":{"duration":0.023933,"end_time":"2022-07-07T19:25:02.897708","exception":false,"start_time":"2022-07-07T19:25:02.873775","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:40:52.788101Z","iopub.execute_input":"2022-07-07T19:40:52.788504Z","iopub.status.idle":"2022-07-07T19:40:52.797570Z","shell.execute_reply.started":"2022-07-07T19:40:52.788467Z","shell.execute_reply":"2022-07-07T19:40:52.796151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X","metadata":{"papermill":{"duration":0.04132,"end_time":"2022-07-07T19:25:02.952876","exception":false,"start_time":"2022-07-07T19:25:02.911556","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:40:52.799995Z","iopub.execute_input":"2022-07-07T19:40:52.800411Z","iopub.status.idle":"2022-07-07T19:40:52.831981Z","shell.execute_reply.started":"2022-07-07T19:40:52.800367Z","shell.execute_reply":"2022-07-07T19:40:52.830960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mi_scores","metadata":{"papermill":{"duration":0.024264,"end_time":"2022-07-07T19:25:02.991241","exception":false,"start_time":"2022-07-07T19:25:02.966977","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:40:52.834008Z","iopub.execute_input":"2022-07-07T19:40:52.834529Z","iopub.status.idle":"2022-07-07T19:40:52.844230Z","shell.execute_reply.started":"2022-07-07T19:40:52.834477Z","shell.execute_reply":"2022-07-07T19:40:52.842738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X[\"Pclass\"].isnull().sum()","metadata":{"papermill":{"duration":0.026119,"end_time":"2022-07-07T19:25:03.031569","exception":false,"start_time":"2022-07-07T19:25:03.005450","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:40:52.846358Z","iopub.execute_input":"2022-07-07T19:40:52.846941Z","iopub.status.idle":"2022-07-07T19:40:52.858953Z","shell.execute_reply.started":"2022-07-07T19:40:52.846877Z","shell.execute_reply":"2022-07-07T19:40:52.857612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Creating `PclassRatio` from count of each `Pclass`","metadata":{"papermill":{"duration":0.014032,"end_time":"2022-07-07T19:25:03.060437","exception":false,"start_time":"2022-07-07T19:25:03.046405","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def create_pclass_ratios(data):\n    raw_pclassratio = pd.DataFrame((data.groupby(\"Pclass\")[\"Pclass\"].agg(\"count\") / data.Pclass.count()) * 100)\n    raw_pclassratio.rename(columns={\"Pclass\": \"PclassRatio\"}, inplace=True)\n    \n    pclass_ratios = []\n    for i in data[\"Pclass\"]:\n        pclass_ratios.append(raw_pclassratio.loc[i, \"PclassRatio\"])\n    return pclass_ratios","metadata":{"papermill":{"duration":0.025127,"end_time":"2022-07-07T19:25:03.099801","exception":false,"start_time":"2022-07-07T19:25:03.074674","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:40:52.861154Z","iopub.execute_input":"2022-07-07T19:40:52.861993Z","iopub.status.idle":"2022-07-07T19:40:52.870311Z","shell.execute_reply.started":"2022-07-07T19:40:52.861938Z","shell.execute_reply":"2022-07-07T19:40:52.869337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X[\"PclassRatio\"] = create_pclass_ratios(X)\nX","metadata":{"papermill":{"duration":0.059379,"end_time":"2022-07-07T19:25:03.173688","exception":false,"start_time":"2022-07-07T19:25:03.114309","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:40:52.872039Z","iopub.execute_input":"2022-07-07T19:40:52.872816Z","iopub.status.idle":"2022-07-07T19:40:52.927712Z","shell.execute_reply.started":"2022-07-07T19:40:52.872753Z","shell.execute_reply":"2022-07-07T19:40:52.926769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Creating `PclassAvgFare` column based on `Fare` as per `Pclass`","metadata":{"papermill":{"duration":0.014352,"end_time":"2022-07-07T19:25:03.202743","exception":false,"start_time":"2022-07-07T19:25:03.188391","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def create_avg_pclass_fares(data):\n    average_fares_pclass = pd.DataFrame(data.groupby(\"Pclass\")[\"Fare\"].mean())\n    average_fares_pclass\n\n    fares = []\n    for i in data[\"Pclass\"]:\n        fares.append(average_fares_pclass.loc[i, \"Fare\"])\n    return fares","metadata":{"papermill":{"duration":0.023331,"end_time":"2022-07-07T19:25:03.240557","exception":false,"start_time":"2022-07-07T19:25:03.217226","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:40:52.929080Z","iopub.execute_input":"2022-07-07T19:40:52.929659Z","iopub.status.idle":"2022-07-07T19:40:52.936163Z","shell.execute_reply.started":"2022-07-07T19:40:52.929616Z","shell.execute_reply":"2022-07-07T19:40:52.934793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X[\"PclassAvgFare\"] = create_avg_pclass_fares(X)\nX","metadata":{"papermill":{"duration":0.066263,"end_time":"2022-07-07T19:25:03.321365","exception":false,"start_time":"2022-07-07T19:25:03.255102","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:40:52.938545Z","iopub.execute_input":"2022-07-07T19:40:52.938928Z","iopub.status.idle":"2022-07-07T19:40:52.993856Z","shell.execute_reply.started":"2022-07-07T19:40:52.938895Z","shell.execute_reply":"2022-07-07T19:40:52.992362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### checking MI score again on new columns","metadata":{"papermill":{"duration":0.014815,"end_time":"2022-07-07T19:25:03.352407","exception":false,"start_time":"2022-07-07T19:25:03.337592","status":"completed"},"tags":[]}},{"cell_type":"code","source":"mi_scores = get_mi_score(X, Y)\nmi_scores","metadata":{"papermill":{"duration":0.103745,"end_time":"2022-07-07T19:25:03.471088","exception":false,"start_time":"2022-07-07T19:25:03.367343","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:40:52.995770Z","iopub.execute_input":"2022-07-07T19:40:52.996129Z","iopub.status.idle":"2022-07-07T19:40:53.083376Z","shell.execute_reply.started":"2022-07-07T19:40:52.996098Z","shell.execute_reply":"2022-07-07T19:40:53.082242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Creating same features for our `test_data`","metadata":{"papermill":{"duration":0.023343,"end_time":"2022-07-07T19:25:03.517989","exception":false,"start_time":"2022-07-07T19:25:03.494646","status":"completed"},"tags":[]}},{"cell_type":"code","source":"test_data = test_data.join(split_tickets(test_data[\"Ticket\"]))","metadata":{"papermill":{"duration":0.037739,"end_time":"2022-07-07T19:25:03.581439","exception":false,"start_time":"2022-07-07T19:25:03.543700","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:40:53.085112Z","iopub.execute_input":"2022-07-07T19:40:53.085849Z","iopub.status.idle":"2022-07-07T19:40:53.095102Z","shell.execute_reply.started":"2022-07-07T19:40:53.085812Z","shell.execute_reply":"2022-07-07T19:40:53.093827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data[\"PclassRatio\"] = create_pclass_ratios(test_data)\ntest_data[\"PclassAvgFare\"] = create_avg_pclass_fares(test_data)","metadata":{"papermill":{"duration":0.053495,"end_time":"2022-07-07T19:25:03.658677","exception":false,"start_time":"2022-07-07T19:25:03.605182","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:40:53.101399Z","iopub.execute_input":"2022-07-07T19:40:53.102162Z","iopub.status.idle":"2022-07-07T19:40:53.129320Z","shell.execute_reply.started":"2022-07-07T19:40:53.102109Z","shell.execute_reply":"2022-07-07T19:40:53.127694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data[\"Total_members\"] = test_data[[\"SibSp\", \"Parch\"]].sum(axis=1)","metadata":{"papermill":{"duration":0.034391,"end_time":"2022-07-07T19:25:03.717666","exception":false,"start_time":"2022-07-07T19:25:03.683275","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:40:53.131407Z","iopub.execute_input":"2022-07-07T19:40:53.132011Z","iopub.status.idle":"2022-07-07T19:40:53.143644Z","shell.execute_reply.started":"2022-07-07T19:40:53.131943Z","shell.execute_reply":"2022-07-07T19:40:53.142244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data.head()","metadata":{"papermill":{"duration":0.053403,"end_time":"2022-07-07T19:25:03.795302","exception":false,"start_time":"2022-07-07T19:25:03.741899","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:40:53.145955Z","iopub.execute_input":"2022-07-07T19:40:53.146341Z","iopub.status.idle":"2022-07-07T19:40:53.174044Z","shell.execute_reply.started":"2022-07-07T19:40:53.146306Z","shell.execute_reply":"2022-07-07T19:40:53.172266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 6. Selecting useful features","metadata":{"papermill":{"duration":0.023935,"end_time":"2022-07-07T19:25:03.845803","exception":false,"start_time":"2022-07-07T19:25:03.821868","status":"completed"},"tags":[]}},{"cell_type":"code","source":"useful_features = mi_scores[mi_scores > 0.045].index\nX = X[useful_features]\ntest_data = test_data[useful_features]\nuseful_features","metadata":{"papermill":{"duration":0.035929,"end_time":"2022-07-07T19:25:03.905599","exception":false,"start_time":"2022-07-07T19:25:03.869670","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:40:53.175933Z","iopub.execute_input":"2022-07-07T19:40:53.176849Z","iopub.status.idle":"2022-07-07T19:40:53.191716Z","shell.execute_reply.started":"2022-07-07T19:40:53.176808Z","shell.execute_reply":"2022-07-07T19:40:53.190730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 7. Extracting numerical and categorical columns","metadata":{"papermill":{"duration":0.023773,"end_time":"2022-07-07T19:25:03.953314","exception":false,"start_time":"2022-07-07T19:25:03.929541","status":"completed"},"tags":[]}},{"cell_type":"code","source":"num_cols = X.select_dtypes(exclude=\"object\").columns\ncat_cols = X.select_dtypes(\"object\").columns\nnum_cols, cat_cols","metadata":{"papermill":{"duration":0.037381,"end_time":"2022-07-07T19:25:04.014857","exception":false,"start_time":"2022-07-07T19:25:03.977476","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:40:53.193152Z","iopub.execute_input":"2022-07-07T19:40:53.194259Z","iopub.status.idle":"2022-07-07T19:40:53.207720Z","shell.execute_reply.started":"2022-07-07T19:40:53.194219Z","shell.execute_reply":"2022-07-07T19:40:53.206094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 8. Making Pipeline","metadata":{"papermill":{"duration":0.024697,"end_time":"2022-07-07T19:25:04.064807","exception":false,"start_time":"2022-07-07T19:25:04.040110","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class CreatePipeline:\n    \"\"\"Create Pipeline\n    methods:\n        pipeline: Create Final Pipeline\n        \n        create_model: Create the provided model\n        \n        numerical_transformer: Transform numerical cols\n        \n        categorical_transformer: Transform categorical cols \\\n        OneHotEncoding / OrdinalEncoding\n        \n        data_preprocessor: Preprocess the data using ColumnTransformer     \n        \"\"\"\n    \n    def pipeline(self, *, preprocessor, model, verbose=False):\n        \"\"\"Creates pipeline\n        params:\n            preprocessor\n            model\n        \"\"\"\n        steps = [(\"preprocessor\", preprocessor),\n                 (\"model\", model)]\n        return Pipeline(steps=steps, verbose=verbose)\n    \n    \n    def numerical_transformer(self, *, strategy=\"mean\", **params):\n        \"\"\"Transform numerical columns using `SimpleImputer`.\n        params:\n            strategy: \"mean\" | \"median\" | \"most_frequent\" | \"constant\"\n            **params: extra keyword args for SimpleImputer\"\"\"\n        \n        transformer = SimpleImputer(strategy=strategy, **params)\n        return transformer\n\n    \n    def categorical_transformer(self, *, \n                                imp_strategy=\"most_frequent\", \n                                encoder_type=\"Ordinal\", \n                                imp_params={}, encoder_params={}):\n        \"\"\"Transform categorical columns by making Pipeline\n        `SimpleImputer` | `OneHotEncoder` | `OrdinalEncoder`.\n        args:\n            imp_strategy: strategy for imputer values can be\n                \"most_frequent\" | \"constant\"\n            encoder_type: encoder type,\n                \"Ordinal\" | \"OneHot\"\n        kwargs:\n            imp_params: keyword args for `SimpleImputer`.\n            encoder_params: keyword args for encoder.`\n        \"\"\"\n        if not encoder_type in (\"Ordinal\", \"OneHot\"):\n            raise ValueError(f\"Inappropriate value for encoder_type passed: {encoder_type}\\\n            Takes one of 'Ordinal' | 'OneHot'.\")\n        \n        encoder = OrdinalEncoder if encoder_type==\"Ordinal\" else OneHotEncoder\n        transformer = Pipeline(steps=[\n            (\"imputer\", SimpleImputer(strategy=imp_strategy, **imp_params)),\n            (encoder_type, encoder(**encoder_params))\n        ])\n        return transformer\n    \n    \n    def data_preprocessor(self, *, transformers):\n        \"\"\"Preprocess the data using `ColumnTransformer`.\n        Pass extact list of transformers\n        to be passed in `ColumnTransformer`.\n        each tuple consist of: (transformer_name,\n                                transformer,\n                                list_of_columns).\"\"\"\n        preprocessor = ColumnTransformer(transformers=transformers)\n        return preprocessor\n    \n    \n    def create_model(self, *, model, random_state=0, n_estimators=1000, **kwargs):\n        \"\"\"Creates the model.\n        **kwargs: keyword args for model.\"\"\"\n        my_model = model(random_state=random_state, n_estimators=n_estimators, **kwargs)\n        return my_model","metadata":{"papermill":{"duration":0.046379,"end_time":"2022-07-07T19:25:04.136889","exception":false,"start_time":"2022-07-07T19:25:04.090510","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:40:53.209442Z","iopub.execute_input":"2022-07-07T19:40:53.210058Z","iopub.status.idle":"2022-07-07T19:40:53.223373Z","shell.execute_reply.started":"2022-07-07T19:40:53.210021Z","shell.execute_reply":"2022-07-07T19:40:53.222462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cp = CreatePipeline()\nnum_transformer = cp.numerical_transformer(strategy=\"median\")\ncat_transformer = cp.categorical_transformer(encoder_type=\"OneHot\", encoder_params={\"handle_unknown\": \"ignore\"})\npreprocessor = cp.data_preprocessor(\n                    transformers=[(\"num\", num_transformer, num_cols),\n                                  (\"cat\", cat_transformer, cat_cols)\n                                 ])","metadata":{"papermill":{"duration":0.034866,"end_time":"2022-07-07T19:25:04.197395","exception":false,"start_time":"2022-07-07T19:25:04.162529","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:40:53.224687Z","iopub.execute_input":"2022-07-07T19:40:53.225332Z","iopub.status.idle":"2022-07-07T19:40:53.240549Z","shell.execute_reply.started":"2022-07-07T19:40:53.225289Z","shell.execute_reply":"2022-07-07T19:40:53.239335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 9. Doing hyperparameter tuning with cross-validation (`XGBClassifier`)","metadata":{"papermill":{"duration":0.023517,"end_time":"2022-07-07T19:25:04.244925","exception":false,"start_time":"2022-07-07T19:25:04.221408","status":"completed"},"tags":[]}},{"cell_type":"code","source":"n_estimators = [500, 750, 1000]\nmax_depths = [5, 10]\nlearning_rate = [0.05, 0.1]\nmaes = {}\ni = 0\nfor n in n_estimators:\n    for md in max_depths:\n        for rate in learning_rate:\n            i += 1\n            model = cp.create_model(model=XGBClassifier, n_estimators=n, max_depth=md, learning_rate=rate)\n            pipeline = cp.pipeline(preprocessor=preprocessor, model=model)\n            scores = -1 * cross_val_score(pipeline, X, Y, cv=10, verbose=True,\n                                    scoring=\"neg_mean_absolute_error\")\n            mae = scores.mean()\n            maes[i] = [n, md, rate, mae]","metadata":{"papermill":{"duration":347.994742,"end_time":"2022-07-07T19:30:52.263956","exception":false,"start_time":"2022-07-07T19:25:04.269214","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:40:53.241843Z","iopub.execute_input":"2022-07-07T19:40:53.242862Z","iopub.status.idle":"2022-07-07T19:49:05.967383Z","shell.execute_reply.started":"2022-07-07T19:40:53.242811Z","shell.execute_reply":"2022-07-07T19:49:05.966394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 10. Checking MAE scores","metadata":{"papermill":{"duration":0.02542,"end_time":"2022-07-07T19:30:52.314893","exception":false,"start_time":"2022-07-07T19:30:52.289473","status":"completed"},"tags":[]}},{"cell_type":"code","source":"for i in maes:\n    n, md, rate, mae = maes[i]\n    print(f\"{i}.\\tN_estimators: {n}\\tmax_depth: {md}\\tlearning_rate: {rate}\\tMAE: {mae}\")","metadata":{"papermill":{"duration":0.033813,"end_time":"2022-07-07T19:30:52.374011","exception":false,"start_time":"2022-07-07T19:30:52.340198","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:49:05.969256Z","iopub.execute_input":"2022-07-07T19:49:05.970056Z","iopub.status.idle":"2022-07-07T19:49:05.977535Z","shell.execute_reply.started":"2022-07-07T19:49:05.970010Z","shell.execute_reply":"2022-07-07T19:49:05.976481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min(maes, key=lambda x: maes[x][3])    # but selecting 5th","metadata":{"papermill":{"duration":0.033871,"end_time":"2022-07-07T19:30:52.433440","exception":false,"start_time":"2022-07-07T19:30:52.399569","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:49:05.979807Z","iopub.execute_input":"2022-07-07T19:49:05.980624Z","iopub.status.idle":"2022-07-07T19:49:05.998036Z","shell.execute_reply.started":"2022-07-07T19:49:05.980567Z","shell.execute_reply":"2022-07-07T19:49:05.996711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 11. Selecting best parameters\n#### `n_estimators: 1000`\n#### `learning_rate: 0.05`\n#### `max_depth: 5`\n#### `MAE: 0.18739076154806492`","metadata":{"papermill":{"duration":0.025049,"end_time":"2022-07-07T19:30:52.483736","exception":false,"start_time":"2022-07-07T19:30:52.458687","status":"completed"},"tags":[]}},{"cell_type":"code","source":"best_n_estimators = 1000\nbest_max_depth = 5\nbest_rate = 0.05","metadata":{"papermill":{"duration":0.032668,"end_time":"2022-07-07T19:30:52.541974","exception":false,"start_time":"2022-07-07T19:30:52.509306","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:50:00.830482Z","iopub.execute_input":"2022-07-07T19:50:00.831864Z","iopub.status.idle":"2022-07-07T19:50:00.837480Z","shell.execute_reply.started":"2022-07-07T19:50:00.831781Z","shell.execute_reply":"2022-07-07T19:50:00.836501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 12. Training model on best parameters","metadata":{"papermill":{"duration":0.025062,"end_time":"2022-07-07T19:30:52.592648","exception":false,"start_time":"2022-07-07T19:30:52.567586","status":"completed"},"tags":[]}},{"cell_type":"code","source":"model = cp.create_model(model=XGBClassifier, n_estimators=best_n_estimators, learning_rate=best_rate, max_depth=best_max_depth)\npipeline = cp.pipeline(preprocessor=preprocessor, model=model)\npipeline.fit(X, Y)","metadata":{"papermill":{"duration":2.644684,"end_time":"2022-07-07T19:30:55.263620","exception":false,"start_time":"2022-07-07T19:30:52.618936","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:50:02.090938Z","iopub.execute_input":"2022-07-07T19:50:02.092160Z","iopub.status.idle":"2022-07-07T19:50:08.402153Z","shell.execute_reply.started":"2022-07-07T19:50:02.092104Z","shell.execute_reply":"2022-07-07T19:50:08.400889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 13. Predicting `Survived` on `test_data`","metadata":{"papermill":{"duration":0.025683,"end_time":"2022-07-07T19:30:55.315092","exception":false,"start_time":"2022-07-07T19:30:55.289409","status":"completed"},"tags":[]}},{"cell_type":"code","source":"test_preds = pipeline.predict(test_data)\ntest_preds","metadata":{"papermill":{"duration":0.066546,"end_time":"2022-07-07T19:30:55.407546","exception":false,"start_time":"2022-07-07T19:30:55.341000","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:50:08.404055Z","iopub.execute_input":"2022-07-07T19:50:08.405701Z","iopub.status.idle":"2022-07-07T19:50:08.478638Z","shell.execute_reply.started":"2022-07-07T19:50:08.405653Z","shell.execute_reply":"2022-07-07T19:50:08.476559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 14. Submitting predictions","metadata":{"papermill":{"duration":0.025785,"end_time":"2022-07-07T19:30:55.459852","exception":false,"start_time":"2022-07-07T19:30:55.434067","status":"completed"},"tags":[]}},{"cell_type":"code","source":"output = pd.DataFrame({\"PassengerId\": test_data.index, \"Survived\": test_preds})\noutput.to_csv(\"./submission_7.csv\", index=False)","metadata":{"papermill":{"duration":0.040151,"end_time":"2022-07-07T19:30:55.525792","exception":false,"start_time":"2022-07-07T19:30:55.485641","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T19:50:19.969270Z","iopub.execute_input":"2022-07-07T19:50:19.970309Z","iopub.status.idle":"2022-07-07T19:50:19.980253Z","shell.execute_reply.started":"2022-07-07T19:50:19.970263Z","shell.execute_reply":"2022-07-07T19:50:19.979166Z"},"trusted":true},"execution_count":null,"outputs":[]}]}