{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt # plots\nfrom scipy import stats\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-27T17:49:20.060534Z","iopub.execute_input":"2022-07-27T17:49:20.060914Z","iopub.status.idle":"2022-07-27T17:49:21.178425Z","shell.execute_reply.started":"2022-07-27T17:49:20.060812Z","shell.execute_reply":"2022-07-27T17:49:21.177425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def normality(x,alpha = 1e-3) -> bool:\n    \"\"\"\n    This is a normality test, comparing p-value to alpha\n    \"\"\"\n    k2, p = stats.normaltest(x.dropna().to_numpy())\n    #print(\"p = {:g}\".format(p))\n    if p < alpha:  # null hypothesis: x comes from a normal distribution\n        #The null hypothesis can be rejected - it seems to be normal\n        return True\n    else:\n        #The null hypothesis cannot be rejected - it is non-normally distributed\n        return False","metadata":{"execution":{"iopub.status.busy":"2022-07-27T17:51:40.860584Z","iopub.execute_input":"2022-07-27T17:51:40.860977Z","iopub.status.idle":"2022-07-27T17:51:40.867218Z","shell.execute_reply.started":"2022-07-27T17:51:40.860936Z","shell.execute_reply":"2022-07-27T17:51:40.866290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the dataset to dataframes\ndstrain = pd.read_csv('../input/titanic/train.csv')\ndstest = pd.read_csv('../input/titanic/test.csv')","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-07-27T17:51:45.406932Z","iopub.execute_input":"2022-07-27T17:51:45.407300Z","iopub.status.idle":"2022-07-27T17:51:45.440838Z","shell.execute_reply.started":"2022-07-27T17:51:45.407254Z","shell.execute_reply":"2022-07-27T17:51:45.439699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's take a look at the feaures of the training dataset\ndstrain.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T17:51:48.045987Z","iopub.execute_input":"2022-07-27T17:51:48.046872Z","iopub.status.idle":"2022-07-27T17:51:48.077043Z","shell.execute_reply.started":"2022-07-27T17:51:48.046831Z","shell.execute_reply":"2022-07-27T17:51:48.076006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dstrain.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-07-27T17:51:50.657921Z","iopub.execute_input":"2022-07-27T17:51:50.658203Z","iopub.status.idle":"2022-07-27T17:51:50.665302Z","shell.execute_reply.started":"2022-07-27T17:51:50.658164Z","shell.execute_reply":"2022-07-27T17:51:50.664527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's take a look at every numerical feature\ndstrain.drop(['Survived'],axis=1).hist(figsize=(20, 15));","metadata":{"execution":{"iopub.status.busy":"2022-07-27T17:51:53.436133Z","iopub.execute_input":"2022-07-27T17:51:53.436893Z","iopub.status.idle":"2022-07-27T17:51:54.476210Z","shell.execute_reply.started":"2022-07-27T17:51:53.436848Z","shell.execute_reply":"2022-07-27T17:51:54.475503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**__Data Encoding:__**\n\nsurvival\tSurvival\t0 = No, 1 = Yes\npclass\tTicket class\t1 = 1st, 2 = 2nd, 3 = 3rd\nsex\tSex\t\nAge\tAge in years\t\nsibsp\t# of siblings / spouses aboard the Titanic\t\nparch\t# of parents / children aboard the Titanic\t\nticket\tTicket number\t\nfare\tPassenger fare\t\ncabin\tCabin number\t\nembarked\tPort of Embarkation\tC = Cherbourg, Q = Queenstown, S = Southampton","metadata":{}},{"cell_type":"code","source":"#Let's inspect the correlation matrix of features in terms of Pearson correlation heatmap\nimport seaborn as sns\ncorrelation_mat = dstrain.drop(['Survived'],axis=1).iloc[:].corr()\nsns.heatmap(correlation_mat, annot = True, cmap=plt.cm.Reds);","metadata":{"execution":{"iopub.status.busy":"2022-07-27T17:51:59.973353Z","iopub.execute_input":"2022-07-27T17:51:59.973700Z","iopub.status.idle":"2022-07-27T17:52:00.704045Z","shell.execute_reply.started":"2022-07-27T17:51:59.973666Z","shell.execute_reply":"2022-07-27T17:52:00.703079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Correlation with the target variable in general is relatively weak and mostly related to the Pclass and Fare\ncor_target = dstrain.corr()\n#sns.heatmap(cor_target, annot=True, cmap=plt.cm.Reds)\n#plt.show()\ncor_target = abs(cor_target[\"Survived\"])\ncor_target[cor_target>0.2]","metadata":{"execution":{"iopub.status.busy":"2022-07-27T17:52:04.128824Z","iopub.execute_input":"2022-07-27T17:52:04.129152Z","iopub.status.idle":"2022-07-27T17:52:04.142921Z","shell.execute_reply.started":"2022-07-27T17:52:04.129118Z","shell.execute_reply":"2022-07-27T17:52:04.141650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Pclass is more strongly correlated with the target","metadata":{}},{"cell_type":"markdown","source":"- Pclass and Fare are negatively corellated with above 0.5 coeff. May be it worth to build a common feature out of both of them\n- Age is weakly negatively correlated to Pclass and SibSp (number of siblings), but not to the Fare\n- SibSp (sibling ans sposes) is positively correlated to Parch (parents and children) which is logical and once again may be used to construct a common feature.","metadata":{}},{"cell_type":"code","source":"normality(dstrain['Age'])","metadata":{"execution":{"iopub.status.busy":"2022-07-27T17:52:07.761460Z","iopub.execute_input":"2022-07-27T17:52:07.764662Z","iopub.status.idle":"2022-07-27T17:52:07.777598Z","shell.execute_reply.started":"2022-07-27T17:52:07.764578Z","shell.execute_reply":"2022-07-27T17:52:07.776051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 3 Classes, non-evenly distributed\ndstrain.groupby(\"Pclass\")[\"Pclass\"].count()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T17:52:10.382984Z","iopub.execute_input":"2022-07-27T17:52:10.383311Z","iopub.status.idle":"2022-07-27T17:52:10.392608Z","shell.execute_reply.started":"2022-07-27T17:52:10.383280Z","shell.execute_reply":"2022-07-27T17:52:10.391806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's see how many passengers had a zero Fare value:\nsum(dstrain['Fare'] == 0)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T17:52:15.105957Z","iopub.execute_input":"2022-07-27T17:52:15.106348Z","iopub.status.idle":"2022-07-27T17:52:15.115345Z","shell.execute_reply.started":"2022-07-27T17:52:15.106296Z","shell.execute_reply":"2022-07-27T17:52:15.114481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Box-Cox transform the Fare to remove the skewness (if it is positive, so OK)\n\ncrim_boxcox = stats.boxcox(dstrain[dstrain['Fare'] != 0]['Fare'])[0]\npd.Series(crim_boxcox).skew()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T17:52:17.952156Z","iopub.execute_input":"2022-07-27T17:52:17.952983Z","iopub.status.idle":"2022-07-27T17:52:17.967354Z","shell.execute_reply.started":"2022-07-27T17:52:17.952938Z","shell.execute_reply":"2022-07-27T17:52:17.966234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Age is equally distributed. Need to be normalized for models, affected by non-normalized features\n* Fare is numerical value, but is skewed right.\n- Unteresting point. 15 out of training dataset passangers had a free fare ticket. They might be a crew members or just an error. I would set their fare to some special value (like 0.1) and than perform the box-cox transform\n* Pclass has only 3 values. 1-hot encoding seems to be an option.\n* Names are worthless and shall be dropped.\n* Sex shall be changed to [0,1] values - Male = 1, Female = 0\n* Cabin has many NaNs and inadequate data. Need to be dropped or specially treated. Some hidden information may be there.\n- The number of cabin consists of a letter (probably indicating a deck) and the number. NaNs are probably for the people that have traveled without the cabin.\n- Let's reformat the data in such a way, that the Cabin deck will be extracted as A=1,B=2,etc and NaN=0, while the numbers of rooms will be dropped.\n* Ticket number is worhtless as well - drop it\n* Passenger ID - drop it, evely distributed, uncorrelated and probably has no effect on any result\n* \"Embarked\" feature might have some correlation with the Fare. May be unified by it or just one-hot encode it\n* Lasso reularization might be used on the set of features (despite the fact the number of features is relatively small and might be evaluated manually)","metadata":{}},{"cell_type":"code","source":"#List of all decks:\nset(dstrain['Cabin'].str[0].tolist())","metadata":{"execution":{"iopub.status.busy":"2022-07-27T17:52:26.834633Z","iopub.execute_input":"2022-07-27T17:52:26.834983Z","iopub.status.idle":"2022-07-27T17:52:26.845408Z","shell.execute_reply.started":"2022-07-27T17:52:26.834953Z","shell.execute_reply":"2022-07-27T17:52:26.843774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_constructor(df):\n    \"\"\"\n    This function will remove the unnecessary features, encode the parametric values and construct the rest\n    \"\"\"\n    #Age - 177 NaNs in the training set. It is too much to drop. I'd take an average or median age\n    df['Age'] = df['Age'].fillna(df['Age'].median())\n\n    # Embarked - convert to numeric value\n    df = df.join(pd.get_dummies(df['Embarked'],prefix='EmbarkedIn', prefix_sep='_'))\n    df = df.drop('Embarked',axis = 1)\n    \n    medianF = df['Fare'].median()\n    # Fare - replace the 0 with median fare and apply Box-Cox transform\n    #df['Fare'].replace(to_replace = 0,value = medianF, inplace=True)\n    \n    # Fare - replace the 0 with 0.1 Fare and apply Box-Cox transform\n    df['Fare'].replace(to_replace = 0,value = 0.1, inplace=True)\n\n    # Constuct feature for Pclass/Fare (can cause a loss of information?)\n    df['Fare_per_class'] = df['Fare']/df['Pclass']\n    df['Fare_per_class'] = stats.boxcox(df['Fare_per_class'])[0]\n    df = df.drop('Fare',axis = 1)\n    \n    # Extract the Decks from Cabin numebers and one-hot encode them. Drop the Cabin\n    df['Deck'] = df['Cabin'].astype(str).str.upper().str[0]\n    #df.loc[df[\"Deck\"] == \"A\", \"Deck\"] = 1\n    df = df.join(pd.get_dummies(df['Deck']))\n    df = df.drop('Deck',axis = 1)\n    df = df.drop('Cabin',axis = 1)\n    try:\n        df = df.drop('T',axis = 1) # This is a misstake value, not needed.\n    except:\n        pass #Non-existing feature\n\n    # Dropping 'PassengerId','Name','Ticket'\n    df = df.drop(['PassengerId','Name','Ticket'],axis=1)\n    \n    # Applying one-hot encoding for 'Sex' [\"Female,Male\"] -> [0,1]\n    df = df.join(pd.get_dummies(df['Sex']))\n    df = df.drop('Sex',axis = 1)\n    \n    # Remove the remaining NaNs, just in case\n    pass\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-07-27T17:52:31.121065Z","iopub.execute_input":"2022-07-27T17:52:31.121851Z","iopub.status.idle":"2022-07-27T17:52:31.134968Z","shell.execute_reply.started":"2022-07-27T17:52:31.121793Z","shell.execute_reply":"2022-07-27T17:52:31.134266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Construct features and targets for Train and Validation datasets and split them accordingly\nfrom sklearn.model_selection import train_test_split\n\ny = dstrain['Survived']\nX = feature_constructor(dstrain).drop('Survived',axis=1)\n\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2022-05-22T12:25:04.161613Z","iopub.execute_input":"2022-05-22T12:25:04.161860Z","iopub.status.idle":"2022-05-22T12:25:04.204951Z","shell.execute_reply.started":"2022-05-22T12:25:04.161831Z","shell.execute_reply":"2022-05-22T12:25:04.204057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n# Try the alternative models, as provided by auto-sklearn:\nimport sklearn\nfrom sklearn.model_selection import train_test_split\ny = dstrain['Survived']\nX = feature_constructor(dstrain).drop('Survived',axis=1)\n\nX_train, X_test, y_train, y_test = \\\n    sklearn.model_selection.train_test_split(X, y, random_state=1)\n    \n# Fit the auto-sklearn\nimport autosklearn.regression\nimport sklearn.metrics\nmodel = autosklearn.classification.AutoSklearnClassifier(time_left_for_this_task=120, per_run_time_limit=30)\n\nmodel.fit(X_train, y_train, dataset_name='Titanic')\nensemble_dict = model.show_models()\n\ndf_cv_results = pd.DataFrame(model.cv_results_).sort_values(by = 'mean_test_score', ascending = False)\ndf_cv_results\n\nmodel.leaderboard(detailed = True, ensemble_only=False)\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2022-07-27T17:54:56.456948Z","iopub.execute_input":"2022-07-27T17:54:56.457298Z","iopub.status.idle":"2022-07-27T17:54:56.494778Z","shell.execute_reply.started":"2022-07-27T17:54:56.457264Z","shell.execute_reply":"2022-07-27T17:54:56.493730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will try: \n- Decision Tree classifier\n- RandomForestClassifier\n- AdaBoostClassifier\n- GradientBoostingClassifier\n- XGBClassifier with k-fold cross-validation\n\nAccuracy will be used as The Metrics","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score, confusion_matrix\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.ensemble import RandomForestClassifier,AdaBoostClassifier,GradientBoostingClassifier\nfrom xgboost import XGBClassifier\nfrom sklearn.model_selection import cross_val_score, KFold","metadata":{"execution":{"iopub.status.busy":"2022-05-22T12:25:04.206163Z","iopub.execute_input":"2022-05-22T12:25:04.206667Z","iopub.status.idle":"2022-05-22T12:25:04.210977Z","shell.execute_reply.started":"2022-05-22T12:25:04.206636Z","shell.execute_reply":"2022-05-22T12:25:04.210400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Decision Tree Classifier\nclf = DecisionTreeClassifier()\nclf.fit(X_train, y_train)\ny_pred = clf.predict(X_val)\ndecision_tree_accuracy = accuracy_score(y_val, y_pred)","metadata":{"execution":{"iopub.status.busy":"2022-05-22T12:25:04.212377Z","iopub.execute_input":"2022-05-22T12:25:04.212665Z","iopub.status.idle":"2022-05-22T12:25:04.234550Z","shell.execute_reply.started":"2022-05-22T12:25:04.212627Z","shell.execute_reply":"2022-05-22T12:25:04.233517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Randoom Forest Classifier\nclf = RandomForestClassifier(n_estimators=50, max_features=\"auto\",random_state=0)\nclf.fit(X_train, y_train)\ny_pred = clf.predict(X_val)\nrandom_forest_accuracy = accuracy_score(y_val, y_pred)","metadata":{"execution":{"iopub.status.busy":"2022-05-22T12:25:04.236041Z","iopub.execute_input":"2022-05-22T12:25:04.236450Z","iopub.status.idle":"2022-05-22T12:25:04.366792Z","shell.execute_reply.started":"2022-05-22T12:25:04.236312Z","shell.execute_reply":"2022-05-22T12:25:04.365920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Ada Boost Classifier\nclf = AdaBoostClassifier(n_estimators=50)\nclf.fit(X_train, y_train)\ny_pred = clf.predict(X_val)\nadaboost_accuracy = accuracy_score(y_val, y_pred)","metadata":{"execution":{"iopub.status.busy":"2022-05-22T12:25:04.367929Z","iopub.execute_input":"2022-05-22T12:25:04.368154Z","iopub.status.idle":"2022-05-22T12:25:04.486779Z","shell.execute_reply.started":"2022-05-22T12:25:04.368126Z","shell.execute_reply":"2022-05-22T12:25:04.485983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Gradient Boosting Classifier\nclf = GradientBoostingClassifier(n_estimators=50)\nclf.fit(X_train, y_train)\ny_pred = clf.predict(X_val)\ngradient_boosting_accuracy = accuracy_score(y_val, y_pred)","metadata":{"execution":{"iopub.status.busy":"2022-05-22T12:25:04.487951Z","iopub.execute_input":"2022-05-22T12:25:04.488181Z","iopub.status.idle":"2022-05-22T12:25:04.565401Z","shell.execute_reply.started":"2022-05-22T12:25:04.488152Z","shell.execute_reply":"2022-05-22T12:25:04.564512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# XGB Classifier\nxgbc = XGBClassifier(max_depth=2,n_estimators=150)\nxgbc.fit(X_train, y_train)\nscores = cross_val_score(xgbc, X_train, y_train, cv=5)\nprint(\"Mean cross-validation score: %.2f\" % scores.mean())\nkfold = KFold(n_splits=10, shuffle=True)\nkf_cv_scores = cross_val_score(xgbc, X_train, y_train, cv=kfold )\nprint(\"K-fold CV average score: %.2f\" % kf_cv_scores.mean())\nypred = xgbc.predict(X_val)\nprint(confusion_matrix(y_val,y_pred))\nXGB_accuracy = accuracy_score(y_val, y_pred)","metadata":{"execution":{"iopub.status.busy":"2022-05-22T13:22:25.419484Z","iopub.execute_input":"2022-05-22T13:22:25.420092Z","iopub.status.idle":"2022-05-22T13:22:31.638189Z","shell.execute_reply.started":"2022-05-22T13:22:25.420051Z","shell.execute_reply":"2022-05-22T13:22:31.636947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Decision Tree Classifier = ', round(decision_tree_accuracy,4))\nprint('Randoom Forest Classifier = ', round(random_forest_accuracy,4))\nprint('Ada Boost Classifier = ', round(adaboost_accuracy,4))\nprint('Gradient Boosting Classifier Accuracy = ', round(gradient_boosting_accuracy,4))\nprint('XGB Classifier Accuracy = ', round(XGB_accuracy,4))","metadata":{"execution":{"iopub.status.busy":"2022-05-22T12:25:10.652840Z","iopub.execute_input":"2022-05-22T12:25:10.653365Z","iopub.status.idle":"2022-05-22T12:25:10.666505Z","shell.execute_reply.started":"2022-05-22T12:25:10.653324Z","shell.execute_reply":"2022-05-22T12:25:10.665491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"XGB seems to be a good one. Let's fine-tune it's hyperparemeters","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import GridSearchCV\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2022-05-22T12:25:10.667944Z","iopub.execute_input":"2022-05-22T12:25:10.668759Z","iopub.status.idle":"2022-05-22T12:25:10.680433Z","shell.execute_reply.started":"2022-05-22T12:25:10.668718Z","shell.execute_reply":"2022-05-22T12:25:10.679764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def algorithm_pipeline(X_train_data, X_test_data, y_train_data, y_test_data, \n                       model, param_grid, cv=10, scoring_fit='neg_mean_squared_error',\n                       do_probabilities = False):\n    gs = GridSearchCV(\n        estimator=model,\n        param_grid=param_grid, \n        cv=cv, \n        n_jobs=-1, \n        scoring=scoring_fit,\n        verbose=2\n    )\n    fitted_model = gs.fit(X_train_data, y_train_data)\n    \n    if do_probabilities:\n        pred = fitted_model.predict_proba(X_test_data)\n    else:\n        pred = fitted_model.predict(X_test_data)\n    \n    return fitted_model, pred","metadata":{"execution":{"iopub.status.busy":"2022-05-22T12:25:10.681787Z","iopub.execute_input":"2022-05-22T12:25:10.682436Z","iopub.status.idle":"2022-05-22T12:25:10.694060Z","shell.execute_reply.started":"2022-05-22T12:25:10.682395Z","shell.execute_reply":"2022-05-22T12:25:10.693358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = XGBClassifier()\n\nparam_grid = {\n        'min_child_weight': [1, 5, 10],\n        'gamma': [1, 1.5, 2, 5, 10],\n        'subsample': [1.0],\n        'colsample_bytree': [0.6, 0.8, 1.0],\n        'max_depth': [4, 5, 7],\n        'n_estimators': [100],\n        'alpha': [0.2, 0.5, 1],\n        'lambda': [0.2, 0.5, 1],\n        'num_parallel_tree' : [5]\n        }\n\nmodel, y_pred = algorithm_pipeline(X_train, X_val, y_train, y_val, model, \n                                 param_grid, cv=10)","metadata":{"execution":{"iopub.status.busy":"2022-05-22T12:25:10.695383Z","iopub.execute_input":"2022-05-22T12:25:10.696248Z","iopub.status.idle":"2022-05-22T13:07:15.677750Z","shell.execute_reply.started":"2022-05-22T12:25:10.696175Z","shell.execute_reply":"2022-05-22T13:07:15.676938Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Best XGB model is:\nprint(model.best_params_)\n\nbest_model = XGBClassifier()\nbest_model.set_params(**model.best_params_)\nbest_model.fit(X_train, y_train)\nscores = cross_val_score(best_model, X_train, y_train, cv=10)\nprint(\"Mean cross-validation score: %.2f\" % scores.mean())\nkfold = KFold(n_splits=10, shuffle=True)\nkf_cv_scores = cross_val_score(best_model, X_train, y_train, cv=kfold )\nprint(\"K-fold CV average score: %.2f\" % kf_cv_scores.mean())\ny_pred = best_model.predict(X_val)\nprint(confusion_matrix(y_val,y_pred))\nbest_model_accuracy = accuracy_score(y_val, y_pred)\nprint('Best Model Accuracy = ', round(best_model_accuracy,4))","metadata":{"execution":{"iopub.status.busy":"2022-05-22T13:07:15.685662Z","iopub.execute_input":"2022-05-22T13:07:15.689260Z","iopub.status.idle":"2022-05-22T13:07:23.067565Z","shell.execute_reply.started":"2022-05-22T13:07:15.689180Z","shell.execute_reply":"2022-05-22T13:07:23.066783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Alternative Best Model\nparams = {\n    'alpha': 0.1, \n    'colsample_bytree': 0.8, \n    'gamma': 1.4, \n    'lambda': 0.1, \n    'max_depth': 5, \n    'min_child_weight': 5, \n    'n_estimators': 1000, \n    'num_parallel_tree': 5, \n    'subsample': 1}\nmodel1 = XGBClassifier()\nmodel1.set_params(**params)\nmodel1.fit(X_train, y_train)\n#scores = cross_val_score(model1, X_train, y_train, cv=10)\ny_pred = model1.predict(X_val)\nprint(confusion_matrix(y_val,y_pred))\naccuracy_score(y_val, y_pred)","metadata":{"execution":{"iopub.status.busy":"2022-05-22T13:18:37.866764Z","iopub.execute_input":"2022-05-22T13:18:37.867882Z","iopub.status.idle":"2022-05-22T13:18:43.234145Z","shell.execute_reply.started":"2022-05-22T13:18:37.867829Z","shell.execute_reply":"2022-05-22T13:18:43.231356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Make a prediction with the best model on test data\nX_test = feature_constructor(dstest)\ny_pred = best_model.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T18:26:12.324527Z","iopub.execute_input":"2022-07-27T18:26:12.324879Z","iopub.status.idle":"2022-07-27T18:26:12.382100Z","shell.execute_reply.started":"2022-07-27T18:26:12.324845Z","shell.execute_reply":"2022-07-27T18:26:12.380791Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Export the predicted results\npd.DataFrame(dstest['PassengerId']).join(pd.DataFrame({\"Survived\":y_pred})).to_csv('/kaggle/working/TitanicPredict.csv')","metadata":{"execution":{"iopub.status.busy":"2022-05-22T13:19:07.355958Z","iopub.execute_input":"2022-05-22T13:19:07.356621Z","iopub.status.idle":"2022-05-22T13:19:07.367245Z","shell.execute_reply.started":"2022-05-22T13:19:07.356568Z","shell.execute_reply":"2022-05-22T13:19:07.366475Z"},"trusted":true},"execution_count":null,"outputs":[]}]}