{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-11T14:30:17.255775Z","iopub.execute_input":"2022-07-11T14:30:17.256171Z","iopub.status.idle":"2022-07-11T14:30:17.265476Z","shell.execute_reply.started":"2022-07-11T14:30:17.256139Z","shell.execute_reply":"2022-07-11T14:30:17.264443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import GridSearchCV, cross_val_score\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import f1_score, roc_auc_score, precision_score, classification_report, precision_recall_curve, confusion_matrix\nfrom sklearn.metrics import precision_score, recall_score, f1_score, accuracy_score\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.ensemble import GradientBoostingClassifier\nfrom lightgbm import LGBMClassifier\nfrom sklearn.model_selection import GridSearchCV, StratifiedKFold\nimport random\nfrom sklearn.model_selection import train_test_split\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:34:55.028674Z","iopub.execute_input":"2022-07-11T14:34:55.029071Z","iopub.status.idle":"2022-07-11T14:34:55.037002Z","shell.execute_reply.started":"2022-07-11T14:34:55.029039Z","shell.execute_reply":"2022-07-11T14:34:55.035543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data preparation ","metadata":{}},{"cell_type":"code","source":"train=pd.read_csv('../input/titanic/train.csv')\ntest=pd.read_csv('../input/titanic/test.csv')\ntrain.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:30:17.279342Z","iopub.execute_input":"2022-07-11T14:30:17.279722Z","iopub.status.idle":"2022-07-11T14:30:17.307028Z","shell.execute_reply.started":"2022-07-11T14:30:17.279689Z","shell.execute_reply":"2022-07-11T14:30:17.305939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:30:17.309703Z","iopub.execute_input":"2022-07-11T14:30:17.310042Z","iopub.status.idle":"2022-07-11T14:30:17.328117Z","shell.execute_reply.started":"2022-07-11T14:30:17.310012Z","shell.execute_reply":"2022-07-11T14:30:17.327373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['Cabin'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:30:17.329562Z","iopub.execute_input":"2022-07-11T14:30:17.329868Z","iopub.status.idle":"2022-07-11T14:30:17.339805Z","shell.execute_reply.started":"2022-07-11T14:30:17.329818Z","shell.execute_reply":"2022-07-11T14:30:17.338593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:30:17.341162Z","iopub.execute_input":"2022-07-11T14:30:17.341636Z","iopub.status.idle":"2022-07-11T14:30:17.364052Z","shell.execute_reply.started":"2022-07-11T14:30:17.341590Z","shell.execute_reply":"2022-07-11T14:30:17.362924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Bootstrap for Train dataset","metadata":{}},{"cell_type":"code","source":"train_2 = train.sample(replace=True, n=500)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:30:17.365969Z","iopub.execute_input":"2022-07-11T14:30:17.366365Z","iopub.status.idle":"2022-07-11T14:30:17.373244Z","shell.execute_reply.started":"2022-07-11T14:30:17.366316Z","shell.execute_reply":"2022-07-11T14:30:17.372262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def resample_data_linearly(df_main, df_new, n_samples, sample_size):\n    df_new = df_main.sample(replace=True, n=sample_size)\n    for i in range (n_samples):\n         #print(i)\n         df_new = pd.concat([df_new, df_main.sample(replace=True, n=sample_size)])\n    return df_new","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:30:17.374807Z","iopub.execute_input":"2022-07-11T14:30:17.375406Z","iopub.status.idle":"2022-07-11T14:30:17.385020Z","shell.execute_reply.started":"2022-07-11T14:30:17.375360Z","shell.execute_reply":"2022-07-11T14:30:17.383757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_2 = resample_data_linearly(train, train_2, 20, 600)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:30:17.390030Z","iopub.execute_input":"2022-07-11T14:30:17.391185Z","iopub.status.idle":"2022-07-11T14:30:17.459377Z","shell.execute_reply.started":"2022-07-11T14:30:17.391132Z","shell.execute_reply":"2022-07-11T14:30:17.458030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_2.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:30:17.460811Z","iopub.execute_input":"2022-07-11T14:30:17.461296Z","iopub.status.idle":"2022-07-11T14:30:17.483968Z","shell.execute_reply.started":"2022-07-11T14:30:17.461261Z","shell.execute_reply":"2022-07-11T14:30:17.482450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['Survived'].mean()-train_2['Survived'].mean()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:30:17.485199Z","iopub.execute_input":"2022-07-11T14:30:17.486534Z","iopub.status.idle":"2022-07-11T14:30:17.496475Z","shell.execute_reply.started":"2022-07-11T14:30:17.486481Z","shell.execute_reply":"2022-07-11T14:30:17.495202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature engineerinng","metadata":{}},{"cell_type":"code","source":"train['Sex'] = pd.factorize(train.Sex)[0]\ntest['Sex'] = pd.factorize(test.Sex)[0]","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:30:17.497700Z","iopub.execute_input":"2022-07-11T14:30:17.498561Z","iopub.status.idle":"2022-07-11T14:30:17.508185Z","shell.execute_reply.started":"2022-07-11T14:30:17.498523Z","shell.execute_reply":"2022-07-11T14:30:17.506975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# choose categorical and continuous features from data\n\ncategorical_columns = [c for c in test.columns \n                       if test[c].dtype.name == 'object' \n                       and test[c].name != 'Name' \n                       and test[c].name != 'Ticket' \n                       and test[c].name != 'Cabin'\n                       or test[c].name == 'Pclass']\nnumerical_columns = [c for c in test.columns \n                     if test[c].dtype.name != 'object' \n                     and test[c].name != 'PassengerId' \n                     and test[c].name != 'Pclass']\n\nprint('categorical_columns:', categorical_columns)\nprint('numerical_columns:', numerical_columns)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:30:17.509632Z","iopub.execute_input":"2022-07-11T14:30:17.510250Z","iopub.status.idle":"2022-07-11T14:30:17.522029Z","shell.execute_reply.started":"2022-07-11T14:30:17.510211Z","shell.execute_reply":"2022-07-11T14:30:17.520752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fill missing data\n\nfor c in categorical_columns:\n    train[c].fillna(train[c].mode()[0], inplace=True)\n    test[c].fillna(train[c].mode()[0], inplace=True)\n    \nfor c in numerical_columns:\n    train[c].fillna(train[c].median(), inplace=True)\n    test[c].fillna(train[c].median(), inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:30:17.523813Z","iopub.execute_input":"2022-07-11T14:30:17.524310Z","iopub.status.idle":"2022-07-11T14:30:17.547857Z","shell.execute_reply.started":"2022-07-11T14:30:17.524252Z","shell.execute_reply":"2022-07-11T14:30:17.546413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = pd.concat([train[numerical_columns],\n    pd.get_dummies(train[categorical_columns])], axis=1)\n\nX_test = pd.concat([test[numerical_columns],\n    pd.get_dummies(test[categorical_columns])], axis=1)\n\ny_train = train['Survived']","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:30:17.549578Z","iopub.execute_input":"2022-07-11T14:30:17.549954Z","iopub.status.idle":"2022-07-11T14:30:17.571572Z","shell.execute_reply.started":"2022-07-11T14:30:17.549924Z","shell.execute_reply":"2022-07-11T14:30:17.570219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('categorical_columns:', categorical_columns)\nprint('numerical_columns:', numerical_columns)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:30:17.572880Z","iopub.execute_input":"2022-07-11T14:30:17.573546Z","iopub.status.idle":"2022-07-11T14:30:17.579653Z","shell.execute_reply.started":"2022-07-11T14:30:17.573497Z","shell.execute_reply":"2022-07-11T14:30:17.578409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:30:17.581272Z","iopub.execute_input":"2022-07-11T14:30:17.582395Z","iopub.status.idle":"2022-07-11T14:30:17.611247Z","shell.execute_reply.started":"2022-07-11T14:30:17.582328Z","shell.execute_reply":"2022-07-11T14:30:17.610042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = []\n\ndef evaluate_results(name, y_train, y_predict):\n    print('Classification results:')\n    f1 = f1_score(y_train, y_predict)\n    print(\"f1: %.2f%%\" % (f1 * 100.0)) \n    roc = roc_auc_score(y_train, y_predict)\n    print(\"roc: %.2f%%\" % (roc * 100.0)) \n    gini = 2*roc_auc_score(y_train, y_predict)-1\n    print(\"gini: %.2f%%\" % (gini * 100.0)) \n    rec = recall_score(y_train, y_predict, average='binary')\n    print(\"recall: %.2f%%\" % (rec * 100.0)) \n    prc = precision_score(y_train, y_predict, average='binary')\n    print(\"precision: %.2f%%\" % (prc * 100.0)) \n    acc = accuracy_score(y_train, y_predict)\n    print(\"accuracy: %.2f%%\" % (acc * 100.0)) \n    mean_fact = y_train.mean()\n    print(\"mean_fact: %.2f%%\" % (mean_fact * 100.0)) \n    mean_pred = y_predict.mean()\n    print(\"mean_pred: %.2f%%\" % (mean_pred * 100.0)) \n\n\n    result = [name, prc, rec, roc, gini,  f1, acc, mean_fact, mean_pred]\n    results.append(result)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:30:17.612936Z","iopub.execute_input":"2022-07-11T14:30:17.613647Z","iopub.status.idle":"2022-07-11T14:30:17.624877Z","shell.execute_reply.started":"2022-07-11T14:30:17.613596Z","shell.execute_reply":"2022-07-11T14:30:17.623802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Decision tree","metadata":{}},{"cell_type":"code","source":"%%time\ntree_params = {'max_depth': range(2, 11), \"max_features\": range(2, 10)}\n\ntree_1 = GridSearchCV(DecisionTreeClassifier(random_state=17),\n                                 tree_params, cv=5)                  \n\ntree_1.fit(X_train, y_train)\nprint(\"Best params:\", tree_1.best_params_)\nprint(\"Best cross validaton score\", tree_1.best_score_)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:30:17.626305Z","iopub.execute_input":"2022-07-11T14:30:17.626915Z","iopub.status.idle":"2022-07-11T14:30:19.730503Z","shell.execute_reply.started":"2022-07-11T14:30:17.626883Z","shell.execute_reply":"2022-07-11T14:30:19.729278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evaluate_results('tuned_tree_1_w_cv', y_train, tree_1.predict(X_train))","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:30:19.735997Z","iopub.execute_input":"2022-07-11T14:30:19.736417Z","iopub.status.idle":"2022-07-11T14:30:19.757745Z","shell.execute_reply.started":"2022-07-11T14:30:19.736376Z","shell.execute_reply":"2022-07-11T14:30:19.756669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tuned_tree = DecisionTreeClassifier(random_state=17, max_depth=10, max_features= 9)\ntuned_tree.fit(X_train, y_train);","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:30:19.759259Z","iopub.execute_input":"2022-07-11T14:30:19.759636Z","iopub.status.idle":"2022-07-11T14:30:19.769454Z","shell.execute_reply.started":"2022-07-11T14:30:19.759602Z","shell.execute_reply":"2022-07-11T14:30:19.768443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evaluate_results('tuned_tree_2_w_cv', y_train, tuned_tree.predict(X_train))","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:30:19.771201Z","iopub.execute_input":"2022-07-11T14:30:19.771992Z","iopub.status.idle":"2022-07-11T14:30:19.792173Z","shell.execute_reply.started":"2022-07-11T14:30:19.771951Z","shell.execute_reply":"2022-07-11T14:30:19.791323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#feature importance\nfor importance, name in sorted(zip(tuned_tree.feature_importances_, X_train.columns),reverse=True)[:5]:\n    print (name, importance)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:30:19.793859Z","iopub.execute_input":"2022-07-11T14:30:19.794582Z","iopub.status.idle":"2022-07-11T14:30:19.801107Z","shell.execute_reply.started":"2022-07-11T14:30:19.794541Z","shell.execute_reply":"2022-07-11T14:30:19.799961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# KNN","metadata":{}},{"cell_type":"code","source":"from sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.preprocessing import StandardScaler","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:30:19.802853Z","iopub.execute_input":"2022-07-11T14:30:19.803615Z","iopub.status.idle":"2022-07-11T14:30:19.811404Z","shell.execute_reply.started":"2022-07-11T14:30:19.803562Z","shell.execute_reply":"2022-07-11T14:30:19.810413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for kNN, we need to scale features\n\nknn = KNeighborsClassifier(n_neighbors=10)\nscaler = StandardScaler()\nX_train_scaled = scaler.fit_transform(X_train)\nX_test_scaled = scaler.transform(X_test)\nknn.fit(X_train_scaled, y_train)\n\nknn_pred = knn.predict(X_train)\naccuracy_score(y_train, knn_pred)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:30:19.813027Z","iopub.execute_input":"2022-07-11T14:30:19.813784Z","iopub.status.idle":"2022-07-11T14:30:19.881141Z","shell.execute_reply.started":"2022-07-11T14:30:19.813739Z","shell.execute_reply":"2022-07-11T14:30:19.880387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.pipeline import Pipeline\n\nknn_pipe = Pipeline(\n    [(\"scaler\", StandardScaler()), (\"knn\", KNeighborsClassifier(n_jobs=-1))]\n)\n\nknn_params = {\"knn__n_neighbors\": range(1, 10)}\n\nknn_grid = GridSearchCV(knn_pipe, knn_params, cv=5, n_jobs=-1, verbose=True)\n\nknn_grid.fit(X_train, y_train)\n\nknn_grid.best_params_, knn_grid.best_score_","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:30:19.882590Z","iopub.execute_input":"2022-07-11T14:30:19.883175Z","iopub.status.idle":"2022-07-11T14:30:21.364988Z","shell.execute_reply.started":"2022-07-11T14:30:19.883142Z","shell.execute_reply":"2022-07-11T14:30:21.363802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evaluate_results('knn_grid', y_train, knn_grid.predict(X_train))","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:30:21.366509Z","iopub.execute_input":"2022-07-11T14:30:21.367298Z","iopub.status.idle":"2022-07-11T14:30:21.514065Z","shell.execute_reply.started":"2022-07-11T14:30:21.367262Z","shell.execute_reply":"2022-07-11T14:30:21.512919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Random Forest","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\n#grid search for RF\nforest_params = {'max_depth': range(5, 16),\n                 'max_features': range(2, 20)}\n\nlocally_best_forest = GridSearchCV(\n    RandomForestClassifier(n_estimators=10, random_state=17, \n                           n_jobs=4),\n    forest_params, cv=3, verbose=1, n_jobs=4, scoring='roc_auc')\n\nlocally_best_forest.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:35:08.210014Z","iopub.execute_input":"2022-07-11T14:35:08.210814Z","iopub.status.idle":"2022-07-11T14:35:32.327681Z","shell.execute_reply.started":"2022-07-11T14:35:08.210756Z","shell.execute_reply":"2022-07-11T14:35:32.326252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Best params:\", locally_best_forest.best_params_)\nprint(\"Best cross validaton score\", locally_best_forest.best_score_)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:30:45.713332Z","iopub.execute_input":"2022-07-11T14:30:45.713673Z","iopub.status.idle":"2022-07-11T14:30:45.719327Z","shell.execute_reply.started":"2022-07-11T14:30:45.713642Z","shell.execute_reply":"2022-07-11T14:30:45.718191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evaluate_results('RF', y_train, locally_best_forest.predict(X_train))","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:30:45.720534Z","iopub.execute_input":"2022-07-11T14:30:45.720838Z","iopub.status.idle":"2022-07-11T14:30:45.844375Z","shell.execute_reply.started":"2022-07-11T14:30:45.720810Z","shell.execute_reply":"2022-07-11T14:30:45.843547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"locally_best_forest_2 = RandomForestClassifier(random_state=17, max_depth= 5, max_features= 5)\nlocally_best_forest_2.fit(X_train, y_train);","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:30:45.845766Z","iopub.execute_input":"2022-07-11T14:30:45.846365Z","iopub.status.idle":"2022-07-11T14:30:46.063153Z","shell.execute_reply.started":"2022-07-11T14:30:45.846316Z","shell.execute_reply":"2022-07-11T14:30:46.061784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evaluate_results('RF_2', y_train, locally_best_forest_2.predict(X_train))","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:30:46.064787Z","iopub.execute_input":"2022-07-11T14:30:46.065505Z","iopub.status.idle":"2022-07-11T14:30:46.108765Z","shell.execute_reply.started":"2022-07-11T14:30:46.065453Z","shell.execute_reply":"2022-07-11T14:30:46.107536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(locally_best_forest_2.feature_importances_,\n             index=X_train.columns, columns=['Importance']).sort_values(\n    by='Importance', ascending=False)[:30]","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:30:46.110565Z","iopub.execute_input":"2022-07-11T14:30:46.111734Z","iopub.status.idle":"2022-07-11T14:30:46.138089Z","shell.execute_reply.started":"2022-07-11T14:30:46.111675Z","shell.execute_reply":"2022-07-11T14:30:46.136842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Random Forest with optuna","metadata":{}},{"cell_type":"markdown","source":"# LogisticRegression","metadata":{}},{"cell_type":"code","source":"%%time\nlogit = LogisticRegression(solver=\"lbfgs\", n_jobs=-1, random_state=7)\nlogit.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:52:09.067132Z","iopub.execute_input":"2022-07-01T06:52:09.06823Z","iopub.status.idle":"2022-07-01T06:52:09.129239Z","shell.execute_reply.started":"2022-07-01T06:52:09.068168Z","shell.execute_reply":"2022-07-01T06:52:09.12807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evaluate_results('logreg_1', y_train, logit.predict(X_train))","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:52:09.131357Z","iopub.execute_input":"2022-07-01T06:52:09.132254Z","iopub.status.idle":"2022-07-01T06:52:09.154216Z","shell.execute_reply.started":"2022-07-01T06:52:09.132187Z","shell.execute_reply":"2022-07-01T06:52:09.15289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# LogReg and tunning","metadata":{}},{"cell_type":"code","source":"lr = LogisticRegression(random_state=5)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:52:09.156111Z","iopub.execute_input":"2022-07-01T06:52:09.156837Z","iopub.status.idle":"2022-07-01T06:52:09.162446Z","shell.execute_reply.started":"2022-07-01T06:52:09.156794Z","shell.execute_reply":"2022-07-01T06:52:09.16129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"parameters = {'C': (0.0001, 0.001, 0.01, 0.1, 1, 10)}","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:52:09.16949Z","iopub.execute_input":"2022-07-01T06:52:09.170987Z","iopub.status.idle":"2022-07-01T06:52:09.176816Z","shell.execute_reply.started":"2022-07-01T06:52:09.170934Z","shell.execute_reply":"2022-07-01T06:52:09.175663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=5)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:52:09.178625Z","iopub.execute_input":"2022-07-01T06:52:09.179351Z","iopub.status.idle":"2022-07-01T06:52:09.188104Z","shell.execute_reply.started":"2022-07-01T06:52:09.179306Z","shell.execute_reply":"2022-07-01T06:52:09.186971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grid_search = GridSearchCV(lr, parameters, n_jobs=-1, scoring='roc_auc', cv=skf)\ngrid_search = grid_search.fit(X_train, y_train)\ngrid_search.best_estimator_","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:52:09.189757Z","iopub.execute_input":"2022-07-01T06:52:09.190448Z","iopub.status.idle":"2022-07-01T06:52:09.784878Z","shell.execute_reply.started":"2022-07-01T06:52:09.190405Z","shell.execute_reply":"2022-07-01T06:52:09.783439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr_2 = LogisticRegression(C=1, dual=False,\n                          intercept_scaling=1, max_iter=100, n_jobs=None, penalty='l2', random_state=5,\n                          tol=0.0001, verbose=0, warm_start=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:52:09.787364Z","iopub.execute_input":"2022-07-01T06:52:09.788084Z","iopub.status.idle":"2022-07-01T06:52:09.794658Z","shell.execute_reply.started":"2022-07-01T06:52:09.788039Z","shell.execute_reply":"2022-07-01T06:52:09.793482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr_2.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:52:09.796346Z","iopub.execute_input":"2022-07-01T06:52:09.797021Z","iopub.status.idle":"2022-07-01T06:52:09.847521Z","shell.execute_reply.started":"2022-07-01T06:52:09.79698Z","shell.execute_reply":"2022-07-01T06:52:09.846724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evaluate_results('logreg_3', y_train, lr_2.predict(X_train))","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:52:09.848744Z","iopub.execute_input":"2022-07-01T06:52:09.849259Z","iopub.status.idle":"2022-07-01T06:52:09.863969Z","shell.execute_reply.started":"2022-07-01T06:52:09.849229Z","shell.execute_reply":"2022-07-01T06:52:09.863188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# LGBM","metadata":{}},{"cell_type":"code","source":"lgb_clf = LGBMClassifier(random_state=17)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:52:09.865298Z","iopub.execute_input":"2022-07-01T06:52:09.86563Z","iopub.status.idle":"2022-07-01T06:52:09.871235Z","shell.execute_reply.started":"2022-07-01T06:52:09.8656Z","shell.execute_reply":"2022-07-01T06:52:09.870387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nlgb_clf.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:52:09.873039Z","iopub.execute_input":"2022-07-01T06:52:09.873396Z","iopub.status.idle":"2022-07-01T06:52:10.194776Z","shell.execute_reply.started":"2022-07-01T06:52:09.873366Z","shell.execute_reply":"2022-07-01T06:52:10.192935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evaluate_results('lgbm_1', y_train, lgb_clf.predict(X_train))","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:52:10.196392Z","iopub.execute_input":"2022-07-01T06:52:10.19698Z","iopub.status.idle":"2022-07-01T06:52:10.220935Z","shell.execute_reply.started":"2022-07-01T06:52:10.196942Z","shell.execute_reply":"2022-07-01T06:52:10.220088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(lgb_clf.feature_importances_,\n             index=X_train.columns, columns=['Importance']).sort_values(\n    by='Importance', ascending=False)[:10]","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:52:10.22227Z","iopub.execute_input":"2022-07-01T06:52:10.222807Z","iopub.status.idle":"2022-07-01T06:52:10.236402Z","shell.execute_reply.started":"2022-07-01T06:52:10.222773Z","shell.execute_reply":"2022-07-01T06:52:10.235296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# LGBM with gridsearch","metadata":{}},{"cell_type":"code","source":"param_grid = {'num_leaves': [7, 15, 31, 63], \n              'max_depth': [3, 4, 5, 6, -1], \n              'learning_rate': np.logspace(-3, 0, 10)}","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:52:10.237581Z","iopub.execute_input":"2022-07-01T06:52:10.23806Z","iopub.status.idle":"2022-07-01T06:52:10.243271Z","shell.execute_reply.started":"2022-07-01T06:52:10.238031Z","shell.execute_reply":"2022-07-01T06:52:10.242167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grid_searcher = GridSearchCV(estimator=lgb_clf, param_grid=param_grid, \n                             cv=5, verbose=1, n_jobs=4, scoring='roc_auc')","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:52:10.244508Z","iopub.execute_input":"2022-07-01T06:52:10.245013Z","iopub.status.idle":"2022-07-01T06:52:10.256568Z","shell.execute_reply.started":"2022-07-01T06:52:10.244984Z","shell.execute_reply":"2022-07-01T06:52:10.255496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ngrid_searcher.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:52:10.257853Z","iopub.execute_input":"2022-07-01T06:52:10.258743Z","iopub.status.idle":"2022-07-01T06:52:27.794175Z","shell.execute_reply.started":"2022-07-01T06:52:10.258711Z","shell.execute_reply":"2022-07-01T06:52:27.79325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grid_searcher.best_params_, grid_searcher.best_score_","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:52:27.798046Z","iopub.execute_input":"2022-07-01T06:52:27.798922Z","iopub.status.idle":"2022-07-01T06:52:27.805244Z","shell.execute_reply.started":"2022-07-01T06:52:27.79888Z","shell.execute_reply":"2022-07-01T06:52:27.804357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evaluate_results('lgbm_2', y_train, grid_searcher.predict(X_train))","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:52:27.806644Z","iopub.execute_input":"2022-07-01T06:52:27.807265Z","iopub.status.idle":"2022-07-01T06:52:27.832419Z","shell.execute_reply.started":"2022-07-01T06:52:27.807224Z","shell.execute_reply":"2022-07-01T06:52:27.831377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_lgb = LGBMClassifier(learning_rate= 0.15, max_depth= 3, max_features= 'log2', min_samples_leaf= 12, \n                           min_samples_split= 0.1, n_estimators= 10, subsample= 0.5, n_jobs=4)\nfinal_lgb.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:52:27.836515Z","iopub.execute_input":"2022-07-01T06:52:27.836877Z","iopub.status.idle":"2022-07-01T06:52:27.858258Z","shell.execute_reply.started":"2022-07-01T06:52:27.836845Z","shell.execute_reply":"2022-07-01T06:52:27.857183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evaluate_results('lgbm_3', y_train, final_lgb.predict(X_train))","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:52:27.861327Z","iopub.execute_input":"2022-07-01T06:52:27.862827Z","iopub.status.idle":"2022-07-01T06:52:27.885006Z","shell.execute_reply.started":"2022-07-01T06:52:27.862783Z","shell.execute_reply":"2022-07-01T06:52:27.884007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(final_lgb.feature_importances_,\n             index=X_train.columns, columns=['Importance']).sort_values(\n    by='Importance', ascending=False)[:10]","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:52:27.889606Z","iopub.execute_input":"2022-07-01T06:52:27.891797Z","iopub.status.idle":"2022-07-01T06:52:27.904991Z","shell.execute_reply.started":"2022-07-01T06:52:27.891755Z","shell.execute_reply":"2022-07-01T06:52:27.903508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# XGBoost","metadata":{}},{"cell_type":"code","source":"xgb = GradientBoostingClassifier (random_state=42)\nxgb.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:52:27.912781Z","iopub.execute_input":"2022-07-01T06:52:27.913209Z","iopub.status.idle":"2022-07-01T06:52:28.059086Z","shell.execute_reply.started":"2022-07-01T06:52:27.913164Z","shell.execute_reply":"2022-07-01T06:52:28.058051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evaluate_results('xgb', y_train, xgb.predict(X_train))","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:52:28.060672Z","iopub.execute_input":"2022-07-01T06:52:28.061107Z","iopub.status.idle":"2022-07-01T06:52:28.080636Z","shell.execute_reply.started":"2022-07-01T06:52:28.061059Z","shell.execute_reply":"2022-07-01T06:52:28.079379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(xgb.feature_importances_,\n             index=X_train.columns, columns=['Importance']).sort_values(\n    by='Importance', ascending=False)[:10]","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:52:28.082087Z","iopub.execute_input":"2022-07-01T06:52:28.082553Z","iopub.status.idle":"2022-07-01T06:52:28.097063Z","shell.execute_reply.started":"2022-07-01T06:52:28.082522Z","shell.execute_reply":"2022-07-01T06:52:28.096227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# XGBoosting with gridsearch","metadata":{}},{"cell_type":"code","source":"param_grid = {'learning_rate':[0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2], \n              'min_samples_split':[0.1, 0.5, 12],\n              'max_depth':[3, 5, 8],\n              'min_samples_leaf':[0.1, 0.5, 12],\n              'max_features':[\"log2\",\"sqrt\"], \n              'subsample':[0.5, 0.618, 0.8, 0.85, 0.9, 0.95, 1.0],\n              'n_estimators':[10]}","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:52:28.098177Z","iopub.execute_input":"2022-07-01T06:52:28.098974Z","iopub.status.idle":"2022-07-01T06:52:28.105332Z","shell.execute_reply.started":"2022-07-01T06:52:28.098939Z","shell.execute_reply":"2022-07-01T06:52:28.104421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grid_searcher = GridSearchCV(estimator=xgb, param_grid=param_grid, \n                             cv=6, verbose=1, n_jobs=4)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:52:28.106391Z","iopub.execute_input":"2022-07-01T06:52:28.107492Z","iopub.status.idle":"2022-07-01T06:52:28.118014Z","shell.execute_reply.started":"2022-07-01T06:52:28.107444Z","shell.execute_reply":"2022-07-01T06:52:28.117036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ngrid_searcher.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:52:28.119163Z","iopub.execute_input":"2022-07-01T06:52:28.119744Z","iopub.status.idle":"2022-07-01T06:54:09.56379Z","shell.execute_reply.started":"2022-07-01T06:52:28.119705Z","shell.execute_reply":"2022-07-01T06:54:09.562629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grid_searcher.best_params_, grid_searcher.best_score_","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:54:09.565634Z","iopub.execute_input":"2022-07-01T06:54:09.566848Z","iopub.status.idle":"2022-07-01T06:54:09.575928Z","shell.execute_reply.started":"2022-07-01T06:54:09.566799Z","shell.execute_reply":"2022-07-01T06:54:09.574868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evaluate_results('xgb_2', y_train, grid_searcher.predict(X_train))","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:54:09.577603Z","iopub.execute_input":"2022-07-01T06:54:09.578172Z","iopub.status.idle":"2022-07-01T06:54:09.59861Z","shell.execute_reply.started":"2022-07-01T06:54:09.578127Z","shell.execute_reply":"2022-07-01T06:54:09.597284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_xgb = GradientBoostingClassifier(learning_rate= 0.15, max_depth= 8, max_features= 'log2', \n                                       min_samples_leaf= 12, min_samples_split= 12, n_estimators= 10,\n                                       subsample= 0.9, random_state=42)\nfinal_xgb.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:54:09.60045Z","iopub.execute_input":"2022-07-01T06:54:09.601152Z","iopub.status.idle":"2022-07-01T06:54:09.638906Z","shell.execute_reply.started":"2022-07-01T06:54:09.601109Z","shell.execute_reply":"2022-07-01T06:54:09.637792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evaluate_results('final_xgb', y_train, final_xgb.predict(X_train))","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:54:09.640236Z","iopub.execute_input":"2022-07-01T06:54:09.640574Z","iopub.status.idle":"2022-07-01T06:54:09.657994Z","shell.execute_reply.started":"2022-07-01T06:54:09.640545Z","shell.execute_reply":"2022-07-01T06:54:09.656383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(final_xgb.feature_importances_,\n             index=X_train.columns, columns=['Importance']).sort_values(\n    by='Importance', ascending=False)[:10]","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:54:09.659799Z","iopub.execute_input":"2022-07-01T06:54:09.660304Z","iopub.status.idle":"2022-07-01T06:54:09.675031Z","shell.execute_reply.started":"2022-07-01T06:54:09.660261Z","shell.execute_reply":"2022-07-01T06:54:09.673872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# XGBoost and shap","metadata":{}},{"cell_type":"code","source":"#!pip install shap\nimport shap\nimport xgboost\n\n# load JS visualization code to notebook\nshap.initjs()","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:54:09.677025Z","iopub.execute_input":"2022-07-01T06:54:09.677347Z","iopub.status.idle":"2022-07-01T06:54:13.475976Z","shell.execute_reply.started":"2022-07-01T06:54:09.677319Z","shell.execute_reply":"2022-07-01T06:54:13.474314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"explainer = shap.TreeExplainer(final_xgb)\nshap_values = explainer.shap_values(X_train)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:54:13.47791Z","iopub.execute_input":"2022-07-01T06:54:13.479566Z","iopub.status.idle":"2022-07-01T06:54:13.552223Z","shell.execute_reply.started":"2022-07-01T06:54:13.479509Z","shell.execute_reply":"2022-07-01T06:54:13.55093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shap.summary_plot(shap_values, X_train)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:54:13.554148Z","iopub.execute_input":"2022-07-01T06:54:13.554802Z","iopub.status.idle":"2022-07-01T06:54:14.229662Z","shell.execute_reply.started":"2022-07-01T06:54:13.554768Z","shell.execute_reply":"2022-07-01T06:54:14.228276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shap.summary_plot(shap_values, X_train, plot_type=\"bar\")","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:54:14.231019Z","iopub.execute_input":"2022-07-01T06:54:14.231361Z","iopub.status.idle":"2022-07-01T06:54:14.502667Z","shell.execute_reply.started":"2022-07-01T06:54:14.231332Z","shell.execute_reply":"2022-07-01T06:54:14.501803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plain NN","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedShuffleSplit, cross_validate\nfrom sklearn.neural_network import MLPClassifier","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:54:14.503684Z","iopub.execute_input":"2022-07-01T06:54:14.504655Z","iopub.status.idle":"2022-07-01T06:54:14.519377Z","shell.execute_reply.started":"2022-07-01T06:54:14.504608Z","shell.execute_reply":"2022-07-01T06:54:14.518117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mlp = MLPClassifier(random_state=42)\n\nmlp.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:54:14.520772Z","iopub.execute_input":"2022-07-01T06:54:14.521088Z","iopub.status.idle":"2022-07-01T06:54:15.904688Z","shell.execute_reply.started":"2022-07-01T06:54:14.52106Z","shell.execute_reply":"2022-07-01T06:54:15.903287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evaluate_results('mlp', y_train, mlp.predict(X_train))","metadata":{"execution":{"iopub.status.busy":"2022-07-01T06:54:15.911241Z","iopub.execute_input":"2022-07-01T06:54:15.915411Z","iopub.status.idle":"2022-07-01T06:54:15.946012Z","shell.execute_reply.started":"2022-07-01T06:54:15.91534Z","shell.execute_reply":"2022-07-01T06:54:15.944668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Results","metadata":{}},{"cell_type":"code","source":"from pickle import FALSE\ndf_results = pd.DataFrame(np.array(results),\n                    columns=[\"model\", \"precision\", \"recall\", \"roc_auc\", \"gini\", \"f_score\", \"accuracy\", \"mean_fact\", \"mean_pred\"])\n\ndf_results = df_results.apply(pd.to_numeric,errors='ignore')\n\ndf_results.drop_duplicates(inplace = True)\ndf_results.sort_values(by=\"gini\", ascending = False, inplace = True)\n\n\n\ndf_results","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:35:56.494419Z","iopub.execute_input":"2022-07-11T14:35:56.494815Z","iopub.status.idle":"2022-07-11T14:35:56.523684Z","shell.execute_reply.started":"2022-07-11T14:35:56.494782Z","shell.execute_reply":"2022-07-11T14:35:56.522452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_pred = locally_best_forest.predict(X_test)\n\nsubmission = pd.DataFrame({\n        \"PassengerId\": test[\"PassengerId\"],\n        \"Survived\": Y_pred\n    })\nsubmission.to_csv(\"../working/submission.csv\", index=False)\n#all_data.to_csv(\"../working/predictions.csv\",index=False,sep=\",\")","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:36:00.593918Z","iopub.execute_input":"2022-07-11T14:36:00.595103Z","iopub.status.idle":"2022-07-11T14:36:00.709268Z","shell.execute_reply.started":"2022-07-11T14:36:00.595064Z","shell.execute_reply":"2022-07-11T14:36:00.708135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}