{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport warnings\nimport matplotlib.pyplot as plt\nimport scipy.stats as stats\nfrom lightgbm import LGBMClassifier\nfrom xgboost import XGBClassifier\nfrom catboost import CatBoostClassifier\n\nimport sklearn.linear_model as linear_model\nimport sklearn.ensemble as ensemble\nimport sklearn.naive_bayes as naive_bayes\nimport sklearn.tree as tree\n\nfrom sklearn.svm import LinearSVC, SVC\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.naive_bayes import GaussianNB \nfrom sklearn.gaussian_process import GaussianProcessClassifier \nfrom scipy.stats import skew\n\nfrom sklearn.model_selection import cross_val_score, GridSearchCV, RandomizedSearchCV\nfrom sklearn.pipeline import make_pipeline, Pipeline\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import OneHotEncoder, StandardScaler, LabelEncoder, PolynomialFeatures\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.metrics import roc_auc_score, f1_score, plot_confusion_matrix, \\\n    plot_roc_curve, plot_precision_recall_curve, classification_report, \\\n    precision_score, recall_score, accuracy_score\n\nwarnings.filterwarnings('ignore')\ntrain = pd.read_csv(\"/kaggle/input/titanic/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/titanic/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-07-06T10:21:45.504011Z","iopub.execute_input":"2022-07-06T10:21:45.504399Z","iopub.status.idle":"2022-07-06T10:21:45.529918Z","shell.execute_reply.started":"2022-07-06T10:21:45.504369Z","shell.execute_reply":"2022-07-06T10:21:45.529157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Report:\n    def __init__(self, X, y):\n        self.X = X\n        self.y = y\n         \n    def _score(self, model, name):\n        X = self.X\n        y = self.y\n        \n        predictions = model.predict(X)\n\n        report = pd.DataFrame(columns={'accuracy'}, data=[0])\n        report['accuracy'] = accuracy_score(y, predictions)\n#         report['ROC-AUC'] = roc_auc_score(y, model.predict_proba(X)[:, 1])\n        report['F1'] = f1_score(y_test, predictions)\n        report['precision_0'] = precision_score(y, predictions, pos_label=0)\n        report['precision_1'] = precision_score(y, predictions, pos_label=1)\n        report['recall_0'] = recall_score(y, predictions, pos_label=0)\n        report['recall_1'] = recall_score(y, predictions, pos_label=1)\n\n        report.index = [name]\n\n        return report\n\n    def scoreModels(self, modelsDict: dict) -> pd.DataFrame:\n        scores = pd.DataFrame()\n        for modelName in modelsDict:\n            model = modelsDict[modelName]\n            scores = scores.append(self._score(model, modelName))\n        return scores\n    \n    def scoreSearches(self, searches: list) -> None:\n        for search in searches:\n            cls = search.best_estimator_.get_params()['classifier']\n            print(f\"Best parameter {cls}: \\n(CV score={search.best_score_}):\\n\")\n            \n    def write_result_to_csv(self, model, df: pd.DataFrame) -> None:\n        result_df = pd.DataFrame(test['PassengerId'], columns=['PassengerId'])\n        result_df['Survived'] = model.predict(df)\n        result_df.to_csv('result.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T10:21:46.019275Z","iopub.execute_input":"2022-07-06T10:21:46.019827Z","iopub.status.idle":"2022-07-06T10:21:46.033722Z","shell.execute_reply.started":"2022-07-06T10:21:46.019797Z","shell.execute_reply":"2022-07-06T10:21:46.032906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MainPipeline():\n    searches = {}\n    best_trained_models = {}\n    preprocessors = {}\n    models_config = []\n    \n    def __init__(self, X, y, preprocessors: dict, models_config: dict):\n        self.preprocessors = preprocessors\n        self.models_config = models_config\n        \n        self.X = self._preprocess_data(X)\n        self.y = y\n\n        \n        self.searches = []\n        self.best_estimators = []\n    \n    def train_single_model(self, params: dict):\n        model = params['model']\n        grid = params['grid']\n        \n        model_classname = model.__class__.__name__\n        if model_classname in self.best_trained_models:\n            self.best_trained_models.pop(model_classname)\n        \n\n        search, best_estimator = self._search_for_best_model(model, grid)\n        self.best_trained_models[model_classname] = model\n        \n        return best_estimator\n        \n    def train_all_models(self) -> None:\n        for config_item in self.models_config:\n            model = config_item['model']\n            model_classname = model.__class__.__name__\n\n            grid = config_item['grid']\n            \n            search, best_estimator = self._search_for_best_model(model, grid)\n            self.best_trained_models[model_classname] = best_estimator\n            \n    def get_searches(self):\n        return self.searches\n    \n    def get_best_trained_models(self):\n        return self.best_trained_models\n    \n    def build_ensemble_from(self, names: list, voting='soft'):\n        models = []\n        \n        for name in names:\n            if name not in list(self.best_trained_models):\n                raise Exception(name + \" is not in the list of trained models\")\n            models.append((name, self.best_trained_models[name]))\n\n        return ensemble.VotingClassifier(estimators=models, voting='soft');\n        \n    def _pipeline(self, estimator):\n        numerical_transformer = Pipeline(\n            steps=[\n                (\"numerical_imputer\", self.preprocessors['numerical_imputer']),\n                ('polynomials', PolynomialFeatures(degree=1, interaction_only=True)),\n                (\"numerical_scaler\", self.preprocessors['numerical_scaler']),\n            ]\n        )\n        categorical_transformer = Pipeline(\n            steps=[\n                (\"categorical_imputer\", self.preprocessors['categorical_imputer']),\n                (\"categorical_encoder\", self.preprocessors['categorical_encoder']),\n            ]\n        )\n        preprocessor = ColumnTransformer(\n            transformers=[\n                (\"num\", numerical_transformer, self.preprocessors['numerical_features']),\n                (\"cat\", categorical_transformer, self.preprocessors['categorical_features']),\n            ]\n        )\n        clf = Pipeline(  # or just Pipeline if we don't care about PMML format\n            steps=[(\"preprocessor\", preprocessor), (\"classifier\", estimator)]\n        )\n        return clf\n\n    def _search_for_best_model(self, model, grid: dict):\n        pipe = self._pipeline(model)\n        search = RandomizedSearchCV(pipe, grid, n_jobs=-1).fit(self.X, self.y)\n        \n        return search, search.best_estimator_\n\n    def _preprocess_data(self, df: pd.DataFrame):\n        df['Title'] = df['Name'].str.extract(' ([A-Za-z]+)\\.', expand=False)\n        df['Title'] = df['Title'].replace(['Lady', 'Countess','Capt', 'Col',\\\n                                  'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'],\n                                 'Rare')\n        df['Title'] = df['Title'].replace('Mlle', 'Miss')\n        df['Title'] = df['Title'].replace('Ms', 'Miss')\n        df['Title'] = df['Title'].replace('Mme', 'Mrs')\n\n        df['Fare2'] = df['Fare'] ** 2\n        df['Age2'] = df['Age'] ** 2\n\n        df['FamilySize'] = df['SibSp'] + df['Parch'] + 1\n        df['FamilySize2'] = df['FamilySize'] ** 2 \n\n        df['FamilyFare'] = df['FamilySize'] * df['Fare']\n\n        df['FamilyFareClass'] = df['FamilySize'] * df['Fare'] * df['Pclass'] \n\n        df = df.drop(columns=['Name'], axis=1)\n        if('Name' in self.preprocessors['categorical_features']):\n            self.preprocessors['categorical_features'].remove('Name')\n\n        return df","metadata":{"execution":{"iopub.status.busy":"2022-07-06T10:29:49.745145Z","iopub.execute_input":"2022-07-06T10:29:49.745527Z","iopub.status.idle":"2022-07-06T10:29:49.772677Z","shell.execute_reply.started":"2022-07-06T10:29:49.745456Z","shell.execute_reply":"2022-07-06T10:29:49.771143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = train['Survived']\nX = train.drop(columns=['Survived', 'Cabin', 'PassengerId', 'Ticket'])\n\nX_test = test.drop(columns=['Cabin', 'PassengerId', 'Ticket'])\n\nX_train, x_test, y_train, y_test = train_test_split(X,y, test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T10:29:50.701673Z","iopub.execute_input":"2022-07-06T10:29:50.702050Z","iopub.status.idle":"2022-07-06T10:29:50.716229Z","shell.execute_reply.started":"2022-07-06T10:29:50.702021Z","shell.execute_reply":"2022-07-06T10:29:50.715338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preprocessors = {\n    \"numerical_features\": X.dtypes[X.dtypes != 'object'].index.values.tolist(),\n    \"categorical_features\": X.dtypes[X.dtypes == 'object'].index.values.tolist(),\n    \"numerical_imputer\": SimpleImputer(strategy='mean'),\n    \"numerical_scaler\": StandardScaler(),\n    \"categorical_imputer\": SimpleImputer(strategy='most_frequent'),\n    \"categorical_encoder\": OneHotEncoder(handle_unknown='ignore'),\n}\n\nmodels_config = [\n    {\n        'model': ensemble.GradientBoostingClassifier(random_state = 42),\n        'grid': {\n            \"classifier__n_estimators\": [50, 100, 300, 500],\n            \"classifier__max_depth\": [5, 20, 40, 60],\n            'classifier__min_samples_leaf': [5, 20, 40, 60, 100],\n            'classifier__max_features': ['auto', 'sqrt'],\n        }, \n    },\n    {\n        'model': linear_model.LogisticRegression(),\n        'grid': {\n            \"classifier__penalty\": ['l2', 'l1', 'elasticnet'],\n            'classifier__C': [0.0001, 0.001, 0.01, 1.0, 10.0, 100, 1000],\n            'classifier__class_weight': ['balanced', None]\n        }, \n    },\n    {\n        'model': ensemble.RandomForestClassifier(random_state = 42, criterion = 'entropy'),\n        'grid': {\n            \"classifier__max_depth\": [3, 5, 20, 50],\n            'classifier__min_samples_leaf': [3, 5, 20, 40],\n            'classifier__max_features': ['auto', 'sqrt'],\n            \"classifier__n_estimators\": [30, 50, 100, 300, 400, 500],\n            'classifier__bootstrap': [True, False]\n        }, \n    },\n    {\n        'model': LGBMClassifier(class_weight='balanced'),\n        'grid': {\n             'classifier__learning_rate': [0.001, 0.01, 0.1],\n             'classifier__num_leaves': [30, 50],\n             'classifier__n_estimators': [50, 100, 300]\n        }, \n    },\n    {\n        'model': CatBoostClassifier(),\n        'grid': {\n            'classifier__learning_rate': [0.001, 0.01, 0.1],\n#             'classifier__num_leaves': [30, 50],\n            'classifier__n_estimators': [50, 100, 300, 400, 500]\n        }, \n    },\n    {\n        'model': XGBClassifier(class_weight='balanced', categorical_feature='auto'),\n        'grid': {\n             'classifier__learning_rate': [0.001, 0.01, 0.1],\n             'classifier__num_leaves': [30, 50, 70, 100],\n             'classifier__n_estimators': [50, 100, 300, 500]\n        }, \n    },\n    {\n        'model': SVC(random_state=42, probability=True),\n        'grid': {\n             'classifier__C': [0.001, 0.01, 0.1, 1, 100],\n             'classifier__gamma': ['scale', 'auto'],\n             'classifier__shrinking': [False, True],\n             'classifier__class_weight': ['balanced', None],\n        }, \n    },\n    {\n        'model': ensemble.AdaBoostClassifier(),\n        'grid': {}, \n    },\n    {\n        'model': ensemble.BaggingClassifier(),\n        'grid': {}, \n    },\n    {\n        'model': ensemble.ExtraTreesClassifier(),\n        'grid': {}, \n    },\n    {\n        'model': GaussianProcessClassifier(),\n        'grid': {}, \n    },\n    {\n        'model': linear_model.LogisticRegressionCV(),\n        'grid': {}, \n    },\n    {\n        'model': naive_bayes.BernoulliNB(),\n        'grid': {}, \n    },\n    {\n        'model': naive_bayes.GaussianNB(),\n        'grid': {}, \n    },\n    {\n        'model': KNeighborsClassifier(),\n        'grid': {}, \n    },\n]","metadata":{"execution":{"iopub.status.busy":"2022-07-06T10:29:50.883326Z","iopub.execute_input":"2022-07-06T10:29:50.884333Z","iopub.status.idle":"2022-07-06T10:29:50.905056Z","shell.execute_reply.started":"2022-07-06T10:29:50.884288Z","shell.execute_reply":"2022-07-06T10:29:50.903880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pipeline = MainPipeline(X_train, y_train, preprocessors, models_config)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T10:29:53.189798Z","iopub.execute_input":"2022-07-06T10:29:53.190220Z","iopub.status.idle":"2022-07-06T10:29:53.207669Z","shell.execute_reply.started":"2022-07-06T10:29:53.190191Z","shell.execute_reply":"2022-07-06T10:29:53.206670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training single model","metadata":{}},{"cell_type":"code","source":"single_model = {\n    'model': linear_model.LogisticRegression(),\n    'grid': {\n        \"classifier__penalty\": ['l2', 'l1', 'elasticnet'],\n        'classifier__C': [0.0001, 0.001, 0.01, 1.0, 10.0, 100, 1000],\n        'classifier__class_weight': ['balanced', None]\n    }, \n}\n\nlogreg = pipeline.train_single_model(single_model)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T10:22:12.492317Z","iopub.execute_input":"2022-07-06T10:22:12.492731Z","iopub.status.idle":"2022-07-06T10:22:14.345039Z","shell.execute_reply.started":"2022-07-06T10:22:12.492699Z","shell.execute_reply":"2022-07-06T10:22:14.344211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"logreg.score(x_test, y_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T10:22:15.668640Z","iopub.execute_input":"2022-07-06T10:22:15.669035Z","iopub.status.idle":"2022-07-06T10:22:15.684243Z","shell.execute_reply.started":"2022-07-06T10:22:15.669004Z","shell.execute_reply":"2022-07-06T10:22:15.683479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train all models","metadata":{}},{"cell_type":"code","source":"pipeline.train_all_models()","metadata":{"execution":{"iopub.status.busy":"2022-07-06T10:22:41.669085Z","iopub.execute_input":"2022-07-06T10:22:41.669521Z","iopub.status.idle":"2022-07-06T10:23:31.864109Z","shell.execute_reply.started":"2022-07-06T10:22:41.669489Z","shell.execute_reply":"2022-07-06T10:23:31.862574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"big_stack = pipeline.build_ensemble_from([\n    ensemble.GradientBoostingClassifier().__class__.__name__,\n    ensemble.RandomForestClassifier().__class__.__name__,\n    LGBMClassifier().__class__.__name__,\n    CatBoostClassifier().__class__.__name__,\n    XGBClassifier().__class__.__name__,\n    SVC().__class__.__name__,\n    ensemble.AdaBoostClassifier().__class__.__name__,\n    ensemble.BaggingClassifier().__class__.__name__,\n    ensemble.ExtraTreesClassifier().__class__.__name__,\n    GaussianProcessClassifier().__class__.__name__,\n    linear_model.LogisticRegressionCV().__class__.__name__,\n    naive_bayes.BernoulliNB().__class__.__name__,\n    naive_bayes.GaussianNB().__class__.__name__,\n    KNeighborsClassifier().__class__.__name__,\n])\n\nbig_stack.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T10:23:31.865716Z","iopub.execute_input":"2022-07-06T10:23:31.865973Z","iopub.status.idle":"2022-07-06T10:23:36.215010Z","shell.execute_reply.started":"2022-07-06T10:23:31.865946Z","shell.execute_reply":"2022-07-06T10:23:36.214211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ensemble","metadata":{}},{"cell_type":"code","source":"big_stack = pipeline.build_ensemble_from([\n    ensemble.GradientBoostingClassifier().__class__.__name__,\n    ensemble.RandomForestClassifier().__class__.__name__,\n    LGBMClassifier().__class__.__name__,\n    CatBoostClassifier().__class__.__name__,\n    XGBClassifier().__class__.__name__,\n    SVC().__class__.__name__,\n    ensemble.AdaBoostClassifier().__class__.__name__,\n    ensemble.BaggingClassifier().__class__.__name__,\n    ensemble.ExtraTreesClassifier().__class__.__name__,\n    GaussianProcessClassifier().__class__.__name__,\n    linear_model.LogisticRegressionCV().__class__.__name__,\n    naive_bayes.BernoulliNB().__class__.__name__,\n    naive_bayes.GaussianNB().__class__.__name__,\n    KNeighborsClassifier().__class__.__name__,\n])\n\nbig_stack.fit(X_train, y_train);","metadata":{"execution":{"iopub.status.busy":"2022-07-06T10:23:55.716019Z","iopub.execute_input":"2022-07-06T10:23:55.716360Z","iopub.status.idle":"2022-07-06T10:23:59.059131Z","shell.execute_reply.started":"2022-07-06T10:23:55.716335Z","shell.execute_reply":"2022-07-06T10:23:59.058091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Report","metadata":{}},{"cell_type":"code","source":"report = Report(x_test, y_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T10:24:20.245886Z","iopub.execute_input":"2022-07-06T10:24:20.246209Z","iopub.status.idle":"2022-07-06T10:24:20.250773Z","shell.execute_reply.started":"2022-07-06T10:24:20.246185Z","shell.execute_reply":"2022-07-06T10:24:20.249782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pipeline.get_best_trained_models()['LogisticRegression'].predict(x_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T10:24:24.226979Z","iopub.execute_input":"2022-07-06T10:24:24.227340Z","iopub.status.idle":"2022-07-06T10:24:24.242636Z","shell.execute_reply.started":"2022-07-06T10:24:24.227312Z","shell.execute_reply":"2022-07-06T10:24:24.241657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models_to_score = pipeline.get_best_trained_models()\nmodels_to_score['BigStack'] = big_stack\n\nscores = report.scoreModels(models_to_score)\nscores","metadata":{"execution":{"iopub.status.busy":"2022-07-06T10:24:38.159237Z","iopub.execute_input":"2022-07-06T10:24:38.159684Z","iopub.status.idle":"2022-07-06T10:24:38.821078Z","shell.execute_reply.started":"2022-07-06T10:24:38.159649Z","shell.execute_reply":"2022-07-06T10:24:38.820099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Results","metadata":{}},{"cell_type":"code","source":"report.write_result_to_csv(big_stack, pipeline._preprocess_data(X_test))","metadata":{"execution":{"iopub.status.busy":"2022-07-06T10:29:58.258584Z","iopub.execute_input":"2022-07-06T10:29:58.258957Z","iopub.status.idle":"2022-07-06T10:29:58.638541Z","shell.execute_reply.started":"2022-07-06T10:29:58.258927Z","shell.execute_reply":"2022-07-06T10:29:58.637307Z"},"trusted":true},"execution_count":null,"outputs":[]}]}