{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":50160,"databundleVersionId":7602123,"sourceType":"competition"}],"dockerImageVersionId":30646,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os, glob\nimport gc\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom pathlib import Path\nfrom typing import Literal\nimport polars as pl\nimport polars.selectors as cs\nfrom sklearn.model_selection import train_test_split, cross_validate, StratifiedGroupKFold\nfrom sklearn.metrics import roc_auc_score\nfrom catboost import CatBoostClassifier\nimport lightgbm as lgb","metadata":{"execution":{"iopub.status.busy":"2024-02-23T17:52:17.737936Z","iopub.execute_input":"2024-02-23T17:52:17.738351Z","iopub.status.idle":"2024-02-23T17:52:17.745057Z","shell.execute_reply.started":"2024-02-23T17:52:17.738323Z","shell.execute_reply":"2024-02-23T17:52:17.743891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if os.path.exists('/kaggle'):\n    PATH_DATASET = Path(\"/kaggle/input/home-credit-credit-risk-model-stability\")\nelse:\n    PATH_DATASET = Path(\"C:/Users/fatik/PycharmProjects/kaggle_bank/data/\")\nPATH_PARQUETS = PATH_DATASET / \"parquet_files\"\nPATH_TRAIN = PATH_PARQUETS / \"train\"\nPATH_TEST = PATH_PARQUETS / \"test\"\n\npd.set_option('display.max_columns', 1000)\npd.set_option('display.max_rows', 1000)","metadata":{"execution":{"iopub.status.busy":"2024-02-23T17:52:18.862961Z","iopub.execute_input":"2024-02-23T17:52:18.863385Z","iopub.status.idle":"2024-02-23T17:52:18.869711Z","shell.execute_reply.started":"2024-02-23T17:52:18.863346Z","shell.execute_reply":"2024-02-23T17:52:18.868839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DatasetConstructor:\n    def __init__(self, mode: Literal['train', 'test']):\n        self.mode = mode\n        self.path = PATH_PARQUETS / mode\n\n    @staticmethod\n    def reduce_memory_usage_pl(df):\n        \"\"\" Reduce memory usage by polars dataframe {df} with name {name} by changing its data types.\n            Original pandas version of this function: https://www.kaggle.com/code/arjanso/reducing-dataframe-memory-size-by-65 \"\"\"\n        print(f\"Memory usage of dataframe is {round(df.estimated_size('mb'), 2)} MB\")\n        Numeric_Int_types = [pl.Int8,pl.Int16,pl.Int32,pl.Int64]\n        Numeric_Float_types = [pl.Float32,pl.Float64]\n        for col in df.columns:\n            try:\n                col_type = df[col].dtype\n                if col_type == pl.Categorical:\n                    continue\n                c_min = df[col].min()\n                c_max = df[col].max()\n                if col_type in Numeric_Int_types:\n                    if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                        df = df.with_columns(df[col].cast(pl.Int8))\n                    elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                        df = df.with_columns(df[col].cast(pl.Int16))\n                    elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                        df = df.with_columns(df[col].cast(pl.Int32))\n                    elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                        df = df.with_columns(df[col].cast(pl.Int64))\n                elif col_type in Numeric_Float_types:\n                    if c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                        df = df.with_columns(df[col].cast(pl.Float32))\n                    else:\n                        pass\n                # elif col_type == pl.Utf8:\n                #     df = df.with_columns(df[col].cast(pl.Categorical))\n                else:\n                    pass\n            except:\n                pass\n        print(f\"Memory usage of dataframe became {round(df.estimated_size('mb'), 2)} MB\")\n        return df\n\n    @staticmethod\n    def detect_datetime_cols(df):\n        return df.select_dtypes(object).apply(lambda x: pd.to_datetime(x, errors='ignore'), axis=0).select_dtypes(np.datetime64).columns.tolist()\n\n    def _to_pandas(self, df):\n        df = df.to_pandas().set_index('case_id')\n        df = df.replace([np.inf, -np.inf], np.nan)\n        return df\n\n    def merge_static(self, df):\n        df_static = (\n            pl.concat([pl.scan_parquet(p, low_memory=True) for p in glob.glob(str(self.path / f\"{self.mode}_static_0_*\"))],how=\"vertical_relaxed\",)\n            .with_columns(\n                [\n                    (pl.col(col).cast(pl.String).str.to_date(strict=False))\n                    for col in [\n                        'datefirstoffer_1144D',\n                        'datelastinstal40dpd_247D',\n                        'datelastunpaid_3546854D',\n                        'dtlastpmtallstes_4499206D',\n                        'firstclxcampaign_1125D',\n                        'firstdatedue_489D',\n                        'lastactivateddate_801D',\n                       'lastapplicationdate_877D',\n                        'lastapprdate_640D',\n                        'lastdelinqdate_224D',\n                       'lastrejectdate_50D',\n                        'lastrepayingdate_696D',\n                       'maxdpdinstldate_3546855D',\n                        'payvacationpostpone_4187118D',\n                       'validfrom_1069D'\n                    ]\n                ] + [\n                    (pl.col(col).cast(pl.String).cast(pl.Categorical))\n                    for col in [\n                        'bankacctype_710L', 'cardtype_51L', 'credtype_322L',\n                       'disbursementtype_67L', 'equalitydataagreement_891L',\n                       'equalityempfrom_62L', 'inittransactioncode_186L',\n                       'isbidproductrequest_292L', 'isdebitcard_729L',\n                       'lastapprcommoditycat_1041M', 'lastapprcommoditytypec_5251766M',\n                       'lastcancelreason_561M', 'lastrejectcommoditycat_161M',\n                       'lastrejectcommodtypec_5251769M', 'lastrejectreason_759M',\n                       'lastrejectreasonclient_4145040M', 'lastst_736L', 'opencred_647L',\n                       'paytype1st_925L', 'paytype_783L', 'previouscontdistrict_112M',\n                       'twobodfilling_608L', 'typesuite_864L'\n                    ]\n                ]\n            )\n        )\n        return df.join(df_static, how=\"left\", on=\"case_id\")\n\n    def merge_static_cb(self, df):\n        df_static_cb = (\n            pl.scan_parquet(self.path / f\"{self.mode}_static_cb_0.parquet\", low_memory=True)\n            .with_columns(\n                [\n                    (pl.col(col).cast(pl.String).str.to_date(strict=False))\n                    for col in [\n                        'assignmentdate_238D',\n                        'assignmentdate_4527235D',\n                        'assignmentdate_4955616D',\n                        'birthdate_574D',\n                        'dateofbirth_337D',\n                        'dateofbirth_342D',\n                        'responsedate_1012D',\n                        'responsedate_4527233D',\n                        'responsedate_4917613D'\n                    ]\n                ] + [\n                    (pl.col(col).cast(pl.String).cast(pl.Categorical))\n                    for col in [\n                        'description_5085714M', 'education_1103M', 'education_88M',\n                       'maritalst_385M', 'maritalst_893M', 'requesttype_4525192L',\n                       'riskassesment_302T'\n                    ]\n                ]\n            )\n        )\n        return df.join(df_static_cb, how=\"left\", on=\"case_id\")\n\n    def load(self):\n        df = pl.scan_parquet(self.path / f\"{self.mode}_base.parquet\", low_memory=True).with_columns(\n            pl.col(\"date_decision\").str.to_date()\n        )\n        # Depth=0\n        df = self.merge_static(df)\n        df = self.merge_static_cb(df)\n\n        df =(\n            df\n            .with_columns(\n                pl.col(pl.Float64).cast(pl.Float32),\n                pl.col(pl.Int64).cast(pl.Int32),\n            )\n        )\n        df = df.select(~cs.date())\n\n        # Drop categorical large-dimension columns\n        df = df.drop([\n            'lastapprcommoditytypec_5251766M',\n             'previouscontdistrict_112M',\n             'district_544M',\n             'profession_152M',\n             'name_4527232M',\n             'name_4917606M',\n             'employername_160M',\n             'classificationofcontr_400M',\n             'financialinstitution_382M',\n             'contaddr_district_15M',\n             'contaddr_zipcode_807M',\n             'empladdr_district_926M',\n             'empladdr_zipcode_114M',\n             'registaddr_district_1083M',\n             'registaddr_zipcode_184M',\n             'addres_district_368M',\n             'addres_zip_823M'])\n        df = df.collect()\n        df = self.reduce_memory_usage_pl(df)\n        df = self._to_pandas(df)\n        return df","metadata":{"execution":{"iopub.status.busy":"2024-02-23T17:52:21.533681Z","iopub.execute_input":"2024-02-23T17:52:21.534345Z","iopub.status.idle":"2024-02-23T17:52:21.565710Z","shell.execute_reply.started":"2024-02-23T17:52:21.534311Z","shell.execute_reply":"2024-02-23T17:52:21.564455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_constructor = DatasetConstructor('train')\ndf_train = train_constructor.load()\ndf_train.info()","metadata":{"execution":{"iopub.status.busy":"2024-02-23T17:52:24.713949Z","iopub.execute_input":"2024-02-23T17:52:24.714400Z","iopub.status.idle":"2024-02-23T17:52:35.788428Z","shell.execute_reply.started":"2024-02-23T17:52:24.714369Z","shell.execute_reply":"2024-02-23T17:52:35.787327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X, y = df_train.drop(columns='target'), df_train['target']\nX = X.drop(columns=['WEEK_NUM'] )\nX.shape","metadata":{"execution":{"iopub.status.busy":"2024-02-23T17:52:35.790534Z","iopub.execute_input":"2024-02-23T17:52:35.790978Z","iopub.status.idle":"2024-02-23T17:52:36.427010Z","shell.execute_reply.started":"2024-02-23T17:52:35.790923Z","shell.execute_reply":"2024-02-23T17:52:36.425968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nparams = {\n    \"boosting_type\": \"gbdt\",\n    \"objective\": \"binary\",\n    \"metric\": \"auc\",\n    \"max_depth\": 12,\n    \"num_leaves\": 32,\n    \"min_data_in_leaf\": 10,\n    \"learning_rate\": 0.05,\n    \"feature_fraction\": 0.8,\n    \"bagging_fraction\": 0.8,\n    \"bagging_freq\": 5,\n    \"n_estimators\": 100,\n    'min_data_in_bin':1,\n    'max_bin': 64,\n    \"verbose\": -1,\n    \"random_state\": 42,\n    'n_jobs': -1\n}\ncv = StratifiedGroupKFold(n_splits=10, shuffle=False)\ncv_results = cross_validate(\n    lgb.LGBMClassifier(**params),\n    X, y,\n    groups=df_train['WEEK_NUM'],\n    scoring='roc_auc',\n    cv=cv,\n    verbose=3,\n    return_estimator=True,\n    #return_indices=True\n)\nprint(f\"AUC: {cv_results['test_score'].mean():.3f}\", f\"+-{cv_results['test_score'].std():.3f}\")","metadata":{"execution":{"iopub.status.busy":"2024-02-23T17:53:31.844491Z","iopub.execute_input":"2024-02-23T17:53:31.844876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params_cat = {\n    \"iterations\":1000,\n    \"depth\":14,\n    \"learning_rate\":0.01,\n    \"eval_metric\":'AUC',\n    \"random_seed\":1,\n    \"bootstrap_type\":'Bayesian',\n    \"bagging_temperature\":1,\n    \"od_type\":'Iter',\n    \"od_wait\":50,\n    \"task_type\":'GPU'\n}\n\ncat_model = CatBoostClassifier(**params_cat)\ncat_features = [col for col in X.columns if X[col].dtype.name == 'category' or X[col].dtype.name == 'object']\nfor col in cat_features:\n    X[col] = X[col].cat.add_categories('Missing').fillna('Missing')\n    #X_test[col] = X_test[col].cat.add_categories('Missing').fillna('Missing')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_model.fit(\n    X_train, y_train,\n    eval_set=(X_test, y_test),\n    cat_features=cat_features,\n    use_best_model=True,\n    verbose=True\n )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_constructor = DatasetConstructor('test')\ndf_test=test_constructor.load()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = cv_results['estimator']\ntest = df_test[models[0].feature_name_]\npreds_proba = [model.predict_proba(test)[:, 1] for model in models]\ndf_test[\"score\"] = np.average(preds_proba, axis=0)\ndf_test[[\"score\"]].to_csv(\"submission_0.csv\")\ntest","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_cat = df_test.copy()\ndf_test_cat = df_test_cat.drop(\"WEEK_NUM\", axis=1)\ncat_features = [col for col in df_test_cat.columns if df_test_cat[col].dtype.name == 'category' or df_test_cat[col].dtype.name == 'object']\nfor col in cat_features:\n    df_test_cat[col] = df_test_cat[col].cat.add_categories('Missing').fillna('Missing')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_cat = df_test_cat[cat_model.feature_names_]\npreds_proba_cat = cat_model.predict_proba(test_cat)\nprint(preds_proba_cat)\ndf_test_cat[\"score\"] = 1 - preds_proba_cat\ndf_test_cat[[\"score\"]].to_csv(\"submission_0_cat.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ensemble = df_test.copy()\ndf_ensemble[\"score_cat\"] = df_test_cat[\"score\"]\ndf_ensemble[\"score_lgb\"] = df_test[\"score\"]\n#df_ensemble[\"score_lgb\"]\ndf_ensemble[\"score\"] = df_ensemble.apply(lambda x: x.score_lgb *0.5 + x.score_cat*0.5, axis=1)\n\ndf_ensemble[[\"score\"]].to_csv(\"submission_0_all.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nsubmission = pd.DataFrame({\n    \"case_id\": df_ensemble[\"case_id\"].to_numpy(),\n    \"score\": df_ensemble[[\"score\"]]\n}).set_index('case_id')\nsubmission.to_csv(\"./submission_end.csv\")'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}