{"metadata":{"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30664,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false},"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"papermill":{"default_parameters":{},"duration":145.018038,"end_time":"2024-02-07T21:30:18.166484","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-02-07T21:27:53.148446","version":"2.4.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import polars as pl\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score \nimport pandas as pd\nfrom catboost import CatBoostClassifier\nfrom sklearn.model_selection import GridSearchCV\nimport numpy as np\n\nfrom tqdm.notebook import tqdm\n\n\ndataPath = \"/kaggle/input/home-credit-credit-risk-model-stability/\"\n# dataPath = '/Users/captainbanana/Downloads/home-credit-credit-risk-model-stability/'","metadata":{"papermill":{"duration":4.227484,"end_time":"2024-02-07T21:28:01.739621","exception":false,"start_time":"2024-02-07T21:27:57.512137","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-03-18T21:14:28.018306Z","iopub.execute_input":"2024-03-18T21:14:28.018810Z","iopub.status.idle":"2024-03-18T21:14:30.487894Z","shell.execute_reply.started":"2024-03-18T21:14:28.018765Z","shell.execute_reply":"2024-03-18T21:14:30.486392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_table_dtypes(df: pl.DataFrame) -> pl.DataFrame:\n    # implement here all desired dtypes for tables\n    # the following is just an example\n    for col in df.columns:\n        # last letter of column name will help you determine the type\n        if col[-1] in (\"P\", \"A\"):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n\n    return df\n\ndef convert_strings(df: pd.DataFrame) -> pd.DataFrame:\n    for col in df.columns:  \n        if df[col].dtype.name in ['object', 'string']:\n            df[col] = df[col].astype(\"string\").astype('category')\n            current_categories = df[col].cat.categories\n            new_categories = current_categories.to_list() + [\"Unknown\"]\n            new_dtype = pd.CategoricalDtype(categories=new_categories, ordered=True)\n            df[col] = df[col].astype(new_dtype)\n    return df","metadata":{"papermill":{"duration":0.020544,"end_time":"2024-02-07T21:28:01.767089","exception":false,"start_time":"2024-02-07T21:28:01.746545","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-03-18T21:14:30.492159Z","iopub.execute_input":"2024-03-18T21:14:30.493427Z","iopub.status.idle":"2024-03-18T21:14:30.505084Z","shell.execute_reply.started":"2024-03-18T21:14:30.493356Z","shell.execute_reply":"2024-03-18T21:14:30.503676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_basetable = pl.read_csv(dataPath + \"csv_files/train/train_base.csv\")\ntrain_static = pl.concat(\n    [\n        pl.read_csv(dataPath + \"csv_files/train/train_static_0_0.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/train/train_static_0_1.csv\").pipe(set_table_dtypes),\n    ],\n    how=\"vertical_relaxed\",\n)\ntrain_static_cb = pl.read_csv(dataPath + \"csv_files/train/train_static_cb_0.csv\").pipe(set_table_dtypes)\ntrain_person_1 = pl.read_csv(dataPath + \"csv_files/train/train_person_1.csv\").pipe(set_table_dtypes)\ntrain_person_2 = pl.read_csv(dataPath + \"csv_files/train/train_person_2.csv\").pipe(set_table_dtypes)\ntrain_applprev_1 = pl.read_csv(dataPath + \"csv_files/train/train_applprev_1_0.csv\")\ntrain_a_1_2 = pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_a_1_2.csv\")\ntrain_a_2_1 = pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_a_2_1.csv\")","metadata":{"papermill":{"duration":19.207089,"end_time":"2024-02-07T21:28:20.980847","exception":false,"start_time":"2024-02-07T21:28:01.773758","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-03-18T21:14:30.506976Z","iopub.execute_input":"2024-03-18T21:14:30.507491Z","iopub.status.idle":"2024-03-18T21:15:09.506240Z","shell.execute_reply.started":"2024-03-18T21:14:30.507447Z","shell.execute_reply":"2024-03-18T21:15:09.504822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_basetable = pl.read_csv(dataPath + \"csv_files/test/test_base.csv\")\ntest_static = pl.concat(\n    [\n        pl.read_csv(dataPath + \"csv_files/test/test_static_0_0.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/test/test_static_0_1.csv\").pipe(set_table_dtypes),\n    ],\n    how=\"vertical_relaxed\",\n)\n\ntest_static_cb = pl.read_csv(dataPath + \"csv_files/test/test_static_cb_0.csv\").pipe(set_table_dtypes)\ntest_person_1 = pl.read_csv(dataPath + \"csv_files/test/test_person_1.csv\").pipe(set_table_dtypes)\ntest_person_2 = pl.read_csv(dataPath + \"csv_files/test/test_person_2.csv\").pipe(set_table_dtypes)\ntest_applprev_1 = pl.read_csv(dataPath + \"csv_files/test/test_applprev_1_0.csv\")\ntest_a_1_2 = pl.read_csv(dataPath + \"csv_files/test/test_credit_bureau_a_1_2.csv\")\ntest_a_2_1 = pl.read_csv(dataPath + \"csv_files/test/test_credit_bureau_a_2_1.csv\")","metadata":{"papermill":{"duration":0.079736,"end_time":"2024-02-07T21:28:21.067361","exception":false,"start_time":"2024-02-07T21:28:20.987625","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-03-18T21:15:09.509097Z","iopub.execute_input":"2024-03-18T21:15:09.509482Z","iopub.status.idle":"2024-03-18T21:15:09.612826Z","shell.execute_reply.started":"2024-03-18T21:15:09.509449Z","shell.execute_reply":"2024-03-18T21:15:09.611533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nstatic_columns = ['clientscnt_157L', 'totaldebt_9A', 'applications30d_658L', 'case_id']\nperson_1_columns = ['registaddr_district_1083M', 'sex_738L', 'childnum_185L', 'incometype_1044T', 'birth_259D', 'case_id']\nperson_2_columns = ['case_id', 'empls_economicalst_849M']\napplprev_columns = ['creationdate_885D', 'avgpmtlast12m_4525200A', 'case_id']\nstatic_cb_columns = ['education_88M', 'case_id']\na_1_2_columns = ['instlamount_768A', 'case_id']\napplprev_columns = ['rejectreasonclient_4145042M', 'case_id']\nbase_columns = ['case_id', 'date_decision']","metadata":{"execution":{"iopub.status.busy":"2024-03-18T21:15:09.614377Z","iopub.execute_input":"2024-03-18T21:15:09.615402Z","iopub.status.idle":"2024-03-18T21:15:09.622668Z","shell.execute_reply.started":"2024-03-18T21:15:09.615362Z","shell.execute_reply":"2024-03-18T21:15:09.621122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def merge_data(base, static, static_cb, person_1, person_2, credit_a_1_2, applprev):\n    base = base[base_columns]\n    static = static[static_columns]\n    static_cb = static_cb[static_cb_columns]\n    person_1 = person_1[person_1_columns]\n    person_2 = person_2[person_2_columns]\n    credit_a_1_2 = credit_a_1_2[a_1_2_columns]\n    applprev = applprev[applprev_columns]\n\n    person_1 = person_1.group_by('case_id').agg(childnum_185L=pl.max('childnum_185L'), sex_738L=pl.max('sex_738L'), \n                                                incometype_1044T=pl.max('incometype_1044T'), birth_259D=pl.max('birth_259D'))\n    base = base.join(person_1, how='left', on='case_id')\n    \n    person_2 = person_2.group_by('case_id').agg(empls_economicalst_849M=pl.col('empls_economicalst_849M'))\n    base = base.join(person_2, how='left', on='case_id')\n    \n    credit_a_1_2 = credit_a_1_2.group_by('case_id').agg(instlamount_768A=pl.sum('instlamount_768A'))\n    base = base.join(credit_a_1_2, how='left', on='case_id')\n    \n    applprev = applprev.group_by('case_id').agg(rejectreasonclient_4145042M=pl.col('rejectreasonclient_4145042M'))\n    base = base.join(applprev, how='left', on='case_id')\n\n    return base\n    \n    ","metadata":{"execution":{"iopub.status.busy":"2024-03-18T21:15:09.624544Z","iopub.execute_input":"2024-03-18T21:15:09.624932Z","iopub.status.idle":"2024-03-18T21:15:09.643883Z","shell.execute_reply.started":"2024-03-18T21:15:09.624901Z","shell.execute_reply":"2024-03-18T21:15:09.642593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = merge_data(base=train_basetable, applprev=train_applprev_1, credit_a_1_2=train_a_1_2,\n           person_1=train_person_1, \n           person_2=train_person_2, static=train_static, static_cb=train_static_cb)\ntrain = train.join(train_basetable[['case_id', 'target']], how='left', on='case_id')","metadata":{"execution":{"iopub.status.busy":"2024-03-18T21:15:09.645568Z","iopub.execute_input":"2024-03-18T21:15:09.646218Z","iopub.status.idle":"2024-03-18T21:15:16.242323Z","shell.execute_reply.started":"2024-03-18T21:15:09.646179Z","shell.execute_reply":"2024-03-18T21:15:16.241177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = merge_data(base=test_basetable, applprev=test_applprev_1, credit_a_1_2=test_a_1_2,\n           person_1=test_person_1, \n           person_2=test_person_2, static=test_static, static_cb=test_static_cb)\n# test = test.join(test_basetable[['case_id']], how='left', on='case_id')","metadata":{"execution":{"iopub.status.busy":"2024-03-18T21:15:16.243434Z","iopub.execute_input":"2024-03-18T21:15:16.243792Z","iopub.status.idle":"2024-03-18T21:15:16.254912Z","shell.execute_reply.started":"2024-03-18T21:15:16.243765Z","shell.execute_reply":"2024-03-18T21:15:16.253297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_list(df, column_name):\n    df_exploded = df.explode(column_name)\n\n    group_mean = df_exploded.group_by(column_name).agg(pl.mean('target'))\n    \n    value_to_mean = {row[column_name]: row['target'] for row in group_mean.rows(named=True)}\n    return value_to_mean","metadata":{"execution":{"iopub.status.busy":"2024-03-18T21:15:16.257180Z","iopub.execute_input":"2024-03-18T21:15:16.258300Z","iopub.status.idle":"2024-03-18T21:15:16.265473Z","shell.execute_reply.started":"2024-03-18T21:15:16.258253Z","shell.execute_reply":"2024-03-18T21:15:16.264421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PrepareFeatures:\n    def __init__(self):\n        pass\n\n    def fit(self, df):\n        self.column_to_dict = {}\n        self.column_to_dict['empls_economicalst_849M'] = preprocess_list(df, 'empls_economicalst_849M')\n        self.column_to_dict['rejectreasonclient_4145042M'] = preprocess_list(df, 'rejectreasonclient_4145042M')\n        self.column_to_mean = {}\n        for col in df.columns:\n            if df[col].dtype == pl.String:\n                self.column_to_mean[col] = df[col].mode()[0]\n            elif df[col].dtype in (pl.Float32, pl.Float64):\n                self.column_to_mean[col] = df[col].mean()\n\n    def transform(self, df):\n        df = df.clone()\n        df = df.with_columns((df['date_decision'].str.to_datetime() - df['birth_259D'].str.to_datetime()).dt.total_seconds().alias('age'))\n        df = df.drop(columns=['date_decision', 'birth_259D'])\n\n        for col in self.column_to_dict.keys():\n            df = df.with_columns(df[col].apply(lambda x: np.mean([self.column_to_dict[col][s] for s in x]))\\\n                                 .alias(col))\n        \n        for col in df.columns:\n            try:\n                if col == 'case_id':\n                    continue\n                if col not in self.column_to_mean:\n                    continue\n                if df[col].dtype == pl.String:\n                    df = df.with_columns(df[col].fill_null(value=self.column_to_mean[col]).alias(col))\n                else:\n                    df = df.with_columns(df[col].fill_null(self.column_to_mean[col]).alias(col))\n            except:\n                print(f'{col} is bad')\n                df = df.with_columns(pl.lit(0).alias(col))\n        return df.to_pandas()","metadata":{"execution":{"iopub.status.busy":"2024-03-18T21:15:16.270756Z","iopub.execute_input":"2024-03-18T21:15:16.271273Z","iopub.status.idle":"2024-03-18T21:15:16.287695Z","shell.execute_reply.started":"2024-03-18T21:15:16.271233Z","shell.execute_reply":"2024-03-18T21:15:16.286513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preprocessor = PrepareFeatures()\npreprocessor.fit(train)\ntrain_ready = preprocessor.transform(train)","metadata":{"execution":{"iopub.status.busy":"2024-03-18T21:15:16.289723Z","iopub.execute_input":"2024-03-18T21:15:16.291427Z","iopub.status.idle":"2024-03-18T21:16:12.800955Z","shell.execute_reply.started":"2024-03-18T21:15:16.291364Z","shell.execute_reply":"2024-03-18T21:16:12.799709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_features = [col for col in train_ready.columns if train_ready[col].dtype == 'object']\nY = train_ready['target']\nX = train_ready.drop(columns=['target'])\nX_test = test","metadata":{"execution":{"iopub.status.busy":"2024-03-18T21:16:12.803196Z","iopub.execute_input":"2024-03-18T21:16:12.804151Z","iopub.status.idle":"2024-03-18T21:16:12.899444Z","shell.execute_reply.started":"2024-03-18T21:16:12.804103Z","shell.execute_reply":"2024-03-18T21:16:12.898101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = CatBoostClassifier(cat_features=cat_features, verbose=False)\n\n# Определите диапазон гиперпараметров для Grid Search\nparams = {'depth': [6, 8, 10],\n          'learning_rate' : [0.01, 0.05, 0.1],\n          'iterations': [30, 50, 80]}\n\n","metadata":{"execution":{"iopub.status.busy":"2024-03-18T21:16:12.901438Z","iopub.execute_input":"2024-03-18T21:16:12.901947Z","iopub.status.idle":"2024-03-18T21:16:12.911552Z","shell.execute_reply.started":"2024-03-18T21:16:12.901899Z","shell.execute_reply":"2024-03-18T21:16:12.910623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# grid = {'learning_rate': [0.03, 0.1],\n#         'depth': [4, 6, 10],\n#         'iterations': [30, 50, 80]}\n\n# grid_search_result = model.grid_search(grid, \n#                                        X=X, \n#                                        y=Y)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2024-03-18T21:16:12.912898Z","iopub.execute_input":"2024-03-18T21:16:12.914241Z","iopub.status.idle":"2024-03-18T21:16:12.930460Z","shell.execute_reply.started":"2024-03-18T21:16:12.914205Z","shell.execute_reply":"2024-03-18T21:16:12.929155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {'depth': 10, 'iterations': 80, 'learning_rate': 0.1}","metadata":{"execution":{"iopub.status.busy":"2024-03-18T21:16:12.931864Z","iopub.execute_input":"2024-03-18T21:16:12.932252Z","iopub.status.idle":"2024-03-18T21:16:12.943730Z","shell.execute_reply.started":"2024-03-18T21:16:12.932218Z","shell.execute_reply":"2024-03-18T21:16:12.942494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = CatBoostClassifier(**params)\nmodel.fit(X, Y, cat_features=cat_features)\n","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2024-03-18T21:16:12.945255Z","iopub.execute_input":"2024-03-18T21:16:12.945650Z","iopub.status.idle":"2024-03-18T21:16:49.854371Z","shell.execute_reply.started":"2024-03-18T21:16:12.945616Z","shell.execute_reply":"2024-03-18T21:16:49.853054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({\n    \"case_id\": X_test['case_id'],\n    \"score\":   [0] * len(X_test),\n}).set_index('case_id')\nsubmission.to_csv(\"./submission.csv\")","metadata":{"papermill":{"duration":0.026081,"end_time":"2024-02-07T21:30:16.813857","exception":false,"start_time":"2024-02-07T21:30:16.787776","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-03-18T21:16:49.856087Z","iopub.execute_input":"2024-03-18T21:16:49.856537Z","iopub.status.idle":"2024-03-18T21:16:49.871651Z","shell.execute_reply.started":"2024-03-18T21:16:49.856502Z","shell.execute_reply":"2024-03-18T21:16:49.870587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.predict(train_ready)","metadata":{"execution":{"iopub.status.busy":"2024-03-18T21:18:13.532962Z","iopub.execute_input":"2024-03-18T21:18:13.533401Z","iopub.status.idle":"2024-03-18T21:18:14.744603Z","shell.execute_reply.started":"2024-03-18T21:18:13.533371Z","shell.execute_reply":"2024-03-18T21:18:14.743653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}