{"metadata":{"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30684,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false},"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"papermill":{"default_parameters":{},"duration":145.018038,"end_time":"2024-02-07T21:30:18.166484","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-02-07T21:27:53.148446","version":"2.4.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Initial Submission\n","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.007009,"end_time":"2024-02-07T21:27:57.505355","exception":false,"start_time":"2024-02-07T21:27:57.498346","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import polars as pl\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport gc\n\nimport lightgbm as lgb\nfrom sklearn.model_selection import train_test_split\n# from sklearn.metrics import roc_auc_score, f1_score\nfrom imblearn.over_sampling import RandomOverSampler\n\nfrom catboost import CatBoostClassifier\n\n\n# SET HERE\ntrain_directory = '/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/'\ntest_directory = '/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/'","metadata":{"papermill":{"duration":4.227484,"end_time":"2024-02-07T21:28:01.739621","exception":false,"start_time":"2024-02-07T21:27:57.512137","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-07T21:02:26.205707Z","iopub.execute_input":"2024-05-07T21:02:26.206093Z","iopub.status.idle":"2024-05-07T21:02:28.317184Z","shell.execute_reply.started":"2024-05-07T21:02:26.206062Z","shell.execute_reply":"2024-05-07T21:02:28.315594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def aggregate(df):\n    '''\n    Defines aggregation style for group 1 and 2 datasets. take mean of numerics and max + min of strings\n    '''\n    \n    num_cols = [c for c in df.columns if c[-1] in ['P', 'A']]\n    other_cols = [c for c in df.columns if c[-1] not in ['P', 'A']]\n\n    num_agg_mean = [pl.mean(c).alias('mean_' + c) for c in num_cols]\n    num_agg_max = [pl.max(c).alias('max_' + c) for c in num_cols]\n    num_agg_min = [pl.min(c).alias('min_' + c) for c in num_cols]\n    str_agg_max = [pl.max(c).alias('max_' + c) for c in other_cols if c not in ['case_id', 'num_group1', 'num_group2']]\n    str_agg_min = [pl.min(c).alias('min_' + c) for c in other_cols if c not in ['case_id', 'num_group1', 'num_group2']]\n    \n#     agg = num_agg_mean + num_agg_max + num_agg_min + str_agg_max + str_agg_min\n    agg = num_agg_mean + str_agg_max\n    return agg\n\n    \ndef set_datatypes(df):\n    '''\n    Tests column data type and reformats.\n    '''\n    \n    for c in df.columns:\n        if c in ['case_id', 'WEEK_NUM', 'num_group1', 'num_group2']: # excl MONTH\n            df = df.with_columns(pl.col(c).cast(pl.Int64))\n        # elif col in ['date_decision']:\n        #     df = df.with_columns(pl.col(c).cast(pl.Date))\n        elif c[-1] in ['P', 'A'] or c == 'target':\n            df = df.with_columns(pl.col(c).cast(pl.Float64))\n        elif c[-1] == 'M' or c == 'MONTH':\n            df = df.with_columns(pl.col(c).cast(pl.String))\n        elif c[-1] == 'D' or c == 'date_decision':\n            df = df.with_columns(pl.col(c).cast(pl.Date).dt.truncate('1mo'))\n        # else:\n        #     print('column {c} is unknown datatype'.format(c=c))\n\n    return df\n\n\ndef reduce_columns(df):\n    '''\n    Tests for columns with many nulls or string columns with only 1 or many many values.\n    '''\n    \n    for c in df.columns:\n        p_null = df[c].is_null().mean() >= 0.70\n        uniq = df[c].n_unique() == 1 or df[c].n_unique() > 200\n        \n        if c in ['target', 'case_id', 'MONTH']:\n            pass\n        elif p_null:\n            df = df.drop(c)\n        elif c[-1] == 'M' and uniq:\n            df = df.drop(c)\n        elif c[-1] == 'D' or c in ['WEEK_NUM', 'date_decision']:\n            df = df.drop(c) # for now\n    \n    return df\n\n\ndef load_from_parquet(path, source):\n    '''\n    Loads a parquet file at a path and does some formatting. If path includes a set of tables,\n    load each and then concat them together. Also determine if the table is of depth 0. if \n    not, perform aggregation.\n    '''\n\n    # if split into multiple tables, first combine. tested, and separate files shouldn't have\n    # any overlap with case_id\n    if type(path) == list:\n        d0 = 'static_0' in path[0] or 'static_cb_0' in path[0] or '_base' in path[0]\n        tot = []\n        for t in path:\n            d = pl.read_parquet(source+t)\n            d = set_datatypes(d)\n            if not d0:\n                d = d.group_by('case_id').agg(aggregate(d))\n            tot.append(d)\n\n        # combine\n        df = pl.concat(tot, how='vertical_relaxed')\n\n    else:\n        d0 = 'static_0' in path or 'static_cb_0' in path or '_base' in path\n        df = pl.read_parquet(source+path)\n        df = set_datatypes(df)\n        if not d0:\n            df = df.group_by('case_id').agg(aggregate(df))\n\n    # only need to do this for training. later, i'll make sure train and test have the same cols\n    if 'train' in source:\n        df = reduce_columns(df) # do this after aggregation, if it occurs\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-05-07T21:02:32.945933Z","iopub.execute_input":"2024-05-07T21:02:32.946540Z","iopub.status.idle":"2024-05-07T21:02:32.966849Z","shell.execute_reply.started":"2024-05-07T21:02:32.946509Z","shell.execute_reply":"2024-05-07T21:02:32.965700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_tables = [\n    'train_base.parquet',\n    ['train_applprev_1_0.parquet', 'train_applprev_1_1.parquet'],\n    'train_applprev_2.parquet',\n    ['train_credit_bureau_a_1_0.parquet', 'train_credit_bureau_a_1_1.parquet', 'train_credit_bureau_a_1_2.parquet', 'train_credit_bureau_a_1_3.parquet'],\n    ['train_credit_bureau_a_2_0.parquet', 'train_credit_bureau_a_2_1.parquet', 'train_credit_bureau_a_2_2.parquet', 'train_credit_bureau_a_2_3.parquet', 'train_credit_bureau_a_2_4.parquet', 'train_credit_bureau_a_2_5.parquet', 'train_credit_bureau_a_2_6.parquet', 'train_credit_bureau_a_2_7.parquet', 'train_credit_bureau_a_2_8.parquet', 'train_credit_bureau_a_2_9.parquet', 'train_credit_bureau_a_2_10.parquet'],\n    'train_credit_bureau_b_1.parquet',\n    'train_credit_bureau_b_2.parquet',\n    'train_debitcard_1.parquet',\n    'train_deposit_1.parquet',\n    'train_other_1.parquet',\n    'train_person_1.parquet',\n    'train_person_2.parquet',\n    ['train_static_0_0.parquet', 'train_static_0_1.parquet'],\n    'train_static_cb_0.parquet',\n    'train_tax_registry_a_1.parquet',\n    'train_tax_registry_b_1.parquet',\n    'train_tax_registry_c_1.parquet'\n]\n\ntest_tables = [\n    'test_base.parquet',\n    ['test_applprev_1_0.parquet', 'test_applprev_1_1.parquet', 'test_applprev_1_2.parquet'],\n    'test_applprev_2.parquet',\n    ['test_credit_bureau_a_1_0.parquet', 'test_credit_bureau_a_1_1.parquet', 'test_credit_bureau_a_1_2.parquet', 'test_credit_bureau_a_1_3.parquet', 'test_credit_bureau_a_1_4.parquet'],\n    ['test_credit_bureau_a_2_0.parquet', 'test_credit_bureau_a_2_1.parquet', 'test_credit_bureau_a_2_2.parquet', 'test_credit_bureau_a_2_3.parquet', 'test_credit_bureau_a_2_4.parquet', 'test_credit_bureau_a_2_5.parquet', 'test_credit_bureau_a_2_6.parquet', 'test_credit_bureau_a_2_7.parquet', 'test_credit_bureau_a_2_8.parquet', 'test_credit_bureau_a_2_9.parquet', 'test_credit_bureau_a_2_10.parquet', 'test_credit_bureau_a_2_11.parquet'],\n    'test_credit_bureau_b_1.parquet',\n    'test_credit_bureau_b_2.parquet',\n    'test_debitcard_1.parquet',\n    'test_deposit_1.parquet',\n    'test_other_1.parquet',\n    'test_person_1.parquet',\n    'test_person_2.parquet',\n    ['test_static_0_0.parquet', 'test_static_0_1.parquet', 'test_static_0_2.parquet'],\n    'test_static_cb_0.parquet',\n    'test_tax_registry_a_1.parquet',\n    'test_tax_registry_b_1.parquet',\n    'test_tax_registry_c_1.parquet'\n]","metadata":{"execution":{"iopub.status.busy":"2024-05-07T21:02:35.708968Z","iopub.execute_input":"2024-05-07T21:02:35.709634Z","iopub.status.idle":"2024-05-07T21:02:35.717324Z","shell.execute_reply.started":"2024-05-07T21:02:35.709588Z","shell.execute_reply":"2024-05-07T21:02:35.715885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# actually load data from location\n\n# start with training data\ntrain_data = load_from_parquet(train_tables[0], train_directory)\nfor t in train_tables[1:]:\n    # print('\\n', t)\n    train_data = train_data.join(load_from_parquet(t, train_directory), on='case_id', how='left')\n\ngc.collect()\n    \n# test data\ntest_data = load_from_parquet(test_tables[0], test_directory)\nfor t in test_tables[1:]:\n    # print('\\n', t)\n    test_data = test_data.join(load_from_parquet(t, test_directory), on='case_id', how='left')\n\n# make sure test and training have same columns\ntest_data = test_data.select([c for c in train_data.columns if c != 'target'])\n\ngc.collect()\n\nprint('train data shape:\\t', train_data.shape)\nprint('test data shape:\\t', test_data.shape)","metadata":{"execution":{"iopub.status.busy":"2024-05-07T21:02:36.948758Z","iopub.execute_input":"2024-05-07T21:02:36.949157Z","iopub.status.idle":"2024-05-07T21:05:14.721173Z","shell.execute_reply.started":"2024-05-07T21:02:36.949129Z","shell.execute_reply":"2024-05-07T21:05:14.719642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# convert datasets to pandas, using category dtype where relevant\n\ntrain_data = train_data.to_pandas()#.sample(750000)\nobj_cols = list(train_data.select_dtypes('object').columns)\ntrain_data[obj_cols] = train_data[obj_cols].astype('category')\n\ntest_data = test_data.to_pandas()\ntest_data[obj_cols] = test_data[obj_cols].astype('category')","metadata":{"execution":{"iopub.status.busy":"2024-05-07T21:05:14.723416Z","iopub.execute_input":"2024-05-07T21:05:14.723784Z","iopub.status.idle":"2024-05-07T21:05:14.728230Z","shell.execute_reply.started":"2024-05-07T21:05:14.723755Z","shell.execute_reply":"2024-05-07T21:05:14.727296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fit model\n# train_data = train_data.to_pandas()\n# test_data = test_data.to_pandas()\n# cat_features = train_data.select_dtypes(include=['object', 'category']).columns\n\n# # Fill NaN values with a placeholder string such as 'missing'\n# train_data[cat_features] = train_data[cat_features].fillna('missing')\n# test_data[cat_features] = test_data[cat_features].fillna('missing')\n\n# # Convert all categorical features to type 'category'\n# train_data[cat_features] = train_data[cat_features].astype('category')\n# test_data[cat_features] = test_data[cat_features].astype('category')\n\nx_train, x_val, y_train, y_val = train_test_split(\n    train_data.drop(columns=['target', 'case_id']),\n    train_data['target'], \n    test_size=0.3, \n    random_state=0\n)\n\ndel train_data\ngc.collect()\n\n# ros = RandomOverSampler(random_state=0, sampling_strategy=0.3)\n# x_train, y_train = ros.fit_resample(x_train, y_train)\n\n# params = {\n#     \"boosting_type\": \"gbdt\",\n#     \"metric\": \"auc\",\n#     \"max_depth\": 10,  \n#     \"learning_rate\": 0.05,\n#     \"n_estimators\": 2000,  \n#     \"colsample_bytree\": 0.8,\n#     \"colsample_bynode\": 0.8,\n#     \"reg_alpha\": 0.1,\n#     \"reg_lambda\": 10,\n#     'num_leaves':64\n# }\n\nm = lgb.LGBMClassifier()\nm.fit(x_train, y_train, eval_set=[(x_val, y_val)])\n\n# cat_features_indices = [train_data.columns.get_loc(c) for c in cat_features if c in train_data]\n\n# m = CatBoostClassifier(\n#     iterations=3000, \n#     learning_rate=0.03, \n#     depth=6,\n#     cat_features=cat_features_indices,\n#     eval_metric='AUC',\n#     verbose=300\n# )\n# m.fit(x_train, y_train, eval_set=[(x_val, y_val)])\n\n# print('mean AUC score: {s}'.format(s=np.mean([np.mean(m.evals_result_['valid_0']['auc']) for m in models])))\n#print('mean LL score: {s}'.format(s=np.mean(m.evals_result_['valid_0']['binary_logloss'])))","metadata":{"execution":{"iopub.status.busy":"2024-05-07T21:05:14.729542Z","iopub.execute_input":"2024-05-07T21:05:14.730078Z","iopub.status.idle":"2024-05-07T21:05:58.951087Z","shell.execute_reply.started":"2024-05-07T21:05:14.730047Z","shell.execute_reply":"2024-05-07T21:05:58.949131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# lgb.plot_importance(m, importance_type=\"split\", figsize=(10,50))\n# plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# prepare to make predictions\ninds = test_data['case_id']\ntest = test_data.drop(columns=['case_id'])\n\n# make predictions on trained model\npredictions = m.predict_proba(test)[:, 1]\n\nout = pd.DataFrame(columns=['case_id', 'score'])\nout['case_id'] = inds\nout['score'] = predictions\nout = out.set_index('case_id')\n\nout.to_csv('submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}