{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30673,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-03-25T02:12:13.177015Z","iopub.execute_input":"2024-03-25T02:12:13.177631Z","iopub.status.idle":"2024-03-25T02:12:13.197079Z","shell.execute_reply.started":"2024-03-25T02:12:13.177575Z","shell.execute_reply":"2024-03-25T02:12:13.195871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import polars as pl\nimport lightgbm as lgb\nfrom sklearn.model_selection import train_test_split\n# from sklearn import metrics\nimport glob\n# import shutil\n\nfrom sklearnex import patch_sklearn\n\npatch_sklearn()","metadata":{"execution":{"iopub.status.busy":"2024-03-25T02:12:13.199577Z","iopub.execute_input":"2024-03-25T02:12:13.200260Z","iopub.status.idle":"2024-03-25T02:12:13.206909Z","shell.execute_reply.started":"2024-03-25T02:12:13.200217Z","shell.execute_reply":"2024-03-25T02:12:13.205933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_path = \"/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_\"\ndata_path_te = \"/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/test_\"\nkey_col = \"case_id\"","metadata":{"execution":{"iopub.status.busy":"2024-03-25T02:12:13.208847Z","iopub.execute_input":"2024-03-25T02:12:13.209632Z","iopub.status.idle":"2024-03-25T02:12:13.223576Z","shell.execute_reply.started":"2024-03-25T02:12:13.209595Z","shell.execute_reply":"2024-03-25T02:12:13.222136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_table_dtypes(df: pl.DataFrame) -> pl.DataFrame:\n    # implement here all desired dtypes for tables\n    # the following is just an example\n    for col in df.columns:\n        # last letter of column name will help you determine the type\n        if col[-1] in (\"P\", \"A\"):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n\n        if col[-1] in (\"D\"):\n            df = df.with_columns(\n                pl.col(col).str.strptime(pl.Date, \"%Y-%m-%d\").cast(pl.Int64).alias(col)\n            )\n    return df\n\n\ndef convert_strings(df: pd.DataFrame) -> pd.DataFrame:\n    for col in df.columns:\n        if df[col].dtype.name in [\"object\", \"string\"]:\n            df[col] = df[col].astype(\"string\").astype(\"category\")\n            current_categories = df[col].cat.categories\n            new_categories = current_categories.to_list() + [\"Unknown\"]\n            new_dtype = pd.CategoricalDtype(categories=new_categories, ordered=True)\n            df[col] = df[col].astype(new_dtype)\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-03-25T02:12:13.225202Z","iopub.execute_input":"2024-03-25T02:12:13.225870Z","iopub.status.idle":"2024-03-25T02:12:13.239178Z","shell.execute_reply.started":"2024-03-25T02:12:13.225831Z","shell.execute_reply":"2024-03-25T02:12:13.238110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_max_min(path, fi_rank=[], group1=None):\n    print(path)\n    fi_rank_d = fi_rank.copy()\n    fi_rank_d.append(key_col)\n    try:\n        df = pl.scan_parquet(path).pipe(set_table_dtypes)\n        if group1 != None:\n            df = df.filter(pl.col(\"num_group1\") == group1)\n            df = df.select(pl.col(set(df.columns) & set(fi_rank_d)))\n            df = df.select(\n                pl.col(key_col),\n                pl.col(set(df.columns) - {key_col}).name.suffix(\"_\" + str(group1)),\n            )\n        else:\n            df = df.select(pl.col(set(df.columns) & set(fi_rank_d)))\n        df = df.collect()\n        \n    except:\n        df = pl.from_pandas(\n            pd.concat([pd.read_parquet(fname) for fname in glob.glob(path)])\n        )\n        if group1 != None:\n            df = df.filter(pl.col(\"num_group1\") == group1)\n            df = df.select(pl.col(set(df.columns) & set(fi_rank_d)))\n            df = df.select(\n                pl.col(key_col),\n                pl.col(set(df.columns) - {key_col}).name.suffix(\"_\" + str(group1)),\n            )\n        else:\n            df = df.select(pl.col(set(df.columns) & set(fi_rank_d)))\n    \n    if group1 != None:\n        df = df.filter(pl.col(\"num_group1\") == group1)\n        df = df.select(pl.col(set(df.columns) & set(fi_rank_d)))\n        df = df.select(\n            pl.col(key_col),\n            pl.col(set(df.columns) - {key_col}).name.suffix(\"_\" + str(group1)),\n        )\n    else:\n        df = df.select(pl.col(set(df.columns) & set(fi_rank_d)))\n\n    _bool_list = [pl.DataFrame()]\n    for col in df.columns:\n        if df[col].dtype == pl.Utf8:\n            _bool_list.append(df[col].to_dummies().cast(pl.Boolean))\n            df = df.drop(col)\n    if len(_bool_list) > 0:\n        dfmax = (\n            df.with_columns(pl.concat(_bool_list, how=\"horizontal\"))\n            .group_by(key_col)\n            .max()\n        )\n    else:\n        dfmax = df.group_by(key_col).max()\n\n    dfmax = dfmax.select(\n        pl.col(key_col),\n        pl.col(set(df.columns) - {key_col}).name.suffix(\"_max\"),\n    )\n    dfmin = df.group_by(key_col).min()\n    dfmin = dfmin.select(\n        pl.col(key_col),\n        pl.col(set(df.columns) - {key_col}).name.suffix(\"_min\"),\n    )\n    # return dfmin\n    return dfmin.join(dfmax, how=\"outer_coalesce\", on=\"case_id\")","metadata":{"execution":{"iopub.status.busy":"2024-03-25T02:12:13.242134Z","iopub.execute_input":"2024-03-25T02:12:13.242826Z","iopub.status.idle":"2024-03-25T02:12:13.268516Z","shell.execute_reply.started":"2024-03-25T02:12:13.242788Z","shell.execute_reply":"2024-03-25T02:12:13.267129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_features(train_feat, data_path, table_list):\n    features1 = []\n    for table_name in table_list:\n        features = get_max_min(data_path + table_name, fi_rank=fi_rank)\n        feature_list = features.columns\n        feature_list.remove(key_col)\n        features1.extend(feature_list)\n        train_feat = train_feat.join(features, how=\"left\", on=key_col)\n    return train_feat","metadata":{"execution":{"iopub.status.busy":"2024-03-25T02:12:13.269909Z","iopub.execute_input":"2024-03-25T02:12:13.270240Z","iopub.status.idle":"2024-03-25T02:12:13.287071Z","shell.execute_reply.started":"2024-03-25T02:12:13.270212Z","shell.execute_reply":"2024-03-25T02:12:13.285839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fi = pd.read_csv(\"./feature_importance.csv\", index_col=0).reset_index()\n# fi[\"index\"] = [indexer.rstrip(\"_max\") for indexer in fi[\"index\"]]\n# fi[\"index\"] = [indexer.rstrip(\"_min\") for indexer in fi[\"index\"]]\n# fi = fi.sort_values(\"importance\", ascending=False)\n# fi.set_index(\"index\", inplace = True)\n# fi_rank = list(fi.index[0:30])","metadata":{"execution":{"iopub.status.busy":"2024-03-25T02:12:13.288351Z","iopub.execute_input":"2024-03-25T02:12:13.289124Z","iopub.status.idle":"2024-03-25T02:12:13.300562Z","shell.execute_reply.started":"2024-03-25T02:12:13.289092Z","shell.execute_reply":"2024-03-25T02:12:13.299330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sorted by importance\nfi_rank = ['sex_738L',\n 'annuity_780A',\n 'validfrom_1069D',\n 'pmtnum_254L',\n 'incometype_1044T',\n 'price_1097A',\n 'mobilephncnt_593L',\n 'avgdpdtolclosure24_3658938P',\n 'lastrejectdate_50D',\n 'financialinstitution_591M',\n 'numberofcontrsvalue_358L',\n 'birth_259D',\n 'credamount_770A',\n 'numberofoverdueinstlmax_1039L',\n 'empl_employedfrom_271D',\n 'cntpmts24_3658933L',\n 'interestrate_311L',\n 'totaldebt_9A',\n 'residualamount_856A',\n 'totalamount_996A',\n 'dateofcredstart_739D',\n 'totalamount_6A',\n 'numberofoutstandinstls_59L',\n 'relationshiptoclient_415T',\n 'overdueamountmax_155A',\n 'collater_valueofguarantee_1124L',\n 'employedfrom_700D',\n 'inittransactionamount_650A',\n 'relationshiptoclient_415T',\n 'familystate_726L',\n 'education_927M',\n 'numincomingpmts_3546848L',\n 'dateofcredstart_181D',\n 'cntincpaycont9m_3716944L',\n 'dpdmax_757P',\n 'maxannuity_159A',\n 'mindbddpdlast24m_3658935P',\n 'days180_256L',\n 'dpdmax_139P',\n 'numberofinstls_320L',\n 'numberofoverdueinstlmaxdat_148D',\n 'homephncnt_628L',\n 'familystate_447L',\n 'financialinstitution_382M',\n 'contractst_964M',\n 'numinstpaidearly3d_3546850L',\n 'numberofoutstandinstls_59L',\n 'maxdpdtolerance_577P',\n 'totalsettled_863A',\n 'education_1138M',\n 'language1_981M',\n 'tenor_203L',\n 'days120_123L',\n 'dtlastpmtallstes_4499206D',\n 'numberofinstls_229L',\n 'annuity_853A',\n 'days90_310L',\n 'dateofbirth_337D',\n 'financialinstitution_591M',\n 'overdueamountmax2_398A',\n 'approvaldate_319D',\n 'numinstunpaidmax_3546851L',\n 'lastrejectcommoditycat_161M',\n 'nominalrate_281L',\n 'dtlastpmt_581D',\n 'annualeffectiverate_199L',\n 'lastapprcommoditycat_1041M',\n 'language1_981M',\n 'clientscnt12m_3712952L',\n 'numinstlsallpaid_934L']","metadata":{"execution":{"iopub.status.busy":"2024-03-25T02:12:13.301824Z","iopub.execute_input":"2024-03-25T02:12:13.302194Z","iopub.status.idle":"2024-03-25T02:12:13.312781Z","shell.execute_reply.started":"2024-03-25T02:12:13.302165Z","shell.execute_reply":"2024-03-25T02:12:13.311468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tables = glob.glob(data_path + \"*.parquet\")\ntables.remove(data_path + \"base.parquet\")\ntables.sort()\n_list = []\nfor table in tables:\n    cols = pl.scan_parquet(table).columns\n    for col in cols:\n        if col != key_col:\n            _list.append([col, table.split(\"/\")[-1].split(\"_\", 1)[1]])\nmap = pd.DataFrame(_list, columns=[\"Variable\", \"table\"]).drop_duplicates(\n    subset=\"Variable\"\n)","metadata":{"execution":{"iopub.status.busy":"2024-03-25T02:12:13.386251Z","iopub.execute_input":"2024-03-25T02:12:13.386940Z","iopub.status.idle":"2024-03-25T02:12:13.802497Z","shell.execute_reply.started":"2024-03-25T02:12:13.386893Z","shell.execute_reply":"2024-03-25T02:12:13.801538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_rank = pd.DataFrame(fi_rank, columns=[\"Variable\"])\nmap = pd.merge(map, df_rank, on=\"Variable\")\ntable_list = map[\"table\"].unique()","metadata":{"execution":{"iopub.status.busy":"2024-03-25T02:12:13.804215Z","iopub.execute_input":"2024-03-25T02:12:13.805014Z","iopub.status.idle":"2024-03-25T02:12:13.823892Z","shell.execute_reply.started":"2024-03-25T02:12:13.804981Z","shell.execute_reply":"2024-03-25T02:12:13.822799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"table_list","metadata":{"execution":{"iopub.status.busy":"2024-03-25T02:12:13.825666Z","iopub.execute_input":"2024-03-25T02:12:13.826062Z","iopub.status.idle":"2024-03-25T02:12:13.839584Z","shell.execute_reply.started":"2024-03-25T02:12:13.826013Z","shell.execute_reply":"2024-03-25T02:12:13.838427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"table_dict = {\n    \"applprev_1_0.parquet\": \"applprev_1_*.parquet\",\n    \"applprev_2.parquet\": \"applprev_2.parquet\",\n    \"credit_bureau_a_1_0.parquet\": \"credit_bureau_a_1_*.parquet\",\n    \"credit_bureau_a_2_0.parquet\": \"credit_bureau_a_2_*.parquet\",\n    \"credit_bureau_b_1.parquet\": \"credit_bureau_b_1*.parquet\",\n    \"credit_bureau_b_2.parquet\": \"credit_bureau_b_2*.parquet\",\n    \"person_1.parquet\": \"person_1.parquet\",\n    \"person_2.parquet\": \"person_2.parquet\",\n    \"static_0_0.parquet\": \"static_0_*.parquet\",\n    \"static_cb_0.parquet\": \"static_cb_*.parquet\",\n}","metadata":{"execution":{"iopub.status.busy":"2024-03-25T02:12:13.841676Z","iopub.execute_input":"2024-03-25T02:12:13.842261Z","iopub.status.idle":"2024-03-25T02:12:13.848972Z","shell.execute_reply.started":"2024-03-25T02:12:13.842211Z","shell.execute_reply":"2024-03-25T02:12:13.847795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"table_list = [table_dict[key] for key in table_list]","metadata":{"execution":{"iopub.status.busy":"2024-03-25T02:12:13.851739Z","iopub.execute_input":"2024-03-25T02:12:13.852317Z","iopub.status.idle":"2024-03-25T02:12:13.862603Z","shell.execute_reply.started":"2024-03-25T02:12:13.852287Z","shell.execute_reply":"2024-03-25T02:12:13.861470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_base = pl.read_parquet(data_path + \"base.parquet\")\ntrain_base = train_base.with_columns(\n    pl.col(\"date_decision\").str.strptime(pl.Date, \"%Y-%m-%d\").alias(\"date_decision\")\n)\ntrain = (\n    add_features(train_base, data_path, table_list)\n    .fill_null(0)\n    .fill_nan(False)\n    .sort(\"date_decision\")\n)","metadata":{"execution":{"iopub.status.busy":"2024-03-25T02:12:13.864113Z","iopub.execute_input":"2024-03-25T02:12:13.864852Z","iopub.status.idle":"2024-03-25T02:14:27.414053Z","shell.execute_reply.started":"2024-03-25T02:12:13.864813Z","shell.execute_reply":"2024-03-25T02:14:27.412842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2024-03-25T02:14:27.416205Z","iopub.execute_input":"2024-03-25T02:14:27.418324Z","iopub.status.idle":"2024-03-25T02:14:27.448976Z","shell.execute_reply.started":"2024-03-25T02:14:27.418280Z","shell.execute_reply":"2024-03-25T02:14:27.447681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label = \"target\"\nignore_colums = [\"case_id\", \"date_decision\", \"WEEK_NUM\"]\nX = train.drop(ignore_colums).to_pandas().fillna(0)\nX.sort_index(axis=1, inplace=True)\ny = X.pop(label)","metadata":{"execution":{"iopub.status.busy":"2024-03-25T02:14:27.450335Z","iopub.execute_input":"2024-03-25T02:14:27.450696Z","iopub.status.idle":"2024-03-25T02:14:29.153998Z","shell.execute_reply.started":"2024-03-25T02:14:27.450666Z","shell.execute_reply":"2024-03-25T02:14:29.152701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tpot import TPOTClassifier\n\npredictor = TPOTClassifier(\n    generations=1,\n    early_stop=5,\n    population_size=10,\n    cv=5,\n    random_state=42,\n    verbosity=2,\n    memory=\"./temp/\",\n)\npredictor.fit(X, y)","metadata":{"execution":{"iopub.status.busy":"2024-03-25T02:14:29.155318Z","iopub.execute_input":"2024-03-25T02:14:29.155674Z","iopub.status.idle":"2024-03-25T02:14:34.748060Z","shell.execute_reply.started":"2024-03-25T02:14:29.155645Z","shell.execute_reply":"2024-03-25T02:14:34.747042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_base = pl.read_parquet(data_path_te + \"base.parquet\")\ntest_base = test_base.with_columns(\n    pl.col(\"date_decision\").str.strptime(pl.Date, \"%Y-%m-%d\").alias(\"date_decision\")\n)\ntest = add_features(test_base, data_path_te, table_list).fill_null(0).fill_nan(False)\nXte = test.drop(ignore_colums).to_pandas()\nmissing_col = list(set(X.columns)-set(Xte.columns))\nXte = pd.concat([Xte, pd.DataFrame([], columns=missing_col, index = Xte.index).fillna(0)], axis = 1)\nXte.sort_index(axis=1, inplace=True)\n# df = pd.DataFrame([], columns=missing_col, index = Xte.index).fillna(0)","metadata":{"execution":{"iopub.status.busy":"2024-03-25T02:14:34.749308Z","iopub.execute_input":"2024-03-25T02:14:34.749812Z","iopub.status.idle":"2024-03-25T02:14:35.236682Z","shell.execute_reply.started":"2024-03-25T02:14:34.749783Z","shell.execute_reply":"2024-03-25T02:14:35.235281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pred = predictor.predict(Xte)\n# pred = pd.DataFrame(predictor.predict(Xte), columns=[\"score\"])\npred = pd.DataFrame(predictor.predict_proba(Xte)[1], columns=[\"score\"])\npd.concat([test[key_col].to_pandas(), pred], axis=1).to_csv(\n    \"submission.csv\", index=None\n)","metadata":{"execution":{"iopub.status.busy":"2024-03-25T02:14:35.238484Z","iopub.execute_input":"2024-03-25T02:14:35.239031Z","iopub.status.idle":"2024-03-25T02:14:35.268490Z","shell.execute_reply.started":"2024-03-25T02:14:35.238953Z","shell.execute_reply":"2024-03-25T02:14:35.267340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.concat([test[key_col].to_pandas(), pred], axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-03-25T02:14:35.270019Z","iopub.execute_input":"2024-03-25T02:14:35.270695Z","iopub.status.idle":"2024-03-25T02:14:35.292464Z","shell.execute_reply.started":"2024-03-25T02:14:35.270651Z","shell.execute_reply":"2024-03-25T02:14:35.291277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}