{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30664,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        os.path.join(dirname, filename)\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-04-05T20:27:28.278418Z","iopub.execute_input":"2024-04-05T20:27:28.278838Z","iopub.status.idle":"2024-04-05T20:27:28.789066Z","shell.execute_reply.started":"2024-04-05T20:27:28.278806Z","shell.execute_reply":"2024-04-05T20:27:28.787569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import polars as pl\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nimport seaborn as sns\ndataPath = \"/kaggle/input/home-credit-credit-risk-model-stability/\"","metadata":{"execution":{"iopub.status.busy":"2024-04-05T20:27:28.791481Z","iopub.execute_input":"2024-04-05T20:27:28.792025Z","iopub.status.idle":"2024-04-05T20:27:29.980446Z","shell.execute_reply.started":"2024-04-05T20:27:28.791991Z","shell.execute_reply":"2024-04-05T20:27:29.978955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_table_dtypes(df: pl.DataFrame) -> pl.DataFrame:\n    # implement here all desired dtypes for tables\n    # the following is just an example\n    for col in df.columns:\n        # last letter of column name will help you determine the type\n        if col[-1] in (\"P\", \"A\"):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n\n    return df\n\ndef convert_strings(df: pd.DataFrame) -> pd.DataFrame:\n    for col in df.columns:  \n        if df[col].dtype.name in ['object', 'string']:\n            df[col] = df[col].astype(\"string\").astype('category')\n            current_categories = df[col].cat.categories\n            new_categories = current_categories.to_list() + [\"Unknown\"]\n            new_dtype = pd.CategoricalDtype(categories=new_categories, ordered=True)\n            df[col] = df[col].astype(new_dtype)\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-04-05T20:27:29.982912Z","iopub.execute_input":"2024-04-05T20:27:29.983471Z","iopub.status.idle":"2024-04-05T20:27:29.995909Z","shell.execute_reply.started":"2024-04-05T20:27:29.983423Z","shell.execute_reply":"2024-04-05T20:27:29.994259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_basetable = pl.read_parquet(dataPath + \"parquet_files/train/train_base.parquet\")\ntrain_static = pl.concat(\n    [\n        pl.read_parquet(dataPath + \"parquet_files/train/train_static_0_0.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/train/train_static_0_1.parquet\").pipe(set_table_dtypes),\n    ],\n    how=\"vertical_relaxed\",\n)\ntrain_static_cb = pl.read_parquet(dataPath + \"parquet_files/train/train_static_cb_0.parquet\").pipe(set_table_dtypes)\ntrain_person_1 = pl.read_parquet(dataPath + \"parquet_files/train/train_person_1.parquet\").pipe(set_table_dtypes) \ntrain_credit_bureau_b_2 = pl.read_parquet(dataPath + \"parquet_files/train/train_credit_bureau_b_2.parquet\").pipe(set_table_dtypes) ","metadata":{"execution":{"iopub.status.busy":"2024-04-05T20:27:29.999691Z","iopub.execute_input":"2024-04-05T20:27:30.001202Z","iopub.status.idle":"2024-04-05T20:27:37.911505Z","shell.execute_reply.started":"2024-04-05T20:27:30.001141Z","shell.execute_reply":"2024-04-05T20:27:37.909918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_person_1_feats_2 = train_person_1.select([\"case_id\", \"num_group1\", \"housetype_905L\"]).filter(\n    pl.col(\"num_group1\") == 0\n).drop(\"num_group1\").rename({\"housetype_905L\": \"person_housetype\"})\n\ntrain_person_1_feats_2=train_person_1_feats_2.with_columns(pl.col(\"person_housetype\").fill_null(\"not_owned\"))\n\ntrain_person_1_feats_1 = train_person_1.group_by(\"case_id\").agg(\n    pl.col(\"mainoccupationinc_384A\").max().alias(\"mainoccupationinc_384A_max\"),\n    (pl.col(\"incometype_1044T\") == \"SELFEMPLOYED\").max().alias(\"mainoccupationinc_384A_any_selfemployed\")\n)\ntrain_person_1_feats_1=train_person_1_feats_1.with_columns(pl.col(\"mainoccupationinc_384A_any_selfemployed\").cast(pl.Int16))\n\ntrain_credit_bureau_b_2_feats = train_credit_bureau_b_2.group_by(\"case_id\").agg(\n    pl.col(\"pmts_pmtsoverdue_635A\").max().alias(\"pmts_pmtsoverdue_635A_max\"),\n    (pl.col(\"pmts_dpdvalue_108P\") > 31).max().alias(\"pmts_dpdvalue_108P_over31\")\n)\ntrain_credit_bureau_b_2_feats=train_credit_bureau_b_2_feats.with_columns(pl.col(\"pmts_dpdvalue_108P_over31\").cast(pl.Int16))","metadata":{"execution":{"iopub.status.busy":"2024-04-05T20:27:37.917850Z","iopub.execute_input":"2024-04-05T20:27:37.919816Z","iopub.status.idle":"2024-04-05T20:27:38.413932Z","shell.execute_reply.started":"2024-04-05T20:27:37.919754Z","shell.execute_reply":"2024-04-05T20:27:38.412911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"date_features_cb=[col for col in train_static_cb.columns if col[-1]==\"D\"]\ncategorical_features_cb=[col for col in train_static.columns if(train_static[col].dtype==\"O\" and col not in date_features_cb)]\nnumerical_features_cb=[col for col in train_static.columns if (train_static[col].dtype!=\"O\")]","metadata":{"execution":{"iopub.status.busy":"2024-04-05T20:27:38.417598Z","iopub.execute_input":"2024-04-05T20:27:38.418091Z","iopub.status.idle":"2024-04-05T20:27:38.427039Z","shell.execute_reply.started":"2024-04-05T20:27:38.418041Z","shell.execute_reply":"2024-04-05T20:27:38.425808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_cb=train_static_cb.with_columns(pl.col(\"requesttype_4525192L\").fill_null(\"DEDUCTION_6\"))\ntrain_static_cb=train_static_cb.drop(\"riskassesment_302T\")\ntrain_static_cb=train_static_cb.drop(['forweek_1077L', 'formonth_206L', 'foryear_850L', 'foryear_818L','foryear_618L', 'forweek_601L', 'forweek_528L', 'fortoday_1092L',\n       'forquarter_634L', 'forquarter_462L', 'forquarter_1017L','formonth_535L', 'formonth_118L', 'for3years_584L', 'for3years_504L','for3years_128L', 'riskassesment_940T',\n        'pmtcount_4955617L','pmtaverage_4955615A', 'pmtaverage_4527227A', 'pmtcount_4527229L','pmtaverage_3A', 'pmtcount_693L', 'contractssum_5085716L'])\ntrain_static_cb=train_static_cb.with_columns(pl.col('pmtssum_45A').fill_null(0))\ntrain_static_cb=train_static_cb.with_columns(pl.col('pmtscount_423L').fill_null(0))\ntrain_static_cb=train_static_cb.with_columns(pl.col('days90_310L').fill_null(0))\ntrain_static_cb=train_static_cb.with_columns(pl.col('days360_512L').fill_null(0))\ntrain_static_cb=train_static_cb.with_columns(pl.col('days30_165L').fill_null(0))\ntrain_static_cb=train_static_cb.with_columns(pl.col('days180_256L').fill_null(0))\ntrain_static_cb=train_static_cb.with_columns(pl.col('days120_123L').fill_null(0))\ntrain_static_cb=train_static_cb.with_columns(pl.col('firstquarter_103L').fill_null(0))\ntrain_static_cb=train_static_cb.with_columns(pl.col('secondquarter_766L').fill_null(0))\ntrain_static_cb=train_static_cb.with_columns(pl.col('thirdquarter_1082L').fill_null(0))\ntrain_static_cb=train_static_cb.with_columns(pl.col('fourthquarter_440L').fill_null(0))\ntrain_static_cb=train_static_cb.with_columns(pl.col('numberofqueries_373L').fill_null(0))\ntrain_static_cb=train_static_cb.drop(date_features_cb)","metadata":{"execution":{"iopub.status.busy":"2024-04-05T20:27:38.428983Z","iopub.execute_input":"2024-04-05T20:27:38.429792Z","iopub.status.idle":"2024-04-05T20:27:38.858389Z","shell.execute_reply.started":"2024-04-05T20:27:38.429748Z","shell.execute_reply":"2024-04-05T20:27:38.856646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#depth0\ncategorical_features=['bankacctype_710L','cardtype_51L','credtype_322L'\n    ,'disbursementtype_67L','equalitydataagreement_891L','equalityempfrom_62L','inittransactioncode_186L'\n    ,'isbidproductrequest_292L','isdebitcard_729L','lastapprcommoditycat_1041M','lastapprcommoditytypec_5251766M'\n    ,'lastcancelreason_561M','lastrejectcommoditycat_161M','lastrejectcommodtypec_5251769M','lastrejectreason_759M'\n    ,'lastrejectreasonclient_4145040M','lastst_736L','opencred_647L','paytype1st_925L','paytype_783L','previouscontdistrict_112M','twobodfilling_608L','typesuite_864L']\nnumerical_features=['actualdpdtolerance_344P','amtinstpaidbefduel24m_4187115A','annuity_780A','annuitynextmonth_57A','applicationcnt_361L','applications30d_658L','applicationscnt_1086L','applicationscnt_464L','applicationscnt_629L','applicationscnt_867L','avgdbddpdlast24m_3658932P','avgdbddpdlast3m_4187120P','avgdbdtollast24m_4525197P','avgdpdtolclosure24_3658938P','avginstallast24m_3658937A','avglnamtstart24m_4525187A','avgmaxdpdlast9m_3716943P','avgoutstandbalancel6m_4187114A','avgpmtlast12m_4525200A'\n    ,'clientscnt12m_3712952L','clientscnt3m_3712950L','clientscnt6m_3712949L','clientscnt_100L','clientscnt_1022L','clientscnt_1071L','clientscnt_1130L','clientscnt_136L','clientscnt_157L','clientscnt_257L','clientscnt_304L','clientscnt_360L','clientscnt_493L','clientscnt_533L','clientscnt_887L','clientscnt_946L','cntincpaycont9m_3716944L','cntpmts24_3658933L','commnoinclast6m_3546845L','credamount_770A','currdebt_22A','currdebtcredtyperange_828A'\n    ,'daysoverduetolerancedd_3976961L','deferredmnthsnum_166L','disbursedcredamount_1113A','downpmt_116A'\n    ,'eir_270L','homephncnt_628L'\n    ,'inittransactionamount_650A','interestrate_311L','interestrategrace_34L','isbidproduct_1095L'\n    ,'lastapprcredamount_781A','lastdependentsnum_448L','lastotherinc_902A','lastotherlnsexpense_631A','lastrejectcredamount_222A'\n    ,'maininc_215A','mastercontrelectronic_519L','mastercontrexist_109L','maxannuity_159A','maxannuity_4075009A','maxdbddpdlast1m_3658939P','maxdbddpdtollast12m_3658940P','maxdbddpdtollast6m_4187119P','maxdebt4_972A','maxdpdfrom6mto36m_3546853P','maxdpdinstlnum_3546846P','maxdpdlast12m_727P','maxdpdlast24m_143P','maxdpdlast3m_392P','maxdpdlast6m_474P','maxdpdlast9m_1059P','maxdpdtolerance_374P','maxinstallast24m_3658928A','maxlnamtstart6m_4525199A','maxoutstandbalancel12m_4187113A','maxpmtlast3m_4525190A','mindbddpdlast24m_3658935P','mindbdtollast24m_4525191P','mobilephncnt_593L','monthsannuity_845L'\n    ,'numactivecreds_622L','numactivecredschannel_414L','numactiverelcontr_750L','numcontrs3months_479L','numincomingpmts_3546848L','numinstlallpaidearly3d_817L','numinstls_657L','numinstlsallpaid_934L','numinstlswithdpd10_728L','numinstlswithdpd5_4187116L','numinstlswithoutdpd_562L','numinstmatpaidtearly2d_4499204L','numinstpaid_4499208L','numinstpaidearly3d_3546850L','numinstpaidearly3dest_4493216L','numinstpaidearly5d_1087L','numinstpaidearly5dest_4493211L','numinstpaidearly5dobd_4499205L','numinstpaidearly_338L','numinstpaidearlyest_4493214L','numinstpaidlastcontr_4325080L','numinstpaidlate1d_3546852L','numinstregularpaid_973L','numinstregularpaidest_4493210L','numinsttopaygr_769L','numinsttopaygrest_4493213L','numinstunpaidmax_3546851L','numinstunpaidmaxest_4493212L','numnotactivated_1143L','numpmtchanneldd_318L','numrejects9m_859L'\n    ,'pctinstlsallpaidearl3d_427L','pctinstlsallpaidlat10d_839L','pctinstlsallpaidlate1d_3546856L','pctinstlsallpaidlate4d_3546849L','pctinstlsallpaidlate6d_3546844L','pmtnum_254L','posfpd10lastmonth_333P','posfpd30lastmonth_3976960P','posfstqpd30lastmonth_3976962P','price_1097A'\n    ,'sellerplacecnt_915L','sellerplacescnt_216L','sumoutstandtotal_3546847A','sumoutstandtotalest_4493215A'\n    ,'totaldebt_9A','totalsettled_863A','totinstallast1m_4525188A']\ndate_features=[\"datefirstoffer_1144D\",\"datelastinstal40dpd_247D\",\"datelastunpaid_3546854D\",\"dtlastpmtallstes_4499206D\",\"firstclxcampaign_1125D\",\"firstdatedue_489D\",'payvacationpostpone_4187118D',\n           \"lastactivateddate_801D\",\"lastapplicationdate_877D\",\"lastapprdate_640D\",\"lastdelinqdate_224D\",\"lastrejectdate_50D\",\"maxdpdinstldate_3546855D\",\"lastrepayingdate_696D\",'validfrom_1069D']","metadata":{"execution":{"iopub.status.busy":"2024-04-05T20:27:38.860234Z","iopub.execute_input":"2024-04-05T20:27:38.860798Z","iopub.status.idle":"2024-04-05T20:27:38.879379Z","shell.execute_reply.started":"2024-04-05T20:27:38.860737Z","shell.execute_reply":"2024-04-05T20:27:38.877697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static=train_static.drop(\"paytype_783L\")\ntrain_static=train_static.drop(\"paytype1st_925L\")\ntrain_static=train_static.drop(\"equalitydataagreement_891L\")\ntrain_static=train_static.drop(\"isbidproductrequest_292L\")\ntrain_static=train_static.drop(\"bankacctype_710L\")\ntrain_static=train_static.drop(\"typesuite_864L\")\ntrain_static=train_static.drop(\"cardtype_51L\")\ntrain_static=train_static.with_columns(pl.col(\"credtype_322L\").fill_null(pl.col(\"credtype_322L\").mode()))\ntrain_static=train_static.with_columns(pl.col(\"inittransactioncode_186L\").fill_null(pl.col(\"inittransactioncode_186L\").mode()))\ntrain_static=train_static.with_columns(pl.col(\"twobodfilling_608L\").fill_null(pl.col(\"twobodfilling_608L\").mode()))\ntrain_static=train_static.with_columns(pl.col(\"disbursementtype_67L\").fill_null(pl.col(\"disbursementtype_67L\").mode()))\ntrain_static=train_static.with_columns(pl.col(\"isdebitcard_729L\").fill_null(True).cast(pl.Int16))\ntrain_static=train_static.with_columns(pl.col(\"equalityempfrom_62L\").fill_null(False).cast(pl.Int16))\ntrain_static=train_static.with_columns(pl.col(\"lastst_736L\").fill_null(\"no_info\"))\ntrain_static=train_static.with_columns(pl.col(\"opencred_647L\").fill_null(False).cast(pl.Int16))","metadata":{"execution":{"iopub.status.busy":"2024-04-05T20:27:38.880945Z","iopub.execute_input":"2024-04-05T20:27:38.881370Z","iopub.status.idle":"2024-04-05T20:27:39.519841Z","shell.execute_reply.started":"2024-04-05T20:27:38.881327Z","shell.execute_reply":"2024-04-05T20:27:39.518170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_num=train_static[numerical_features]\ntrain_static_num=train_static_num.with_columns(pl.col(\"lastotherinc_902A\").fill_null(pl.col(\"lastotherinc_902A\").drop_nulls().mode()))","metadata":{"execution":{"iopub.status.busy":"2024-04-05T20:27:39.524778Z","iopub.execute_input":"2024-04-05T20:27:39.525358Z","iopub.status.idle":"2024-04-05T20:27:39.559880Z","shell.execute_reply.started":"2024-04-05T20:27:39.525315Z","shell.execute_reply":"2024-04-05T20:27:39.558373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.impute import SimpleImputer\nimputer = SimpleImputer(strategy=\"mean\")\n# Assuming your DataFrame is called 'df'\ntrain_static_num.drop(\"target\").to_numpy()\nimputer.fit(train_static_num.drop(\"target\").to_numpy())\nimputed_df=imputer.transform(train_static_num.drop(\"target\").to_numpy())\nimputed_df = pl.DataFrame(imputed_df, train_static_num.columns) ","metadata":{"execution":{"iopub.status.busy":"2024-04-05T20:27:39.561902Z","iopub.execute_input":"2024-04-05T20:27:39.562351Z","iopub.status.idle":"2024-04-05T20:27:52.814247Z","shell.execute_reply.started":"2024-04-05T20:27:39.562314Z","shell.execute_reply":"2024-04-05T20:27:52.812845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static=train_static.with_columns(imputed_df).drop(date_features)","metadata":{"execution":{"iopub.status.busy":"2024-04-05T20:27:52.816309Z","iopub.execute_input":"2024-04-05T20:27:52.816852Z","iopub.status.idle":"2024-04-05T20:27:52.853672Z","shell.execute_reply.started":"2024-04-05T20:27:52.816817Z","shell.execute_reply":"2024-04-05T20:27:52.852516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_static_num","metadata":{"execution":{"iopub.status.busy":"2024-04-05T20:27:52.855689Z","iopub.execute_input":"2024-04-05T20:27:52.856125Z","iopub.status.idle":"2024-04-05T20:27:52.994970Z","shell.execute_reply.started":"2024-04-05T20:27:52.856083Z","shell.execute_reply":"2024-04-05T20:27:52.993287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_basetable=train_basetable.join(\n    train_static, how=\"inner\", on=\"case_id\"\n).join(\n    train_static_cb, how=\"inner\", on=\"case_id\"\n).join(\n    train_person_1_feats_1, how=\"inner\", on=\"case_id\"\n).join(\n    train_person_1_feats_2, how=\"inner\", on=\"case_id\"\n).drop(\"date_decision\",\"MONTH\",\"WEEK_NUM\")\ntrain_basetable","metadata":{"execution":{"iopub.status.busy":"2024-04-05T20:27:52.997692Z","iopub.execute_input":"2024-04-05T20:27:52.998313Z","iopub.status.idle":"2024-04-05T20:28:01.561614Z","shell.execute_reply.started":"2024-04-05T20:27:52.998248Z","shell.execute_reply":"2024-04-05T20:28:01.560349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_basetable = pl.read_parquet(dataPath + \"parquet_files/test/test_base.parquet\")\ntest_static = pl.concat(\n    [\n        pl.read_parquet(dataPath + \"parquet_files/test/test_static_0_0.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/test/test_static_0_1.parquet\").pipe(set_table_dtypes),\n        pl.read_parquet(dataPath + \"parquet_files/test/test_static_0_2.parquet\").pipe(set_table_dtypes),\n    ],\n    how=\"vertical_relaxed\",\n)\ntest_static_cb = pl.read_parquet(dataPath + \"parquet_files/test/test_static_cb_0.parquet\").pipe(set_table_dtypes)\ntest_person_1 = pl.read_parquet(dataPath + \"parquet_files/test/test_person_1.parquet\").pipe(set_table_dtypes) \ntest_credit_bureau_b_2 = pl.read_parquet(dataPath + \"parquet_files/test/test_credit_bureau_b_2.parquet\").pipe(set_table_dtypes) ","metadata":{"execution":{"iopub.status.busy":"2024-04-05T20:28:01.563144Z","iopub.execute_input":"2024-04-05T20:28:01.563541Z","iopub.status.idle":"2024-04-05T20:28:01.630314Z","shell.execute_reply.started":"2024-04-05T20:28:01.563509Z","shell.execute_reply":"2024-04-05T20:28:01.628931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_person_1_feats_2 = test_person_1.select([\"case_id\", \"num_group1\", \"housetype_905L\"]).filter(\n    pl.col(\"num_group1\") == 0\n).drop(\"num_group1\").rename({\"housetype_905L\": \"person_housetype\"})\ntest_person_1_feats_2=test_person_1_feats_2.with_columns(pl.col(\"person_housetype\").fill_null(\"not_owned\"))\ntest_person_1_feats_1 = test_person_1.group_by(\"case_id\").agg(\n    pl.col(\"mainoccupationinc_384A\").max().alias(\"mainoccupationinc_384A_max\"),\n    (pl.col(\"incometype_1044T\") == \"SELFEMPLOYED\").max().alias(\"mainoccupationinc_384A_any_selfemployed\")\n)\ntest_person_1_feats_1=test_person_1_feats_1.with_columns(pl.col(\"mainoccupationinc_384A_any_selfemployed\").cast(pl.Int16))\ntest_credit_bureau_b_2_feats = test_credit_bureau_b_2.group_by(\"case_id\").agg(\n    pl.col(\"pmts_pmtsoverdue_635A\").max().alias(\"pmts_pmtsoverdue_635A_max\"),\n    (pl.col(\"pmts_dpdvalue_108P\") > 31).max().alias(\"pmts_dpdvalue_108P_over31\")\n)\ntest_credit_bureau_b_2_feats=test_credit_bureau_b_2_feats.with_columns(pl.col(\"pmts_dpdvalue_108P_over31\").cast(pl.Int16))","metadata":{"execution":{"iopub.status.busy":"2024-04-05T20:28:01.632119Z","iopub.execute_input":"2024-04-05T20:28:01.633360Z","iopub.status.idle":"2024-04-05T20:28:01.645970Z","shell.execute_reply.started":"2024-04-05T20:28:01.633317Z","shell.execute_reply":"2024-04-05T20:28:01.644502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_static_cb=test_static_cb.with_columns(pl.col(\"requesttype_4525192L\").fill_null(\"DEDUCTION_6\"))\ntest_static_cb=test_static_cb.drop(\"riskassesment_302T\")\ntest_static_cb=test_static_cb.drop(['forweek_1077L', 'formonth_206L', 'foryear_850L', 'foryear_818L','foryear_618L', 'forweek_601L', 'forweek_528L', 'fortoday_1092L',\n       'forquarter_634L', 'forquarter_462L', 'forquarter_1017L','formonth_535L', 'formonth_118L', 'for3years_584L', 'for3years_504L','for3years_128L', 'riskassesment_940T',\n        'pmtcount_4955617L','pmtaverage_4955615A', 'pmtaverage_4527227A', 'pmtcount_4527229L','pmtaverage_3A', 'pmtcount_693L', 'contractssum_5085716L'])\ntest_static_cb=test_static_cb.with_columns(pl.col('pmtssum_45A').fill_null(0))\ntest_static_cb=test_static_cb.with_columns(pl.col('pmtscount_423L').fill_null(0))\ntest_static_cb=test_static_cb.with_columns(pl.col('days90_310L').fill_null(0))\ntest_static_cb=test_static_cb.with_columns(pl.col('days360_512L').fill_null(0))\ntest_static_cb=test_static_cb.with_columns(pl.col('days30_165L').fill_null(0))\ntest_static_cb=test_static_cb.with_columns(pl.col('days180_256L').fill_null(0))\ntest_static_cb=test_static_cb.with_columns(pl.col('days120_123L').fill_null(0))\ntest_static_cb=test_static_cb.with_columns(pl.col('firstquarter_103L').fill_null(0))\ntest_static_cb=test_static_cb.with_columns(pl.col('secondquarter_766L').fill_null(0))\ntest_static_cb=test_static_cb.with_columns(pl.col('thirdquarter_1082L').fill_null(0))\ntest_static_cb=test_static_cb.with_columns(pl.col('fourthquarter_440L').fill_null(0))\ntest_static_cb=test_static_cb.with_columns(pl.col('numberofqueries_373L').fill_null(0))\ntest_static_cb=test_static_cb.drop(date_features_cb)","metadata":{"execution":{"iopub.status.busy":"2024-04-05T20:28:01.650246Z","iopub.execute_input":"2024-04-05T20:28:01.650725Z","iopub.status.idle":"2024-04-05T20:28:01.670503Z","shell.execute_reply.started":"2024-04-05T20:28:01.650691Z","shell.execute_reply":"2024-04-05T20:28:01.668734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_static=test_static.drop(\"paytype_783L\")\ntest_static=test_static.drop(\"paytype1st_925L\")\ntest_static=test_static.drop(\"equalitydataagreement_891L\")\ntest_static=test_static.drop(\"isbidproductrequest_292L\")\ntest_static=test_static.drop(\"bankacctype_710L\")\ntest_static=test_static.drop(\"typesuite_864L\")\ntest_static=test_static.drop(\"cardtype_51L\")\ntest_static=test_static.with_columns(pl.col(\"credtype_322L\").fill_null(pl.col(\"credtype_322L\").mode()))\ntest_static=test_static.with_columns(pl.col(\"inittransactioncode_186L\").fill_null(pl.col(\"inittransactioncode_186L\").mode()))\ntest_static=test_static.with_columns(pl.col(\"twobodfilling_608L\").fill_null(pl.col(\"twobodfilling_608L\").mode()))\ntest_static=test_static.with_columns(pl.col(\"disbursementtype_67L\").fill_null(pl.col(\"disbursementtype_67L\").mode()))\ntest_static=test_static.with_columns(pl.col(\"isdebitcard_729L\").fill_null(True).cast(pl.Int16))\ntest_static=test_static.with_columns(pl.col(\"equalityempfrom_62L\").cast(pl.Boolean).fill_null(False).cast(pl.Int16))\ntest_static=test_static.with_columns(pl.col(\"lastst_736L\").fill_null(\"no_info\"))\ntest_static=test_static.with_columns(pl.col(\"opencred_647L\").fill_null(False).cast(pl.Int16))","metadata":{"execution":{"iopub.status.busy":"2024-04-05T20:28:01.672457Z","iopub.execute_input":"2024-04-05T20:28:01.673018Z","iopub.status.idle":"2024-04-05T20:28:01.695432Z","shell.execute_reply.started":"2024-04-05T20:28:01.672971Z","shell.execute_reply":"2024-04-05T20:28:01.694074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_static_num=test_static[numerical_features]\ntest_static_num=test_static_num.with_columns(pl.col(\"lastotherinc_902A\").fill_null(0.2))","metadata":{"execution":{"iopub.status.busy":"2024-04-05T20:28:01.697043Z","iopub.execute_input":"2024-04-05T20:28:01.698402Z","iopub.status.idle":"2024-04-05T20:28:01.705824Z","shell.execute_reply.started":"2024-04-05T20:28:01.698343Z","shell.execute_reply":"2024-04-05T20:28:01.704875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imputed_data=imputer.transform(test_static_num.to_numpy())\nimputed_data = pl.DataFrame(imputed_data, test_static_num.columns) ","metadata":{"execution":{"iopub.status.busy":"2024-04-05T20:28:01.707058Z","iopub.execute_input":"2024-04-05T20:28:01.708743Z","iopub.status.idle":"2024-04-05T20:28:01.722812Z","shell.execute_reply.started":"2024-04-05T20:28:01.708682Z","shell.execute_reply":"2024-04-05T20:28:01.720847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_static=test_static.with_columns(imputed_data).drop(date_features)\ntest_static","metadata":{"execution":{"iopub.status.busy":"2024-04-05T20:28:01.724823Z","iopub.execute_input":"2024-04-05T20:28:01.725884Z","iopub.status.idle":"2024-04-05T20:28:01.781611Z","shell.execute_reply.started":"2024-04-05T20:28:01.725848Z","shell.execute_reply":"2024-04-05T20:28:01.780385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del imputed_data\ndel test_static_num","metadata":{"execution":{"iopub.status.busy":"2024-04-05T20:28:01.783227Z","iopub.execute_input":"2024-04-05T20:28:01.784491Z","iopub.status.idle":"2024-04-05T20:28:01.790877Z","shell.execute_reply.started":"2024-04-05T20:28:01.784443Z","shell.execute_reply":"2024-04-05T20:28:01.789408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_basetable=test_basetable.join(\n    test_static, how=\"left\", on=\"case_id\"\n).join(\n    test_static_cb, how=\"left\", on=\"case_id\"\n).join(\n    test_person_1_feats_1, how=\"left\", on=\"case_id\"\n).join(\n    test_person_1_feats_2, how=\"left\", on=\"case_id\"\n).drop(\"date_decision\",\"MONTH\",\"WEEK_NUM\")\ntest_basetable","metadata":{"execution":{"iopub.status.busy":"2024-04-05T20:28:01.792855Z","iopub.execute_input":"2024-04-05T20:28:01.793426Z","iopub.status.idle":"2024-04-05T20:28:01.832915Z","shell.execute_reply.started":"2024-04-05T20:28:01.793389Z","shell.execute_reply":"2024-04-05T20:28:01.831441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_basetable=test_basetable.with_columns(pl.col(\"person_housetype\").fill_null(\"not_owned\"))\ntest_basetable=test_basetable.with_columns(pl.col(\"mainoccupationinc_384A_any_selfemployed\").fill_null(0))\ntest_basetable=test_basetable.with_columns(pl.col(\"mainoccupationinc_384A_any_selfemployed\").fill_null(1))\ntest_basetable=test_basetable.with_columns(pl.col(\"mainoccupationinc_384A_max\").fill_null(pl.col(\"mainoccupationinc_384A_max\").mean()))","metadata":{"execution":{"iopub.status.busy":"2024-04-05T20:28:01.834810Z","iopub.execute_input":"2024-04-05T20:28:01.835623Z","iopub.status.idle":"2024-04-05T20:28:01.845552Z","shell.execute_reply.started":"2024-04-05T20:28:01.835576Z","shell.execute_reply":"2024-04-05T20:28:01.844378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_basetable=train_basetable.fill_null(strategy=\"forward\")\ntest_basetable=test_basetable.fill_null(strategy=\"forward\")","metadata":{"execution":{"iopub.status.busy":"2024-04-05T20:30:19.660339Z","iopub.execute_input":"2024-04-05T20:30:19.660919Z","iopub.status.idle":"2024-04-05T20:30:19.670849Z","shell.execute_reply.started":"2024-04-05T20:30:19.660874Z","shell.execute_reply":"2024-04-05T20:30:19.669552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_basetable","metadata":{"execution":{"iopub.status.busy":"2024-04-05T20:30:21.197835Z","iopub.execute_input":"2024-04-05T20:30:21.198302Z","iopub.status.idle":"2024-04-05T20:30:21.227562Z","shell.execute_reply.started":"2024-04-05T20:30:21.198255Z","shell.execute_reply":"2024-04-05T20:30:21.226149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n# Assuming 'df' is your DataFrame containing string columns\ntrain_basetable=train_basetable.to_pandas()\ntest_basetable=test_basetable.to_pandas()\n# Identify string columns in the DataFrame\nstring_columns = train_basetable.select_dtypes(include=['object']).columns\n\n# Initialize LabelEncoder\nlabel_encoder = LabelEncoder()\n\n# Iterate over each string column and label encode its values\nfor col in string_columns:\n    label_encoder.fit(train_basetable[col])\n    train_basetable[col] = label_encoder.transform(train_basetable[col])\n    test_basetable[col] = label_encoder.transform(test_basetable[col])","metadata":{"execution":{"iopub.status.busy":"2024-04-05T20:30:22.504384Z","iopub.execute_input":"2024-04-05T20:30:22.505323Z","iopub.status.idle":"2024-04-05T20:30:33.251204Z","shell.execute_reply.started":"2024-04-05T20:30:22.505271Z","shell.execute_reply":"2024-04-05T20:30:33.249558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.decomposition import PCA\nfrom imblearn.over_sampling import SMOTE","metadata":{"execution":{"iopub.status.busy":"2024-04-05T20:30:33.253558Z","iopub.execute_input":"2024-04-05T20:30:33.253937Z","iopub.status.idle":"2024-04-05T20:30:33.338672Z","shell.execute_reply.started":"2024-04-05T20:30:33.253908Z","shell.execute_reply":"2024-04-05T20:30:33.337354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"smote=SMOTE(sampling_strategy=\"minority\")\nX_sm_train,y_sm_train=smote.fit_resample(train_basetable.drop(\"target\",axis=1),train_basetable[\"target\"])","metadata":{"execution":{"iopub.status.busy":"2024-04-05T20:30:33.341003Z","iopub.execute_input":"2024-04-05T20:30:33.342461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_for_pca_smote=X_sm_train.drop([\"case_id\"],axis=1)\ny_for_pca_smote=y_sm_train\npca_static_smote=PCA(n_components=20)\npca_static_smote.fit(X_for_pca_smote)\npca_depth0_smote=pca_static_smote.transform(X_for_pca_smote)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X=pd.DataFrame(pca_depth0_smote)\ny=y_for_pca_smote\ndel train_basetable\ndel X_sm_train\ndel y_sm_train\ndel X_for_pca_smote","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\n\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42,stratify=y_for_pca_smote)\n# Create a StandardScaler object\nscaler = StandardScaler()\n\n# Fit the scaler to the data (calculate mean and standard deviation)\nscaler.fit(X_train)\n\n# Transform the training data\nX_train_scaled = scaler.transform(X_train)\n\n# Transform the testing data (using the parameters calculated from the training data)\nX_test_scaled = scaler.transform(X_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.tree import DecisionTreeClassifier\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import confusion_matrix,classification_report\n\n\n# Initialize the Decision Tree Classifier\nclf = DecisionTreeClassifier()\n\n# Train the classifier on the training data\nclf.fit(X_train_scaled, y_train)\n\n# Make predictions on the testing data\ny_pred = clf.predict(X_test_scaled)\nprint(classification_report(y_test,y_pred))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\nn_errors = (y_pred != y_test).sum()\n# Run Classification Metrics\nprint(\"{}: {}\".format(\"DecisionTreeClassifier\",n_errors))\nprint(\"Accuracy Score :\")\nprint(accuracy_score(y_test,y_pred))\nprint(\"Confusion matrix :\")\nprint(confusion_matrix(y_test, y_pred))\nprint(\"Classification Report :\")\nprint(classification_report(y_test,y_pred))\nprint(\"ROC AUC score is: \",roc_auc_score(y_test,y_pred))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_for_pca_smote_test= test_basetable.drop(\"case_id\",axis=1)\npca_depth0_smote_test=pca_static_smote.transform(X_for_pca_smote_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_test_scaled=scaler.transform(pca_depth0_smote_test)\ny_submit=clf.predict(final_test_scaled)\ny_submit","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca_test_data=pd.DataFrame(final_test_scaled,columns=['1','2','3','4','5','6','7','8','9','10','11','12','13','14','15','16','17','18','19','20'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_basetable[\"score\"]= y_submit\nsubmission=test_basetable[[\"case_id\",\"score\"]]\nsubmission","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"/kaggle/working/submission.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}