{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":8166831,"sourceType":"datasetVersion","datasetId":4832722},{"sourceId":8450234,"sourceType":"datasetVersion","datasetId":4496896}],"dockerImageVersionId":30635,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Install packages","metadata":{}},{"cell_type":"code","source":"!pip install /kaggle/input/kaggle-home-credit-risk-model-stability-lib/kaggle_home_credit_risk_model_stability-0.3-py3-none-any.whl --force-reinstall","metadata":{"execution":{"iopub.status.busy":"2024-05-18T13:22:37.417400Z","iopub.execute_input":"2024-05-18T13:22:37.418675Z","iopub.status.idle":"2024-05-18T13:23:12.129782Z","shell.execute_reply.started":"2024-05-18T13:22:37.418598Z","shell.execute_reply":"2024-05-18T13:23:12.128169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install /kaggle/input/polars/polars-0.20.15-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl","metadata":{"execution":{"iopub.status.busy":"2024-05-18T13:23:12.133213Z","iopub.execute_input":"2024-05-18T13:23:12.133720Z","iopub.status.idle":"2024-05-18T13:24:08.053987Z","shell.execute_reply.started":"2024-05-18T13:23:12.133678Z","shell.execute_reply":"2024-05-18T13:24:08.052278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import packages","metadata":{}},{"cell_type":"code","source":"import polars as pl\nimport gc\nimport pickle\n\nimport kaggle_home_credit_risk_model_stability.libs as hcr\nfrom kaggle_home_credit_risk_model_stability.libs.env import Env\nfrom kaggle_home_credit_risk_model_stability.libs.input.dataset import Dataset\nfrom kaggle_home_credit_risk_model_stability.libs.input.data_loader import DataLoader\nfrom kaggle_home_credit_risk_model_stability.libs.preprocessor.preprocessor import Preprocessor\nfrom kaggle_home_credit_risk_model_stability.libs.preprocessor.steps import *\nfrom kaggle_home_credit_risk_model_stability.libs.preprocessor.columns_info import ColumnsInfo","metadata":{"execution":{"iopub.status.busy":"2024-05-18T13:24:08.056182Z","iopub.execute_input":"2024-05-18T13:24:08.056572Z","iopub.status.idle":"2024-05-18T13:24:12.419654Z","shell.execute_reply.started":"2024-05-18T13:24:08.056542Z","shell.execute_reply":"2024-05-18T13:24:12.417917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"env = Env(\n    \"/kaggle/input/\",\n    \"/kaggle/working/\"\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-18T13:24:12.422131Z","iopub.execute_input":"2024-05-18T13:24:12.422655Z","iopub.status.idle":"2024-05-18T13:24:12.427388Z","shell.execute_reply.started":"2024-05-18T13:24:12.422609Z","shell.execute_reply":"2024-05-18T13:24:12.426424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_loader = DataLoader(env, tables = [\n    \"base\", \"static_cb_0\", \"static_0\", \"person_1\", \"tax_registry_a_1\", \"tax_registry_b_1\", \"tax_registry_c_1\", \n    \"credit_bureau_a_2\", \n    \"credit_bureau_a_1\", \n    \"applprev_1\",\n    #\"debitcard_1\", \"other_1\", \"deposit_1\"\n    #\"credit_bureau_b_1\", \"credit_bureau_b_2\", <- low amount of data\n    #\"applprev_2\", \"person_2\"\n])\n\npreprocessor = Preprocessor({\n    #\"sort_raw_tables\": SortRawTablesStep(),\n    \"set_column_info_step\": SetColumnsInfoStep(),\n    \"set_types\": SetTypesStep(),\n#    \"restore_date_decision\": RestoreDateDecisionStep(env),\n    \"drop_composite_features\": DropCompositeFeaturesStep(),\n    \"create_day_feature\": CreateDayFeatureStep(),\n    \"process_categorical\": ProcessCategoricalStep(),\n    \"process_person_table\": ProcessPersonTableStep(),\n#    \"process_applprev_table\": ProcessApplprevTableStep(),\n    \"process_static_0_table\": ProcessStatic0TableStep(),\n    \"process_tax_regestry_a1_table\": ProcessTaxRegestryA1TableStep(),\n    \"process_tax_regestry_b1_table\": ProcessTaxRegestryB1TableStep(),\n    \"process_tax_regestry_c1_table\": ProcessTaxRegestryC1TableStep(),\n\n    **{ # credit_burea_a_1\n        \"rename_finantial_institution_for_credit_burea_a_1_table\": RenameFinancialInstitutionForCreditBureauA1TableStep(),\n        \"split_active_close_credit_burea_1_table\": SplitActiveCloseCreditBureaua1TableStep(),\n        \"split_active_credit_bureau_a_1_by_credit_duration\": SplitTableByCreditDurationStep(\n            table_name = \"active_credit_bureau_a_1\",\n            intervals = {\n                \"short\": [0, 135],\n                \"medium\": [135, 270],\n                \"medium_long\": [270, 450],\n                \"long\": [450, 100000],\n            }\n        ),\n        \"split_close_credit_bureau_a_1_by_credit_duration\": SplitTableByCreditDurationStep(\n            table_name = \"close_credit_bureau_a_1\",\n            intervals = {\n                \"short\": [0, 135],\n                \"medium\": [135, 270],\n                \"medium_long\": [270, 450],\n                \"long\": [450, 100000],\n            }\n        ),\n        **{\n            f\"split_{period}_active_credit_bureau_a_1_step\": SplitTableByCategoricalFeatureStep(f\"{period}_active_credit_bureau_a_1\", \"financialinstitution_591M\", [[\"Home Credit\", \"P150_136_157\"]])\n            for period in [\"short\", \"medium\", \"medium_long\", \"long\"]\n        },\n        **{\n            f\"split_{period}_close_credit_bureau_a_1_step\": SplitTableByCategoricalFeatureStep(f\"{period}_close_credit_bureau_a_1\", \"financialinstitution_382M\", [[\"P150_136_157\", \"P133_127_114\"]])\n            for period in [\"short\", \"medium\", \"medium_long\", \"long\"]\n        },\n    },\n    \n    \"process_credit_burea_2_table\": ProcessCreditBureaua2TableStep(),\n    \"reduce_memory_usage_for_dataset\": ReduceMemoryUsageForDatasetStep(),\n    #\"one_hot_encoding\": OneHotEncodingStep(),\n    #\"pairwise_diff_raw_dates\": PairwiseDateDiffStep(),\n    \"aggregate_depth_table\": AggregateDepthTableStep(),\n    \"join_table\": JoinTablesStep(),\n    \"merge_chunked_table\": MergeChunkedTablesStep(),\n    \"drop_almost_null_features\": DropAlmostNullFeaturesStep(0.999),\n    #\"generate_age_feature\": GenerateAgeFeatureStep(),\n    \"generate_base_date_diff\": GenerateBaseDateDiffStep(base_column=\"date_decision\"),\n    \"fill_nulls_in_categorical_features\": FillNullsInCategoricalFeaturesStep(),\n    \"reduce_dimention_for_categorical_features\": ReduceDimentionForCategoricalFeaturesStep(),\n    \"reduce_memory_usage_for_dataframe\": ReduceMemoryUsageForDataFrameStep(),\n    **{\n        f\"create_money_feature_fraction_{base_column}\": CreateMoneyFeatureFractionStep(base_column)\n        for base_column in [\"credamount_770A\", \"mainoccupationinc_384A\", \"maininc_215A\", \"annuity_780A\"] # mean_amount_416A\n    },\n    \"drop_single_value_features\": DropSingleValueFeaturesStep(),\n    \"drop_variable_enum_features\": DropVariableEnumFeaturesStep(),\n    #\"generate_mismatch_features\": GenerateMismatchFeaturesStep(),\n#     **{\n#         f\"generate_anomaly_feature_{use_w}_{quantile}_{threashold}\": GenerateAnomalyFeatureStep(quantile=quantile, threashold=threashold)\n#         for quantile in [0.99, 0.97, 0.95, 0.9, 0.8, 0.7]\n#         for threashold in [3, 2, 1.7, 1.5, 1.3]\n#         for use_w in [False]\n#     },\n#    \"window_normalize_features\": WindowNormalizeFeaturesStep(\"3mo\", [\"eir_270L\", \"interestrate_311L\"]),\n    \"reduce_memory_usage_for_dataframe_final\": ReduceMemoryUsageForDataFrameStep(),\n})","metadata":{"execution":{"iopub.status.busy":"2024-05-18T13:24:12.429883Z","iopub.execute_input":"2024-05-18T13:24:12.431085Z","iopub.status.idle":"2024-05-18T13:24:12.449746Z","shell.execute_reply.started":"2024-05-18T13:24:12.431022Z","shell.execute_reply":"2024-05-18T13:24:12.448474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Process train dataset","metadata":{}},{"cell_type":"code","source":"train_dataset_generator = data_loader.load_train_dataset(chunk_size=100000)\ngc.collect()\ntrain_df, columns_info = preprocessor.process_train_dataset(train_dataset_generator)\ndel train_dataset_generator\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-18T13:24:12.450966Z","iopub.execute_input":"2024-05-18T13:24:12.451337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.write_parquet(env.output_directory / \"train_df.parquet\")\npickle.dump(columns_info, open(env.output_directory / \"columns_info.pkl\", \"wb\"))\nprint(train_df.estimated_size() / 1024 / 1024)\nprint(train_df)\n\ndel train_df\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pickle.dump(data_loader, open(env.output_directory / \"data_loader.pkl\", \"wb\"))\npickle.dump(preprocessor, open(env.output_directory / \"preprocessor.pkl\", \"wb\"))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%reset -f","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Process test dataset","metadata":{}},{"cell_type":"code","source":"import polars as pl\nimport gc\nimport pickle\n\nimport kaggle_home_credit_risk_model_stability.libs as hcr\nfrom kaggle_home_credit_risk_model_stability.libs.env import Env\nfrom kaggle_home_credit_risk_model_stability.libs.input.dataset import Dataset\nfrom kaggle_home_credit_risk_model_stability.libs.input.data_loader import DataLoader\nfrom kaggle_home_credit_risk_model_stability.libs.preprocessor.preprocessor import Preprocessor\nfrom kaggle_home_credit_risk_model_stability.libs.preprocessor.steps import *\nfrom kaggle_home_credit_risk_model_stability.libs.preprocessor.columns_info import ColumnsInfo","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"env = Env(\n    \"/kaggle/input/\",\n    \"/kaggle/working/\"\n)\ndata_loader = pickle.load(open(env.output_directory / \"data_loader.pkl\", \"rb\"))\npreprocessor = pickle.load(open(env.output_directory / \"preprocessor.pkl\", \"rb\"))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset_generator = data_loader.load_test_dataset(chunk_size=100000)\ngc.collect()\ntest_df, columns_info = preprocessor.process_test_dataset(test_dataset_generator)\ndel test_dataset_generator\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.write_parquet(env.output_directory / \"test_df.parquet\")\nprint(test_df.estimated_size() / 1024 / 1024)\nprint(test_df)\n\ndel test_df\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}