{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30673,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd # 数据读取、数据统计\nimport numpy as np # 向量计算、矩阵\nimport os, glob\n\nimport matplotlib.pyplot as plt\n%config InlineBackend.figure_format = 'svg'\n\nDATA_PATH = '/kaggle/input/home-credit-credit-risk-model-stability/'\nos.listdir(DATA_PATH)","metadata":{"_uuid":"051d70d956493feee0c6d64651c6a088724dca2a","_execution_state":"idle","execution":{"iopub.status.busy":"2024-03-27T08:02:59.520651Z","iopub.execute_input":"2024-03-27T08:02:59.521039Z","iopub.status.idle":"2024-03-27T08:02:59.539028Z","shell.execute_reply.started":"2024-03-27T08:02:59.521010Z","shell.execute_reply":"2024-03-27T08:02:59.537273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_person_1 = pd.read_parquet(os.path.join(DATA_PATH, \"parquet_files\", 'train', 'train_person_1.parquet'))","metadata":{"execution":{"iopub.status.busy":"2024-03-27T08:03:21.748165Z","iopub.execute_input":"2024-03-27T08:03:21.748741Z","iopub.status.idle":"2024-03-27T08:03:26.160501Z","shell.execute_reply.started":"2024-03-27T08:03:21.748698Z","shell.execute_reply":"2024-03-27T08:03:26.159541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\ndef reduce_mem_usage(df):\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n\n    for col in df.columns:\n        if col in [\"case_id\", \"WEEK_NUM\", \"num_group1\", \"num_group2\"]:\n            df[col] = df[col].astype(np.int64)\n        elif col in [\"date_decision\"]:\n            df[col] = pd.to_datetime(df[col])\n        elif col[-1] in [\"D\"]:\n            df[col] = pd.to_datetime(df[col])\n        elif col[-1] in (\"P\", \"A\"):\n            df[col] = df[col].astype(np.float64)\n\n        col_type = df[col].dtype\n        if col_type == \"category\":\n            continue\n\n        if col_type not in [object, \"datetime64[ns]\",]:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)\n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-03-27T08:05:36.794396Z","iopub.execute_input":"2024-03-27T08:05:36.794892Z","iopub.status.idle":"2024-03-27T08:05:36.814900Z","shell.execute_reply.started":"2024-03-27T08:05:36.794859Z","shell.execute_reply":"2024-03-27T08:05:36.813636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_person_1 = reduce_mem_usage(train_person_1)","metadata":{"execution":{"iopub.status.busy":"2024-03-27T08:05:56.950304Z","iopub.execute_input":"2024-03-27T08:05:56.950966Z","iopub.status.idle":"2024-03-27T08:06:06.758490Z","shell.execute_reply.started":"2024-03-27T08:05:56.950926Z","shell.execute_reply":"2024-03-27T08:06:06.757030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}