{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7602123,"sourceType":"competition"}],"dockerImageVersionId":30646,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from cpuinfo import get_cpu_info\nimport numpy as np\nimport pandas as pd\nimport polars as pl","metadata":{"execution":{"iopub.status.busy":"2024-02-14T11:40:10.505693Z","iopub.execute_input":"2024-02-14T11:40:10.506087Z","iopub.status.idle":"2024-02-14T11:40:11.424119Z","shell.execute_reply.started":"2024-02-14T11:40:10.506045Z","shell.execute_reply":"2024-02-14T11:40:11.422584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_PATH = '/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_base.parquet'\nDATA_PATH = '/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_credit_bureau_a_2_5.parquet'","metadata":{"execution":{"iopub.status.busy":"2024-02-14T11:40:11.426434Z","iopub.execute_input":"2024-02-14T11:40:11.427148Z","iopub.status.idle":"2024-02-14T11:40:11.435825Z","shell.execute_reply.started":"2024-02-14T11:40:11.427105Z","shell.execute_reply":"2024-02-14T11:40:11.434047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_cpu_info()","metadata":{"execution":{"iopub.status.busy":"2024-02-14T11:40:12.114509Z","iopub.execute_input":"2024-02-14T11:40:12.115006Z","iopub.status.idle":"2024-02-14T11:40:13.397198Z","shell.execute_reply.started":"2024-02-14T11:40:12.114950Z","shell.execute_reply":"2024-02-14T11:40:13.395984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.__version__","metadata":{"execution":{"iopub.status.busy":"2024-02-14T11:40:16.127439Z","iopub.execute_input":"2024-02-14T11:40:16.128248Z","iopub.status.idle":"2024-02-14T11:40:16.136214Z","shell.execute_reply.started":"2024-02-14T11:40:16.128208Z","shell.execute_reply":"2024-02-14T11:40:16.134521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pl.__version__","metadata":{"execution":{"iopub.status.busy":"2024-02-14T11:40:16.426345Z","iopub.execute_input":"2024-02-14T11:40:16.427081Z","iopub.status.idle":"2024-02-14T11:40:16.436584Z","shell.execute_reply.started":"2024-02-14T11:40:16.427044Z","shell.execute_reply":"2024-02-14T11:40:16.435219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 0. Load data from parquet","metadata":{}},{"cell_type":"code","source":"%%time\nbase_pd = pd.read_parquet(BASE_PATH)\nprint(base_pd.shape)","metadata":{"execution":{"iopub.status.busy":"2024-02-14T11:40:19.583189Z","iopub.execute_input":"2024-02-14T11:40:19.583609Z","iopub.status.idle":"2024-02-14T11:40:20.030318Z","shell.execute_reply.started":"2024-02-14T11:40:19.583576Z","shell.execute_reply":"2024-02-14T11:40:20.029003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nbase_pl = pl.read_parquet(BASE_PATH)","metadata":{"execution":{"iopub.status.busy":"2024-02-14T11:40:20.373161Z","iopub.execute_input":"2024-02-14T11:40:20.373672Z","iopub.status.idle":"2024-02-14T11:40:20.560256Z","shell.execute_reply.started":"2024-02-14T11:40:20.373637Z","shell.execute_reply":"2024-02-14T11:40:20.559296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nbase_pd.sample(7)","metadata":{"execution":{"iopub.status.busy":"2024-02-14T11:40:20.713047Z","iopub.execute_input":"2024-02-14T11:40:20.713431Z","iopub.status.idle":"2024-02-14T11:40:20.802071Z","shell.execute_reply.started":"2024-02-14T11:40:20.713402Z","shell.execute_reply":"2024-02-14T11:40:20.800712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nbase_pl.sample(7)","metadata":{"execution":{"iopub.status.busy":"2024-02-14T11:40:20.946085Z","iopub.execute_input":"2024-02-14T11:40:20.946463Z","iopub.status.idle":"2024-02-14T11:40:20.992124Z","shell.execute_reply.started":"2024-02-14T11:40:20.946435Z","shell.execute_reply":"2024-02-14T11:40:20.991258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndata_pd = pd.read_parquet(DATA_PATH)\nprint(data_pd.shape)","metadata":{"execution":{"iopub.status.busy":"2024-02-14T11:40:21.130626Z","iopub.execute_input":"2024-02-14T11:40:21.131103Z","iopub.status.idle":"2024-02-14T11:40:34.597702Z","shell.execute_reply.started":"2024-02-14T11:40:21.131055Z","shell.execute_reply":"2024-02-14T11:40:34.595524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndata_pl = pl.read_parquet(DATA_PATH)","metadata":{"execution":{"iopub.status.busy":"2024-02-14T11:40:34.600360Z","iopub.execute_input":"2024-02-14T11:40:34.600826Z","iopub.status.idle":"2024-02-14T11:40:40.067356Z","shell.execute_reply.started":"2024-02-14T11:40:34.600784Z","shell.execute_reply":"2024-02-14T11:40:40.066414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_pd.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-14T11:40:40.068289Z","iopub.execute_input":"2024-02-14T11:40:40.068620Z","iopub.status.idle":"2024-02-14T11:40:40.096749Z","shell.execute_reply.started":"2024-02-14T11:40:40.068594Z","shell.execute_reply":"2024-02-14T11:40:40.095847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_pl.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-14T11:40:40.098976Z","iopub.execute_input":"2024-02-14T11:40:40.099599Z","iopub.status.idle":"2024-02-14T11:40:40.120149Z","shell.execute_reply.started":"2024-02-14T11:40:40.099568Z","shell.execute_reply":"2024-02-14T11:40:40.119157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Filter","metadata":{}},{"cell_type":"code","source":"%%time\ndata_pd[data_pd['pmts_month_706T'] > 3]","metadata":{"execution":{"iopub.status.busy":"2024-02-14T11:38:35.502391Z","iopub.execute_input":"2024-02-14T11:38:35.502710Z","iopub.status.idle":"2024-02-14T11:38:39.625633Z","shell.execute_reply.started":"2024-02-14T11:38:35.502674Z","shell.execute_reply":"2024-02-14T11:38:39.624766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndata_pl.filter(pl.col('pmts_month_706T') > 3)","metadata":{"execution":{"iopub.status.busy":"2024-02-14T11:38:39.626891Z","iopub.execute_input":"2024-02-14T11:38:39.627503Z","iopub.status.idle":"2024-02-14T11:38:43.226811Z","shell.execute_reply.started":"2024-02-14T11:38:39.627471Z","shell.execute_reply":"2024-02-14T11:38:43.225677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# GroupBy","metadata":{}},{"cell_type":"code","source":"%%time\nkey = ['case_id', 'num_group1', 'num_group2', 'pmts_year_507T']\nvalue = 'pmts_month_706T'\ndata_pd.groupby(key)[value].mean()","metadata":{"execution":{"iopub.status.busy":"2024-02-14T11:35:46.936541Z","iopub.execute_input":"2024-02-14T11:35:46.937684Z","iopub.status.idle":"2024-02-14T11:36:07.849909Z","shell.execute_reply.started":"2024-02-14T11:35:46.937643Z","shell.execute_reply":"2024-02-14T11:36:07.848554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nkey = ['case_id', 'num_group1', 'num_group2', 'pmts_year_507T']\nvalue = 'pmts_month_706T'\ndata_pl.group_by(key, maintain_order=True).agg(pl.col(value).mean())","metadata":{"execution":{"iopub.status.busy":"2024-02-14T11:36:07.851955Z","iopub.execute_input":"2024-02-14T11:36:07.852420Z","iopub.status.idle":"2024-02-14T11:36:18.600977Z","shell.execute_reply.started":"2024-02-14T11:36:07.852375Z","shell.execute_reply":"2024-02-14T11:36:18.600037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Join","metadata":{}},{"cell_type":"code","source":"%%time\ndata_pd = data_pd.merge(base_pd, how='left', on='case_id')","metadata":{"execution":{"iopub.status.busy":"2024-02-14T11:38:43.228504Z","iopub.execute_input":"2024-02-14T11:38:43.228869Z","iopub.status.idle":"2024-02-14T11:38:50.275780Z","shell.execute_reply.started":"2024-02-14T11:38:43.228826Z","shell.execute_reply":"2024-02-14T11:38:50.274544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndata_pl = data_pl.join(base_pl, how='left', on='case_id')","metadata":{"execution":{"iopub.status.busy":"2024-02-14T11:38:50.277507Z","iopub.execute_input":"2024-02-14T11:38:50.277873Z","iopub.status.idle":"2024-02-14T11:38:52.371662Z","shell.execute_reply.started":"2024-02-14T11:38:50.277842Z","shell.execute_reply":"2024-02-14T11:38:52.370327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# New columns","metadata":{}},{"cell_type":"code","source":"%%time\ndata_pd['sin_706T'] = np.sin(data_pd['pmts_month_706T'].values)","metadata":{"execution":{"iopub.status.busy":"2024-02-14T11:24:55.227495Z","iopub.execute_input":"2024-02-14T11:24:55.227928Z","iopub.status.idle":"2024-02-14T11:24:55.906370Z","shell.execute_reply.started":"2024-02-14T11:24:55.227886Z","shell.execute_reply":"2024-02-14T11:24:55.905189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_pd['sin_706T'].head()","metadata":{"execution":{"iopub.status.busy":"2024-02-14T11:27:27.785001Z","iopub.execute_input":"2024-02-14T11:27:27.785448Z","iopub.status.idle":"2024-02-14T11:27:27.794743Z","shell.execute_reply.started":"2024-02-14T11:27:27.785410Z","shell.execute_reply":"2024-02-14T11:27:27.793373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndata_pl = data_pl.with_columns((pl.col('pmts_month_706T').sin()).alias('sin_706T'))","metadata":{"execution":{"iopub.status.busy":"2024-02-14T11:26:52.020597Z","iopub.execute_input":"2024-02-14T11:26:52.021908Z","iopub.status.idle":"2024-02-14T11:26:52.668548Z","shell.execute_reply.started":"2024-02-14T11:26:52.021863Z","shell.execute_reply":"2024-02-14T11:26:52.667192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_pl['sin_706T'].head(5)","metadata":{"execution":{"iopub.status.busy":"2024-02-14T11:27:35.919652Z","iopub.execute_input":"2024-02-14T11:27:35.920064Z","iopub.status.idle":"2024-02-14T11:27:35.927815Z","shell.execute_reply.started":"2024-02-14T11:27:35.920030Z","shell.execute_reply":"2024-02-14T11:27:35.926623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# To Pandas","metadata":{}},{"cell_type":"code","source":"%%time\ndata_pl.to_pandas()","metadata":{"execution":{"iopub.status.busy":"2024-02-14T11:40:40.121330Z","iopub.execute_input":"2024-02-14T11:40:40.121697Z","iopub.status.idle":"2024-02-14T11:40:47.133863Z","shell.execute_reply.started":"2024-02-14T11:40:40.121668Z","shell.execute_reply":"2024-02-14T11:40:47.132749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Lazy evaluation","metadata":{}},{"cell_type":"code","source":"%%time\nq = (\n    pl.scan_parquet(DATA_PATH)\n    .filter(pl.col(\"pmts_month_706T\") > 3)\n    .group_by(['case_id', 'num_group1', 'num_group2'])\n    .agg(pl.col('pmts_month_706T').mean())\n)\n\ndata_lazy = q.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-14T11:32:15.978331Z","iopub.execute_input":"2024-02-14T11:32:15.978784Z","iopub.status.idle":"2024-02-14T11:32:20.893561Z","shell.execute_reply.started":"2024-02-14T11:32:15.978750Z","shell.execute_reply":"2024-02-14T11:32:20.892335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q","metadata":{"execution":{"iopub.status.busy":"2024-02-14T11:32:32.798024Z","iopub.execute_input":"2024-02-14T11:32:32.798483Z","iopub.status.idle":"2024-02-14T11:32:33.008898Z","shell.execute_reply.started":"2024-02-14T11:32:32.798447Z","shell.execute_reply":"2024-02-14T11:32:33.007602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q.show_graph()","metadata":{"execution":{"iopub.status.busy":"2024-02-14T11:32:50.913455Z","iopub.execute_input":"2024-02-14T11:32:50.913857Z","iopub.status.idle":"2024-02-14T11:32:50.946039Z","shell.execute_reply.started":"2024-02-14T11:32:50.913824Z","shell.execute_reply":"2024-02-14T11:32:50.945119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_lazy","metadata":{"execution":{"iopub.status.busy":"2024-02-14T11:32:27.958204Z","iopub.execute_input":"2024-02-14T11:32:27.958624Z","iopub.status.idle":"2024-02-14T11:32:27.968095Z","shell.execute_reply.started":"2024-02-14T11:32:27.958588Z","shell.execute_reply":"2024-02-14T11:32:27.967117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}