{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport polars as pl\n\nimport time","metadata":{"execution":{"iopub.status.busy":"2024-04-29T09:47:13.261134Z","iopub.execute_input":"2024-04-29T09:47:13.261661Z","iopub.status.idle":"2024-04-29T09:47:14.955857Z","shell.execute_reply.started":"2024-04-29T09:47:13.261603Z","shell.execute_reply":"2024-04-29T09:47:14.954119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Reading csv using pandas","metadata":{}},{"cell_type":"code","source":"start_time = time.time()\n\napplication_train_pd=pd.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_credit_bureau_a_2_6.csv')\n\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\n### ignorre memoru usage in this cell","metadata":{"execution":{"iopub.status.busy":"2024-04-29T09:47:17.068964Z","iopub.execute_input":"2024-04-29T09:47:17.070069Z","iopub.status.idle":"2024-04-29T09:48:55.672693Z","shell.execute_reply.started":"2024-04-29T09:47:17.069996Z","shell.execute_reply":"2024-04-29T09:48:55.670443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"application_train_pd.info()","metadata":{"execution":{"iopub.status.busy":"2024-04-29T09:48:55.676306Z","iopub.execute_input":"2024-04-29T09:48:55.676865Z","iopub.status.idle":"2024-04-29T09:48:55.714141Z","shell.execute_reply.started":"2024-04-29T09:48:55.676818Z","shell.execute_reply":"2024-04-29T09:48:55.712871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### reading csv using polars","metadata":{}},{"cell_type":"code","source":"start_time = time.time()\n\napplication_train_pl=pl.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_credit_bureau_a_2_6.csv')\n\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\nprint(\"memory size is  \"+str(application_train_pl.estimated_size(\"mb\"))+\" MB\")","metadata":{"execution":{"iopub.status.busy":"2024-04-29T09:48:55.715810Z","iopub.execute_input":"2024-04-29T09:48:55.716268Z","iopub.status.idle":"2024-04-29T09:49:11.108381Z","shell.execute_reply.started":"2024-04-29T09:48:55.716222Z","shell.execute_reply":"2024-04-29T09:49:11.107107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### First reading with polars and then converting to pandas dataframe","metadata":{}},{"cell_type":"code","source":"start_time = time.time()\n\napplication_train_pl_to_pd=pl.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_credit_bureau_a_2_6.csv').to_pandas()\n\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\n### ignorre memoru usage in this cell","metadata":{"execution":{"iopub.status.busy":"2024-04-29T09:44:37.291561Z","iopub.execute_input":"2024-04-29T09:44:37.292831Z","iopub.status.idle":"2024-04-29T09:45:29.186608Z","shell.execute_reply.started":"2024-04-29T09:44:37.292770Z","shell.execute_reply":"2024-04-29T09:45:29.184989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"application_train_pl_to_pd.info()","metadata":{"execution":{"iopub.status.busy":"2024-04-29T09:45:29.188179Z","iopub.execute_input":"2024-04-29T09:45:29.188601Z","iopub.status.idle":"2024-04-29T09:45:29.216876Z","shell.execute_reply.started":"2024-04-29T09:45:29.188564Z","shell.execute_reply":"2024-04-29T09:45:29.214849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### pandas 2.0 (engine='pyarrow',dtype_backend='pyarrow') ","metadata":{}},{"cell_type":"code","source":"start_time = time.time()\n\napplication_train_pd_2_0=pd.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_credit_bureau_a_2_6.csv',engine='pyarrow',dtype_backend='pyarrow')\n\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\nprint(\"memory usage is \"+str(application_train_pd_2_0.info(verbose = False, memory_usage = 'deep')))","metadata":{"execution":{"iopub.status.busy":"2024-04-29T09:49:11.111192Z","iopub.execute_input":"2024-04-29T09:49:11.111616Z","iopub.status.idle":"2024-04-29T09:49:23.306605Z","shell.execute_reply.started":"2024-04-29T09:49:11.111581Z","shell.execute_reply":"2024-04-29T09:49:23.305114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"application_train_pd_2_0.info()","metadata":{"execution":{"iopub.status.busy":"2024-04-29T09:49:23.308204Z","iopub.execute_input":"2024-04-29T09:49:23.308628Z","iopub.status.idle":"2024-04-29T09:49:23.719519Z","shell.execute_reply.started":"2024-04-29T09:49:23.308592Z","shell.execute_reply":"2024-04-29T09:49:23.717711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"application_train_pd_2_0","metadata":{"execution":{"iopub.status.busy":"2024-04-29T09:12:57.304159Z","iopub.execute_input":"2024-04-29T09:12:57.304562Z","iopub.status.idle":"2024-04-29T09:13:00.569414Z","shell.execute_reply.started":"2024-04-29T09:12:57.304524Z","shell.execute_reply":"2024-04-29T09:13:00.567538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"application_train_pd_2_0.describe()","metadata":{"execution":{"iopub.status.busy":"2024-04-29T09:25:21.118227Z","iopub.execute_input":"2024-04-29T09:25:21.118681Z","iopub.status.idle":"2024-04-29T09:25:28.438503Z","shell.execute_reply.started":"2024-04-29T09:25:21.118640Z","shell.execute_reply":"2024-04-29T09:25:28.437373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nums=['case_id', 'collater_valueofguarantee_1124L',\n       'collater_valueofguarantee_876L', 'num_group1', 'num_group2',\n       'pmts_dpd_1073P', 'pmts_dpd_303P', 'pmts_month_158T', 'pmts_month_706T',\n       'pmts_overdue_1140A', 'pmts_overdue_1152A', 'pmts_year_1139T',\n       'pmts_year_507T']\ncats=['collater_typofvalofguarant_298M','collater_typofvalofguarant_407M','collaterals_typeofguarante_359M',\n      'collaterals_typeofguarante_669M','subjectroles_name_541M','subjectroles_name_838M']","metadata":{"execution":{"iopub.status.busy":"2024-04-29T09:28:30.953479Z","iopub.execute_input":"2024-04-29T09:28:30.953888Z","iopub.status.idle":"2024-04-29T09:28:30.960901Z","shell.execute_reply.started":"2024-04-29T09:28:30.953859Z","shell.execute_reply":"2024-04-29T09:28:30.959508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"application_train_pd_2_0['collater_typofvalofguarant_298M'].unique()","metadata":{"execution":{"iopub.status.busy":"2024-04-29T09:21:32.744782Z","iopub.execute_input":"2024-04-29T09:21:32.745254Z","iopub.status.idle":"2024-04-29T09:21:33.239829Z","shell.execute_reply.started":"2024-04-29T09:21:32.745220Z","shell.execute_reply":"2024-04-29T09:21:33.238417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start_time = time.time()\nprint(application_train_pd_2_0['num_group2'].agg(['min','max','mean','median','std']))\napplication_train_pd_2_0['collater_typofvalofguarant_298M'].agg(['nunique'])\n\nprint(\"--- %s seconds ---\" % (time.time() - start_time))","metadata":{"execution":{"iopub.status.busy":"2024-04-29T09:21:46.384948Z","iopub.execute_input":"2024-04-29T09:21:46.385324Z","iopub.status.idle":"2024-04-29T09:21:47.158015Z","shell.execute_reply.started":"2024-04-29T09:21:46.385296Z","shell.execute_reply":"2024-04-29T09:21:47.156908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start_time = time.time()\napplication_train_pl.with_columns([\n    pl.col('num_group2').min().name.suffix('_min'),\n    pl.col('num_group2').max().name.suffix('_max'),\n    pl.col('num_group2').mean().name.suffix('_mean'),\n    pl.col('num_group2').median().name.suffix('_median'),\n    pl.col('num_group2').std().name.suffix('_std'),\n    pl.col('collater_typofvalofguarant_298M').n_unique().name.suffix('_unique'),\n])\n\nprint(\"--- %s seconds ---\" % (time.time() - start_time))","metadata":{"execution":{"iopub.status.busy":"2024-04-29T09:21:48.789681Z","iopub.execute_input":"2024-04-29T09:21:48.790894Z","iopub.status.idle":"2024-04-29T09:21:50.266897Z","shell.execute_reply.started":"2024-04-29T09:21:48.790854Z","shell.execute_reply":"2024-04-29T09:21:50.265380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Min","metadata":{}},{"cell_type":"code","source":"start_time = time.time()\nprint(application_train_pd['num_group2'].agg(['min']))\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\n\nstart_time = time.time()\nprint(application_train_pd_2_0['num_group2'].agg(['min']))\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\n\nstart_time = time.time()\nprint(application_train_pl.select(pl.min(\"num_group2\")))\nprint(\"--- %s seconds ---\" % (time.time() - start_time))","metadata":{"execution":{"iopub.status.busy":"2024-04-29T09:22:24.113234Z","iopub.execute_input":"2024-04-29T09:22:24.114390Z","iopub.status.idle":"2024-04-29T09:22:24.232230Z","shell.execute_reply.started":"2024-04-29T09:22:24.114315Z","shell.execute_reply":"2024-04-29T09:22:24.230637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Median","metadata":{}},{"cell_type":"code","source":"start_time = time.time()\nprint(application_train_pd['num_group2'].agg(['median']))\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\n\nstart_time = time.time()\nprint(application_train_pd_2_0['num_group2'].agg(['median']))\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\n\nstart_time = time.time()\nprint(application_train_pl.select(pl.median(\"num_group2\")))\nprint(\"--- %s seconds ---\" % (time.time() - start_time))","metadata":{"execution":{"iopub.status.busy":"2024-04-29T09:22:58.890275Z","iopub.execute_input":"2024-04-29T09:22:58.890725Z","iopub.status.idle":"2024-04-29T09:22:59.928778Z","shell.execute_reply.started":"2024-04-29T09:22:58.890691Z","shell.execute_reply":"2024-04-29T09:22:59.927550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Mean","metadata":{}},{"cell_type":"code","source":"start_time = time.time()\nprint(application_train_pd['num_group2'].agg(['mean']))\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\n\nstart_time = time.time()\nprint(application_train_pd_2_0['num_group2'].agg(['mean']))\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\n\nstart_time = time.time()\napplication_train_pl.select(pl.mean(\"num_group2\"))\nprint(\"--- %s seconds ---\" % (time.time() - start_time))","metadata":{"execution":{"iopub.status.busy":"2024-04-29T09:24:05.152420Z","iopub.execute_input":"2024-04-29T09:24:05.153582Z","iopub.status.idle":"2024-04-29T09:24:05.283510Z","shell.execute_reply.started":"2024-04-29T09:24:05.153529Z","shell.execute_reply":"2024-04-29T09:24:05.282138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Std","metadata":{}},{"cell_type":"code","source":"start_time = time.time()\nprint(application_train_pd['num_group2'].agg(['std']))\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\n\nstart_time = time.time()\nprint(application_train_pd_2_0['num_group2'].agg(['std']))\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\n\nstart_time = time.time()\napplication_train_pl.select(pl.std(\"num_group2\"))\nprint(\"--- %s seconds ---\" % (time.time() - start_time))","metadata":{"execution":{"iopub.status.busy":"2024-04-29T09:24:21.604303Z","iopub.execute_input":"2024-04-29T09:24:21.604726Z","iopub.status.idle":"2024-04-29T09:24:22.108432Z","shell.execute_reply.started":"2024-04-29T09:24:21.604696Z","shell.execute_reply":"2024-04-29T09:24:22.106902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Filtering and Selection operations comparison","metadata":{}},{"cell_type":"code","source":"start_time = time.time()\napplication_train_pd[application_train_pd['collater_typofvalofguarant_298M']=='9a0c095e'][nums].mean()\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\n\nstart_time = time.time()\napplication_train_pd_2_0[application_train_pd_2_0['collater_typofvalofguarant_298M']=='9a0c095e'][nums].mean()\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\n\nstart_time = time.time()\napplication_train_pl.filter(pl.col(\"collater_typofvalofguarant_298M\") == '9a0c095e').select(pl.col(nums).mean())\nprint(\"--- %s seconds ---\" % (time.time() - start_time))","metadata":{"execution":{"iopub.status.busy":"2024-04-29T09:29:58.982245Z","iopub.execute_input":"2024-04-29T09:29:58.983305Z","iopub.status.idle":"2024-04-29T09:30:07.297791Z","shell.execute_reply.started":"2024-04-29T09:29:58.983260Z","shell.execute_reply":"2024-04-29T09:30:07.296419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Grouping Operations Comparison","metadata":{}},{"cell_type":"code","source":"nums=[ 'collater_valueofguarantee_1124L',\n       'collater_valueofguarantee_876L', 'num_group1', 'num_group2',\n       'pmts_dpd_1073P', 'pmts_dpd_303P', 'pmts_month_158T', 'pmts_month_706T',\n       'pmts_overdue_1140A', 'pmts_overdue_1152A', 'pmts_year_1139T',\n       'pmts_year_507T']\ncats=['collater_typofvalofguarant_298M','collater_typofvalofguarant_407M','collaterals_typeofguarante_359M',\n      'collaterals_typeofguarante_669M','subjectroles_name_541M','subjectroles_name_838M']\n\n# Pandas numpy Functions\nstart_time = time.time()\nFunction_1= application_train_pd.groupby(['case_id'])['collater_typofvalofguarant_298M'].agg('count')   #Function 1\nprint(\"pandas with numpy backend \"+str((time.time() - start_time)))\nstart_time = time.time()\nFunction_2= application_train_pd.groupby(['case_id'])['collater_valueofguarantee_1124L'].agg('mean')    #Function 2\nprint(\"pandas with numpy backend \"+str((time.time() - start_time)))\nstart_time = time.time()\nFunction_3= application_train_pd.groupby(['case_id'])[nums].agg('mean')       #Function 3\nprint(\"pandas with numpy backend \"+str((time.time() - start_time)))\nstart_time = time.time()\nFunction_4= application_train_pd.groupby(['case_id'])[cats].agg('count')      #Function 4\nprint(\"pandas with numpy backend \"+str((time.time() - start_time)))\n\n\n# Pandas apache arrow Functions\nstart_time = time.time()\nFunction_2_0_1= application_train_pd_2_0.groupby(['case_id'])['collater_typofvalofguarant_298M'].agg('count')   #Function 1\nprint(\"pandas with apache arrow backend \"+str((time.time() - start_time)))\nstart_time = time.time()\nFunction_2_0_2= application_train_pd_2_0.groupby(['case_id'])['collater_valueofguarantee_1124L'].agg('mean')    #Function 2\nprint(\"pandas with apache arrow backend \"+str((time.time() - start_time)))\nstart_time = time.time()\nFunction_2_0_3= application_train_pd_2_0.groupby(['case_id'])[nums].agg('mean')       #Function 3\nprint(\"pandas with apache arrow backend \"+str((time.time() - start_time)))\nstart_time = time.time()\nFunction_2_0_4= application_train_pd_2_0.groupby(['case_id'])[cats].agg('count')      #Function 4\nprint(\"pandas with apache arrow backend \"+str((time.time() - start_time)))\n\n\n# Polars Functions\nstart_time = time.time()\nFunction_1= application_train_pl.group_by('case_id').agg(pl.col('collater_typofvalofguarant_298M').count()) #Function 1\nprint(\"polars \"+str((time.time() - start_time)))\nstart_time = time.time()\nFunction_2= application_train_pl.group_by('case_id').agg(pl.col('collater_valueofguarantee_1124L').mean())  #Function 2\nprint(\"polars \"+str((time.time() - start_time)))\nstart_time = time.time()\nFunction_3= application_train_pl.group_by('case_id').agg(pl.col(nums).mean())     #Function 3\nprint(\"polars \"+str((time.time() - start_time)))\nstart_time = time.time()\nFunction_4= application_train_pl.group_by('case_id').agg(pl.col(cats).count())    #Function 4\nprint(\"polars \"+str((time.time() - start_time)))","metadata":{"execution":{"iopub.status.busy":"2024-04-29T09:37:53.139506Z","iopub.execute_input":"2024-04-29T09:37:53.139950Z","iopub.status.idle":"2024-04-29T09:38:36.416367Z","shell.execute_reply.started":"2024-04-29T09:37:53.139917Z","shell.execute_reply":"2024-04-29T09:38:36.415054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Sorting Operation Comparison","metadata":{}},{"cell_type":"code","source":"cols=['case_id','num_group1'] # columns to be used for sorting\n\nstart_time = time.time()\napplication_train_pd.sort_values(by=cols,ascending=True)\nprint(\"pandas with numpy backend \"+str((time.time() - start_time)))","metadata":{"execution":{"iopub.status.busy":"2024-04-29T09:51:50.717727Z","iopub.execute_input":"2024-04-29T09:51:50.719534Z","iopub.status.idle":"2024-04-29T09:51:58.245852Z","shell.execute_reply.started":"2024-04-29T09:51:50.719435Z","shell.execute_reply":"2024-04-29T09:51:58.243732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start_time = time.time()\napplication_train_pd_2_0.sort_values(by=cols,ascending=True)\nprint(\"pandas with apache arrow backend \"+str((time.time() - start_time)))\n\nstart_time = time.time()\napplication_train_pl.sort(cols,descending=False)\nprint(\"polars \"+str((time.time() - start_time)))","metadata":{"execution":{"iopub.status.busy":"2024-04-29T09:51:25.211563Z","iopub.execute_input":"2024-04-29T09:51:25.213090Z","iopub.status.idle":"2024-04-29T09:51:45.831784Z","shell.execute_reply.started":"2024-04-29T09:51:25.213027Z","shell.execute_reply":"2024-04-29T09:51:45.830562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}