{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\nfrom IPython.display import FileLink\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nimport glob\nfrom tqdm import tqdm\nfrom time import time\n\nfile_names = []\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        #print(os.path.join(dirname, filename))\n        file_names.append(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-10-01T18:23:04.519015Z","iopub.execute_input":"2022-10-01T18:23:04.519446Z","iopub.status.idle":"2022-10-01T18:23:04.528228Z","shell.execute_reply.started":"2022-10-01T18:23:04.519414Z","shell.execute_reply":"2022-10-01T18:23:04.527041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#credits for this \n#https://gist.github.com/fujiyuu75/748bc168c9ca8a49f86e144a08849893\n\ndef reduce_mem_usage(df, keep_float16=True):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                #adding condition to remove float16\n                if keep_float16:\n                    if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                        df[col] = df[col].astype(np.float16)\n                    elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                        df[col] = df[col].astype(np.float32)\n                    else:\n                        df[col] = df[col].astype(np.float64)\n                else:\n                    if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                        df[col] = df[col].astype(np.float32) \n                    elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                        df[col] = df[col].astype(np.float32)\n                    else:\n                        df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df\n\n\ndef import_data(file, keep_float16=True):\n    \"\"\"create a dataframe and optimize its memory usage\"\"\"\n    df = pd.read_csv(file, parse_dates=True, keep_date_col=True)\n    df = reduce_mem_usage(df, keep_float16)\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-10-01T18:23:05.269659Z","iopub.execute_input":"2022-10-01T18:23:05.270893Z","iopub.status.idle":"2022-10-01T18:23:05.287845Z","shell.execute_reply.started":"2022-10-01T18:23:05.270851Z","shell.execute_reply":"2022-10-01T18:23:05.286535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check whether the specified path exists or not\n# if not then create \n\ndef makedir_check(path):\n    exist_dir = os.path.exists(path)\n    if exist_dir:\n        print(\"The directory already exists!\")\n    else:\n        os.makedirs(path)\n        print(f\"The directory, {path}, has been created!\")","metadata":{"execution":{"iopub.status.busy":"2022-10-01T18:23:05.662329Z","iopub.execute_input":"2022-10-01T18:23:05.662773Z","iopub.status.idle":"2022-10-01T18:23:05.669315Z","shell.execute_reply.started":"2022-10-01T18:23:05.662735Z","shell.execute_reply":"2022-10-01T18:23:05.668012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#delete files\ndef delete_files(path, removal='file'):\n    \"\"\"\n    Takes `path` as input and deletes that file\n    \"\"\"\n    if removal=='file':\n        files = glob.glob(path)\n        for f in files:\n            os.remove(f)\n        print('Files deleted successfully.')\n    if removal=='directory':\n        os.removedirs(path)\n        print('Directory deleted successfully.')","metadata":{"execution":{"iopub.status.busy":"2022-10-01T18:23:06.127801Z","iopub.execute_input":"2022-10-01T18:23:06.128215Z","iopub.status.idle":"2022-10-01T18:23:06.134887Z","shell.execute_reply.started":"2022-10-01T18:23:06.128179Z","shell.execute_reply":"2022-10-01T18:23:06.133607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_paths = []\nfor k in file_names:\n    if 'train_' in k and 'dtypes' not in k:\n        train_paths.append(k)\n        \ntrain_paths","metadata":{"execution":{"iopub.status.busy":"2022-10-01T18:23:08.607861Z","iopub.execute_input":"2022-10-01T18:23:08.608268Z","iopub.status.idle":"2022-10-01T18:23:08.620824Z","shell.execute_reply.started":"2022-10-01T18:23:08.608237Z","shell.execute_reply":"2022-10-01T18:23:08.619489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"makedir_check('feather_data')\nmakedir_check('parquet_data')","metadata":{"execution":{"iopub.status.busy":"2022-10-01T18:23:09.474356Z","iopub.execute_input":"2022-10-01T18:23:09.474814Z","iopub.status.idle":"2022-10-01T18:23:09.482637Z","shell.execute_reply.started":"2022-10-01T18:23:09.474776Z","shell.execute_reply":"2022-10-01T18:23:09.481228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"time_read = {}\nsize_file = {}\nfor file_path in tqdm(train_paths):\n    #ext can be parquet or feather\n    times = {}\n    sizes = {}\n    #stores name of file to be saved as \n    name_files_ftr =  (\n        file_path.split('/')[-1]\n        .replace('.', '_compressed.')\n        .replace('csv','ftr')\n    )\n    name_files_parquet = name_files_ftr.replace('ftr', 'parquet')\n    #store only file name without extension\n    file_only = name_files_ftr.split('_c')[0]\n\n    #read file csv\n    t1 = time()\n    df = pd.read_csv(file_path)\n    sizes['memory_csv'] = df.memory_usage(deep=True).sum()/(1024**2)\n    times['read_csv'] = time() - t1\n    df = reduce_mem_usage(df)\n    \n    #save to feather\n    df.to_feather(\"feather_data/\"+name_files_ftr)\n    #calculate reading time for feather\n    t1 = time()\n    df = pd.read_feather(\"feather_data/\"+name_files_ftr)\n    sizes['memory_feather'] = df.memory_usage(deep=True).sum()/(1024**2)\n    times['read_feather'] = time() - t1\n    \n    #save to parquet\n    df = pd.read_csv(file_path)\n    df= reduce_mem_usage(df, keep_float16=False)\n    df.to_parquet(\"parquet_data/\"+name_files_parquet)\n    #calculate reading time of parquet\n    t1 = time()\n    df = pd.read_parquet(\"parquet_data/\"+name_files_parquet)\n    times['read_parquet'] = time() - t1\n    sizes['memory_parquet'] = df.memory_usage(deep=True).sum()/(1024**2)\n    #store size and time for a particular file\n    time_read[file_only] = times\n    size_file[file_only] = sizes","metadata":{"execution":{"iopub.status.busy":"2022-10-01T18:23:10.135851Z","iopub.execute_input":"2022-10-01T18:23:10.136265Z","iopub.status.idle":"2022-10-01T18:45:04.730814Z","shell.execute_reply.started":"2022-10-01T18:23:10.136233Z","shell.execute_reply":"2022-10-01T18:45:04.728812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_feather = import_data('/kaggle/input/tabular-playground-series-oct-2022/test.csv')\ndf_test_parquet = import_data('/kaggle/input/tabular-playground-series-oct-2022/test.csv',keep_float16=False)\n#saving test data\ndf_test_feather.to_feather('feather_data/test_compressed.ftr')\ndf_test_parquet.to_parquet('parquet_data/test_compressed.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-10-01T18:45:04.734792Z","iopub.execute_input":"2022-10-01T18:45:04.735306Z","iopub.status.idle":"2022-10-01T18:45:42.621596Z","shell.execute_reply.started":"2022-10-01T18:45:04.735262Z","shell.execute_reply":"2022-10-01T18:45:42.619897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"time_taken_read = pd.DataFrame(time_read).T\n#rounding of time to 3 decimal places\ntime_taken_read = time_taken_read.apply(lambda x: x.round(3))\ntime_taken_read.columns = ['time_to_read_csv(seconds)', 'time_to_read_feather(seconds)', 'time_to_read_parquet(seconds)']\n","metadata":{"execution":{"iopub.status.busy":"2022-10-01T18:45:42.623639Z","iopub.execute_input":"2022-10-01T18:45:42.624817Z","iopub.status.idle":"2022-10-01T18:45:42.634625Z","shell.execute_reply.started":"2022-10-01T18:45:42.624773Z","shell.execute_reply":"2022-10-01T18:45:42.633353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"time_taken_read","metadata":{"execution":{"iopub.status.busy":"2022-10-01T18:45:42.637241Z","iopub.execute_input":"2022-10-01T18:45:42.637606Z","iopub.status.idle":"2022-10-01T18:45:42.678900Z","shell.execute_reply.started":"2022-10-01T18:45:42.637574Z","shell.execute_reply":"2022-10-01T18:45:42.677766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"size_of_files = pd.DataFrame(size_file).T\nsize_of_files = size_of_files.apply(lambda x: x.round(2))\nsize_of_files.columns = ['memory_csv(MB)', 'memory_feather(MB)', 'memory_parquet(MB)']\nsize_of_files","metadata":{"execution":{"iopub.status.busy":"2022-10-01T18:45:42.680771Z","iopub.execute_input":"2022-10-01T18:45:42.681552Z","iopub.status.idle":"2022-10-01T18:45:42.702097Z","shell.execute_reply.started":"2022-10-01T18:45:42.681499Z","shell.execute_reply":"2022-10-01T18:45:42.700800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"time_taken_read.sort_index(inplace=True)\nsize_of_files.sort_index(inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-10-01T18:45:42.704060Z","iopub.execute_input":"2022-10-01T18:45:42.704896Z","iopub.status.idle":"2022-10-01T18:45:42.717825Z","shell.execute_reply.started":"2022-10-01T18:45:42.704837Z","shell.execute_reply":"2022-10-01T18:45:42.716801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"time_taken_read.reset_index(inplace=True)\nsize_of_files.reset_index(inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-10-01T18:45:42.719496Z","iopub.execute_input":"2022-10-01T18:45:42.721087Z","iopub.status.idle":"2022-10-01T18:45:42.731170Z","shell.execute_reply.started":"2022-10-01T18:45:42.720904Z","shell.execute_reply":"2022-10-01T18:45:42.730048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"time_taken_read.columns= ['file_name', 'time_to_read_csv(seconds)', 'time_to_read_feather(seconds)',\n       'time_to_read_parquet(seconds)']\nsize_of_files.columns = ['file_name', 'memory_csv(MB)', 'memory_feather(MB)', 'memory_parquet(MB)']","metadata":{"execution":{"iopub.status.busy":"2022-10-01T18:45:42.734956Z","iopub.execute_input":"2022-10-01T18:45:42.735817Z","iopub.status.idle":"2022-10-01T18:45:42.744015Z","shell.execute_reply.started":"2022-10-01T18:45:42.735765Z","shell.execute_reply":"2022-10-01T18:45:42.743169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#checking how much proportion of data is relative to csv files\nproportion_files = (size_of_files[['memory_csv(MB)', 'memory_feather(MB)','memory_parquet(MB)']]).div(size_of_files['memory_csv(MB)'], axis=0)\nproportion_files['file_name'] = size_of_files['file_name']","metadata":{"execution":{"iopub.status.busy":"2022-10-01T18:45:42.746080Z","iopub.execute_input":"2022-10-01T18:45:42.746936Z","iopub.status.idle":"2022-10-01T18:45:42.762253Z","shell.execute_reply.started":"2022-10-01T18:45:42.746878Z","shell.execute_reply":"2022-10-01T18:45:42.761308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"proportion_files.plot(x='file_name',\n                      y=['memory_csv(MB)', 'memory_feather(MB)','memory_parquet(MB)'],\n                      kind='bar',\n                     figsize=(10,6))\nplt.xlabel(\"File Name\")\nplt.ylabel(\"Size of File (Proportion)\")\nplt.xticks(rotation=45);","metadata":{"execution":{"iopub.status.busy":"2022-10-01T18:45:42.766919Z","iopub.execute_input":"2022-10-01T18:45:42.767826Z","iopub.status.idle":"2022-10-01T18:45:43.238572Z","shell.execute_reply.started":"2022-10-01T18:45:42.767776Z","shell.execute_reply":"2022-10-01T18:45:43.237595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(\n    proportion_files[['memory_feather(MB)','memory_parquet(MB)']]\n    .apply(lambda x: round((1-x)*100,2))\n    .mean(axis=0).round(2)\n)","metadata":{"execution":{"iopub.status.busy":"2022-10-01T18:45:43.240017Z","iopub.execute_input":"2022-10-01T18:45:43.240401Z","iopub.status.idle":"2022-10-01T18:45:43.256143Z","shell.execute_reply.started":"2022-10-01T18:45:43.240365Z","shell.execute_reply":"2022-10-01T18:45:43.254802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- We can infer from above that our **memory optimization** resulted in:\n    1. `77.45%` compression for **feather** files\n    2. `56.78%` compression for **parquet** files\n ","metadata":{}},{"cell_type":"code","source":"#calculating\nprint(f\"Mean speed-up in time taken to read feather compared to csv: {round((time_taken_read['time_to_read_csv(seconds)']/time_taken_read['time_to_read_feather(seconds)']).mean(),2)} times\")\nprint(f\"Mean speed-up in time taken to read feather compared to csv: {round((time_taken_read['time_to_read_csv(seconds)']/time_taken_read['time_to_read_parquet(seconds)']).mean(),2)} times\")","metadata":{"execution":{"iopub.status.busy":"2022-10-01T18:45:43.257987Z","iopub.execute_input":"2022-10-01T18:45:43.258385Z","iopub.status.idle":"2022-10-01T18:45:43.271531Z","shell.execute_reply.started":"2022-10-01T18:45:43.258348Z","shell.execute_reply":"2022-10-01T18:45:43.270063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- On an average reading the `feather` files is **~94 times faster** than csv.\n- On an average reading the `parquet` files is **~62 times faster** than csv.","metadata":{}},{"cell_type":"code","source":"time_taken_read.plot(x='file_name', \n                     y =  ['time_to_read_feather(seconds)',\n                          'time_to_read_parquet(seconds)'], \n                     kind='bar', figsize=(15,4))\nplt.xlabel('File Names')\nplt.ylabel('Time Taken (in seconds) to read the file')\nplt.title('Comparison of time taken to read feather and parquet files.')\nplt.xticks(rotation=90);","metadata":{"execution":{"iopub.status.busy":"2022-10-01T18:45:43.273996Z","iopub.execute_input":"2022-10-01T18:45:43.274424Z","iopub.status.idle":"2022-10-01T18:45:43.775212Z","shell.execute_reply.started":"2022-10-01T18:45:43.274386Z","shell.execute_reply":"2022-10-01T18:45:43.773930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!du -h /kaggle/working","metadata":{"execution":{"iopub.status.busy":"2022-10-01T19:02:16.681930Z","iopub.execute_input":"2022-10-01T19:02:16.684117Z","iopub.status.idle":"2022-10-01T19:02:17.876425Z","shell.execute_reply.started":"2022-10-01T19:02:16.684054Z","shell.execute_reply":"2022-10-01T19:02:17.873775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!du -h /kaggle/input/tabular-playground-series-oct-2022/","metadata":{"execution":{"iopub.status.busy":"2022-10-01T19:02:18.690403Z","iopub.execute_input":"2022-10-01T19:02:18.691004Z","iopub.status.idle":"2022-10-01T19:02:19.844451Z","shell.execute_reply.started":"2022-10-01T19:02:18.690957Z","shell.execute_reply":"2022-10-01T19:02:19.843018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!tar -czvf parquet_data.tar.gz parquet_data","metadata":{"execution":{"iopub.status.busy":"2022-10-01T19:02:20.442845Z","iopub.execute_input":"2022-10-01T19:02:20.443334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!tar -czvf feather_data.tar.gz feather_data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# delete_files('parquet_data/*')\n\n# os.removedirs(\"parquet_data\")\n\n#this command helps to download file\n# FileLink(r'./parquet_data.tar.gz')\n\n\n# delete_files('feather_data/*')\n\n# os.removedirs(\"feather_data\")\n\n# FileLink(r'./feather_data.tar.gz')\n\n# file_names\n\n# FileLink(r'./feather_data/test_compressed.ftr')\n\n# FileLink(r'./parquet_data/test_compressed.parquet')\n\n# delete_files('feather_data/*')\n# delete_files('parquet_data/*')\n# os.removedirs('feather_data')\n# os.removedirs('parquet_data')","metadata":{"execution":{"iopub.status.busy":"2022-10-01T17:08:29.844398Z","iopub.execute_input":"2022-10-01T17:08:29.844879Z","iopub.status.idle":"2022-10-01T17:08:29.854598Z","shell.execute_reply.started":"2022-10-01T17:08:29.844839Z","shell.execute_reply":"2022-10-01T17:08:29.853119Z"},"trusted":true},"execution_count":null,"outputs":[]}]}