{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-09-01T13:11:41.261496Z","iopub.execute_input":"2022-09-01T13:11:41.261916Z","iopub.status.idle":"2022-09-01T13:11:41.274855Z","shell.execute_reply.started":"2022-09-01T13:11:41.261881Z","shell.execute_reply":"2022-09-01T13:11:41.273569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are instances when we simply want to estimate the number of rows in a very large csv file before we decide what to do about it. But loading that much data to the memory take ages, and often results in system failure. Here I present a very simple way to estimate the number of rows in a csv file of any size. The numbers of rows found this way won't be exact but only an estimation with at least 99% accuracy. However, it will take only a fraction of second to estimate.","metadata":{}},{"cell_type":"code","source":"def short(n, a=1):\n    if n<=1000:\n        return n\n    elif n>=1000 and n<10**6:\n        return str(round(n/1000, a)) + ' thousand'\n    elif n>=10**6 and n<10**9:\n        return str(round(n/10**6, a)) + ' million'\n    elif n>=10**9 and n<10**12:\n        return str(round(n/10**9, a)) + ' billion'\n    elif n>=10**12:\n        return str(round(n/10**12, a)) + ' triillion'\n\ndef est_rows(location):\n    import os\n    df = pd.read_csv(location, nrows=1000, skiprows=1)\n    df.to_csv('test.csv', index=False)\n    sample_size = os.path.getsize('test.csv')\n    full_size = os.path.getsize(location)\n    rows = int(full_size/sample_size*df.shape[0])\n    os.remove(\"test.csv\")\n    print('Estimated rows are:', short(rows))\n    print('Total columns are:', df.shape[1])","metadata":{"execution":{"iopub.status.busy":"2022-09-01T13:26:35.501273Z","iopub.execute_input":"2022-09-01T13:26:35.501734Z","iopub.status.idle":"2022-09-01T13:26:35.511292Z","shell.execute_reply.started":"2022-09-01T13:26:35.501696Z","shell.execute_reply":"2022-09-01T13:26:35.510395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"est_rows('/kaggle/input/amex-default-prediction/test_data.csv')","metadata":{"execution":{"iopub.status.busy":"2022-09-01T13:26:39.682554Z","iopub.execute_input":"2022-09-01T13:26:39.682935Z","iopub.status.idle":"2022-09-01T13:26:39.980890Z","shell.execute_reply.started":"2022-09-01T13:26:39.682904Z","shell.execute_reply":"2022-09-01T13:26:39.979444Z"},"trusted":true},"execution_count":null,"outputs":[]}]}