{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# https://www.kaggle.com/onodera/riiid-read-csv-in-cudf\n\nimport sys\n!cp ../input/rapids/rapids.0.15.0 /opt/conda/envs/rapids.tar.gz\n!cd /opt/conda/envs/ && tar -xzvf rapids.tar.gz > /dev/null\nsys.path = [\"/opt/conda/envs/rapids/lib/python3.7/site-packages\"] + sys.path\nsys.path = [\"/opt/conda/envs/rapids/lib/python3.7\"] + sys.path\nsys.path = [\"/opt/conda/envs/rapids/lib\"] + sys.path \n!cp /opt/conda/envs/rapids/lib/libxgboost.so /opt/conda/lib/\nimport cudf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\n#import dask.dataframe as dd\n#!pip install modin > /dev/null\n#import modin.pandas as modin\n!pip install feather-format > /dev/null\nimport feather\nimport joblib","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from time import time\nfrom contextlib import contextmanager\n\n@contextmanager\ndef timer(name,times):\n    t0 = time()\n    yield\n    t1 = time() - t0\n    times.append(t1)\n    print(f'[{name}] done in {t1:.2f} s')\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pandas_times = []\nfeather_times = []\njoblib_times = []\ncudf_times = []\n\nnrows = 10**7\nwith timer('pd.read_csv', pandas_times):\n    df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv',\n                        #nrows=nrows,\n                        usecols=[1, 2, 3, 4, 7, 8, 9],\n                        dtype={'timestamp': 'int64',\n                              'user_id': 'int32',\n                              'content_id': 'int16',\n                              'content_type_id': 'int8',\n                              'answered_correctly':'int8',\n                              'prior_question_elapsed_time': 'float32',\n                              'prior_question_had_explanation': 'boolean'}\n                       )\n    \ndf.to_feather('./train.feather')\nwith timer('pd.read_feather', feather_times):\n    df = pd.read_feather('./train.feather')\n    \njoblib.dump(df, './train.sav', compress=3)\nwith timer('joblib.load', joblib_times):\n    df = joblib.load('./train.sav')\n    \nwith timer('cudf.read_csv', cudf_times):\n    df = cudf.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv',\n                        #nrows=nrows,\n                        usecols=[1, 2, 3, 4, 7, 8, 9],\n                        dtype={'timestamp': 'int64',\n                              'user_id': 'int32',\n                              'content_id': 'int16',\n                              'content_type_id': 'int8',\n                              'answered_correctly':'int8',\n                              'prior_question_elapsed_time': 'float32',\n                              'prior_question_had_explanation': 'boolean'}\n                       )\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n#plt.rcdefaults()\nfig, ax = plt.subplots()\n\nx = [1, 2, 3, 4]\ny = [pandas_times[0], feather_times[0], joblib_times[0], cudf_times[0]]\nlabel = ['pandas', 'feather', 'joblib', 'cuDF']\n\nax.barh(x, y, align='center')\nax.set_yticks(x)\nax.set_yticklabels(label)\n\nax.set_xlabel('time (s)')\nax.set_title('How long time does it take to read csv?')\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"\n### Pandas >>>> feather >= joblib > cuDF\n* ### cuDF is the fastest. but you need to GPU.\n"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}