{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f743036-f4f6-4e37-8fe1-bf8e89b6531b","_cell_guid":"456b13a2-8c0b-48c2-9cb6-8ef5393fc8f3","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-01-18T14:44:09.270129Z","iopub.execute_input":"2023-01-18T14:44:09.270829Z","iopub.status.idle":"2023-01-18T14:44:09.309082Z","shell.execute_reply.started":"2023-01-18T14:44:09.270734Z","shell.execute_reply":"2023-01-18T14:44:09.307964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Importing libraries","metadata":{"_uuid":"fa493397-d54b-48bf-a2aa-12774f5808c6","_cell_guid":"5864b717-0ecf-4955-ade1-4f96acfa213f","trusted":true}},{"cell_type":"code","source":"import matplotlib.pyplot as plt","metadata":{"_uuid":"45633cd6-2d61-41f8-855c-3621550e918e","_cell_guid":"ca71056d-4b5e-4a67-a328-a3d05713420c","collapsed":false,"execution":{"iopub.status.busy":"2023-01-17T16:41:36.148104Z","iopub.execute_input":"2023-01-17T16:41:36.149027Z","iopub.status.idle":"2023-01-17T16:41:36.156848Z","shell.execute_reply.started":"2023-01-17T16:41:36.148958Z","shell.execute_reply":"2023-01-17T16:41:36.155081Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading the data to dataframe","metadata":{"_uuid":"d43c44ed-b300-4dc4-a235-512e7661e6bc","_cell_guid":"4f3f8d53-b10a-425d-992e-21732a1cc0fc","trusted":true}},{"cell_type":"code","source":"import dask.dataframe as dd","metadata":{"_uuid":"d671fe11-e70f-4ec8-b559-de7abf936304","_cell_guid":"b7a622f9-4f95-420a-95ed-33de3104545a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-01-17T17:18:31.951722Z","iopub.execute_input":"2023-01-17T17:18:31.953077Z","iopub.status.idle":"2023-01-17T17:18:31.959458Z","shell.execute_reply.started":"2023-01-17T17:18:31.95302Z","shell.execute_reply":"2023-01-17T17:18:31.957807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ndtypes = {\n    \"row_id\": \"int64\",\n    \"timestamp\": \"int64\",\n    \"user_id\": \"int32\",\n    \"content_id\": \"int16\",\n    \"content_type_id\": \"boolean\",\n    \"task_container_id\": \"int16\",\n    \"user_answer\": \"int8\",\n    \"answered_correctly\": \"int8\",\n    \"prior_question_elapsed_time\": \"float32\", \n    \"prior_question_had_explanation\": \"boolean\"\n}","metadata":{"execution":{"iopub.status.busy":"2023-01-17T17:16:37.443907Z","iopub.execute_input":"2023-01-17T17:16:37.444823Z","iopub.status.idle":"2023-01-17T17:16:37.457705Z","shell.execute_reply.started":"2023-01-17T17:16:37.444752Z","shell.execute_reply":"2023-01-17T17:16:37.455501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = dd.read_csv('/kaggle/input/otto-recommender-system/train.jsonl', dtype=dtypes).compute()\nprint(\"Train size:\", train_df.shape)","metadata":{"_uuid":"eb2f9935-7dc1-46d1-be7c-cb0227fa4da3","_cell_guid":"6a1690af-628c-4390-b29a-86d23617a06b","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-01-17T17:18:31.96438Z","iopub.execute_input":"2023-01-17T17:18:31.965036Z","iopub.status.idle":"2023-01-17T17:18:32.582223Z","shell.execute_reply.started":"2023-01-17T17:18:31.964947Z","shell.execute_reply":"2023-01-17T17:18:32.579919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.columns","metadata":{"_uuid":"8c2f5340-4dd3-4cd5-a307-fbfc64b2691d","_cell_guid":"59df1997-9936-4c65-99bd-703142c7d9ec","collapsed":false,"execution":{"iopub.status.busy":"2023-01-17T16:45:44.155532Z","iopub.execute_input":"2023-01-17T16:45:44.156113Z","iopub.status.idle":"2023-01-17T16:45:44.165957Z","shell.execute_reply.started":"2023-01-17T16:45:44.156065Z","shell.execute_reply":"2023-01-17T16:45:44.163869Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.shape","metadata":{"_uuid":"65eec323-3387-438a-8be1-41c60b62f755","_cell_guid":"0f996275-5e27-49ee-92d6-3b6eac0720a5","collapsed":false,"execution":{"iopub.status.busy":"2023-01-17T16:47:51.098712Z","iopub.execute_input":"2023-01-17T16:47:51.099222Z","iopub.status.idle":"2023-01-17T16:47:51.158122Z","shell.execute_reply.started":"2023-01-17T16:47:51.099178Z","shell.execute_reply":"2023-01-17T16:47:51.156963Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]}]}