{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c74f1674-68b9-7309-1a0b-150ebc5dcf9f" }, "outputs": [], "source": [ "# This Python 3 environment comes with many helpful analytics libraries installed\n", "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n", "# For example, here's several helpful packages to load in \n", "\n", "import numpy as np # linear algebra\n", "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "import seaborn as sns\n", "import random as rand\n", "import matplotlib.pyplot as plt\n", "%matplotlib inline\n", "import sys\n", "if sys.version_info[0] < 3: \n", " from StringIO import StringIO\n", "else:\n", " from io import StringIO\n", "\n", "\n", "\n", "# Input data files are available in the \"../input/\" directory.\n", "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n", "\n", "from subprocess import check_output\n", "import subprocess as sbps\n", "file_name_list = check_output([\"ls\", \"../input\"]).decode(\"utf8\")\n", "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n", "\n", "# Any results you write to the current directory are saved as output." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "e020fe3a-d97f-c239-da66-71e8044fd12a" }, "outputs": [], "source": [ "# p = sbps.Popen(['ls', '-lh', '../input'], stdout=sbps.PIPE)\n", "print(\n", " sbps.check_output(['ls', '-lh', '../input']).decode('utf8') ,\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "4a329991-fbf8-5c0d-c100-4beb31e7006b" }, "outputs": [], "source": [ "def get_file_head(file_name, n_head):\n", " return sbps.check_output(\n", " ['head', '-{}'.format(n_head), '../input/{}'.format(file_name)]\n", " )\n", "\n", "def get_file_length(file_name):\n", " p = sbps.Popen(\n", " ['cat', '../input/{}'.format(file_name)],\n", " stdout=sbps.PIPE\n", " )\n", " return sbps.check_output(\n", " ['wc', '-l'],\n", " stdin=p.stdout\n", " )\n", " " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f55c1eed-c20e-10f6-93de-dbb1e14c0deb" }, "outputs": [], "source": [ "for file_name in file_name_list.split('\\n')[:-1]:\n", " print(\n", " '-------------{}-------------'.format(file_name)\n", " )\n", " print(\n", " '{}'.format(\n", " get_file_head(file_name, n_head=5)\n", " ).split('\\\\n')\n", " )\n", " print(\n", " 'file_length: {}'.format(\n", " get_file_length(file_name)\n", " )\n", " )" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "aa97013a-ce76-958e-32ee-90595e51c9c8" }, "source": [ "## File Metadata\n", "without packing all objects in your memory we could see the file metadata with above functions. What we've got:\n", "\n", "all files include header: \n", "\n", "number of lines:\n", "\n", "- destinations.csv: 62107\n", "- sample_submission.csv: 2528244\n", "- test.csv: 2528244\n", "- train.csv: 37670294\n", "\n", "\n", "Now we should random sample the file so that it would fit into the memory" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "29549b55-b7c3-a381-9b2f-6697f47ed7e9" }, "outputs": [], "source": [ "tl_destinations = 62107\n", "tl_test = 2528244\n", "tl_train = 37670294\n", "\n", "# lets count sample size needed with confidence level 0.99\n", "# and margin of error = 0.01\n", "ts_train = 16580\n", "ts_test = 16480\n", "ts_destinations = 13092" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "5fd76a9e-1665-1be3-7765-c2e265456aaf" }, "outputs": [], "source": [ "def create_file_buffer_from_file(file_name, n_sample, total_lines):\n", " # open file handler\n", " file_path = '../input/{}'.format(file_name)\n", " f = open(file_path)\n", " file_buffer = list()\n", " rows = np.sort(\n", " np.random.randint(\n", " 1, \n", " tl_train,\n", " size=n_sample\n", " )\n", " )\n", " # get the file header\n", " f.seek(0)\n", " header_line = f.readline()\n", " file_buffer.append(header_line)\n", " for row in rows:\n", " f.seek(row)\n", " f.readline() #discard - bound to be partial line\n", " file_buffer.append(\n", " f.readline()\n", " )\n", " \n", " f.close()\n", " return StringIO('\\n'.join(file_buffer))\n", " \n", "def get_random_sample_from_file(file_name, n_sample, total_lines): \n", " \n", " return pd.read_csv(\n", " create_file_buffer_from_file(\n", " file_name, \n", " n_sample, \n", " total_lines\n", " )\n", " )\n", " " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "495d5c8e-b673-b63c-eb62-c6bd6f5bcfeb" }, "outputs": [], "source": [ "_df_train = get_random_sample_from_file('train.csv', ts_train, tl_train)\n", "_df_train.head()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "1d9b38d5-c897-0876-e4a2-ba9c90e982b1" }, "outputs": [], "source": [ "_df_train.describe()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "eb3e64ea-c76f-a6dc-603c-64077c01cf19" }, "outputs": [], "source": [ "_df_train.columns" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "51f5b7c4-acf9-7fed-6a13-3d9ece400a8b" }, "outputs": [], "source": [ "_df_train.groupby('is_booking')['date_time'].count()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "8ed15ee6-11b3-2e53-9d44-ce79b612f283" }, "outputs": [], "source": [ "_df_train['dt'] = pd.to_datetime(_df_train['date_time'])\n", "_df_train['dt'].dtype" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b6c5046d-802c-364c-9ad6-4b8fc7ec5e39" }, "outputs": [], "source": [ "_df_train['date_time'].dtype" ] } ], "metadata": { "_change_revision": 0, "_is_fork": false, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.5.2" } }, "nbformat": 4, "nbformat_minor": 0 }