{"cells": [{"source": ["**Loading Libraries**"], "cell_type": "markdown", "metadata": {"_uuid": "118a4380c77ce04932dc8e19ee578e11c50aa4c0", "_cell_guid": "6c390e7e-2fc9-49ca-ad6e-2bb38cdb5440"}}, {"source": ["# This Python 3 environment comes with many helpful analytics libraries installed\n", "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n", "# For example, here's several helpful packages to load in \n", "\n", "# This is a set of helper functions to help you fetch data and save it in usable files\n", "\n", "import numpy as np # linear algebra\n", "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "import sklearn \n", "import matplotlib\n", "import matplotlib.pyplot as plt\n", "\n", "from subprocess import check_output\n", "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n", "# Any results you write to the current directory are saved as output."], "cell_type": "code", "execution_count": null, "outputs": [], "metadata": {"collapsed": true, "_uuid": "854a166cc16898ce8271608ae2bbd73ecc80e344", "_cell_guid": "f183c329-8e90-4a05-baea-84a164c8428c"}}, {"source": ["def load_user_logs():\n", "    print(\"Loading training Data\")\n", "    train = pd.read_csv(\"../input/train.csv\")\n", "    print(\"Loading test Data\")\n", "    test = pd.read_csv(\"../input/sample_submission_zero.csv\")\n", "    valid_msno = pd.concat([train, test])['msno'].as_matrix()\n", "    train = []\n", "    test = []\n", "\n", "    user_logs_iter = pd.read_csv(\"../input/user_logs.csv\", chunksize=10000000, iterator=True, low_memory=False, parse_dates=['date'])\n", "    user_logs = pd.DataFrame()\n", "    user_log_counts = pd.DataFrame()\n", "\n", "    i = 0\n", "    for df in user_logs_iter:\n", "        df = df[df['msno'].isin(valid_msno)]\n", "        if i is 0: \n", "            user_logs = df\n", "            user_log_counts = pd.DataFrame(user_logs['msno'].value_counts().reset_index())\n", "            user_log_counts.columns = ['msno','logs_count']\n", "        else:\n", "            temp_user_log_counts = pd.DataFrame(df['msno'].value_counts().reset_index())\n", "            temp_user_log_counts.columns = ['msno','logs_count']\n", "            user_logs = pd.concat([user_logs, df])\n", "            user_log_counts = pd.concat([user_log_counts, temp_user_log_counts])\n", "        \n", "        user_logs = user_logs.groupby([\"msno\"], as_index=False)[\"num_25\", \"num_50\", \"num_75\", \"num_985\", \"num_100\", \"num_unq\", \"total_secs\"].sum()        \n", "        user_log_counts = user_log_counts.groupby([\"msno\"], as_index=False)[\"logs_count\"].sum()        \n", "\n", "        print(\"User Logs {} df: {}\".format(str(i), user_logs.shape))\n", "        print(\"User Log Counts {} df: {}\".format(str(i), user_log_counts.shape))     \n", "        i = i+1\n", "            \n", "    print(user_logs.shape)\n", "    user_logs = pd.merge(user_logs, user_log_counts, how='left', on='msno')\n", "    print(user_logs.shape)\n", "    \n", "    return user_logs\n", "\n", "## Suggest commenting out all logs except for the user_logs when fetching user logs\n", "## and vice versa\n", "def raw_daq():\n", "#     print(\"Loading training Data\")\n", "#     train = pd.read_csv(\"../input/train.csv\")\n", "#     print(\"Loading test Data\")\n", "#     test = pd.read_csv(\"../input/sample_submission_zero.csv\")\n", "#     print(\"Loading transaction Data\")\n", "#     transactions = pd.read_csv(\"../input/transactions.csv\")\n", "#     print(\"Loading members Data\")\n", "#     members = pd.read_csv(\"../input/members.csv\")\n", "    print(\"Loading user Log Data\")\n", "    user_logs = load_user_logs()\n", "\n", "#     print(\"Train set shape {}\".format(train.shape))\n", "#     print(\"Test set shape {}\".format(test.shape))\n", "#     print(\"Transactions set shape {}\".format(transactions.shape))\n", "#     print(\"Members set shape {}\".format(members.shape))\n", "    print(\"User Logs set shape {}\".format(user_logs.shape))\n", "    user_logs.to_csv(\"user_logs_collapsed.csv\",header=True)"], "cell_type": "code", "execution_count": null, "outputs": [], "metadata": {"collapsed": true, "_uuid": "bb0f2f2a3bdeeaa119408cc8b0e99ddea292aabf", "_cell_guid": "2c765c7a-5b01-41bb-869a-52fa5bd21620"}}, {"source": ["**Visualizing Each Data Frame as we build up the train set**"], "cell_type": "markdown", "metadata": {"_uuid": "9ded36d4a51d25d928d25ba586891613e1b6f24c", "_cell_guid": "e2eb2dea-cb46-4222-9877-3140b0a898d3"}}, {"source": ["def check_sets():\n", "    print(\"Test set shape {} and train set shape {}\".format(test_set.shape,train_set.shape))\n", "\n", "# Function for outputting data to csv files: Use as needed\n", "def raw_data_merge():\n", "    print(\"Merging Transactions\")\n", "    transaction_counts = pd.DataFrame(transactions['msno'].value_counts().reset_index())\n", "    transaction_counts.columns = ['msno','trans_count']\n", "    # Merging based on msno keys on the left input dataframe\n", "    train_set = pd.merge(train, transaction_counts, how='left', on='msno') \n", "    test_set = pd.merge(test, transaction_counts, how='left', on='msno')\n", "\n", "    print(\"Merging Member Data\")\n", "    gender = {'male':1, 'female':2}\n", "    members['gender'] = members['gender'].map(gender)\n", "    members.fillna(0)\n", "    train_set = pd.merge(train_set, members, how='left', on='msno')\n", "    test_set = pd.merge(test_set, members, how='left', on='msno')\n", "\n", "    # print(\"Merging User Logs\")\n", "    # train_set = pd.merge(train_set, user_logs, how='left', on='msno')\n", "    # test_set = pd.merge(test_set, user_logs, how='left', on='msno')\n", "    train_set.to_csv(\"train_set_no_logs.csv\",header=True)\n", "    test_set.to_csv(\"test_set_no_logs.csv\",header=True)\n", "    check_sets()"], "cell_type": "code", "execution_count": null, "outputs": [], "metadata": {"collapsed": true, "_uuid": "348cd2d478e1d5be21413a873e9f08dcee518a2d", "_cell_guid": "611dc352-93ff-4e48-9cf5-e18ff0d5885d"}}, {"source": ["print(\"Getting raw data\")\n", "raw_daq()"], "cell_type": "code", "execution_count": null, "outputs": [], "metadata": {"collapsed": true, "_uuid": "94d57d08dd31ee568caed554a8991667d33cbb43", "_cell_guid": "c90fd284-4f37-4015-b0c5-b6de10ead0c2"}}], "metadata": {"kernelspec": {"language": "python", "name": "python3", "display_name": "Python 3"}, "language_info": {"nbconvert_exporter": "python", "name": "python", "pygments_lexer": "ipython3", "file_extension": ".py", "codemirror_mode": {"name": "ipython", "version": 3}, "mimetype": "text/x-python", "version": "3.6.3"}}, "nbformat_minor": 1, "nbformat": 4}