{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "24d63ed6-0de4-f42a-44f2-41cf38281d60"
      },
      "outputs": [],
      "source": [
        "# This Python 3 environment comes with many helpful analytics libraries installed\n",
        "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
        "# For example, here's several helpful packages to load in \n",
        "import os\n",
        "import numpy as np # linear algebra\n",
        "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
        "\n",
        "# Input data files are available in the \"../input/\" directory.\n",
        "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
        "\n",
        "from subprocess import check_output\n",
        "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n",
        "\n",
        "# Any results you write to the current directory are saved as output."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "627469a6-963f-aad4-6487-16f317c0f517"
      },
      "outputs": [],
      "source": [
        "df_train = pd.read_csv(\"../input/train_users_2.csv\")\n",
        " \n",
        "df_train.head(n=5) # Only display a few lines and not the whole dataframe"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "97f2146d-ca14-ffd7-a132-40eb0a72da78"
      },
      "outputs": [],
      "source": [
        "df_test = pd.read_csv('../input/test_users.csv')\n",
        "df_test.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "556b58a3-4e46-b203-453e-1ea8260b9417"
      },
      "outputs": [],
      "source": [
        "df_all = pd.concat((df_train,df_test), axis=0, ignore_index=True)\n",
        "df_all.head(5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "3d8988a9-f2d4-efb3-9e77-03dd4c1b8451"
      },
      "outputs": [],
      "source": [
        "df_all.drop('date_first_booking', axis=1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "5aaa08a0-13fd-6c1b-d220-2234c8b6d8a9"
      },
      "outputs": [],
      "source": [
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "55dd31ec-9a5d-c28f-ef5d-5001295a5d8b"
      },
      "outputs": [],
      "source": [
        "df_all['date_account_created'] = pd.to_datetime(df_all['date_account_created'], format='%Y-%m-%d')\n",
        "df_all.sample(5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "173d415b-c417-5cce-dab9-2314aa687216"
      },
      "outputs": [],
      "source": [
        "df_all['timestamp_first_active'] = pd.to_datetime(df_all['timestamp_first_active'], format='%Y%m%d%H%M%S')\n",
        "df_all.sample(5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "157289c6-b92b-79ff-38aa-9c8cee72d6a1"
      },
      "outputs": [],
      "source": [
        "def remove_age_outliers(x, min_value=15, max_value=90):\n",
        "    if np.logical_or(x<=min_value,x>=max_value):\n",
        "        return np.nan\n",
        "    else:\n",
        "        return x"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "644345af-82f9-7136-8056-6c3b6177a599"
      },
      "outputs": [],
      "source": [
        "df_all['age'].apply(lambda x: remove_age_outliers(x))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "df63eb43-7459-ce3f-38cd-f7b6208ba209"
      },
      "outputs": [],
      "source": [
        "df_all['age'].fillna(-1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "86df84ee-0921-17c8-4ab2-1b2d2865ea5d"
      },
      "outputs": [],
      "source": [
        "df_all_age = df_all.age.astype(int)\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "f1c5158e-f93d-e0ba-ee48-bd8def467bf2"
      },
      "outputs": [],
      "source": [
        "def check_NaN_Values_in_df(df):\n",
        "    for col in df:\n",
        "        nan_count = df[col].isnull().sum()\n",
        "        \n",
        "        if nan_count != 0:\n",
        "            print(col + '=>' + str(nan_count) + 'NaN Values')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "7bc05441-46b5-b092-410e-6c63ecf32e92"
      },
      "outputs": [],
      "source": [
        "check_NaN_Values_in_df(df_all)\n",
        "df_all.sample(5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "0fce42f3-c47d-40a4-2f2d-14076a38abc2"
      },
      "outputs": [],
      "source": [
        "df_all['first_affiliate_tracked'].fillna(1, inplace=True)\n",
        "check_NaN_Values_in_df(df_all)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "b2d8d087-1e27-32ad-d08c-c5b1c5da2441"
      },
      "outputs": [],
      "source": [
        "df_all.drop('timestamp_first_active', axis=1, inplace=True)\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "47ac977e-656e-43f5-2e5f-c9d147f0c383"
      },
      "outputs": [],
      "source": [
        "df_all = df_all[df_all['date_account_created'] > '2013-02-01']\n",
        "df_all.sample(5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "74810160-f6cb-2ae4-8f31-43473d5eb537"
      },
      "outputs": [],
      "source": [
        "if not os.path.exists(\"output\"):\n",
        "    os.makedirs(\"output\")\n",
        "    \n",
        "df_all.to_csv(\"output/cleaned.csv\", sep=',', index=False)"
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}