{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "3dba9a63-deee-4cac-1375-94601477fbf5"
      },
      "outputs": [],
      "source": [
        "import os \n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "92a1a9e7-f03f-d4f6-ea59-f17a9c9001a0"
      },
      "outputs": [],
      "source": [
        "df_train = pd.read_csv(\"../input/train_users_2.csv\")\n",
        "df_train.sample(n=5) # Only display a few lines and not the whole dataframe"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ba9f054b-9d10-6b08-3be2-467b75c44c8c"
      },
      "outputs": [],
      "source": [
        "df_test = pd.read_csv(\"../input/test_users.csv\")\n",
        "df_test.sample(n=5)  # Only display a few lines and not the whole dataframe"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "bb806d4c-0bd5-bb91-6961-78962f0d5514"
      },
      "outputs": [],
      "source": [
        "#Combine into one dataset\n",
        "df_all = pd.concat((df_train, df_test), axis=0, ignore_index=True)\n",
        "df_all.head(n=5) # Only display a few lines and not the whole dataframe"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "e2145ab9-8f68-d360-a5f0-a329157434f6"
      },
      "outputs": [],
      "source": [
        "#Remove date_first_booking column\n",
        "df_all.drop('date_first_booking', axis=1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "dc324577-5093-e8ed-6ab9-4a547fcd1e85"
      },
      "outputs": [],
      "source": [
        "df_all.head(n=5) # Only display a few lines and not the whole dataframe"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "a6bfac8e-7ddc-baf4-65fc-e4a1427a0715"
      },
      "outputs": [],
      "source": [
        "df_all['date_account_created'] = pd.to_datetime(df_all['date_account_created'], format='%Y-%m-%d')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "0998bff4-b01d-99d0-c805-b8736ad6a8f5"
      },
      "outputs": [],
      "source": [
        "df_all.head(n=5) # Only display a few lines and not the whole dataframe"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "cd82401d-67e4-13b8-ce67-4f5d51687e3a"
      },
      "outputs": [],
      "source": [
        "df_all['timestamp_first_active'] = pd.to_datetime(df_all['timestamp_first_active'], format='%Y%m%d%H%M%S')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "d5ae151d-028a-d680-e628-49e73c9107a4"
      },
      "outputs": [],
      "source": [
        "df_all.head(n=5) # Only display a few lines and not the whole dataframe"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "6e0134c1-4060-9d10-bdd1-c1c62ee01b00"
      },
      "outputs": [],
      "source": [
        "def remove_age_outliers(x, min_value=15, max_value=90):\n",
        "    if np.logical_or(x<=min_value, x>=max_value):\n",
        "        return np.nan\n",
        "    else:\n",
        "        return x"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "86750fc3-d350-8091-0a07-9e9af8606fb6"
      },
      "outputs": [],
      "source": [
        "df_all['age'] = df_all['age'].apply(lambda x: remove_age_outliers(x) if (not np.isnan(x)) else x)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "a2555c58-23b2-0eb8-ee20-83fd06f46a71"
      },
      "outputs": [],
      "source": [
        "df_all['age']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "38bc6f26-f1ad-c857-df4f-3da717cbca4e"
      },
      "outputs": [],
      "source": [
        "df_all['age'].fillna(-1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "f4a0f342-85dc-7ea2-761f-66be8fc091e9"
      },
      "outputs": [],
      "source": [
        "df_all.age = df_all.age.astype(int)\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "91e66259-140d-66d0-ca0f-d027ea878768"
      },
      "outputs": [],
      "source": [
        "def check_NaN_Values_in_df(df):\n",
        "    for col in df:\n",
        "        nan_count = df[col].isnull().sum()\n",
        "        \n",
        "        if nan_count != 0:\n",
        "            print (col+ \" => \" + str(nan_count) + \"NaN Values\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "f47decbb-571c-dea5-2469-885ddd523e0d"
      },
      "outputs": [],
      "source": [
        "check_NaN_Values_in_df(df_all)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "8d591a74-dd05-9166-4e72-1cd9bd75aa0f"
      },
      "outputs": [],
      "source": [
        "df_all['first_affiliate_tracked'].fillna(-1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "5ef62c6d-3cab-2aab-d3f0-80eb7da3c454"
      },
      "outputs": [],
      "source": [
        "df_all.drop('timestamp_first_active', axis=1, inplace=True)\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "3ccb45ca-868a-b274-7552-c2cd678dc502"
      },
      "outputs": [],
      "source": [
        "df_all = df_all[df_all['date_account_created']> '2013-02-01']\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "e6cd36ea-5f69-7216-dc7f-58b3ab593dc1"
      },
      "outputs": [],
      "source": [
        "#We create the output directory \n",
        "if not os.path.exists(\"output\"):\n",
        "    os.makedirs(\"output\")\n",
        "    \n",
        "#We export to csv\n",
        "df_all.to_csv(\"output/cleaned.csv\", sep=',',index=False)"
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}