{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "1a4feef7-7e6c-66ea-c5ab-3bf449c5c0ea"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "c78d4daf-f654-b7ec-adb0-7d48cfe5d8c2"
      },
      "outputs": [],
      "source": [
        "df_train = pd.read_csv(\"../input/train_users_2.csv\")\n",
        "df_train.head(n=5) # Only display the n first lines\n",
        "df_train.sample(n=5) # Only display a few lines and not the whole dataframe"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "bf23f2c1-26a1-eba1-47e4-4dd8fa261042"
      },
      "outputs": [],
      "source": [
        "df_test = pd.read_csv(\"../input/test_users.csv\")\n",
        "df_test.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "f94ed1fa-47e6-ee18-e763-3eaf49ecae36"
      },
      "outputs": [],
      "source": [
        "# Combine into one dataset\n",
        "df_all = pd.concat((df_train, df_test), axis=0, ignore_index=True)\n",
        "df_all.head(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "91f25efd-51ae-0c54-5725-a147cd70d101"
      },
      "outputs": [],
      "source": [
        "# Remove data_first_booking column\n",
        "df_all.drop('date_first_booking', axis=1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ef47a82e-f24a-81f0-392f-3ad923363cd3"
      },
      "outputs": [],
      "source": [
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ff0a1b69-0c5e-58e3-a69b-1417ef015054"
      },
      "outputs": [],
      "source": [
        "df_all['date_account_created'] = pd.to_datetime(df_all['date_account_created'], format='%Y-%m-%d')\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "f5edfbb5-3982-7aef-72b3-41d6c9b9ecae"
      },
      "outputs": [],
      "source": [
        "df_all['timestamp_first_active'] = pd.to_datetime(df_all['timestamp_first_active'], format='%Y%m%d%H%M%S')\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "e8e6d1e1-b7af-ff80-2f42-dee709b5c24b"
      },
      "outputs": [],
      "source": [
        "def remove_age_outliers(x, min_value=15, max_value=90):\n",
        "    if np.logical_or(x<=min_value, x>max_value): #plus efficace qu'un calcul math car + efficace sur un tableau\n",
        "        return np.nan\n",
        "    else:\n",
        "        return x"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "9b472189-a2e7-5b77-1297-6e8bd64e8619"
      },
      "outputs": [],
      "source": [
        "df_all['age'] = df_all['age'].apply(lambda x: remove_age_outliers(x) if(not np.isnan(x)) else x)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "e2c25b56-708e-0e37-9695-57400109ee67"
      },
      "outputs": [],
      "source": [
        "df_all['age'].sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "4c04b120-aec6-43da-57c6-967946cec0f3"
      },
      "outputs": [],
      "source": [
        "df_all['age'].fillna(-1, inplace=True)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ca00d2f6-b086-48bc-42c5-837a26dd4601"
      },
      "outputs": [],
      "source": [
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "b5d56687-bae0-acb3-8763-91aaa5a1ce0b"
      },
      "outputs": [],
      "source": [
        "df_all.age = df_all.age.astype(int)\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "3c05fa54-cb76-1ea0-07e3-093ae3830b53"
      },
      "outputs": [],
      "source": [
        "def check_NaN_Values_in_df(df):\n",
        "    for col in df:\n",
        "        nan_count = df[col].isnull().sum()\n",
        "        \n",
        "        if nan_count != 0:\n",
        "            print (col + \" =>\" + str(nan_count) + \"Nan Values\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "9578e47c-4c20-a1a5-8394-0dcf13fd74b4"
      },
      "outputs": [],
      "source": [
        "check_NaN_Values_in_df(df_all)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "0a3b0f30-30ea-fd77-1495-a3a17ea4d207"
      },
      "outputs": [],
      "source": [
        "check_NaN_Values_in_df(df_all)\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "40136c94-30b5-4614-a5a3-7c404ee2b2db"
      },
      "outputs": [],
      "source": [
        "df_all.drop('timestamp_first_active', axis=1, inplace=True)\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "d3bb9fa3-7088-8a24-4bd1-fb50e1b5b6b9"
      },
      "outputs": [],
      "source": [
        "df_all.drop('language', axis=1, inplace=True)\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "eceddacb-d895-ed2c-a85c-d42d7ff39409"
      },
      "outputs": [],
      "source": [
        "df_all = df_all[df_all['date_account_created']>'2013-02-01']\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "7731e726-04fb-1934-a4fc-dc33acdde10f"
      },
      "outputs": [],
      "source": [
        "if not os.path.exists(\"output\"):\n",
        "    os.makedirs(\"output\")\n",
        "    \n",
        "df_all.to_csv(\"output/cleaned.csv\", sep=',',index=False)"
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}