{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "f441b3b6-5646-cda6-7e13-655c6ba148f3"
      },
      "outputs": [],
      "source": [
        "# This Python 3 environment comes with many helpful analytics libraries installed\n",
        "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
        "# For example, here's several helpful packages to load in \n",
        "\n",
        "import numpy as np # linear algebra\n",
        "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
        "\n",
        "# Input data files are available in the \"../input/\" directory.\n",
        "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
        "\n",
        "from subprocess import check_output\n",
        "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n",
        "\n",
        "# Any results you write to the current directory are saved as output."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "76864a29-4c14-0c52-47f7-5083128400e0"
      },
      "outputs": [],
      "source": [
        "import os"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "b5b4b7fc-b482-1861-3f0d-5dd0b517dd44"
      },
      "outputs": [],
      "source": [
        "df_train = pd.read_csv(\"../input/train_users_2.csv\")\n",
        "df_train.head(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "46937ee1-aa45-3e53-4e0a-4a3dbe3e736f"
      },
      "outputs": [],
      "source": [
        "df_test = pd.read_csv(\"../input/test_users.csv\")\n",
        "df_test.head(n=5)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "0ba315b9-a03b-8e78-28bd-dfdef9818c10"
      },
      "source": [
        "La colonne date_first_booking est inutile pour la pr\u00e9diction (uniquement pr\u00e9sente dans les donn\u00e9es de training) -> on la vire"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "a25bc472-601c-19a2-82d1-5c0782fe9a93"
      },
      "outputs": [],
      "source": [
        "df_all = pd.concat((df_train, df_test), axis=0, ignore_index=True)\n",
        "df_all.tail(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "63fa6fe6-d1ec-4636-5c6d-4a0c469a897b"
      },
      "outputs": [],
      "source": [
        "df_all.drop('date_first_booking', axis=1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "12096403-30e2-1e6e-6791-7a64c0aa6224"
      },
      "outputs": [],
      "source": [
        "df_all['date_account_created'] = pd.to_datetime(df_all['date_account_created'], format='%Y-%m-%d')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "9e7a1590-492b-fc63-9ddf-f87f96dc54df"
      },
      "outputs": [],
      "source": [
        "df_all['timestamp_first_active'] = pd.to_datetime(df_all['timestamp_first_active'], format='%Y%m%d%H%M%S')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "a5afe07c-8475-a9c4-8e32-f1c8b519c0fd"
      },
      "outputs": [],
      "source": [
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "cad64f1e-d9c1-229c-be3d-7124d2d8e82d"
      },
      "outputs": [],
      "source": [
        "def remove_age_outliers(x, min_value=18, max_value=90):\n",
        "    if np.logical_or(x <= min_value, x >= max_value):\n",
        "        return np.nan\n",
        "    else:\n",
        "        return x"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "da047ea5-3919-fe2a-57ff-75e72682c94a"
      },
      "outputs": [],
      "source": [
        "df_all['age'] = df_all['age'].apply(remove_age_outliers)\n",
        "# si on veut filtrer les nan : lambda x: remove(...) if not np.isnan(x) else x"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "1f1c967a-b799-15db-93f1-d31268807ba9"
      },
      "outputs": [],
      "source": [
        "df_all['age'].fillna(-1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "b5d050cc-5aef-0770-9b72-b7039d0df675"
      },
      "outputs": [],
      "source": [
        "df_all.age = df_all.age.astype(int)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "d1299ce5-f959-05a6-fa5d-9f4be800ab68"
      },
      "outputs": [],
      "source": [
        "def check_NaN(df):\n",
        "    for col in df:\n",
        "        nan_count = df[col].isnull().sum()\n",
        "        if nan_count:\n",
        "            print(col, '=>', nan_count)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "441bcead-83dc-30d8-12c9-fb2e82fe75a7"
      },
      "outputs": [],
      "source": [
        "check_NaN(df_all)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "f4a397e2-a840-0fd5-d726-4d6791f46931"
      },
      "outputs": [],
      "source": [
        "df_all.first_affiliate_tracked.fillna(-1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "a927f3eb-8206-119a-d297-4405cc71cb3e"
      },
      "outputs": [],
      "source": [
        "check_NaN(df_all)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "92998edb-65e1-db3a-24cb-29ef0bcc00b6"
      },
      "outputs": [],
      "source": [
        "df_all.drop('timestamp_first_active', axis=1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "5d0b3acf-b0a4-61e7-bab0-6969b1d78816"
      },
      "outputs": [],
      "source": [
        "df_all.drop('language', axis=1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ec1c3a86-3955-51cd-c332-6979fef790f6"
      },
      "outputs": [],
      "source": [
        "df_all.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "4d8bfa62-a83a-8414-8122-39c08146f5dd"
      },
      "outputs": [],
      "source": [
        "df_all = df_all[df_all['date_account_created'] > '2013-02-01']\n",
        "df_all.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "3656e9e5-1dfe-f7ae-5e56-4eefaf380b6b"
      },
      "outputs": [],
      "source": [
        "df_all"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "2d5bdcd6-5ac5-514e-ea0d-a520ceb52771"
      },
      "outputs": [],
      "source": [
        "if not os.path.exists('output'):\n",
        "    os.makedirs('output')\n",
        "\n",
        "df_all.to_csv('output/cleaned_users.csv', sep=',', index=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "c6fbe955-f9e8-01eb-b171-6c2e34c1eb56"
      },
      "outputs": [],
      "source": [
        "df_all[['age']].mean()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "1bc6776e-1646-71a1-3e78-a3cc5785e3de"
      },
      "outputs": [],
      "source": ""
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}