{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "198d440d-6311-7aaf-4763-1d29b7a53e06"
      },
      "outputs": [],
      "source": [
        "# This Python 3 environment comes with many helpful analytics libraries installed\n",
        "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
        "# For example, here's several helpful packages to load in \n",
        "import os\n",
        "\n",
        "import numpy as np # linear algebra\n",
        "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "2e8bf58a-eb71-203f-991b-81f4122c2eab"
      },
      "outputs": [],
      "source": [
        "df_train = pd.read_csv('../input/train_users_2.csv')\n",
        "df_train.head(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "d6823608-844c-b0ff-b1a9-1a8b5ce0fb3d"
      },
      "outputs": [],
      "source": [
        "df_test = pd.read_csv('../input/test_users.csv')\n",
        "df_test.head(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "2d25236e-6419-9d30-3fc8-c6b263a57694"
      },
      "outputs": [],
      "source": [
        "# ACHTUNG : Pas clean\n",
        "df_all = pd.concat((df_train, df_test), axis=0, ignore_index=True)\n",
        "df_all.head(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "c252605e-d950-60df-7507-bba687eb559d"
      },
      "outputs": [],
      "source": [
        "df_all.drop('date_first_booking', axis=1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "f9e825ea-511d-a22f-a628-7f5e59967913"
      },
      "outputs": [],
      "source": [
        "df_all.head(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "484aed21-fdf5-0f25-d468-b3031651d663"
      },
      "outputs": [],
      "source": [
        "df_all['date_account_created'] = pd.to_datetime(df_all['date_account_created'], format='%Y-%m-%d')\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "a5ee65a4-1603-8ed9-5115-7cfefe9c4903"
      },
      "outputs": [],
      "source": [
        "# Formatage dates\n",
        "df_all['timestamp_first_active'] = pd.to_datetime(df_all['timestamp_first_active'], format='%Y%m%d%H%M%S')\n",
        "df_all.head(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "474f0316-c746-a1cf-d672-1800d67a3208"
      },
      "outputs": [],
      "source": [
        "# Fonction nettoyage age\n",
        "def remove_age_outliers(x, min_value=15, max_value=90):\n",
        "    if np.logical_or(x<=min_value, x>=max_value):\n",
        "        return np.nan\n",
        "    else:\n",
        "        return x"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "3647aec5-6951-e73f-e110-9d40030a9573"
      },
      "outputs": [],
      "source": [
        "# Nettoyage des ages avec fonction lambda\n",
        "df_all.age = df_all['age'].apply(lambda x: remove_age_outliers(x) if(not np.isnan(x)) else x)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "3bf4ecb3-0c84-44fd-6dbe-e88d2e0c99f6"
      },
      "outputs": [],
      "source": [
        "df_all.age.fillna(-1, inplace=True)\n",
        "df_all.sample(5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "153ee8f0-d65f-924f-cae5-8e6b5b91490e"
      },
      "outputs": [],
      "source": [
        "# Conversion age en entier\n",
        "df_all.age = df_all.age.astype(int)\n",
        "df_all.sample(5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "3e757b11-9f9f-fd9b-ee88-b3511663e10f"
      },
      "outputs": [],
      "source": [
        "df_all.head(5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "f7851800-f193-75aa-7a40-779313129dc9"
      },
      "outputs": [],
      "source": [
        "def check_NaN_Values_in_df(df):\n",
        "    for col in df:\n",
        "        nan_count = df[col].isnull().sum()\n",
        "        \n",
        "        if nan_count != 0:\n",
        "            print(col + ' => ' + str(nan_count) + ' NaN values')\n",
        "check_NaN_Values_in_df(df_all)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "f60106ae-04f8-57fa-4945-c5a5ff2f083a"
      },
      "outputs": [],
      "source": [
        "df_all['first_affiliate_tracked'].fillna(-1, inplace=True)\n",
        "check_NaN_Values_in_df(df_all)\n",
        "df_all.sample(5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "d23e5ae7-4952-50ee-4e9b-80db04836d82"
      },
      "outputs": [],
      "source": [
        "# Enlever les earlybirds < Fev 2013\n",
        "# Choix par id true/false => r\u00e9cup\u00e8re que les true\n",
        "df_all = df_all[df_all['date_account_created'] > '2013-02-010']\n",
        "df_all.sample(5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "9e7673e4-1db1-2ac9-849e-971795a74c73"
      },
      "outputs": [],
      "source": [
        "if not os.path.exists('output'):\n",
        "    os.makedirs('output')\n",
        "    \n",
        "df_all.to_csv('output/cleaned.csv', sep=',', index=False)"
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}