{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "03ee8f7c-4a2b-9afb-fc50-5e9e04c7330d"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import numpy as np\n",
        "import pandas as pd"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "41143104-910e-ef85-471b-ca7b94b43253"
      },
      "outputs": [],
      "source": [
        "df_train=pd.read_csv(\"../input/train_users_2.csv\")#chargement donnes d'entrainement\n",
        "df_train.head(n=5) #Only first lines"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "9bb3cb34-ca52-63ba-214b-72f0524fb488"
      },
      "outputs": [],
      "source": [
        "df_train=pd.read_csv(\"../input/train_users_2.csv\")\n",
        "df_train.sample(n=5) #Only few lines"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "1fe2137b-7d39-25ea-72b8-7718d0d584cd"
      },
      "outputs": [],
      "source": [
        "df_test=pd.read_csv(\"../input/test_users.csv\")#chargement donnes de tests\n",
        "df_test.sample(n=5) "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "70e248c3-ac49-eb48-4fc9-c3ffcc994b4e"
      },
      "outputs": [],
      "source": [
        "#regroupe les deux tableaux ignore index permet de pas avoir deux fois les numeros de lignes\n",
        "df_all=pd.concat((df_train, df_test), axis=0, ignore_index=True)\n",
        "df_all.head(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "eaf0a104-06ca-c0de-f5c4-03a33c70ddc9"
      },
      "outputs": [],
      "source": [
        "#supprime la colonne premiere date de resa\n",
        "df_all.drop('date_first_booking',axis=1,inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "006c2920-3cf4-3aa7-70e9-c57704cb4db3"
      },
      "outputs": [],
      "source": [
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "6896d34f-127d-9a6f-095b-1833fa4ea29e"
      },
      "outputs": [],
      "source": [
        "df_all['date_account_created']=pd.to_datetime(df_all['date_account_created'],format='%Y-%m-%d')\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "3b13db1e-5b59-2aea-f90f-3d3d58531048"
      },
      "outputs": [],
      "source": [
        "#modifie le format de la date et du temps pour unifier\n",
        "df_all['timestamp_first_active']=pd.to_datetime(df_all['timestamp_first_active'], format='%Y%m%d%H%M%S')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "a7bffdea-0b9d-c3b6-b411-92281b4da6fd"
      },
      "outputs": [],
      "source": [
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "65dbd44e-28b7-9798-2b5b-de70414bc88c"
      },
      "outputs": [],
      "source": [
        "#enleve les ages absurdes\n",
        "def remove_age_outliers(x,min_value=15, max_value=90):\n",
        "    if np.logical_or(x<=min_value,x>=max_value):\n",
        "        return np.nan\n",
        "    else:\n",
        "        return x"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "7fbe2f61-0b2c-b698-6377-8b81bfb93738"
      },
      "outputs": [],
      "source": [
        "#si x est un nan on le renvoie sinon on applique la fonction\n",
        "df_all['age'].apply(lambda x: remove_age_outliers(x) if (not np.isnan(x)) else x)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "32af29d4-8df7-7654-5c0d-b50179b60cf4"
      },
      "outputs": [],
      "source": [
        "df_all['age'].fillna(-1,inplace=True) #met -1 au lieu de nan fillna modifie directemetn le dataframe en destructif"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ccd7963a-37f8-c8b6-a9e2-fdc0497fd3bc"
      },
      "outputs": [],
      "source": [
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "d57226d1-30ca-e496-9992-1e7f33bbba17"
      },
      "outputs": [],
      "source": [
        "df_all.age=df_all.age.astype(int) #transforme le type de age en int\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "3973d32d-ead7-c8fe-868b-7fe6c7a23a50"
      },
      "outputs": [],
      "source": [
        "#combien de valeur nan il reste dans la tableau\n",
        "def check_NaN_Values_in_df(df):\n",
        "    for col in df:\n",
        "        nan_count = df[col].isnull().sum()\n",
        "        \n",
        "        if nan_count !=0:\n",
        "            print(col +\"=>\"+str(nan_count)+\" NaN values\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "fcc7637e-0a51-a91f-73d6-551757316cef"
      },
      "outputs": [],
      "source": [
        "check_NaN_Values_in_df(df_all)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "974173df-4f72-72bc-5ea0-34cdacc263f3"
      },
      "outputs": [],
      "source": [
        "df_all['first_affiliate_tracked'].fillna(-1,inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "52cb9036-1ca7-fa46-69f0-0bb807f34e23"
      },
      "outputs": [],
      "source": [
        "check_NaN_Values_in_df(df_all)\n",
        "df_all.sample(5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "067bfba5-3f5e-83fe-89af-00e7371ee6d3"
      },
      "outputs": [],
      "source": [
        "df_all.drop('timestamp_first_active',axis=1,inplace=True)#car meme date entre crea et timestamp\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "40b165b5-234a-4afc-5161-f32fc36a3267"
      },
      "outputs": [],
      "source": [
        "df_all.drop('language',axis=1,inplace=True)#car tous le monde parle anglais attention des fois pas judicieux\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "d2a80fb0-4550-93df-35fa-5092980310dd"
      },
      "outputs": [],
      "source": [
        "df_all = df_all[df_all['date_account_created'] > '2013-02-01']\n",
        "df_all.sample(n=5)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "0a8afb35-98e3-db07-ed24-d286067a0b2e"
      },
      "outputs": [],
      "source": [
        "if not os.path.exists(\"output\"):\n",
        "    os.makedirs(\"output\")\n",
        "    \n",
        "df_all.to_csv(\"output/cleaned.csv\",sep=',',index=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "55a0d616-8c30-0446-bff8-06ff20ff456e"
      },
      "outputs": [],
      "source": [
        "df_all.sample(n=5)"
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}