{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "09fa1269-cc2b-4c05-a3d6-8e4e5584155c"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import numpy as np\n",
        "import pandas as pd"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "4cc676df-e91b-b51d-f927-58f48e452a47"
      },
      "outputs": [],
      "source": [
        "#definition d'un dataframe\n",
        "df_train = pd.read_csv(\"../input/train_users_2.csv\")\n",
        "df_train.sample(n=5) \n",
        "#df_train.head(n=5) "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "19c724aa-4617-423a-2354-a10fc7fe4a87"
      },
      "outputs": [],
      "source": [
        "#charge les data de test\n",
        "#pr traitement simult des data de tests et de train\n",
        "df_test = pd.read_csv(\"../input/test_users.csv\")\n",
        "df_test.sample(n=5) "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "cea6a29c-decc-8365-84be-4c10f027321c"
      },
      "outputs": [],
      "source": [
        "df_all = pd.concat((df_train,df_test),axis=0,ignore_index=True)\n",
        "df_all.head(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "e6dd0ba8-b64d-b6e9-8aaa-c6c85746654e"
      },
      "outputs": [],
      "source": [
        "df_all.drop('date_first_booking',axis=1,inplace=True)# on supprime la colonne\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "973596ef-6119-9476-9d13-0ba0ca20fb7a"
      },
      "outputs": [],
      "source": [
        "#clean format des dates\n",
        "df_all['date_account_created'] = pd.to_datetime(df_all['date_account_created'], format='%Y-%m-%d')\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "581fdc35-ff47-4509-662b-3c832b3da8f0"
      },
      "outputs": [],
      "source": [
        "#format du time stamp\n",
        "df_all['timestamp_first_active'] = pd.to_datetime(df_all['timestamp_first_active'], format='%Y%m%d%H%M%S')\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "4f72fdfe-ff38-0393-6fb5-5e93249c1507"
      },
      "outputs": [],
      "source": [
        "#suppression des data outliers (r\u00e9sa entre 0 et 15 ans par exemple) (fa\u00e7on simple sans se compliquer la vie)\n",
        "def remove_age_outliers(x, min_value=15, max_value=90):\n",
        "    if np.logical_or(x<=min_value, x>=max_value):\n",
        "        return np.nan\n",
        "    else:\n",
        "        return x\n",
        "    "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "cb010ce5-893f-718a-e5c2-043cf08afaad"
      },
      "outputs": [],
      "source": [
        "#df_all['age'].apply(lambda x:   remove_age_outliers(x)) #crash en python 2, on peut comparer un nan avec un num\n",
        "df_all['age']=df_all['age'].apply(lambda x:   remove_age_outliers(x) if(not np.isnan(x))else x)\n",
        "df_all['age'].fillna(-1, inplace=True) #fonctionne ici, mais pas forc\u00e9ment pour un autre projet\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "e6fea43a-2bf8-5287-6409-8285cd6e57e9"
      },
      "outputs": [],
      "source": [
        "#conversion age en entier\n",
        "df_all.age = df_all.age.astype(int)  #equivalent \u00e0 df_all['age']\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "c2fa5034-5cc0-a5b1-e052-0d5331c421ec"
      },
      "outputs": [],
      "source": [
        "def check_Nan_Values_in_df(df):\n",
        "    for col in df:\n",
        "        nan_count = df[col].isnull().sum()\n",
        "        \n",
        "        if nan_count != 0:\n",
        "            print(col + \"=>\"+str(nan_count)+ \" Nan Values\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "e51da7c8-6958-5a3c-41cc-da5ebb2b979c"
      },
      "outputs": [],
      "source": [
        "check_Nan_Values_in_df(df_all)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "d6e60d7b-a6fa-a7c9-0b53-7851506e4ffa"
      },
      "outputs": [],
      "source": [
        "#pas normal d'avoir des Nan sur first affiliate tracked\n",
        "df_all['first_affiliate_tracked'].fillna(-1, inplace=True)\n",
        "check_Nan_Values_in_df(df_all)\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "1847a741-01f3-495c-72a1-62c37af6fc0b"
      },
      "outputs": [],
      "source": [
        "#on d\u00e9gage le time stamp car redondant avec la date et heure min sec inutile (redondance 99% du temps)\n",
        "df_all.drop('timestamp_first_active',axis=1, inplace=True)\n",
        "#on d\u00e9gage la langue, pour essayer\n",
        "df_all.drop('language',axis=1, inplace=True)\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "c20c4c60-6ebb-7c3c-bfe0-6fab063bfaed"
      },
      "outputs": [],
      "source": [
        "#on d\u00e9gage ceux avant f\u00e9vrier 2013 (retrait des early outliers)\n",
        "#on pourrait laisser janvier, potentiellement pour capter les effets saisonnniers\n",
        "df_all = df_all[df_all['date_account_created']>'2013-02-01']\n",
        "df_all.sample(n=5)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ddee250b-d5da-973b-1dbf-d5f28c14f819"
      },
      "outputs": [],
      "source": [
        "#enregistrement du nouveau csv propre\n",
        "if not os.path.exists(\"output\"):\n",
        "    os.makedirs(\"output\")\n",
        "    \n",
        "df_all.to_csv(\"output/cleaned.csv\", sep=',', index=False)"
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}