{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "9e74d56f-085f-752e-b420-0896cbb6db14"
      },
      "outputs": [],
      "source": [
        "# This Python 3 environment comes with many helpful analytics libraries installed\n",
        "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
        "# For example, here's several helpful packages to load in \n",
        "\n",
        "import numpy as np # linear algebra\n",
        "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
        "\n",
        "# Input data files are available in the \"../input/\" directory.\n",
        "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
        "\n",
        "from subprocess import check_output\n",
        "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n",
        "\n",
        "# Any results you write to the current directory are saved as output."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "12875310-26c9-75af-6354-3f09a020557e"
      },
      "outputs": [],
      "source": [
        "#Session 2 : Data Cleansing\n",
        "\n",
        "import os #appel system\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "32d7b29f-7140-1ce8-8834-e1d0b3c0228d"
      },
      "outputs": [],
      "source": [
        "df_train=pd.read_csv(\"../input/train_users_2.csv\") # training data\n",
        "df_train.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ac59183c-2d80-2e81-18f8-83deab77b9f7"
      },
      "outputs": [],
      "source": [
        "df_test=pd.read_csv(\"../input/test_users.csv\") # donn\u00e9es de test\n",
        "df_test.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "6af75c5c-fc5f-aa3b-a779-874d92ba9183"
      },
      "outputs": [],
      "source": [
        "# Concat\u00e9nation du training et test data\n",
        "df_all=pd.concat((df_train,df_test), axis=0, ignore_index=True)\n",
        "df_all.head(n=5)\n",
        "#/!\\ : valeur nulle pour date_first_booking dans test data car pas encore de r\u00e9servation (c'est ce qu'on essaye de pr\u00e9dire)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "3b33ce2b-4645-c325-1aed-933d4c433323"
      },
      "outputs": [],
      "source": [
        "# Remove date_first_booking column\n",
        "df_all.drop('date_first_booking',axis=1,inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "623b5ab7-5231-3983-c075-49901c1d57f3"
      },
      "outputs": [],
      "source": [
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "6840f805-2e53-eaf4-458f-bd7a061b0cab"
      },
      "outputs": [],
      "source": [
        "# Format datetime of date_account_created\n",
        "df_all['date_account_created']=pd.to_datetime(df_all['date_account_created'], format='%Y-%m-%d')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "e1d19665-28ab-d0ab-4d13-a1db051b42ed"
      },
      "outputs": [],
      "source": [
        "df_all.head(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "dc0877f0-6832-c16c-6ee1-220a10a88b36"
      },
      "outputs": [],
      "source": [
        "# Format datetime of date_account_created\n",
        "df_all['timestamp_first_active']=pd.to_datetime(df_all['timestamp_first_active'], format='%Y%m%d%H%M%S')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "f8b26074-78fd-17c7-30d9-5758971ccdd8"
      },
      "outputs": [],
      "source": [
        "# Format datetime of timestamp_first_active\n",
        "df_all.head(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "9498ba79-08ee-6fc3-2b01-b2201b7e6d62"
      },
      "outputs": [],
      "source": [
        "def remove_age_outliers(x, min_value=15, max_value=90):\n",
        "    if np.logical_or(x<=min_value, x>=max_value):\n",
        "        return np.nan\n",
        "    else:\n",
        "        return x"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "226982bf-64e7-6dbc-c19c-899fa354ff8a"
      },
      "outputs": [],
      "source": [
        "# Sort age column\n",
        "df_all['age']=df_all['age'].apply(lambda x: remove_age_outliers(x) if(not np.isnan(x)) else x) #apply : from 1st row to last\n",
        "#si la valeur x n'est pas nan (<=> int) then apply(remove_age_outliers(x))\n",
        "#else return x."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "4dc1f34d-ca1c-3725-6482-28fb16fc0217"
      },
      "outputs": [],
      "source": [
        "df_all['age'].head(50)\n",
        "df_all['age'].fillna(-1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "7e92cce4-998a-2da5-560e-d87d49e5d128"
      },
      "outputs": [],
      "source": [
        "df_all['age'].head(50)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "685c33da-a32d-3157-f50e-14552b716064"
      },
      "outputs": [],
      "source": [
        "# Convert age from float to age\n",
        "df_all.age=df_all.age.astype(int)\n",
        "df_all.sample(5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "2b8f0c57-39b1-8241-911a-6b1d10acf504"
      },
      "outputs": [],
      "source": [
        "#Function to count number of NaN values in df column\n",
        "def check_NaN_Values_in_df(df):\n",
        "    for col in df:\n",
        "        nan_count=df[col].isnull().sum()\n",
        "        \n",
        "        if nan_count !=0:\n",
        "            print(col + \" -> \" + str(nan_count) + \" NaN Values\")\n",
        "        "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "1c9e677f-efb4-966b-d489-b439d0b7b89c"
      },
      "outputs": [],
      "source": [
        "check_NaN_Values_in_df(df_all)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "e39711b4-1c2f-6a46-c650-250c677f0932"
      },
      "outputs": [],
      "source": [
        "df_all['first_affiliate_tracked'].fillna(-1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "affe90cb-6768-837d-8a95-74a6567d4e1d"
      },
      "outputs": [],
      "source": [
        "df_all.sample(10)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "dfe8c6d7-6259-8b48-f77b-f314e1e0f00c"
      },
      "outputs": [],
      "source": [
        "df_all.drop('timestamp_first_active',axis=1, inplace=True)\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "8c4c8134-c159-4eeb-ef27-4cbdd35c84d5"
      },
      "outputs": [],
      "source": [
        "# Drop all row where date_account_created<=2013/02/01\n",
        "\n",
        "df_all=df_all[df_all['date_account_created']>'2013-02-01']\n",
        "df_all.sample(5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "78854220-56cb-c976-1023-a0a84ee87ad3"
      },
      "outputs": [],
      "source": [
        "#Makedir output\n",
        "if not os.path.exists(\"output\"):\n",
        "    os.makedirs(\"output\")\n",
        "\n",
        "#Export to CSV\n",
        "df_all.to_csv(\"output/cleaned.csv\", sep =',', index=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "dbc1d177-9f46-2c2c-1437-e1f0381a4459"
      },
      "outputs": [],
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "5db4c797-d3f4-3190-e248-efdee66b06f3"
      },
      "outputs": [],
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "d3aec624-0563-b1cf-9b85-04f294238bc0"
      },
      "outputs": [],
      "source": [
        ""
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}