{
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0,
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "85349eca-51c0-90a8-d47c-db7057894e54",
        "_active": false
      },
      "outputs": [],
      "source": "# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n\n# Any results you write to the current directory are saved as output.",
      "execution_state": "idle"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "7a462b45-8842-d112-5bc1-e1796b90c66f",
        "_active": false
      },
      "outputs": [],
      "source": "import os",
      "execution_state": "idle"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "32a74afe-a15d-e546-e9fd-bbad4f244f81",
        "_active": false
      },
      "outputs": [],
      "source": "df_train = pd.read_csv(\"../input/train_users_2.csv\")\ndf_train.sample(n = 5) # only display few bunch of lines.",
      "execution_state": "idle"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "81afd53a-ad4c-f43d-935a-ec9fad1b36ff",
        "_active": false
      },
      "outputs": [],
      "source": "df_test = pd.read_csv(\"../input/test_users.csv\")\ndf_test.sample(n = 5) # only display few bunch of lines.",
      "execution_state": "idle"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "c8981317-3fe1-c7ef-83a0-c509414a66c4",
        "_active": false
      },
      "outputs": [],
      "source": "# Concat both table to apply algo \ndf_all = pd.concat((df_train, df_test), axis=0, ignore_index = True)\ndf_all.head(n=5)",
      "execution_state": "idle"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "83b42d45-4101-a0c9-f2ac-69498db48913",
        "_active": false
      },
      "outputs": [],
      "source": "# Remove date_first_booking cause we cant use it from dataset\ndf_all.drop('date_first_booking', axis=1, inplace=True)\ndf_all.head(n=5)",
      "execution_state": "idle"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "923700ee-5018-2f1b-e75b-8fe5ce81250a",
        "_active": false
      },
      "outputs": [],
      "source": "df_all['date_account_created'] = pd.to_datetime(df_all['date_account_created'], format = '%Y-%m-%d')",
      "execution_state": "idle"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "373d9d80-dbfb-70b0-940c-88b64c75ba09",
        "_active": false
      },
      "outputs": [],
      "source": "df_all.sample(n=5)",
      "execution_state": "idle"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "3174af80-306d-bf3c-4413-6ec9bb1e4e67",
        "_active": false
      },
      "outputs": [],
      "source": "df_all['timestamp_first_active'] = pd.to_datetime(df_all['timestamp_first_active'], format = '%Y%m%d%H%M%S')\ndf_all.sample(n=5)",
      "execution_state": "idle"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "299fe7f8-06a1-dfdb-6138-48bdbbba7304",
        "_active": false
      },
      "outputs": [],
      "source": "def remove_age_outliers(x, min_value = 15, max_value = 90):\n    if np.logical_or(x <= min_value, x >= max_value):\n        return np.nan\n    else:\n        return x",
      "execution_state": "idle"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "c16f3fb1-e3e0-7a54-43ba-715e93507f2f",
        "_active": false
      },
      "outputs": [],
      "source": "# Remove unrelevant age data\ndf_all['age'].apply(lambda x: remove_age_outliers(x) if(not np.isnan(x)) else x)",
      "execution_state": "idle"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "399e38de-21f0-42a4-bbfb-359f4f52f3e0",
        "_active": false
      },
      "outputs": [],
      "source": "df_all['age'].fillna(-1, inplace = True)\ndf_all.sample(n = 5)",
      "execution_state": "idle"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "57c199d2-ea12-7a6c-a931-e0c1a6516803",
        "_active": false
      },
      "outputs": [],
      "source": "df_all.age = df_all.age.astype(int)\ndf_all.sample(n = 5)",
      "execution_state": "idle"
    },
    {
      "metadata": {
        "_cell_guid": "0552ebe9-4ed1-1e76-14fe-e3a98e71f15f",
        "_active": false,
        "collapsed": false
      },
      "source": "def check_NaN_Values_in_df(df):\n    for col in df:\n        nan_count = df[col].isnull().sum()\n        \n        if nan_count != 0:\n            print(col + \" => \" + str(nan_count) + \" NaN Values\")",
      "execution_count": null,
      "cell_type": "code",
      "outputs": [],
      "execution_state": "idle"
    },
    {
      "metadata": {
        "_cell_guid": "c9b6db38-9919-3aef-4493-5ac7d8fce41e",
        "_active": false,
        "collapsed": false
      },
      "source": "# Check for NaN values in all the data frame\ncheck_NaN_Values_in_df(df_all)",
      "execution_count": null,
      "cell_type": "code",
      "outputs": [],
      "execution_state": "idle"
    },
    {
      "metadata": {
        "_cell_guid": "3dc6fef6-9c2c-5dc2-7892-da9b01709818",
        "_active": false,
        "collapsed": false
      },
      "source": "# As the whole test dataframe doesnt have any country_destination, this explains the 62096 NaN Values\n",
      "execution_count": null,
      "cell_type": "code",
      "outputs": [],
      "execution_state": "idle"
    },
    {
      "metadata": {
        "_cell_guid": "64543a14-886d-6a95-9bde-7a94fc42c76f",
        "_active": false,
        "collapsed": false
      },
      "source": "# Remove useless col\ndf_all.drop('timestamp_first_active', axis = 1, inplace = True)\ndf_all.sample(n=5)",
      "execution_count": null,
      "cell_type": "code",
      "outputs": [],
      "execution_state": "idle"
    },
    {
      "metadata": {
        "_cell_guid": "361b38ac-2a8f-588c-fe30-ec7b1c9d2d4a",
        "_active": false,
        "collapsed": false
      },
      "source": "# Following previous analysis, we consider the too old users as unrelevant ones\ndf_all = df_all[df_all['date_account_created'] > '2013-02-01']\ndf_all.sample(n = 5)",
      "execution_count": null,
      "cell_type": "code",
      "outputs": [],
      "execution_state": "idle"
    },
    {
      "metadata": {
        "_cell_guid": "dcd618dd-5db5-e86e-5934-28c6440fb51f",
        "_active": false,
        "collapsed": false
      },
      "source": "if not os.path.exists(\"output\"):\n    os.makedirs(\"output\")\n    \ndf_all.to_csv(\"output/cleaned.csv\", sep = ',', index = False)",
      "execution_count": null,
      "cell_type": "code",
      "outputs": [],
      "execution_state": "idle"
    },
    {
      "metadata": {
        "_cell_guid": "932beca1-7afa-d959-de81-3bb48d77e60c",
        "_active": true,
        "collapsed": false
      },
      "source": "df_all['age'].fillna(average_age, inplace = True)",
      "execution_count": null,
      "cell_type": "code",
      "outputs": [],
      "execution_state": "idle"
    }
  ]
}