{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "403dc7e8-6ebb-7e11-0e0b-0a07ff6e8103"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import numpy as np\n",
        "import pandas as pd\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "9a61a159-e031-b9fc-2bdf-66fa9b8f244c"
      },
      "outputs": [],
      "source": [
        "df_train = pd.read_csv(\"../input/train_users_2.csv\")\n",
        "df_train.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "dcf5501c-94c9-c2f3-fc02-cdeb4befdbbe"
      },
      "outputs": [],
      "source": [
        "df_test = pd.read_csv(\"../input/test_users.csv\")\n",
        "df_test.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "cbb5c1d6-255e-9464-aa22-fff34ef59eb4"
      },
      "outputs": [],
      "source": [
        "df_all = pd.concat((df_train, df_test),axis=0,ignore_index=True)\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "d1db274d-5d0e-e1ba-ba19-04d8ee3c783b"
      },
      "outputs": [],
      "source": [
        "df_all.drop('date_first_booking', axis=1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "8d1fca0e-6cee-af17-3cc8-ec59d8dc7eb9"
      },
      "outputs": [],
      "source": [
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "82774db4-ca2c-e5b8-6e6c-fb16d69430db"
      },
      "outputs": [],
      "source": [
        "df_all['date_account_created']= pd.to_datetime(df_all['date_account_created'], format='%Y-%m-%d')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "1de715ae-91a7-ac05-7745-63833f17cc97"
      },
      "outputs": [],
      "source": [
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "b8e53811-449e-13c4-702b-4aebb05e262e"
      },
      "outputs": [],
      "source": [
        "df_all['timestamp_first_active']= pd.to_datetime(df_all['timestamp_first_active'], format='%Y%m%d%H%M%S')\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "c266a697-5a42-1de8-b08f-0cc0cf03dd13"
      },
      "outputs": [],
      "source": [
        "def remove_age_outliers(x, min_value=15, max_value=90):\n",
        "    if np.logical_or(x<=min_value, x>=max_value):\n",
        "        return np.nan\n",
        "    else:\n",
        "        return x"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "f64be3d3-c4b3-c550-0215-d455d69dc2af"
      },
      "outputs": [],
      "source": [
        "df_all['age']=df_all['age'].apply(lambda x: remove_age_outliers(x) if(not np.isnan(x)) else x )\n",
        "##we remove the age outliers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "955e5211-3646-2d32-e1fa-59412e9e7b8d"
      },
      "outputs": [],
      "source": [
        "df_all['age'].fillna(-1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "b6ff4b64-448e-f70f-1f37-3d9c504f2112"
      },
      "outputs": [],
      "source": [
        "df_all.age = df_all.age.astype(int)\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "306a8687-0721-9e3c-31eb-0e63bd297eef"
      },
      "outputs": [],
      "source": [
        "def check_Nan_Values_in_df(df):\n",
        "    for col in df:\n",
        "        nan_count = df[col].isnull().sum()\n",
        "        \n",
        "        if nan_count != 0:\n",
        "            print (col+\" => \"+ str(nan_count)+\" NaN Values\")\n",
        "        "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ad933acb-c347-956b-b9f6-567851724671"
      },
      "outputs": [],
      "source": [
        "check_Nan_Values_in_df(df_all)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "e2cb6a0d-60cb-02ce-d5aa-e2b8c6f75390"
      },
      "outputs": [],
      "source": [
        "df_all['first_affiliate_tracked'].fillna(-1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "b0cbdfb4-579d-075b-e932-d560531b153e"
      },
      "outputs": [],
      "source": [
        "check_Nan_Values_in_df(df_all)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "e7630e98-40a6-f17e-81ed-18ba28376fa6"
      },
      "outputs": [],
      "source": [
        "df_all.drop('timestamp_first_active',axis=1, inplace=True)\n",
        "df_all.drop('language',axis=1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "3a4ffd1e-14cb-aed4-2c60-0b6cb6424d4d"
      },
      "outputs": [],
      "source": [
        "df_all"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "6189bb0b-d517-f13e-395e-24c92d4dfa76"
      },
      "outputs": [],
      "source": [
        "df_all = df_all[df_all['date_account_created'] > '2013-02-01']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "df855bf5-3e5d-f999-616b-ab1afad15691"
      },
      "outputs": [],
      "source": [
        "if not os.path.exists(\"output\"):\n",
        "    os.makedirs(\"output\")\n",
        "    \n",
        "df_all.to_csv(\"output/cleaned.csv\", sep=',',index=False)"
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}