{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "577db8b1-6dd4-b16b-d7e2-b456206808b6"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "print(\":)\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "51e8d2ba-e175-ae73-c48d-42f4aa231a68"
      },
      "outputs": [],
      "source": [
        "df_train = pd.read_csv(\"../input/train_users_2.csv\")\n",
        "df_train.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "6368cfa2-95b2-d0e9-83c2-4a52162a8e6d"
      },
      "outputs": [],
      "source": [
        "df_test = pd.read_csv(\"../input/test_users.csv\")\n",
        "df_train.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "7aee202e-d7f3-0bbb-89ae-2abe4b9e73fb"
      },
      "outputs": [],
      "source": [
        "df_all = pd.concat((df_train, df_test), axis = 0, ignore_index = True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "4e991a40-99ad-6803-1da8-b019e95e9ff1"
      },
      "outputs": [],
      "source": [
        "df_all.sample(n=20)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "90aa6eed-a0bd-c8d4-7139-1aaeb8503c65"
      },
      "outputs": [],
      "source": [
        "# df_all.query('country_destination != \"NaN\"')\n",
        "\n",
        "df_all.drop('date_first_booking', axis = 1, inplace = True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "cdf31a75-507c-292a-b740-e59d2f5d4b97"
      },
      "outputs": [],
      "source": [
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "b6cf8f1d-7a6a-5486-d4d9-be27b185345e"
      },
      "outputs": [],
      "source": [
        "df_all['date_account_created'] = pd.to_datetime(df_all['date_account_created'], format=\"%Y-%m-%d\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "e5992685-b687-46e6-9862-c6d31315bd18"
      },
      "outputs": [],
      "source": [
        "df_all['timestamp_first_active'] = pd.to_datetime(df_all['timestamp_first_active'], format=\"%Y%m%d%H%M%S\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "45bf61db-7ed4-78b5-9efc-10ce354a984a"
      },
      "outputs": [],
      "source": [
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "5ac02a81-46ee-30b6-865c-a5eea81daa37"
      },
      "outputs": [],
      "source": [
        "def suppr_ages_incorrects(x, min_value=15, max_value=105):\n",
        "    if np.logical_or(x<= min_value, x>=max_value):\n",
        "        return np.nan\n",
        "    else:\n",
        "        return x"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "7d58167f-9e18-be36-cdb4-2d1ac65a1bf7"
      },
      "outputs": [],
      "source": [
        "df_all['age'] = df_all['age'].apply(lambda x: suppr_ages_incorrects(x, 15, 100))\n",
        "\n",
        "# df_all.query('age > 100')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "92077d04-0f01-7e7b-6ee4-4dc214b37a52"
      },
      "outputs": [],
      "source": [
        "df_all.age.fillna(-1, inplace = True)\n",
        "\n",
        "df_all.age = df_all.age.astype(int)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "2f2ed37b-165b-b2ff-3252-a74a95c4d6ea"
      },
      "outputs": [],
      "source": [
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "afddb795-8cb1-74b1-88fe-d7d522304d15"
      },
      "outputs": [],
      "source": [
        "def NaN_Values_in_df(df):\n",
        "    for col in df:\n",
        "        nb_nan = df[col].isnull().sum()\n",
        "        if nb_nan != 0:\n",
        "            print(col + \" => \" + str(nb_nan) + \"NaNs\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "7d61b261-e680-9532-2761-6e9edfca32a4"
      },
      "outputs": [],
      "source": [
        "NaN_Values_in_df(df_all)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "6cc0b981-6740-1ff2-930d-a405bf118bf9"
      },
      "outputs": [],
      "source": [
        "df_all.first_affiliate_tracked.fillna(-1, inplace = True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "b144afa6-29bb-4de6-793f-8d16d0e76656"
      },
      "outputs": [],
      "source": [
        "df_all.drop('timestamp_first_active', axis=1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "f57aaa7f-94ea-b7a8-009a-ae752c8d9ba1"
      },
      "outputs": [],
      "source": [
        "df_all.drop('language', axis=1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "f470dee0-89d8-53e8-80e1-e1a863d877de"
      },
      "outputs": [],
      "source": [
        "df_all = df_all[df_all['date_account_created'] > '2013-01-01']\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "d5018353-735e-45af-a20f-6ce81d293815"
      },
      "outputs": [],
      "source": [
        "df_all.count()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "8fd58b83-1d2d-919c-06a5-b21ce45223f8"
      },
      "outputs": [],
      "source": [
        "if not os.path.exists('output'):\n",
        "    os.makedirs('output')\n",
        "\n",
        "df_all.to_csv(\"output/cleaned.csv\", sep=',', index=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "559b8cbb-e70f-af9e-2ae5-52cc09ce5d27"
      },
      "outputs": [],
      "source": ""
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}