{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "5cbf1909-e85b-6a9b-ead7-a265ea90edce"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "69abad50-0d07-61cb-7ab8-95e3decbd6e8"
      },
      "outputs": [],
      "source": [
        "df_train = pd.read_csv(\"../input/train_users_2.csv\")\n",
        "df_train.sample(n=5) "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "41f6b6d0-e153-8985-5638-ff045574b22a"
      },
      "outputs": [],
      "source": [
        "df_test = pd.read_csv(\"../input/test_users.csv\")\n",
        "df_test.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "e04ebc24-07bc-64c7-4d94-fe3f1b5461f1"
      },
      "outputs": [],
      "source": [
        "df_all = pd.concat((df_train, df_test), axis=0, ignore_index=True)\n",
        "df_all.head(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "7c477986-f2d9-5e5d-00fc-0a1e92b8b9df"
      },
      "outputs": [],
      "source": [
        "df_all.drop('date_first_booking', axis=1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "6fd62711-0c04-99b1-9897-230314687171"
      },
      "outputs": [],
      "source": [
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "b870706d-c618-e016-777f-e39cb5099e3d"
      },
      "outputs": [],
      "source": [
        "df_all['date_account_created'] = pd.to_datetime(df_all['date_account_created'],format='%Y-%m-%d')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "28bfab6d-3015-ebbd-cb26-deccc2f1d856"
      },
      "outputs": [],
      "source": [
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "d8bf8e34-784f-e7c6-9559-395a2b37753a"
      },
      "outputs": [],
      "source": [
        "df_all['timestamp_first_active']=pd.to_datetime(df_all['timestamp_first_active'], format='%Y%m%d%H%M%S')\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "4cdc8623-401f-e1ca-c8ac-06407ef4e0b0"
      },
      "outputs": [],
      "source": [
        "def remove_age_outliers(x, min_value=15, max_value=90):\n",
        "    if np.logical_or(x<=min_value, x>=max_value):\n",
        "        return np.nan\n",
        "    else:\n",
        "        return x"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "cf05a834-995a-5078-3b9b-0941b0449f39"
      },
      "outputs": [],
      "source": [
        "df_all['age'] = df_all['age'].apply(lambda x: remove_age_outliers(x) if(not np.isnan(x)) else x)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "943dfb02-4758-48c5-92e5-7c5a5bba19bd"
      },
      "outputs": [],
      "source": [
        "df_all['age'].fillna(-1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ef773378-1281-bb92-db88-9d53b4181a57"
      },
      "outputs": [],
      "source": [
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "a86e8d8c-161e-77d8-5eeb-1774d878e851"
      },
      "outputs": [],
      "source": [
        "df_all.age = df_all.age.astype(int)\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "8c8cf03d-62eb-6448-b80e-3d8b05919776"
      },
      "outputs": [],
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "953c7fb0-3269-16ea-a4ff-b30842ad9146"
      },
      "outputs": [],
      "source": [
        "def check_NaN_Values_in_df(df):\n",
        "    for col in df:\n",
        "        nan_count = df[col].isnull().sum()\n",
        "        \n",
        "        if nan_count != 0:\n",
        "            print (col + \" => \" + str(nan_count) + \" NaN Values\")\n",
        "        "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "2d9c39ab-90af-ef85-49fa-a149fd934c78"
      },
      "outputs": [],
      "source": [
        "check_NaN_Values_in_df(df_all)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "8f8b8f15-0c78-0b84-6942-ebd9b51c4b91"
      },
      "outputs": [],
      "source": [
        "df_all['first_affiliate_tracked'].fillna(-1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "c181dc1b-3bb0-48d9-d0e9-440a6dd97620"
      },
      "outputs": [],
      "source": [
        "check_NaN_Values_in_df(df_all)\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "5289a064-5559-cbc7-86b9-8a2d7979a870"
      },
      "outputs": [],
      "source": [
        "df_all.drop('timestamp_first_active', axis=1, inplace=True)\n",
        "df_all.drop('language', axis=1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "de29f322-7889-c07d-dbac-83c5f87cd4c0"
      },
      "outputs": [],
      "source": [
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "8d104652-34af-1343-76fc-61c8b9aa0e0b"
      },
      "outputs": [],
      "source": [
        "df_all = df_all[df_all['date_account_created'] > '2013-02-01']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "61e2e8bd-4c47-1b73-615d-910f2fc68c8e"
      },
      "outputs": [],
      "source": [
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "7182f9a8-838b-2fb3-4545-4416a3921f9a"
      },
      "outputs": [],
      "source": [
        "if not os.path.exists(\"output\"):\n",
        "    os.makedirs(\"output\")\n",
        "    \n",
        "df_all.to_csv(\"output/cleaned.csv\", sep=',', index=False)"
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}