{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "c8b3bba9-5ab0-1817-2adc-cd9f88263b74"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "6a063288-0e0d-c7ed-9e70-bbe71123dd46"
      },
      "outputs": [],
      "source": [
        "df_train = pd.read_csv(\"../input/train_users_2.csv\")\n",
        "df_train.sample(n=5) #display few lines"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "e3d66569-9a4a-c5b0-b9f1-e8888d27208a"
      },
      "outputs": [],
      "source": [
        "df_test = pd.read_csv(\"../input/test_users.csv\")\n",
        "df_test.head(5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "c9b22fd5-a827-ce02-41f5-1dd9ff6e79ee"
      },
      "outputs": [],
      "source": [
        "df_all = pd.concat((df_train,df_test),axis=0,ignore_index=True)\n",
        "df_all.head(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "0bdb6f2c-61d4-f8c1-4df3-f731f95f96c0"
      },
      "outputs": [],
      "source": [
        "df_all.drop('date_first_booking',axis=1,inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "1a0390be-7004-9c3c-8506-f2a132f90141"
      },
      "outputs": [],
      "source": [
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "3e6093bb-1027-a5f0-243d-40e838e2fe06"
      },
      "outputs": [],
      "source": [
        "df_all['date_account_created'] = pd.to_datetime(df_all['date_account_created'],format='%Y-%m-%d')\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "33efdb47-7c8e-f9ba-b6c9-034537f869c1"
      },
      "outputs": [],
      "source": [
        "df_all['timestamp_first_active'] = pd.to_datetime(df_all['timestamp_first_active'],format='%Y%m%d%H%M%S')\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "4428666e-bc66-adb3-6433-9ed189f5c1da"
      },
      "outputs": [],
      "source": [
        "def remove_age_outliers(x, min_value=15, max_value=90):\n",
        "    if np.logical_or(x<=min_value,x>=max_value):\n",
        "        return np.nan\n",
        "    else:\n",
        "        return x"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "3ce8d38d-9e38-95c2-d2e5-688eca400911"
      },
      "outputs": [],
      "source": [
        "df_all['age'] = df_all['age'].apply(lambda x: remove_age_outliers(x) if(np.isnan(x)) else x)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "b447d5ee-eab8-281e-73ab-897bd2661424"
      },
      "outputs": [],
      "source": [
        "df_all['age'].fillna(-1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "460afbb1-b7ad-4951-59ec-2dc739c6ddbd"
      },
      "outputs": [],
      "source": [
        "df_all['age'].head(10)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "62c98ea5-278e-dd5c-e044-de8d38cefc12"
      },
      "outputs": [],
      "source": [
        "def check_NaN_Values_in_df(df):\n",
        "    for col in df:\n",
        "        nan_count = df[col].isnull().sum()\n",
        "        \n",
        "        if nan_count != 0:\n",
        "            print(col + \" => \" + str(nan_count) + \" NaN Values\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ba380f92-b8d3-a8e3-98e7-a6485d7dc653"
      },
      "outputs": [],
      "source": [
        "check_NaN_Values_in_df(df_all)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "5579a72a-cc25-57d2-8c0c-0e74156e0aeb"
      },
      "outputs": [],
      "source": [
        "df_all.first_affiliate_tracked.fillna(-1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "dbb86ec4-55de-2b69-34d5-6ce1747d3ade"
      },
      "outputs": [],
      "source": [
        "check_NaN_Values_in_df(df_all)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "014bfeb9-ea16-173f-0373-0549232af4cb"
      },
      "outputs": [],
      "source": [
        "df_all.drop('timestamp_first_active',axis=1, inplace=True)\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "c08c78ff-3b5f-67a5-7659-33b24b7abea4"
      },
      "outputs": [],
      "source": [
        "df_all.drop('language',axis=1, inplace=True)\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "f14bc0ab-eeba-40d9-4d4f-550e7491c988"
      },
      "outputs": [],
      "source": [
        "df_all = df_all[df_all['date_account_created']>'2013-02-01']\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "aae30a24-ea05-9e3f-d45a-0696d178378a"
      },
      "outputs": [],
      "source": [
        "if not os.path.exists(\"output\"):\n",
        "    os.makedirs(\"output\")\n",
        "    \n",
        "df_all.to_csv(\"output/cleaned.csv\", sep=',', index=False)"
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}