{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "478377b6-b7a9-daab-53e1-0da3ea3c4cc6"
      },
      "outputs": [],
      "source": [
        "# This Python 3 environment comes with many helpful analytics libraries installed\n",
        "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
        "# For example, here's several helpful packages to load in \n",
        "\n",
        "import numpy as np # linear algebra\n",
        "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
        "\n",
        "# Input data files are available in the \"../input/\" directory.\n",
        "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
        "\n",
        "from subprocess import check_output\n",
        "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n",
        "\n",
        "# Any results you write to the current directory are saved as output."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "08c80116-236b-3074-f6e2-c085de37a1aa"
      },
      "outputs": [],
      "source": [
        "import os \n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "2ac38d0e-82ff-b8fa-8097-0bf86468dc65"
      },
      "outputs": [],
      "source": [
        "df_train = pd.read_csv(\"../input/train_users_2.csv\")\n",
        "df_train.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "7886826b-71dd-56fb-03a6-adf2ef19aa7c"
      },
      "outputs": [],
      "source": [
        "df_test = pd.read_csv ('../input/test_users.csv')\n",
        "df_test.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "21cf0fd3-3538-673b-b784-a4f10fe6fbc8"
      },
      "outputs": [],
      "source": [
        "df_all=pd.concat((df_train,df_test), axis=0, ignore_index=True) #Sticking test data array and train data array\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "63eb57aa-e98d-22c1-bd21-2e7fa0ea6e94"
      },
      "outputs": [],
      "source": [
        "df_all.drop('date_first_booking', axis=1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "46e33553-f166-f1d1-ab49-c5539417010b"
      },
      "outputs": [],
      "source": [
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "e2350862-0c2c-3284-f51a-d055a4850a69"
      },
      "outputs": [],
      "source": [
        "df_all['date_account_created'] = pd.to_datetime(df_all['date_account_created'], format='%Y-%m-%d')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "f4de71b4-c6d6-82bb-da7f-2356b14baea8"
      },
      "outputs": [],
      "source": [
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "3af9bb52-b3dc-c474-09d9-b67d063dac66"
      },
      "outputs": [],
      "source": [
        "df_all['timestamp_first_active'] = pd.to_datetime(df_all['timestamp_first_active'], format='%Y%m%d%H%M%S')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "df887dd2-7cea-e2e6-92c7-4533f385c98f"
      },
      "outputs": [],
      "source": [
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "1c93e797-6d18-a523-40b8-99a5689f4afa"
      },
      "outputs": [],
      "source": [
        "def remove_age_outliers(x, min_value=15, max_value=90):\n",
        "    if np.logical_or(x<=min_value, x>=max_value):\n",
        "        return np.nan\n",
        "    else:\n",
        "        return x"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "90216e69-b583-f82d-2201-46a1dd8cb72b"
      },
      "outputs": [],
      "source": [
        "df_all['age']=df_all['age'].apply(lambda x : remove_age_outliers(x) if (not np.isnan(x)) else x)\n",
        "    "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "2fc23964-9cf7-486d-1336-f65d677c7b16"
      },
      "outputs": [],
      "source": [
        "df_all['age'].fillna(-1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "9adf0233-a337-239e-9994-8041f3ba4f73"
      },
      "outputs": [],
      "source": [
        "df_all.sample(n=5)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "6a9a5dcc-6305-00b1-e3aa-f9262b017513"
      },
      "outputs": [],
      "source": [
        "df_all.age = df_all.age.astype(int)\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "dec5f65b-4c43-ce44-1d3e-a4af134608f8"
      },
      "outputs": [],
      "source": [
        "def check_NaN_values_in_df(df):\n",
        "    for col in df:\n",
        "        nan_count = df[col].isnull().sum()\n",
        "        if nan_count !=0 :\n",
        "            print (col + \" => \" + str(nan_count) + \"NanN Values\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "4af12abb-2831-6ade-47ea-273a3a918b4e"
      },
      "outputs": [],
      "source": [
        "check_NaN_values_in_df(df_all)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "36181145-ebaf-5156-e087-0f0281142916"
      },
      "outputs": [],
      "source": [
        "df_all['first_affiliate_tracked'].fillna(-1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "cf519ec0-93eb-d263-a7df-85aea910ad6a"
      },
      "outputs": [],
      "source": [
        "check_NaN_values_in_df(df_all)\n",
        "df_all.sample(n=5)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "b629d6f0-310f-d1c6-9e21-3eb020c8771e"
      },
      "outputs": [],
      "source": [
        "df_all.drop('timestamp_first_active', axis=1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "b1145bbd-3bae-6772-d7bb-ccdbffb21c06"
      },
      "outputs": [],
      "source": [
        "df_all.drop('language', axis=1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "2f9edfb6-115d-d6c3-7d95-61d51e570b01"
      },
      "outputs": [],
      "source": [
        "df_all = df_all[df_all['date_account_created'] > '2013-02-01']\n",
        "df_all.sample(n=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "2dd2152c-6a01-882d-7488-4b2314640fed"
      },
      "outputs": [],
      "source": [
        "if not os.path.exists(\"output\") :\n",
        "    os.makedirs(\"output\")\n",
        "    \n",
        "df_all.to_csv(\"output/cleaned.csv\", sep=',', index=False)"
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}