{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "d8a52de2-7585-9949-18e7-08e87c75d2fe" }, "outputs": [], "source": [ "import numpy as np # linear algebra\n", "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "import numpy as np\n", "import pandas as pd\n", "from sklearn.feature_extraction.text import CountVectorizer\n", "from sklearn.preprocessing import LabelEncoder\n", "from sklearn.model_selection import StratifiedKFold\n", "import random\n", "from math import exp\n", "import xgboost as xgb\n", "\n", "# Input data files are available in the \"../input/\" directory.\n", "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n", "\n", "from subprocess import check_output\n", "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n", "\n", "# Any results you write to the current directory are saved as output." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b39f58d0-fa46-56f5-0989-bbec9e3b2d82" }, "outputs": [], "source": [ "random.seed(4321)\n", "np.random.seed(4321)\n", "\n", "X_train = pd.read_json(\"../input/train.json\")\n", "X_test = pd.read_json(\"../input/test.json\")\n", "\n", "interest_level_map = {'low': 0, 'medium': 1, 'high': 2}\n", "X_train['interest_level'] = X_train['interest_level'].apply(lambda x: interest_level_map[x])\n", "X_test['interest_level'] = -1" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "bcf43d21-6d11-b3c2-7042-29f1369e5a72" }, "outputs": [], "source": [ "X_test" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "1c928374-e2c1-86bd-c90b-6dcaed45af84" }, "outputs": [], "source": [ "print(X_train.head() )" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "e73307a4-5bf4-1696-fe8d-05257c2c6dc8" }, "outputs": [], "source": [ "\n", "\n", "#add features\n", "feature_transform = CountVectorizer(stop_words='english', max_features=150)\n", "X_train['features'] = X_train[\"features\"].apply(lambda x: \" \".join([\"_\".join(i.lower().split(\" \")) for i in x]))\n", "X_test['features'] = X_test[\"features\"].apply(lambda x: \" \".join([\"_\".join(i.lower().split(\" \")) for i in x]))\n", "feature_transform.fit(list(X_train['features']) + list(X_test['features']))\n", "\n", "train_size = len(X_train)\n", "low_count = len(X_train[X_train['interest_level'] == 0])\n", "medium_count = len(X_train[X_train['interest_level'] == 1])\n", "high_count = len(X_train[X_train['interest_level'] == 2])\n", "\n", "def find_objects_with_only_one_record(feature_name):\n", " temp = pd.concat([X_train[feature_name].reset_index(), \n", " X_test[feature_name].reset_index()])\n", " temp = temp.groupby(feature_name, as_index = False).count()\n", " return temp[temp['index'] == 1]\n", "\n", "managers_with_one_lot = find_objects_with_only_one_record('manager_id')\n", "buildings_with_one_lot = find_objects_with_only_one_record('building_id')\n", "addresses_with_one_lot = find_objects_with_only_one_record('display_address')\n", "\n", "lambda_val = None\n", "k=5.0\n", "f=1.0\n", "r_k=0.01 \n", "g = 1.0\n", "\n", "def categorical_average(variable, y, pred_0, feature_name):\n", " def calculate_average(sub1, sub2):\n", " s = pd.DataFrame(data = {\n", " variable: sub1.groupby(variable, as_index = False).count()[variable], \n", " 'sumy': sub1.groupby(variable, as_index = False).sum()['y'],\n", " 'avgY': sub1.groupby(variable, as_index = False).mean()['y'],\n", " 'cnt': sub1.groupby(variable, as_index = False).count()['y']\n", " })\n", " \n", " tmp = sub2.merge(s.reset_index(), how='left', left_on=variable, right_on=variable) \n", " del tmp['index'] \n", " tmp.loc[pd.isnull(tmp['cnt']), 'cnt'] = 0.0\n", " tmp.loc[pd.isnull(tmp['cnt']), 'sumy'] = 0.0\n", "\n", " def compute_beta(row):\n", " cnt = row['cnt'] if row['cnt'] < 200 else float('inf')\n", " return 1.0 / (g + exp((cnt - k) / f))\n", " \n", " if lambda_val is not None:\n", " tmp['beta'] = lambda_val\n", " else:\n", " tmp['beta'] = tmp.apply(compute_beta, axis = 1)\n", " \n", " tmp['adj_avg'] = tmp.apply(lambda row: (1.0 - row['beta']) * row['avgY'] + row['beta'] * row['pred_0'],\n", " axis = 1)\n", " \n", " tmp.loc[pd.isnull(tmp['avgY']), 'avgY'] = tmp.loc[pd.isnull(tmp['avgY']), 'pred_0']\n", " tmp.loc[pd.isnull(tmp['adj_avg']), 'adj_avg'] = tmp.loc[pd.isnull(tmp['adj_avg']), 'pred_0']\n", " tmp['random'] = np.random.uniform(size = len(tmp))\n", " tmp['adj_avg'] = tmp.apply(lambda row: row['adj_avg'] *(1 + (row['random'] - 0.5) * r_k),\n", " axis = 1)\n", " \n", " return tmp['adj_avg'].ravel()\n", " \n", " #cv for training set \n", " k_fold = StratifiedKFold(5)\n", " X_train[feature_name] = -999 \n", " for (train_index, cv_index) in k_fold.split(np.zeros(len(X_train)),\n", " X_train['interest_level'].ravel()):\n", " sub = pd.DataFrame(data = {variable: X_train[variable],\n", " 'y': X_train[y],\n", " 'pred_0': X_train[pred_0]})\n", " \n", " sub1 = sub.iloc[train_index] \n", " sub2 = sub.iloc[cv_index]\n", " \n", " X_train.loc[cv_index, feature_name] = calculate_average(sub1, sub2)\n", " \n", " #for test set\n", " sub1 = pd.DataFrame(data = {variable: X_train[variable],\n", " 'y': X_train[y],\n", " 'pred_0': X_train[pred_0]})\n", " sub2 = pd.DataFrame(data = {variable: X_test[variable],\n", " 'y': X_test[y],\n", " 'pred_0': X_test[pred_0]})\n", " X_test.loc[:, feature_name] = calculate_average(sub1, sub2) \n", "\n", "def transform_data(X):\n", " #add features \n", " feat_sparse = feature_transform.transform(X[\"features\"])\n", " vocabulary = feature_transform.vocabulary_\n", " del X['features']\n", " X1 = pd.DataFrame([ pd.Series(feat_sparse[i].toarray().ravel()) for i in np.arange(feat_sparse.shape[0]) ])\n", " X1.columns = list(sorted(vocabulary.keys()))\n", " X = pd.concat([X.reset_index(), X1.reset_index()], axis = 1)\n", " del X['index']\n", " \n", " X[\"num_photos\"] = X[\"photos\"].apply(len)\n", " X['created'] = pd.to_datetime(X[\"created\"])\n", " X[\"num_description_words\"] = X[\"description\"].apply(lambda x: len(x.split(\" \")))\n", " X['price_per_bed'] = X['price'] / X['bedrooms'] \n", " X['price_per_bath'] = X['price'] / X['bathrooms']\n", " X['price_per_room'] = X['price'] / (X['bathrooms'] + X['bedrooms'] )\n", " \n", " X['low'] = 0\n", " X.loc[X['interest_level'] == 0, 'low'] = 1\n", " X['medium'] = 0\n", " X.loc[X['interest_level'] == 1, 'medium'] = 1\n", " X['high'] = 0\n", " X.loc[X['interest_level'] == 2, 'high'] = 1\n", " \n", " X['display_address'] = X['display_address'].apply(lambda x: x.lower().strip())\n", " X['street_address'] = X['street_address'].apply(lambda x: x.lower().strip())\n", " \n", " X['pred0_low'] = low_count * 1.0 / train_size\n", " X['pred0_medium'] = medium_count * 1.0 / train_size\n", " X['pred0_high'] = high_count * 1.0 / train_size\n", " \n", " X.loc[X['manager_id'].isin(managers_with_one_lot['manager_id'].ravel()), \n", " 'manager_id'] = \"-1\"\n", " X.loc[X['building_id'].isin(buildings_with_one_lot['building_id'].ravel()), \n", " 'building_id'] = \"-1\"\n", " X.loc[X['display_address'].isin(addresses_with_one_lot['display_address'].ravel()), \n", " 'display_address'] = \"-1\"\n", " \n", " return X\n", "\n", "def normalize_high_cordiality_data():\n", " high_cardinality = [\"building_id\", \"manager_id\"]\n", " for c in high_cardinality:\n", " categorical_average(c, \"medium\", \"pred0_medium\", c + \"_mean_medium\")\n", " categorical_average(c, \"high\", \"pred0_high\", c + \"_mean_high\")\n", "\n", "def transform_categorical_data():\n", " categorical = ['building_id', 'manager_id', \n", " 'display_address', 'street_address']\n", " \n", " for f in categorical:\n", " encoder = LabelEncoder()\n", " encoder.fit(list(X_train[f]) + list(X_test[f])) \n", " X_train[f] = encoder.transform(X_train[f].ravel())\n", " X_test[f] = encoder.transform(X_test[f].ravel())\n", " \n", "\n", "def remove_columns(X):\n", " columns = [\"photos\", \"pred0_high\", \"pred0_low\", \"pred0_medium\",\n", " \"description\", \"low\", \"medium\", \"high\",\n", " \"interest_level\", \"created\"]\n", " for c in columns:\n", " del X[c]\n", "\n", "print(\"Starting transformations\") \n", "X_train = transform_data(X_train) \n", "X_test = transform_data(X_test) \n", "y = X_train['interest_level'].ravel()\n", "\n", "print(\"Normalizing high cordiality data...\")\n", "normalize_high_cordiality_data()\n", "transform_categorical_data()\n", "\n", "remove_columns(X_train)\n", "remove_columns(X_test)\n", "\n", "print(\"Start fitting...\")\n", "\n", "param = {}\n", "param['objective'] = 'multi:softprob'\n", "param['eta'] = 0.02\n", "param['max_depth'] = 4\n", "param['silent'] = 1\n", "param['num_class'] = 3\n", "param['eval_metric'] = \"mlogloss\"\n", "param['min_child_weight'] = 1\n", "param['subsample'] = 0.7\n", "param['colsample_bytree'] = 0.7\n", "param['seed'] = 321\n", "param['nthread'] = 8\n", "num_rounds = 2000\n", "\n", "xgtrain = xgb.DMatrix(X_train, label=y)\n", "clf = xgb.train(param, xgtrain, num_rounds)\n", "\n", "print(\"Fitted\")\n", "\n", "def prepare_submission(model):\n", " xgtest = xgb.DMatrix(X_test)\n", " preds = model.predict(xgtest) \n", " sub = pd.DataFrame(data = {'listing_id': X_test['listing_id'].ravel()})\n", " sub['low'] = preds[:, 0]\n", " sub['medium'] = preds[:, 1]\n", " sub['high'] = preds[:, 2]\n", " sub.to_csv(\"submission.csv\", index = False, header = True)\n", "\n", "prepare_submission(clf)" ] } ], "metadata": { "_change_revision": 0, "_is_fork": false, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.0" } }, "nbformat": 4, "nbformat_minor": 0 }