{"cells":[
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "import numpy as np\nimport pandas as pd \nfrom subprocess import check_output\n#print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\nimport datetime\nimport time"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "destinations = pd.read_csv(\"../input/destinations.csv\")\ntrain1 = pd.read_csv(\"../input/train.csv\", nrows=100000)\ntest1 = pd.read_csv(\"../input/test.csv\", nrows = 100)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "train2 = pd.read_csv(\"../input/train.csv\", nrows=100000)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "train3 = pd.read_csv(\"../input/train.csv\", nrows=200002)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "train4.info()"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "train4 = train3.ix[100001:200002,:]"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "train1[\"date_time\"] = pd.to_datetime(train1[\"date_time\"])\ntrain1[\"year\"] = train1[\"date_time\"].dt.year\ntrain1[\"month\"] = train1[\"date_time\"].dt.month"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "train1 = train1[train1.is_booking == True]"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "train4 = train4[train4.is_booking == True]"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "train4[\"date_time\"] = pd.to_datetime(train4[\"date_time\"])\ntrain4[\"year\"] = train4[\"date_time\"].dt.year\ntrain4[\"month\"] = train4[\"date_time\"].dt.month"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "from sklearn.decomposition import PCA"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "X = destinations.ix[:,1:150]"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "y = destinations.ix[:,0:1]"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "pca = PCA(n_components=3)\nX_r = pca.fit(X).transform(X)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "X_r1 = pd.DataFrame(X_r)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "X_r1[\"srch_destination_id\"] = destinations[\"srch_destination_id\"]"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "print('explained variance ratio (first 3 components): %s' % str(pca.explained_variance_ratio_))"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "def calc_fast_features(df):\n    df[\"date_time\"] = pd.to_datetime(df[\"date_time\"])\n    df[\"srch_ci\"] = pd.to_datetime(df[\"srch_ci\"], format='%Y-%m-%d', errors=\"coerce\")\n    df[\"srch_co\"] = pd.to_datetime(df[\"srch_co\"], format='%Y-%m-%d', errors=\"coerce\")\n    \n    props = {}\n    for prop in [\"month\", \"day\", \"hour\", \"minute\", \"dayofweek\", \"quarter\"]:\n        props[prop] = getattr(df[\"date_time\"].dt, prop)\n    \n    carryover = [p for p in df.columns if p not in [\"date_time\", \"srch_ci\", \"srch_co\"]]\n    for prop in carryover:\n        props[prop] = df[prop]\n    \n    date_props = [\"month\", \"day\", \"dayofweek\", \"quarter\"]\n    for prop in date_props:\n        props[\"ci_{0}\".format(prop)] = getattr(df[\"srch_ci\"].dt, prop)\n        props[\"co_{0}\".format(prop)] = getattr(df[\"srch_co\"].dt, prop)\n    props[\"stay_span\"] = (df[\"srch_co\"] - df[\"srch_ci\"]).astype('timedelta64[h]')\n        \n    ret = pd.DataFrame(props)\n    \n    ret = ret.join(X_r1, on=\"srch_destination_id\", how='left', rsuffix=\"dest\")\n    ret = ret.drop(\"srch_destination_iddest\", axis=1)\n    return ret"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "df = calc_fast_features(train2)\ndf.fillna(-1, inplace=True)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "predictors = [c for c in df.columns if c not in [\"hotel_cluster\"]]\nfrom sklearn import cross_validation\nfrom sklearn.ensemble import RandomForestClassifier"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "clf = RandomForestClassifier(n_estimators=10, min_weight_fraction_leaf=0.1)\nscores = cross_validation.cross_val_score(clf, df[predictors], df['hotel_cluster'], cv=3)\nscores"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "from sklearn.ensemble import RandomForestClassifier\nfrom sklearn.cross_validation import KFold\nfrom itertools import chain"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "all_probs = []\nunique_clusters = df[\"hotel_cluster\"].unique()"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "for cluster in unique_clusters:\n    df[\"target\"] = 1\n    df[\"target\"][df[\"hotel_cluster\"] != cluster] = 0\n    predictors = [col for col in df if col not in ['hotel_cluster', \"target\"]]\n    probs = []\n    cv = KFold(len(df[\"target\"]), n_folds=2)\n    clf = RandomForestClassifier(n_estimators=10, min_weight_fraction_leaf=0.1)\n    for i, (tr, te) in enumerate(cv):\n        clf.fit(df[predictors].iloc[tr], df[\"target\"].iloc[tr])\n        preds = clf.predict_proba(df[predictors].iloc[te])\n        probs.append([p[1] for p in preds])\n    full_probs = chain.from_iterable(probs)\n    all_probs.append(list(full_probs))"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "prediction_frame = pd.DataFrame(all_probs).T\nprediction_frame.columns = unique_clusters"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "import ml_metrics as metrics"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "def find_top_5(row):\n    return list(row.nlargest(5).index)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "preds = [ ]\nfor index, row in prediction_frame.iterrows():\n    preds.append(find_top_5(row))"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "metrics.mapk([[l] for l in train4[\"hotel_cluster\"]], preds, k=5)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "test1[\"srch_ci\"] = pd.to_datetime(test1[\"srch_ci\"], format='%Y-%m-%d', errors=\"coerce\")"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "test1[\"srch_co\"] = pd.to_datetime(test1[\"srch_co\"], format='%Y-%m-%d', errors=\"coerce\")"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "test1[\"stay_span\"] = (test1[\"srch_co\"] - test1[\"srch_ci\"]).astype('timedelta64[D]')"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "test2 = pd.to_datetime(test1[\"date_time\"])"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "test2 = pd.DataFrame(test2)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "test1['year'] = test2['date_time'].dt.year\ntest1['month'] = test2['date_time'].dt.month\ntest1['day_of_week'] = test2['date_time'].dt.dayofweek\ntest1['day'] = test2['date_time'].dt.day\ntest1['hour'] = test2['date_time'].dt.hour"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "test1[\"date_time\"] = pd.to_datetime(test1[\"date_time\"], format='%Y-%m-%d', errors=\"coerce\")\ntest1[\"srch_ci\"] = pd.to_datetime(test1[\"srch_ci\"], format='%Y-%m-%d', errors=\"coerce\")\ntest1[\"search_span\"] = (test1[\"srch_ci\"] - test1[\"date_time\"]).astype('timedelta64[D]')"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "test1.ix[(test1['hour'] >= 10) & (test1['hour'] < 18), 'hour'] = 1\ntest1.ix[(test1['hour'] >= 18) & (test1['hour'] < 22), 'hour'] = 2\ntest1.ix[(test1['hour'] >= 22) & (test1['hour'] == 24), 'hour'] = 3\ntest1.ix[(test1['hour'] >= 1) & (test1['hour'] < 10), 'hour'] = 3"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "test1 = test1.drop('srch_ci', axis=1)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "test1 = test1.drop('srch_co', axis=1)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "test1 = test1.drop('date_time', axis=1)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "test1.fillna(-1, inplace=True)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "test1.info()"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "metrics.mapk([[l] for l in test1[\"stay_span\"]], preds, k=5)"
 }
],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}}, "nbformat": 4, "nbformat_minor": 0}