{"cells":[
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "rom __future__ import division\nimport string\nimport numpy as np\nfrom numpy.random import randn\nfrom pandas import Series, DataFrame\nimport pandas as pd\nimport csv\nimport os\nimport matplotlib\nimport matplotlib.pyplot as plt\nfrom scipy import stats\nimport time\nimport datetime\nfrom sklearn.ensemble import GradientBoostingClassifier\n#%matplotlib inline\n#voterlist:list of list which contains classifiers for one cluster\ndef bicluster(i,j):\n    tix = np.array(trainpart['hotel_cluster'].values==i)+np.array(trainpart['hotel_cluster'].values==j)\n    tGBtraintarget = (trainpart['hotel_cluster'].values==i)*1\n    tGBpara = {'data':GBdata,'feature_names':featurelist,'target':tGBtraintarget,\n    'target_names':np.arange(100)}\n    tmp = tGBpara['target'][tix]\n    if sum(tmp==0)==0:\n        tmp[-1] = 0\n    tclf = GradientBoostingClassifier(n_estimators=20, learning_rate=1,\n    max_depth=2, random_state=0).fit(GBdata[tix], tmp)\n    return tclf\n\ndef oneclus(n):\n    if n<99:\n        return bicluster(n,n+1)\n    else:\n        return bicluster(n,0)\n    \ndef getvoter():\n    voterlist = []\n    for i in range(100):\n        accuracy = []\n        clflist = []\n        clf  = oneclus(i)\n        clflist.append(clf)\n        for j in range(100):\n            tix = np.array(testpart1['hotel_cluster'].values==i)+np.array(testpart1['hotel_cluster'].values==j)\n            accuracy.append( clf.score(testdata1[tix], 1*(testpart1['hotel_cluster'][tix].values==i)) )  \n            #must use a testdata that contains true clusters\n        accuracy = DataFrame([accuracy],index = ['accuracy']).T\n        clusix = accuracy.sort_values( by ='accuracy',ascending = True).index[:4]\n        tclf = clf\n        for ind in clusix:\n            tclf = bicluster(i,ind)    \n            clflist.append(tclf)\n        voterlist.append(clflist)\n    return voterlist\n    \ndef GBvote(testdata,voterlist):\n    clusprob = []\n    now = datetime.datetime.now()\n    path = 'submission_GB_' + str(now.strftime(\"%Y-%m-%d-%H-%M\")) + '.csv'\n    out = open(path, \"w\")\n    out.write(\"id,hotel_cluster\\n\")\n    m = len(voterlist[0])\n    for i in range(100):\n        print('1----'+str(i))\n        clflist = voterlist[i]\n        tmp = np.zeros([len(testdata),2])\n        for j in range(m):    #compute the probability given by evevey clf\n            clf = clflist[j]\n            tmp = tmp + clf.predict_proba(testdata)\n        tmp = tmp/m    \n        tmp = (tmp[:,1]>0.5)*tmp[:,1]    #total probability for belonging to cluster i\n        clusprob.append(tmp)\n    clusprob = np.array(clusprob)\n    for i in range(len(testdata)):\n        if i%20000 == 0:\n            print('2----'+str(i))\n        clus = []\n        a = clusprob[:,i]\n        b=np.argsort(a)[-5:]\n        #clusprob.drop(i,axis = 1)\n        for ind in b:\n            clus.append(str(ind))\n        out.write(str(i)+\",\"+\"\\t\".join(clus)+\"\\n\")"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": ""
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "file = open(\"../input/train.csv\")\nfout = open('subset_datatest.csv','w')\nn = 0\nfor line in file:\n    if n == 0:\n        fout.write(line)\n    if n <400000*5:\n        n +=1\n    elif 400000*5<=n <400000*10:\n        n +=1\n        fout.write(line)\n    else:\n        break\nfout.close()\nfile.close()\nfile = open(\"../input/train.csv\")\nfout = open('subset_datatrain.csv','w')\nn = 0\nfor line in file:\n    if n <400000*5:\n        n +=1\n        fout.write(line)\n    else:\n        break\nfout.close()\nfile.close()\n"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "dategroup = ['2013Jan','2013May','2013Sep','2014Jan','2014May','2014Sep','2015Jan','2015May','2015Sep']\nchingroup = ['2013Jan','2013May','2013Sep','2014Jan','2014May','2014Sep','2015Jan','2015May','2015Sep']\ndateix = [[] for i in range(9)]\nchinix = [[] for i in range(9)]\ntimeframe = pd.read_csv('subset_datatest.csv',na_values=['--  '],usecols = ['date_time','srch_ci'])\ndatetime = pd.to_datetime(timeframe['date_time'].values)\ndatetime = Series(np.arange(len(datetime)),index = datetime)\nfor i in range(36):\n    y = divmod(i,12)[0]\n    r = divmod(i,12)[1]\n    n = divmod(i,4)[0]\n    if r<9:\n        dateix[n].extend(datetime['201'+str(3+y)+'-0'+str(r+1)].values)\n    else:\n        dateix[n].extend(datetime['201'+str(3+y)+'-'+str(r+1)].values)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "1"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "chinix = [[] for i in range(9)]\ncitime = pd.to_datetime(timeframe['srch_ci'].values)\ncitime = Series(np.arange(len(citime)),index = citime)\nfor i in range(36):\n    y = divmod(i,12)[0]\n    r = divmod(i,12)[1]\n    n = divmod(i,4)[0]\n    if r<9:\n        chinix[n].extend(citime['201'+str(3+y)+'-0'+str(r+1)].values)\n    else:\n        chinix[n].extend(citime['201'+str(3+y)+'-'+str(r+1)].values)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "timeframe2 = pd.read_csv('subset_datatrain.csv',na_values=['--  '],usecols = ['date_time','srch_ci'])\nchinix2 = [[] for i in range(9)]\ncitime2 = pd.to_datetime(timeframe2['srch_ci'].values)\ncitime2 = Series(np.arange(len(citime2)),index = citime2)\nfor i in range(36):\n    y = divmod(i,12)[0]\n    r = divmod(i,12)[1]\n    n = divmod(i,4)[0]\n    if r<9:\n        chinix2[n].extend(citime2['201'+str(3+y)+'-0'+str(r+1)].values)\n    else:\n        chinix2[n].extend(citime2['201'+str(3+y)+'-'+str(r+1)].values)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "timeframe2 = pd.read_csv('subset_datatrain.csv',na_values=['--  '],usecols = ['date_time','srch_ci'])\nchinix2 = [[] for i in range(9)]\ncitime2 = pd.to_datetime(timeframe2['srch_ci'].values)\ncitime2 = Series(np.arange(len(citime2)),index = citime2)\nfor i in range(36):\n    y = divmod(i,12)[0]\n    r = divmod(i,12)[1]\n    n = divmod(i,4)[0]\n    if r<9:\n        chinix2[n].extend(citime2['201'+str(3+y)+'-0'+str(r+1)].values)\n    else:\n        chinix2[n].extend(citime2['201'+str(3+y)+'-'+str(r+1)].values)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "1"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "featurelist = ['user_location_city','srch_destination_id','hotel_continent','srch_ci']\nwhlist = ['user_location_city','srch_destination_id','hotel_continent','srch_ci','hotel_cluster']\ntrainpart = pd.read_csv('subset_datatrain.csv',na_values=['--  '],usecols = whlist)\nfor i in range(9):\n    trainpart['srch_ci'].values[chinix2[i]] = i\nGBdata = trainpart[featurelist].values\nGBpara = {'data':GBdata,'feature_names':featurelist,'target':trainpart['hotel_cluster'].values,\n'target_names':np.arange(100)}\n\ntestpart = pd.read_csv('../input/test.csv',na_values=['--  '],usecols = featurelist)\nfor i in range(9):\n    testpart['srch_ci'].values[chinix[i]] = i\ntestdata = testpart[featurelist].values\ntestpart1 = pd.read_csv('subset_datatest.csv',na_values=['--  '],usecols = whlist)\ntestdata1 = testpart1[featurelist].values\n\n"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "voterlist = getvoter()\ndel testpart1\ndel testdata1\ndel GBdata\ndel GBpara\ndel trainpart\ndel testdata\n#GBvote(testpart.values,voterlist)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "testdata = testpart.values\nclusprob = []\nnow = datetime.datetime.now()\npath = 'submission_GB_' + str(now.strftime(\"%Y-%m-%d-%H-%M\")) + '.csv'\nout = open(path, \"w\")\nout.write(\"id,hotel_cluster\\n\")\nm = len(voterlist[0])\nfor i in range(100):\n    print('1----'+str(i))\n    clflist = voterlist[i]\n    tmp = np.zeros([len(testdata),2])\n    for j in range(m):    #compute the probability given by evevey clf\n        clf = clflist[j]\n        tmp = tmp + clf.predict_proba(testdata)\n    tmp = tmp/m    \n    tmp = (tmp[:,0]>0.5)*tmp[:,0]    #total probability for belonging to cluster i\n    clusprob.append(tmp)\nclusprob = np.array(clusprob)\nfor i in range(len(testdata)):\n    if i%1000 == 0:\n        print('2----'+str(i))\n    clus = []\n    a = DataFrame(clusprob[:,i])\n    b=a.sort_values(by = 0,ascending = False).index[:5]\n    #clusprob.drop(i,axis = 1)\n    for ind in b:\n        clus.append(str(ind))\n    out.write(str(i)+\",\"+\"\\t\".join(clus)+\"\\n\")"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "1"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "for i in range(len(testdata)):\n    if i%1000 == 0:\n        print('2----'+str(i))\n    clus = []\n    a = DataFrame(clusprob[:,i])\n    b=a.sort_index(by = 0,ascending = False).index[:5]\n    #clusprob.drop(i,axis = 1)\n    for ind in b:\n        clus.append(str(ind))\n    out.write(str(i)+\",\"+\"\\t\".join(clus)+\"\\n\")"
 }
],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}}, "nbformat": 4, "nbformat_minor": 0}