{"cells":[ { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": "from __future__ import division\nimport string\nimport numpy as np\nfrom numpy.random import randn\nfrom pandas import Series, DataFrame\nimport pandas as pd\nimport csv\nimport os\nimport matplotlib\nimport matplotlib.pyplot as plt\nfrom scipy import stats\nimport time\nimport datetime\nfrom sklearn.ensemble import GradientBoostingClassifier\n#%matplotlib inline\n\n \nfile = open(\"../input/train.csv\")\nfout = open('subset_datatrain.csv','w')\nn = 0\nfout.write(file.readline())\nfor line in file:\n arr = line.strip().split(',')\n is_book = int(arr[-6])\n if is_book == 1:\n fout.write(line)\nfout.close()\nfile.close()\n\n\n\ndategroup = ['2013Jan','2013May','2013Sep','2014Jan','2014May','2014Sep','2015Jan','2015May','2015Sep','2016Jan','2016May','2016Sep']\nchingroup = ['2013Jan','2013May','2013Sep','2014Jan','2014May','2014Sep','2015Jan','2015May','2015Sep','2016Jan','2016May','2016Sep']\ndateix = [[] for i in range(12)]\nchinix = [[] for i in range(12)]\n\ndef datedeal(date):\n n = len(date)\n for i in range(n):\n a = date[i]\n if type(a) == type(0.1):\n a = '2015-01-01'\n if int(a[1])>0 or int(a[2])>1 or int(a[2])<1 or int(a[3])>6 or int(a[3])<3: #大于2016或小于2013的年份全部换成2015\n date[i] = '2015-01-01'\n return pd.to_datetime(date)\n\ndef frameDateDeal(frame, datename):\n frame[datename]=frame[datename].fillna('2015-01-01')\n dateix = [[] for i in range(12)]\n datevalue = datedeal(frame[datename].values)\n datevalue = Series(np.arange(len(datevalue)),index = datevalue)\n for i in range(48):\n y = divmod(i,12)[0]\n r = divmod(i,12)[1]\n n = divmod(i,4)[0]\n if r<9:\n dateix[n].extend(datevalue['201'+str(3+y)+'-0'+str(r+1)].values)\n else:\n dateix[n].extend(datevalue['201'+str(3+y)+'-'+str(r+1)].values)\n for i in range(12):\n frame[datename].values[dateix[i]] = i\n return frame\n\n\nfeaturelist = ['user_id','user_location_city','srch_destination_id','hotel_market','srch_ci']\nwhlist = ['user_id','user_location_city','srch_destination_id','hotel_market','srch_ci','hotel_cluster']\ntrainpart = pd.read_csv('subset_datatrain.csv',na_values=['-- '],usecols = whlist)\n\ntrainpart = frameDateDeal(trainpart,'srch_ci')\nLGBdata = trainpart[featurelist].values\nLGBpara = {'data':LGBdata,'feature_names':featurelist,'target':trainpart['hotel_cluster'].values,\n'target_names':np.arange(100)}" }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": "len(LGBpara['target'])" }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": "# coding: utf-8\n\nimport datetime\nfrom heapq import nlargest\nfrom operator import itemgetter\nfrom collections import defaultdict\n\ndef run_solution():\n print('Preparing arrays...')\n f = open(\"../input/train.csv\", \"r\")\n f.readline()\n best_hotels_od_ulc = defaultdict(lambda: defaultdict(int))\n best_hotels_search_dest = defaultdict(lambda: defaultdict(int))\n best_hotels_search_dest1 = defaultdict(lambda: defaultdict(int))\n best_hotel_country = defaultdict(lambda: defaultdict(int))\n popular_hotel_cluster = defaultdict(int)\n total = 0\n\n # Calc counts\n while 1:\n line = f.readline().strip()\n total += 1\n\n if total % 10000000 == 0:\n print('Read {} lines...'.format(total))\n\n if line == '':\n break\n\n arr = line.split(\",\")\n book_year = int(arr[0][:4])\n user_location_city = arr[5]\n orig_destination_distance = arr[6]\n srch_destination_id = arr[16]\n is_booking = int(arr[18])\n hotel_country = arr[21]\n hotel_market = arr[22]\n hotel_cluster = arr[23]\n\n append_1 = 3 + 17*is_booking\n append_2 = 1 + 5*is_booking\n\n if user_location_city != '' and orig_destination_distance != '':\n best_hotels_od_ulc[(user_location_city, orig_destination_distance)][hotel_cluster] += 1\n\n if srch_destination_id != '' and hotel_country != '' and hotel_market != '' and book_year == 2014:\n best_hotels_search_dest[(srch_destination_id, hotel_country, hotel_market)][hotel_cluster] += append_1\n \n if srch_destination_id != '':\n best_hotels_search_dest1[srch_destination_id][hotel_cluster] += append_1\n \n if hotel_country != '':\n best_hotel_country[hotel_country][hotel_cluster] += append_2\n \n popular_hotel_cluster[hotel_cluster] += 1\n \n f.close()\n\n print('Generate submission...')\n now = datetime.datetime.now()\n #path = 'submission_' + str(now.strftime(\"%Y-%m-%d-%H-%M\")) + '.csv'\n path = 'predict_subtrain.csv'\n out = open(path, \"w\")\n f = open(\"subset_datatrain.csv\", \"r\")\n f.readline()\n total = 0\n #out.write(\"id,hotel_cluster\\n\")\n topclasters = nlargest(10, sorted(popular_hotel_cluster.items()), key=itemgetter(1))\n\n while 1:\n line = f.readline().strip()\n total += 1\n\n if total % 1000000 == 0:\n print('Write {} lines...'.format(total))\n\n if line == '':\n break\n\n arr = line.split(\",\")\n #id = arr[0]\n #user_location_city = arr[6]\n #orig_destination_distance = arr[7]\n #srch_destination_id = arr[17]\n #hotel_country = arr[20]\n #hotel_market = arr[21]\n id = arr[0]\n user_location_city = arr[5]\n orig_destination_distance = arr[6]\n srch_destination_id = arr[16]\n hotel_country = arr[21]\n hotel_market = arr[22]\n out.write(str(id) + ',')\n filled = []\n\n s1 = (user_location_city, orig_destination_distance)\n if s1 in best_hotels_od_ulc:\n d = best_hotels_od_ulc[s1]\n topitems = nlargest(10, sorted(d.items()), key=itemgetter(1))\n for i in range(len(topitems)):\n if topitems[i][0] in filled:\n continue\n if len(filled) == 10:\n break\n out.write(',' + topitems[i][0])\n filled.append(topitems[i][0])\n\n s2 = (srch_destination_id, hotel_country, hotel_market)\n if s2 in best_hotels_search_dest:\n d = best_hotels_search_dest[s2]\n topitems = nlargest(10, d.items(), key=itemgetter(1))\n for i in range(len(topitems)):\n if topitems[i][0] in filled:\n continue\n if len(filled) == 10:\n break\n out.write(',' + topitems[i][0])\n filled.append(topitems[i][0])\n elif srch_destination_id in best_hotels_search_dest1:\n d = best_hotels_search_dest1[srch_destination_id]\n topitems = nlargest(10, d.items(), key=itemgetter(1))\n for i in range(len(topitems)):\n if topitems[i][0] in filled:\n continue\n if len(filled) == 10:\n break\n out.write(',' + topitems[i][0])\n filled.append(topitems[i][0])\n\n if hotel_country in best_hotel_country:\n d = best_hotel_country[hotel_country]\n topitems = nlargest(10, d.items(), key=itemgetter(1))\n for i in range(len(topitems)):\n if topitems[i][0] in filled:\n continue\n if len(filled) == 10:\n break\n out.write(',' + topitems[i][0])\n filled.append(topitems[i][0])\n\n for i in range(len(topclasters)):\n if topclasters[i][0] in filled:\n continue\n if len(filled) == 10:\n break\n out.write(',' + topclasters[i][0])\n filled.append(topclasters[i][0])\n\n out.write(\"\\n\")\n out.close()\n print('Completed!')\n\nrun_solution()" }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": "testpart = pd.read_csv('predict_subtrain.csv',na_values=['-- '])" }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": "LGBpara['target']" }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": "testpart" }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": "file = open('predict_subtrain.csv')\nline = file.readline().strip().split(',')\nline = file.readline().strip().split(',')\nline\n" }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": "file = open('predict_subtrain.csv')\naccu = []\nfor clus in LGBpara['target']:\n\tline = file.readline().strip().split(',')\n\tarr = line[-10:-5]\n\tif str(clus) in arr:\n\t\taccu.append(1)\n\telse:\n\t\taccu.append(0)\naccu = Series(accu)\naccu.hist()" }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": "1" }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": "\naccu.value_counts()" }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": "\ntestpart = pd.read_csv('../input/test.csv',na_values=['-- '],usecols = featurelist)\n# for i in range(9):\n# testpart['srch_ci'].values[chinix[i]] = i\ntestpart = frameDateDeal(testpart,'srch_ci')\ntestdata = testpart[featurelist].values\n\n\nfrom sklearn.ensemble import RandomForestClassifier\nRFclf = RandomForestClassifier(n_estimators=30,\n max_depth=18, random_state=0).fit(RFdata, RFpara['target'])\nprint('RFclf OK!')\n\nnow = datetime.datetime.now()\npath = 'submission_RF_' + str(now.strftime(\"%Y-%m-%d-%H-%M\")) + '.csv'\nout = open(path, \"w\")\nout.write(\"id,hotel_cluster\\n\")\n\nimport random\nmn = divmod(len(testdata),40000)\nm = mn[0]\nn = mn[1]\neventid = 0\nfor i in range(m+1):\n clus = []\n if i