{"cells":[
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "import time\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n%matplotlib inline\nfrom PIL import Image\nfrom PIL import ImageFilter\nimport multiprocessing\nimport random; random.seed(2016);\nimport cv2\nimport re\nimport os, glob\n\nsample_sub = pd.read_csv('../input/sample_submission.csv')\ntrain_files = pd.DataFrame([[f,f.split(\"/\")[3].split(\".\")[0].split(\"_\")[0],f.split(\"/\")[3].split(\".\")[0].split(\"_\")[1]] for f in glob.glob(\"../input/train_sm/*.jpeg\")])\ntrain_files.columns = ['path', 'group', 'pic_no']\ntest_files = pd.DataFrame([[f,f.split(\"/\")[3].split(\".\")[0].split(\"_\")[0],f.split(\"/\")[3].split(\".\")[0].split(\"_\")[1]] for f in glob.glob(\"../input/test_sm/*.jpeg\")])\ntest_files.columns = ['path', 'group', 'pic_no']\nprint(len(train_files),len(test_files),len(sample_sub))\ntrain_images = train_files[train_files[\"group\"]=='set107']\ntrain_images = train_images.sort_values(by=[\"pic_no\"], ascending=[1]).reset_index(drop=True)\nplt.rcParams['figure.figsize'] = (12.0, 12.0)\nplt.subplots_adjust(wspace=0, hspace=0)\ni_ = 0\na = []\nfor l in train_images.path:\n    im = cv2.imread(l)\n    plt.subplot(5, 2, i_+1).set_title(l)\n    plt.hist(im.ravel(),256,[0,256]); plt.axis('off')\n    a.append([im.mean(),im.max(),im.min()])\n    plt.subplot(5, 2, i_+2).set_title(l)\n    plt.imshow(cv2.cvtColor(im, cv2.COLOR_BGR2RGB)); plt.axis('off')\n    i_ += 2\nprint(a)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "kaze = cv2.KAZE_create()\nakaze = cv2.AKAZE_create()\nbrisk = cv2.BRISK_create()\n\nplt.rcParams['figure.figsize'] = (7.0, 18.0)\nplt.subplots_adjust(wspace=0, hspace=0)\ni = 0\nfor detector in [kaze, akaze, brisk]:\n    start_time = time.time()\n    im = cv2.imread(train_images.path[0])\n    gray = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY)\n    (kps, descs) = detector.detectAndCompute(gray, None)       \n    cv2.drawKeypoints(im, kps, im, (0, 255, 0))\n    plt.subplot(3, 1, i+1).set_title(list(['kaze','akaze','brisk'])[i] + \" \" + str(round(((time.time() - start_time)/60),5)))\n    plt.imshow(cv2.cvtColor(im, cv2.COLOR_BGR2RGB)); plt.axis('off')\n    i+=1"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "print(cv2.__version__)\n\nimg1 = cv2.imread(train_images.path[0], 0)\nimg2 = cv2.imread(train_images.path[1], 0)\nbrisk = cv2.BRISK_create()\nkp1, des1 = brisk.detectAndCompute(img1,None)\nkp2, des2 = brisk.detectAndCompute(img2,None)\nbf = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=True)\nmatches = bf.match(des1,des2)\nmatches = sorted(matches, key = lambda x:x.distance)\nimg1 = cv2.imread(train_images.path[0])\nimg2 = cv2.imread(train_images.path[1])\nimg3 = cv2.drawMatches(img1,kp1,img2,kp2,matches[:100], flags=2, outImg=img2, matchColor = (0,255,0))\nplt.rcParams['figure.figsize'] = (14.0, 8.0)\nplt.imshow(cv2.cvtColor(img3, cv2.COLOR_BGR2RGB)); plt.axis('off')"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "brisk = cv2.BRISK_create()\ndm = cv2.DescriptorMatcher_create(\"BruteForce\")\n\ndef c_resize(img, ratio):\n    wh = (int(img.shape[1] * ratio), int(img.shape[0] * ratio))\n    img = cv2.resize(img, wh, interpolation = cv2.INTER_AREA)\n    return img\n    \ndef im_stitcher(imp1, imp2, imsr = 1.0, withTransparency=False):\n    img1 = cv2.imread(imp1, 0)\n    img2 = cv2.imread(imp2, 0)\n    if imsr < 1.0:\n        img1 = c_resize(img1,imsr); img2 = c_resize(img2,imsr)\n    h1,w1 = img1.shape[:2]\n    h2,w2 = img2.shape[:2]\n    kp1, des1 = brisk.detectAndCompute(img1,None)\n    kp2, des2 = brisk.detectAndCompute(img2,None)\n    matches = dm.knnMatch(des1,des2, 2)\n    matches_ = []\n    for m in matches:\n        if len(m) == 2 and m[0].distance < m[1].distance * 0.75:\n            matches_.append((m[0].trainIdx, m[0].queryIdx))\n    kp1_ = np.float32([kp1[m[1]].pt for m in matches_]).reshape(-1,1,2)\n    kp2_ = np.float32([kp2[m[0]].pt for m in matches_]).reshape(-1,1,2)\n    H, mask = cv2.findHomography(kp2_,kp1_, cv2.RANSAC, 4.0)\n    pts1 = np.float32([[0,0],[0,h1],[w1,h1],[w1,0]]).reshape(-1,1,2)\n    pts2 = np.float32([[0,0],[0,h2],[w2,h2],[w2,0]]).reshape(-1,1,2)\n    pts2_ = cv2.perspectiveTransform(pts2, H)\n    pts = np.concatenate((pts1, pts2_), axis=0)\n    [xmin, ymin] = np.int32(pts.min(axis=0).ravel() - 0.5)\n    [xmax, ymax] = np.int32(pts.max(axis=0).ravel() + 0.5)\n    t = [-xmin,-ymin]\n    Ht = np.array([[1,0,t[0]],[0,1,t[1]],[0,0,1]])\n    img1 = cv2.imread(imp1)\n    img2 = cv2.imread(imp2)\n    if imsr < 1.0:\n        img1 = c_resize(img1,imsr); img2 = c_resize(img2,imsr)\n    im = cv2.warpPerspective(img2, Ht.dot(H), (xmax-xmin, ymax-ymin))\n    if withTransparency == True:\n        h3,w3 = im.shape[:2]\n        bim = np.zeros((h3,w3,3), np.uint8)\n        bim[t[1]:h1+t[1],t[0]:w1+t[0]] = img1\n        im = cv2.addWeighted(im,1.0,bim,0.9,0)\n    else:\n        im[t[1]:h1+t[1],t[0]:w1+t[0]] = img1\n    return im"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "img = im_stitcher(train_images.path[0], train_images.path[4], 0.5, True)\nplt.rcParams['figure.figsize'] = (12.0, 12.0)\nimg[np.where((img < [20,20,20]).all(axis = 2))] = [255,255,255]\nplt.imshow(img); plt.axis('off')"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "img = cv2.imread(train_images.path[0])\ncv2.imwrite('panoramic.jpeg',img)\nplt.rcParams['figure.figsize'] = (12.0, 12.0)\nfor i in range(1,5):\n    img = im_stitcher(train_images.path[i], 'panoramic.jpeg', 0.5, False)\n    cv2.imwrite('panoramic.jpeg',img)\nimg[np.where((img < [20,20,20]).all(axis = 2))] = [255,255,255]\nplt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB)); plt.axis('off')"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "train_images = train_files[train_files[\"group\"]=='set4']\ntrain_images = train_images.sort_values(by=[\"pic_no\"], ascending=[1]).reset_index(drop=True)\nimg = cv2.imread(train_images.path[0])\ncv2.imwrite('panoramic2.jpeg',img)\nplt.rcParams['figure.figsize'] = (12.0, 12.0)\nfor i in range(1,5):\n    img = im_stitcher(train_images.path[i], 'panoramic2.jpeg', 0.5, False)\n    cv2.imwrite('panoramic2.jpeg',img)\nimg[np.where((img < [20,20,20]).all(axis = 2))] = [255,255,255]\nplt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB)); plt.axis('off')"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "import time; start_time = time.time()\nimport warnings; warnings.filterwarnings('ignore');\nimport multiprocessing\nfrom sklearn import ensemble\nfrom sklearn import pipeline, grid_search\nfrom sklearn.metrics import label_ranking_average_precision_score as lraps\n\ndef image_features(path, tt, group, pic_no):\n    im = cv2.imread(path)\n    me_ = cv2.mean(im)\n    s=[path, tt, group, pic_no, im.mean(), me_[2], me_[1], me_[0]]\n    f = open(\"data.csv\",\"a\")\n    f.write((',').join(map(str, s)) + '\\n')\n    f.close()\n    return\n\nf = open(\"data.csv\",\"w\");\ncol = ['path','tt', 'group', 'pic_no', 'individual_im_mean','rm','bm','gm']\nf.write((',').join(map(str,col)) + '\\n')\nf.close()\n\nif __name__ == '__main__':\n    cpu = multiprocessing.cpu_count(); print (cpu);\n    \n    j = []\n    for s_ in range(0,len(train_files),cpu):     #train\n        for i in range(cpu):\n            i_=s_+i\n            if (i_)<len(train_files):\n                if i_ % 100 == 0:\n                    print(\"train \", i_)\n                filename = train_files.path[i_]\n                p = multiprocessing.Process(target=image_features, args=(filename,'train', train_files[\"group\"][i_], train_files[\"pic_no\"][i_],))\n                j.append(p)\n                p.start()\n    j = []\n    for s_ in range(0,len(test_files),cpu):     #test\n        for i in range(cpu):\n            i_=s_+i\n            if (i_)<len(test_files):\n                if i_ % 100 == 0:\n                    print(\"test \", i_)\n                filename = test_files.path[i_]\n                p = multiprocessing.Process(target=image_features, args=(filename,'test', test_files[\"group\"][i_], test_files[\"pic_no\"][i_],))\n                j.append(p)\n                p.start()\n    \n    while len(j) > 0: #end all jobs\n        j = [x for x in j if x.is_alive()]\n        time.sleep(1)\n    df_all = pd.read_csv('data.csv', index_col=None)\n    df_all = df_all.reset_index(drop=True)\n    df_all['group_min_im_mean'] = df_all[\"group\"].map(lambda x: df_all[df_all['group']==x]['individual_im_mean'].min())\n    df_all['group_max_im_mean'] = df_all[\"group\"].map(lambda x: df_all[df_all['group']==x]['individual_im_mean'].max())\n    df_all['group_mean'] = df_all[\"group\"].map(lambda x: df_all[df_all['group']==x]['individual_im_mean'].mean())\n    df_all['a'] = df_all['individual_im_mean'] - df_all['group_min_im_mean']\n    df_all['b'] = df_all['group_max_im_mean'] - df_all['individual_im_mean']\n    df_all['c'] = df_all['group_mean'] - df_all['individual_im_mean']\n    #red\n    df_all['group_min_im_mean_r'] = df_all[\"group\"].map(lambda x: df_all[df_all['group']==x]['rm'].min())\n    df_all['group_max_im_mean_r'] = df_all[\"group\"].map(lambda x: df_all[df_all['group']==x]['rm'].max())\n    df_all['group_mean_r'] = df_all[\"group\"].map(lambda x: df_all[df_all['group']==x]['rm'].mean())\n    df_all['a_r'] = df_all['rm'] - df_all['group_min_im_mean_r']\n    df_all['b_r'] = df_all['group_max_im_mean_r'] - df_all['rm']\n    #df_all['c_r'] = df_all['group_mean_r'] - df_all['rm']\n    #green\n    df_all['group_min_im_mean_g'] = df_all[\"group\"].map(lambda x: df_all[df_all['group']==x]['gm'].min())\n    df_all['group_max_im_mean_g'] = df_all[\"group\"].map(lambda x: df_all[df_all['group']==x]['gm'].max())\n    df_all['group_mean_g'] = df_all[\"group\"].map(lambda x: df_all[df_all['group']==x]['gm'].mean())\n    df_all['a_g'] = df_all['gm'] - df_all['group_min_im_mean_g']\n    df_all['b_g'] = df_all['group_max_im_mean_g'] - df_all['gm']\n    #df_all['c_g'] = df_all['group_mean_g'] - df_all['gm']\n    #blue\n    df_all['group_min_im_mean_b'] = df_all[\"group\"].map(lambda x: df_all[df_all['group']==x]['bm'].min())\n    df_all['group_max_im_mean_b'] = df_all[\"group\"].map(lambda x: df_all[df_all['group']==x]['bm'].max())\n    df_all['group_mean_b'] = df_all[\"group\"].map(lambda x: df_all[df_all['group']==x]['bm'].mean())\n    df_all['a_b'] = df_all['bm'] - df_all['group_min_im_mean_b']\n    df_all['b_b'] = df_all['group_max_im_mean_b'] - df_all['bm']\n    #df_all['c_b'] = df_all['group_mean_b'] - df_all['bm']\n    \n    df_all['setId'] = df_all[\"group\"].map(lambda x: x.replace('set',''))\n    df_all.to_csv('data.csv')\n    print(\"Features Ready: \", round(((time.time() - start_time)/60),2))\n    \n    X_train = df_all[df_all['tt'] == 'train']\n    X_train = X_train.sort_values(by=['setId','pic_no'], ascending=[1, 1])\n    X_train = X_train.reset_index(drop=True)\n    y_train = X_train[\"pic_no\"].values\n    X_train = X_train.drop(['path','tt','group','pic_no','setId','individual_im_mean','group_min_im_mean','group_max_im_mean','group_mean', 'rm','group_min_im_mean_r','group_max_im_mean_r','group_mean_r', 'gm','group_min_im_mean_g','group_max_im_mean_g','group_mean_g', 'bm','group_min_im_mean_b','group_max_im_mean_b','group_mean_b'], axis=1)\n    X_test = df_all[df_all['tt'] == 'test']\n    X_test = X_test.sort_values(by=['setId','pic_no'], ascending=[1, 1])\n    #X_test.fillna(0, inplace=True)\n    X_test = X_test.reset_index(drop=True)\n    id_test = X_test[[\"setId\",\"pic_no\"]] #.values\n    X_test = X_test.drop(['path','tt','group','pic_no','setId','individual_im_mean','group_min_im_mean','group_max_im_mean','group_mean', 'rm','group_min_im_mean_r','group_max_im_mean_r','group_mean_r', 'gm','group_min_im_mean_g','group_max_im_mean_g','group_mean_g', 'bm','group_min_im_mean_b','group_max_im_mean_b','group_mean_b'], axis=1)\n    rfr = ensemble.RandomForestClassifier(n_estimators = 50, n_jobs = -1, random_state = 2016, verbose = 0)\n    param_grid = {'max_depth': [6], 'max_features': [1.0]}\n    model = grid_search.GridSearchCV(estimator = rfr, param_grid = param_grid, n_jobs = -1, cv = 2, verbose = 0)\n    model.fit(X_train, y_train)\n    print(\"Best parameters found by grid search:\")\n    print(model.best_params_)\n    print(\"Best CV score:\", model.best_score_)\n    y_pred = model.predict_proba(X_test)\n    #y_pred = model.predict(X_test)\n    df = pd.concat((pd.DataFrame(id_test), pd.DataFrame(y_pred)), axis=1)\n    df.columns = ['setId','pic_no','day1','day2','day3','day4','day5']\n    #df.to_csv('submission2.csv',index=False)\n    f = open('submission.csv', 'w')\n    f.write('setId,day\\n')\n    setID = df.setId.unique()\n    for i in setID:\n        a = []\n        df1 = df[df['setId'] == str(i)].reset_index(drop=True)\n        for j in range(1,6):\n            df1 = df1.sort_values(by=['day'+str(j)], ascending=[0]).reset_index(drop=True)\n            #print(df1)\n            a.append(df1.pic_no[0])\n            df1 = df1[1:]\n        f.write(str(i)+\",\"+\" \".join(map(str,a))+\"\\n\")\n        #break\n    f.close()\n    print(\"Ready to submit: \", round(((time.time() - start_time)/60),2))"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": ""
 }
],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}}, "nbformat": 4, "nbformat_minor": 0}