{"cells":[
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n%matplotlib inline\nfrom PIL import Image\nfrom PIL import ImageFilter\nimport multiprocessing\nimport random; random.seed(2016);\n\n\n\n\nimport cv2\nimport re\nimport os, glob\n\nsample_sub = pd.read_csv('../input/sample_submission.csv')\ntrain_files = pd.DataFrame([[f,f.split(\"/\")[3].split(\".\")[0].split(\"_\")[0],f.split(\"/\")[3].split(\".\")[0].split(\"_\")[1]] for f in glob.glob(\"../input/train_sm/*.jpeg\")])\ntrain_files.columns = ['path', 'group', 'pic_no']\ntest_files = pd.DataFrame([[f,f.split(\"/\")[3].split(\".\")[0].split(\"_\")[0],f.split(\"/\")[3].split(\".\")[0].split(\"_\")[1]] for f in glob.glob(\"../input/test_sm/*.jpeg\")])\ntest_files.columns = ['path', 'group', 'pic_no']\nprint(len(train_files),len(test_files),len(sample_sub))\n\n\n"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": ""
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "test_files[\"group\"].unique()"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "b = []\nfor name in test_files[\"group\"].unique():\n    test_images = test_files[test_files[\"group\"]==name]\n    test_images = test_images.sort_values(by=[\"pic_no\"], ascending=[1])\n    plt.rcParams['figure.figsize'] = (12.0, 12.0)\n    plt.subplots_adjust(wspace=0, hspace=0)\n    i_ = 0\n    a = []\n    for l in test_images.path:\n        im = cv2.imread(l)\n        #plt.subplot(5, 2, i_+1).set_title(l)\n        #plt.hist(im.ravel(),256,[0,256]); plt.axis('off')\n        a.append(im.mean())\n        #plt.subplot(5, 2, i_+2).set_title(l)\n        #plt.imshow(cv2.cvtColor(im, cv2.COLOR_BGR2RGB)); plt.axis('off')\n        i_ += 2\n    b.append(a)\nprint(b)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "import pandas as pd\ndf= pd.DataFrame(b)\ndf"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "df.to_csv(\"immediantest.csv\")"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "im = Image.open(train_images.path.iloc[0])\nim2 = Image.open(train_images.path.iloc[1])\nim2 = im2.rotate(-15)\nim2 = im2.resize((3000, 2000), Image.ANTIALIAS)\nr,g,b = im2.split()\nmask = Image.merge(\"L\", (b,))\nim.paste(im2, (0,0,3000,2000), mask)\nplt.imshow(im); plt.axis('off')"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "train_images = train_files[train_files[\"group\"]=='set12']\ntrain_images = train_images.sort_values(by=[\"pic_no\"], ascending=[1])\nplt.rcParams['figure.figsize'] = (\n12.0, 12.0)\nplt.subplots_adjust(wspace=0, hspace=0)\ni_ = 0\na = []\nfor l in train_images.path:\n    im = cv2.imread(l)\n    plt.subplot(5, 2, i_+1).set_title(l)\n    plt.hist(im.ravel(),256,[0,256]); plt.axis('off')\n    a.append([im.mean(),im.max(),im.min()])\n    plt.subplot(5, 2, i_+2).set_title(l)\n    plt.imshow(cv2.cvtColor(im, cv2.COLOR_BGR2RGB)); plt.axis('off')\n    i_ += 2\nprint(a)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# from https://www.kaggle.com/nigelcarpenter/draper-satellite-image-chronology/akaze-stitching/run/227957\nimport numpy as np\nimport cv2\n\nprint(cv2.__version__)\n\ndef warpImage(img, H):\n\trows, cols = img.shape[:2]\n\n\tlist_of_points_1 = np.float32([[0,0], [0,rows], [cols, rows], [cols,0]]).reshape(-1,1,2)\n\tlist_of_points = cv2.perspectiveTransform(list_of_points_1, H)\n\n\t[x_min, y_min] = np.int32(list_of_points.min(axis=0).ravel() - 0.5)\n\t[x_max, y_max] = np.int32(list_of_points.max(axis=0).ravel() + 0.5)\n\n\tprint(\"x_min: \"+str(x_min)+\" y_min:\"+str(y_min))\n\ttranslation_dist = [-x_min, -y_min]\n\tH_translation = np.array([[1, 0, translation_dist[0]], [0, 1, translation_dist[1]], [0,0,1]])\n\toutput_img = cv2.warpPerspective(img, H_translation.dot(H), (x_max - x_min, y_max - y_min))\n\treturn output_img\n\ndef warpImages(img1, img2, H):\n\trows1, cols1 = img1.shape[:2]\n\trows2, cols2 = img2.shape[:2]\n\n\tlist_of_points_1 = np.float32([[0,0], [0,rows1], [cols1, rows1], [cols1,0]]).reshape(-1,1,2)\n\ttemp_points = np.float32([[0,0], [0,rows2], [cols2, rows2], [cols2,0]]).reshape(-1,1,2)\n\n\tlist_of_points_2 = cv2.perspectiveTransform(temp_points, H)\n\tlist_of_points = np.concatenate((list_of_points_1, list_of_points_2), axis=0)\n\n\t[x_min, y_min] = np.int32(list_of_points.min(axis=0).ravel() - 0.5)\n\t[x_max, y_max] = np.int32(list_of_points.max(axis=0).ravel() + 0.5)\n\n\ttranslation_dist = [-x_min, -y_min]\n\tH_translation = np.array([[1, 0, translation_dist[0]], [0, 1, translation_dist[1]], [0,0,1]])\n\n\toutput_img = cv2.warpPerspective(img2, H_translation.dot(H), (x_max - x_min, y_max - y_min))\n\toutput_img[translation_dist[1]:rows1+translation_dist[1],translation_dist[0]:cols1+translation_dist[0]] = img1\n\treturn output_img\n\n\n\ndef drawMatches(imageA, imageB, kpsA, kpsB, matches, status):\n\t# initialize the output visualization image\n\t(hA, wA) = imageA.shape[:2]\n\t(hB, wB) = imageB.shape[:2]\n\tvis = np.zeros((max(hA, hB), wA + wB, 3), dtype=\"uint8\")\n\tvis[0:hA, 0:wA] = imageA\n\tvis[0:hB, wA:] = imageB\n\n\t# loop over the matches\n\tfor ((trainIdx, queryIdx), s) in zip(matches, status):\n\t\t# only process the match if the keypoint was successfully\n\t\t# matched\n\t\tif s == 1:\n\t\t\t# draw the match\n\t\t\tptA = (int(kpsA[queryIdx][0]), int(kpsA[queryIdx][1]))\n\t\t\tptB = (int(kpsB[trainIdx][0]) + wA, int(kpsB[trainIdx][1]))\n\t\t\tcv2.line(vis, ptA, ptB, (0, 255, 0), 1)\n\n\t# return the visualization\n\treturn vis\n\ndef detectAndDescribe(image):\n\tgray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n\tdescriptor = cv2.AKAZE_create()\n\t(kps, features) = descriptor.detectAndCompute(gray, None)\n\n\t# convert the keypoints from KeyPoint objects to NumPy\n\t# arrays\n\tkps = np.float32([kp.pt for kp in kps])\n\n\t# return a tuple of keypoints and features\n\treturn (kps, features)\n\ndef matchKeypoints(kpsA, kpsB, featuresA, featuresB, ratio, reprojThresh):\n\t# compute the raw matches and initialize the list of actual\n\t# matches\n\tmatcher = cv2.DescriptorMatcher_create(\"BruteForce\")\n\trawMatches = matcher.knnMatch(featuresA, featuresB, 2)\n\tmatches = []\n\n\t# loop over the raw matches\n\tfor m in rawMatches:\n\t\t# ensure the distance is within a certain ratio of each\n\t\t# other (i.e. Lowe's ratio test)\n\t\tif len(m) == 2 and m[0].distance < m[1].distance * ratio:\n\t\t\tmatches.append((m[0].trainIdx, m[0].queryIdx))\n\t\t# computing a homography requires at least 4 matches\n\t\tif len(matches) > 30:\n\t\t\t# construct the two sets of points\n\t\t\tptsA = np.float32([kpsA[i] for (_, i) in matches])\n\t\t\tptsB = np.float32([kpsB[i] for (i, _) in matches])\n \n\t\t\t# compute the homography between the two sets of points\n\t\t\t(H, status) = cv2.findHomography(ptsA, ptsB, cv2.RANSAC,\n\t\t\t\treprojThresh)\n \n\t\t\t# return the matches along with the homograpy matrix\n\t\t\t# and status of each matched point\n\t\t\treturn (matches, H, status)\n \n\t# otherwise, no homograpy could be computed\n\treturn None\n\n"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "import time; start_time = time.time()\nimport warnings; warnings.filterwarnings('ignore');\nimport multiprocessing\nfrom sklearn import ensemble\nfrom sklearn import grid_search\nfrom sklearn.metrics import label_ranking_average_precision_score as lraps\n\ndef image_features(path, tt, group, pic_no):\n    im = cv2.imread(path)\n    s=[path, tt, group, pic_no, im.mean(), 0.0, 0.0, 0.0]\n    f = open(\"data.csv\",\"a\")\n    f.write((',').join(map(str, s)) + '\\n')\n    f.close()\n    return\n\nf = open(\"data.csv\",\"w\");\ncol = ['path','tt', 'group', 'pic_no', 'individual_im_mean', 'group_min_im_mean', 'group_max_im_mean', 'group_mean']\nf.write((',').join(map(str,col)) + '\\n')\nf.close()\n\nif __name__ == '__main__':\n    cpu = multiprocessing.cpu_count(); print (cpu);\n    \n    j = []\n    for s_ in range(0,len(train_files),cpu):     #train\n        for i in range(cpu):\n            i_=s_+i\n            if (i_)<len(train_files):\n                if i_ % 100 == 0:\n                    print(\"train \", i_)\n                filename = train_files.path[i_]\n                p = multiprocessing.Process(target=image_features, args=(filename,'train', train_files[\"group\"][i_], train_files[\"pic_no\"][i_],))\n                j.append(p)\n                p.start()\n    j = []\n    for s_ in range(0,len(test_files),cpu):     #test\n        for i in range(cpu):\n            i_=s_+i\n            if (i_)<len(test_files):\n                if i_ % 100 == 0:\n                    print(\"test \", i_)\n                filename = test_files.path[i_]\n                p = multiprocessing.Process(target=image_features, args=(filename,'test', test_files[\"group\"][i_], test_files[\"pic_no\"][i_],))\n                j.append(p)\n                p.start()\n    \n    while len(j) > 0: #end all jobs\n        j = [x for x in j if x.is_alive()]\n        time.sleep(1)\n        \n    df_all = pd.read_csv('data.csv', index_col=None)\n    df_all = df_all.reset_index(drop=True)\n    df_all['group_min_im_mean'] = df_all[\"group\"].map(lambda x: df_all[df_all['group']==x]['individual_im_mean'].min())\n    df_all['group_max_im_mean'] = df_all[\"group\"].map(lambda x: df_all[df_all['group']==x]['individual_im_mean'].max())\n    df_all['group_mean'] = df_all[\"group\"].map(lambda x: df_all[df_all['group']==x]['individual_im_mean'].mean())\n    df_all['a'] = df_all['individual_im_mean'] - df_all['group_min_im_mean']\n    df_all['b'] = df_all['group_max_im_mean'] - df_all['individual_im_mean']\n    df_all['c'] = df_all['group_mean'] - df_all['individual_im_mean']\n    df_all['setId'] = df_all[\"group\"].map(lambda x: x.replace('set',''))\n    df_all.to_csv('data.csv')\n    print(\"Features Ready: \", round(((time.time() - start_time)/60),2))\n    \n    X_train = df_all[df_all['tt'] == 'train']\n    X_train = X_train.sort_values(by=['setId','pic_no'], ascending=[1, 1])\n    X_train = X_train.reset_index(drop=True)\n    y_train = X_train[\"pic_no\"].values\n    X_train = X_train.drop(['path','tt','group','pic_no','setId','individual_im_mean','group_min_im_mean','group_max_im_mean','group_mean'], axis=1)\n    X_test = df_all[df_all['tt'] == 'test']\n    X_test = X_test.sort_values(by=['setId','pic_no'], ascending=[1, 1])\n    #X_test.fillna(0, inplace=True)\n    X_test = X_test.reset_index(drop=True)\n    id_test = X_test[[\"setId\",\"pic_no\"]] #.values\n    X_test = X_test.drop(['path','tt','group','pic_no','setId','individual_im_mean','group_min_im_mean','group_max_im_mean','group_mean'], axis=1)\n    rfr = ensemble.RandomForestClassifier(n_estimators = 50, n_jobs = -1, random_state = 2016, verbose = 0)\n    param_grid = {'max_depth': [6], 'max_features': [3]}\n    model = grid_search.GridSearchCV(estimator = rfr, param_grid = param_grid, n_jobs = -1, cv = 2, verbose = 0)\n    model.fit(X_train, y_train)\n    print(\"Best parameters found by grid search:\")\n    print(model.best_params_)\n    print(\"Best CV score:\", model.best_score_)\n    y_pred = model.predict_proba(X_test)\n    #y_pred = model.predict(X_test)\n    df = pd.concat((pd.DataFrame(id_test), pd.DataFrame(y_pred)), axis=1)\n    df.columns = ['setId','pic_no','day1','day2','day3','day4','day5']\n    #df.to_csv('submission2.csv',index=False)\n    f = open('submission.csv', 'w')\n    f.write('setId,day\\n')\n    setID = df.setId.unique()\n    for i in setID:\n        a = []\n        df1 = df[df['setId'] == str(i)].reset_index(drop=True)\n        for j in range(1,6):\n            df1 = df1.sort_values(by=['day'+str(j)], ascending=[0]).reset_index(drop=True)\n            #print(df1)\n            a.append(df1.pic_no[0])\n            df1 = df1[1:]\n        f.write(str(i)+\",\"+\" \".join(map(str,a))+\"\\n\")\n        #break\n    f.close()\n    print(\"Ready to submit: \", round(((time.time() - start_time)/60),2))"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": ""
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": ""
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": ""
 }
],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}}, "nbformat": 4, "nbformat_minor": 0}