{"cells":[
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "#https://www.kaggle.com/the1owl/draper-satellite-image-chronology/stitch-and-predict/run/233527\n#original by the1owl:\n\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n%matplotlib inline\nfrom PIL import Image\nfrom PIL import ImageFilter\nimport multiprocessing\nimport random; random.seed(2016);\nimport cv2\nimport re\nimport os, glob\n\nsample_sub = pd.read_csv('../input/sample_submission.csv')\ntrain_files = pd.DataFrame([[f,f.split(\"/\")[3].split(\".\")[0].split(\"_\")[0],f.split(\"/\")[3].split(\".\")[0].split(\"_\")[1]] for f in glob.glob(\"../input/train_sm/*.jpeg\")])\ntrain_files.columns = ['path', 'group', 'pic_no']\ntest_files = pd.DataFrame([[f,f.split(\"/\")[3].split(\".\")[0].split(\"_\")[0],f.split(\"/\")[3].split(\".\")[0].split(\"_\")[1]] for f in glob.glob(\"../input/test_sm/*.jpeg\")])\ntest_files.columns = ['path', 'group', 'pic_no']\nprint(len(train_files),len(test_files),len(sample_sub))\ntrain_images = train_files[train_files[\"group\"]=='set107']\ntrain_images = train_images.sort_values(by=[\"pic_no\"], ascending=[1])\nplt.rcParams['figure.figsize'] = (12.0, 12.0)\nplt.subplots_adjust(wspace=0, hspace=0)\ni_ = 0\na = []\nfor l in train_images.path:\n    im = cv2.imread(l)\n    plt.subplot(5, 2, i_+1).set_title(l)\n    plt.hist(im.ravel(),256,[0,256]); plt.axis('off')\n    a.append([im.mean(),im.max(),im.min()])\n    plt.subplot(5, 2, i_+2).set_title(l)\n    plt.imshow(cv2.cvtColor(im, cv2.COLOR_BGR2RGB)); plt.axis('off')\n    i_ += 2\nprint(a)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "im = Image.open(train_images.path.iloc[0])\nim2 = Image.open(train_images.path.iloc[1])\nim2 = im2.rotate(-15)\nim2 = im2.resize((3000, 2000), Image.ANTIALIAS)\nr,g,b = im2.split()\nmask = Image.merge(\"L\", (b,))\nim.paste(im2, (0,0,3000,2000), mask)\nplt.imshow(im); plt.axis('off')"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "train_images = train_files[train_files[\"group\"]=='set4']\ntrain_images = train_images.sort_values(by=[\"pic_no\"], ascending=[1])\nplt.rcParams['figure.figsize'] = (12.0, 12.0)\nplt.subplots_adjust(wspace=0, hspace=0)\ni_ = 0\na = []\nfor l in train_images.path:\n    im = cv2.imread(l)\n    plt.subplot(5, 2, i_+1).set_title(l)\n    plt.hist(im.ravel(),256,[0,256]); plt.axis('off')\n    a.append([im.mean(),im.max(),im.min()])\n    plt.subplot(5, 2, i_+2).set_title(l)\n    plt.imshow(cv2.cvtColor(im, cv2.COLOR_BGR2RGB)); plt.axis('off')\n    i_ += 2\nprint(a)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "os.path.getsize(\"../input/train_sm/set268_1.jpeg\")"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "import time; start_time = time.time()\nimport warnings; warnings.filterwarnings('ignore');\nimport multiprocessing\nfrom sklearn import ensemble\nfrom sklearn import pipeline, grid_search\nfrom sklearn.metrics import label_ranking_average_precision_score as lraps\n\ndef image_features(path, tt, group, pic_no):\n    im = cv2.imread(path)\n    me_ = cv2.mean(im)\n    s=[path, tt, group, pic_no, im.mean(), me_[2], me_[1], me_[0]]\n    f = open(\"data.csv\",\"a\")\n    f.write((',').join(map(str, s)) + '\\n')\n    f.close()\n    return\n\nf = open(\"data.csv\",\"w\");\ncol = ['path','tt', 'group', 'pic_no', 'individual_im_mean','rm','bm','gm']\nf.write((',').join(map(str,col)) + '\\n')\nf.close()\n\nif __name__ == '__main__':\n    cpu = multiprocessing.cpu_count(); print (cpu);\n    \n    j = []\n    for s_ in range(0,len(train_files),cpu):     #train\n        for i in range(cpu):\n            i_=s_+i\n            if (i_)<len(train_files):\n                if i_ % 100 == 0:\n                    print(\"train \", i_)\n                filename = train_files.path[i_]\n                p = multiprocessing.Process(target=image_features, args=(filename,'train', train_files[\"group\"][i_], train_files[\"pic_no\"][i_],))\n                j.append(p)\n                p.start()\n    j = []\n    for s_ in range(0,len(test_files),cpu):     #test\n        for i in range(cpu):\n            i_=s_+i\n            if (i_)<len(test_files):\n                if i_ % 100 == 0:\n                    print(\"test \", i_)\n                filename = test_files.path[i_]\n                p = multiprocessing.Process(target=image_features, args=(filename,'test', test_files[\"group\"][i_], test_files[\"pic_no\"][i_],))\n                j.append(p)\n                p.start()\n    \n    while len(j) > 0: #end all jobs\n        j = [x for x in j if x.is_alive()]\n        time.sleep(1)\n    df_all = pd.read_csv('data.csv', index_col=None)\n    df_all = df_all.reset_index(drop=True)\n    df_all['group_min_im_mean'] = df_all[\"group\"].map(lambda x: df_all[df_all['group']==x]['individual_im_mean'].min())\n    df_all['group_max_im_mean'] = df_all[\"group\"].map(lambda x: df_all[df_all['group']==x]['individual_im_mean'].max())\n    df_all['group_mean'] = df_all[\"group\"].map(lambda x: df_all[df_all['group']==x]['individual_im_mean'].mean())\n    \n    df_all['filesize'] = df_all[\"path\"].map(lambda x: os.path.getsize(x))\n    \n    df_all['a'] = df_all['individual_im_mean'] - df_all['group_min_im_mean']\n    df_all['b'] = df_all['group_max_im_mean'] - df_all['individual_im_mean']\n    df_all['c'] = df_all['group_mean'] - df_all['individual_im_mean']\n    \n    #red\n    df_all['group_min_im_mean_r'] = df_all[\"group\"].map(lambda x: df_all[df_all['group']==x]['rm'].min())\n    df_all['group_max_im_mean_r'] = df_all[\"group\"].map(lambda x: df_all[df_all['group']==x]['rm'].max())\n    df_all['group_mean_r'] = df_all[\"group\"].map(lambda x: df_all[df_all['group']==x]['rm'].mean())\n    df_all['a_r'] = df_all['rm'] - df_all['group_min_im_mean_r']\n    df_all['b_r'] = df_all['group_max_im_mean_r'] - df_all['rm']\n    df_all['c_r'] = df_all['group_mean_r'] - df_all['rm']\n    \n    #green\n    df_all['group_min_im_mean_g'] = df_all[\"group\"].map(lambda x: df_all[df_all['group']==x]['gm'].min())\n    df_all['group_max_im_mean_g'] = df_all[\"group\"].map(lambda x: df_all[df_all['group']==x]['gm'].max())\n    df_all['group_mean_g'] = df_all[\"group\"].map(lambda x: df_all[df_all['group']==x]['gm'].mean())\n    df_all['a_g'] = df_all['gm'] - df_all['group_min_im_mean_g']\n    df_all['b_g'] = df_all['group_max_im_mean_g'] - df_all['gm']\n    df_all['c_g'] = df_all['group_mean_g'] - df_all['gm']\n    \n    #blue\n    df_all['group_min_im_mean_b'] = df_all[\"group\"].map(lambda x: df_all[df_all['group']==x]['bm'].min())\n    df_all['group_max_im_mean_b'] = df_all[\"group\"].map(lambda x: df_all[df_all['group']==x]['bm'].max())\n    df_all['group_mean_b'] = df_all[\"group\"].map(lambda x: df_all[df_all['group']==x]['bm'].mean())\n    df_all['a_b'] = df_all['bm'] - df_all['group_min_im_mean_b']\n    df_all['b_b'] = df_all['group_max_im_mean_b'] - df_all['bm']\n    df_all['c_b'] = df_all['group_mean_b'] - df_all['bm']\n    \n    df_all['setId'] = df_all[\"group\"].map(lambda x: x.replace('set',''))\n    df_all.to_csv('data.csv')\n    print(\"Features Ready: \", round(((time.time() - start_time)/60),2))\n    \n    X_train = df_all[df_all['tt'] == 'train']\n    X_train = X_train.sort_values(by=['setId','pic_no'], ascending=[1, 1])\n    X_train = X_train.reset_index(drop=True)\n    y_train = X_train[\"pic_no\"].values\n    X_train = X_train.drop(['path','tt','group','pic_no'], axis=1)\n    X_test = df_all[df_all['tt'] == 'test']\n    X_test = X_test.sort_values(by=['setId','pic_no'], ascending=[1, 1])\n    #X_test.fillna(0, inplace=True)\n    X_test = X_test.reset_index(drop=True)\n    id_test = X_test[[\"setId\",\"pic_no\"]] #.values\n    X_test = X_test.drop(['path','tt','group','pic_no'], axis=1)\n    rfr = ensemble.RandomForestClassifier(n_estimators = 50, n_jobs = -1, random_state = 2016, verbose = 0)\n    param_grid = {'max_depth': [6], 'max_features': [1.0]}\n    model = grid_search.GridSearchCV(estimator = rfr, param_grid = param_grid, n_jobs = -1, cv = 3, verbose = 0)\n    model.fit(X_train, y_train)\n    print(\"Best parameters found by grid search:\")\n    print(model.best_params_)\n    print(\"Best CV score:\", model.best_score_)\n    y_pred = model.predict_proba(X_test)\n    #y_pred = model.predict(X_test)\n    df = pd.concat((pd.DataFrame(id_test), pd.DataFrame(y_pred)), axis=1)\n    df.columns = ['setId','pic_no','day1','day2','day3','day4','day5']\n    #df.to_csv('submission2.csv',index=False)\n    f = open('submission.csv', 'w')\n    f.write('setId,day\\n')\n    setID = df.setId.unique()\n    for i in setID:\n        a = []\n        df1 = df[df['setId'] == str(i)].reset_index(drop=True)\n        for j in range(1,6):\n            df1 = df1.sort_values(by=['day'+str(j)], ascending=[0]).reset_index(drop=True)\n            #print(df1)\n            a.append(df1.pic_no[0])\n            df1 = df1[1:]\n        f.write(str(i)+\",\"+\" \".join(map(str,a))+\"\\n\")\n        #break\n    f.close()\n    print(\"Ready to submit: \", round(((time.time() - start_time)/60),2))"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": ""
 }
],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}}, "nbformat": 4, "nbformat_minor": 0}