{"cells":[
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "#https://www.kaggle.com/the1owl/draper-satellite-image-chronology/stitch-and-predict/run/233527\n#original by the1owl:\n\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n%matplotlib inline\nfrom PIL import Image\nfrom PIL import ImageFilter\nimport multiprocessing\nimport random; random.seed(2016);\nimport cv2\nimport re\nimport os, glob\n\nsample_sub = pd.read_csv('../input/sample_submission.csv')\ntrain_files = pd.DataFrame([[f,f.split(\"/\")[3].split(\".\")[0].split(\"_\")[0],f.split(\"/\")[3].split(\".\")[0].split(\"_\")[1]] for f in glob.glob(\"../input/train_sm/*.jpeg\")])\ntrain_files.columns = ['path', 'group', 'pic_no']\ntest_files = pd.DataFrame([[f,f.split(\"/\")[3].split(\".\")[0].split(\"_\")[0],f.split(\"/\")[3].split(\".\")[0].split(\"_\")[1]] for f in glob.glob(\"../input/test_sm/*.jpeg\")])\ntest_files.columns = ['path', 'group', 'pic_no']\nprint(len(train_files),len(test_files),len(sample_sub))\ntrain_images = train_files[train_files[\"group\"]=='set107']\ntrain_images = train_images.sort_values(by=[\"pic_no\"], ascending=[1])\nplt.rcParams['figure.figsize'] = (12.0, 12.0)\nplt.subplots_adjust(wspace=0, hspace=0)\ni_ = 0\na = []\nfor l in train_images.path:\n    im = cv2.imread(l)\n    plt.subplot(5, 2, i_+1).set_title(l)\n    plt.hist(im.ravel(),256,[0,256]); plt.axis('off')\n    a.append([im.mean(),im.max(),im.min()])\n    plt.subplot(5, 2, i_+2).set_title(l)\n    plt.imshow(cv2.cvtColor(im, cv2.COLOR_BGR2RGB)); plt.axis('off')\n    i_ += 2\nprint(a)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "im = Image.open(train_images.path.iloc[0])\nim2 = Image.open(train_images.path.iloc[1])\nim2 = im2.rotate(-15)\nim2 = im2.resize((3000, 2000), Image.ANTIALIAS)\nr,g,b = im2.split()\nmask = Image.merge(\"L\", (b,))\nim.paste(im2, (0,0,3000,2000), mask)\nplt.imshow(im); plt.axis('off')"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "train_images = train_files[train_files[\"group\"]=='set4']\ntrain_images = train_images.sort_values(by=[\"pic_no\"], ascending=[1])\nplt.rcParams['figure.figsize'] = (12.0, 12.0)\nplt.subplots_adjust(wspace=0, hspace=0)\ni_ = 0\na = []\nfor l in train_images.path:\n    im = cv2.imread(l)\n    plt.subplot(5, 2, i_+1).set_title(l)\n    plt.hist(im.ravel(),256,[0,256]); plt.axis('off')\n    a.append([im.mean(),im.max(),im.min()])\n    plt.subplot(5, 2, i_+2).set_title(l)\n    plt.imshow(cv2.cvtColor(im, cv2.COLOR_BGR2RGB)); plt.axis('off')\n    i_ += 2\nprint(a)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "import time; start_time = time.time()\nimport warnings; warnings.filterwarnings('ignore');\nimport multiprocessing\nfrom sklearn import ensemble\nfrom sklearn import pipeline, grid_search\nfrom sklearn.metrics import label_ranking_average_precision_score as lraps\n\ndef image_features(path, tt, group, pic_no):\n    im = cv2.imread(path)\n    me_ = cv2.mean(im)\n    s=[path, tt, group, pic_no, im.mean(), me_[2], me_[1], me_[0]]\n    f = open(\"data.csv\",\"a\")\n    f.write((',').join(map(str, s)) + '\\n')\n    f.close()\n    return\n\nf = open(\"data.csv\",\"w\");\ncol = ['path','tt', 'group', 'pic_no', 'individual_im_mean','rm','bm','gm']\nf.write((',').join(map(str,col)) + '\\n')\nf.close()\n\nif __name__ == '__main__':\n    cpu = multiprocessing.cpu_count(); print (cpu);\n    \n    j = []\n    for s_ in range(0,len(train_files),cpu):     #train\n        for i in range(cpu):\n            i_=s_+i\n            if (i_)<len(train_files):\n                if i_ % 100 == 0:\n                    print(\"train \", i_)\n                filename = train_files.path[i_]\n                \n                p = multiprocessing.Process(target=image_features, args=(filename,'train', train_files[\"group\"][i_], train_files[\"pic_no\"][i_],))\n                j.append(p)\n                p.start()\n    j = []\n    for s_ in range(0,len(test_files),cpu):     #test\n        for i in range(cpu):\n            i_=s_+i\n            if (i_)<len(test_files):\n                if i_ % 100 == 0:\n                    print(\"test \", i_)\n                filename = test_files.path[i_]\n                p = multiprocessing.Process(target=image_features, args=(filename,'test', test_files[\"group\"][i_], test_files[\"pic_no\"][i_],))\n                j.append(p)\n                p.start()\n    \n    while len(j) > 0: #end all jobs\n        j = [x for x in j if x.is_alive()]\n        time.sleep(1)\n    "
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": ""
 }
],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}}, "nbformat": 4, "nbformat_minor": 0}