{"cells": [{"cell_type": "code", "execution_count": null, "outputs": [], "source": ["# This Python 3 environment comes with many helpful analytics libraries installed\n", "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n", "# For example, here's several helpful packages to load in \n", "\n", "import os\n", "from pathlib import Path\n", "import multiprocessing as mp\n", "\n", "import numpy as np # linear algebra\n", "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "\n", "from skimage.data import imread\n", "from sklearn.ensemble import RandomForestClassifier\n", "\n", "# Input data files are available in the \"../input/\" directory.\n", "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n", "\n", "from subprocess import check_output\n", "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n", "\n", "# Any results you write to the current directory are saved as output."], "metadata": {"_cell_guid": "60ad1bf0-11fd-4d10-a38a-8abead246b90", "_uuid": "7dc1c3c83093d804a2f396b14cd73d7c9a1e21fe"}}, {"cell_type": "code", "execution_count": null, "outputs": [], "source": ["input_path = Path('../input')\n", "train_path = input_path / 'train'\n", "test_path = input_path / 'test'"], "metadata": {"_cell_guid": "ef13e211-e719-473c-abca-4a78837b7b1a", "_uuid": "cb14a753d32dacec06c2b9767aa7e9f6befa1fb2", "collapsed": true}}, {"cell_type": "code", "execution_count": null, "outputs": [], "source": ["cameras = os.listdir(train_path)\n", "\n", "train_images = []\n", "for camera in cameras:\n", "    for fname in sorted(os.listdir(train_path / camera)):\n", "        train_images.append((camera, fname))\n", "\n", "train = pd.DataFrame(train_images, columns=['camera', 'fname'])\n", "print(train.shape)\n", "train.sample(5)"], "metadata": {"scrolled": true, "_cell_guid": "36327e90-b9fd-4323-b860-c382a42f61d0", "_uuid": "14cf6f0df305ffa690e1b1068535791b01fbcd68"}}, {"cell_type": "code", "execution_count": null, "outputs": [], "source": ["test_images = []\n", "for fname in sorted(os.listdir(test_path)):\n", "    test_images.append(fname)\n", "\n", "test = pd.DataFrame(test_images, columns=['fname'])\n", "print(test.shape)\n", "test.head(5)"], "metadata": {"_cell_guid": "5293c67a-6040-4125-8362-39bdf153914e", "_uuid": "f1eb110d8b012cdb110a61ba17ca0f23f3502d86"}}, {"cell_type": "code", "execution_count": null, "outputs": [], "source": ["def color_stats(q, iolock):\n", "    \n", "    while True:\n", "        \n", "        img_path = q.get()\n", "        if img_path is None:\n", "            break\n", "            \n", "        if type(img_path) is tuple:\n", "            img = imread(train_path / img_path[0] / img_path[1])\n", "            key = img_path[1]\n", "        else:\n", "            img = imread(test_path / img_path)\n", "            key = img_path\n", "\n", "        # Some images read return info in a 2nd dim. We only want the first dim.\n", "        if img.shape == (2,):\n", "            img = img[0]\n", "\n", "        color_info[key] = (img[:, :, 0].mean(), img[:, :, 1].mean(), img[:, :, 2].mean(),\n", "                           img[:, :, 0].std(),  img[:, :, 1].std(),  img[:, :, 2].std())"], "metadata": {"_cell_guid": "9d325fb5-a180-4bb6-9afe-b0b6beabf09d", "_uuid": "afa04be47c0896c4b848dde201c5d29ac7a9a43c", "collapsed": true}}, {"cell_type": "code", "execution_count": null, "outputs": [], "source": ["cols = ['a0', 'a1', 'a2', 's0', 's1', 's2']\n", "\n", "for col in cols:\n", "    train[col] = None\n", "    test[col] = None"], "metadata": {"_cell_guid": "b31dee68-c9cf-4846-b833-a146fc64ea35", "_uuid": "67665bc0ae916c0bbb09398b74c4bcd6dcf71397", "collapsed": true}}, {"cell_type": "code", "execution_count": null, "outputs": [], "source": ["NCORE = 8\n", "\n", "color_info = mp.Manager().dict()\n", "\n", "# Using a queue since the image read is a bottleneck\n", "q = mp.Queue(maxsize=NCORE)\n", "iolock = mp.Lock()\n", "pool = mp.Pool(NCORE, initializer=color_stats, initargs=(q, iolock))\n", "\n", "for i in train_images:\n", "    q.put(i)  # blocks until q below its max size\n", "\n", "for i in test_images:\n", "    q.put(i)  # blocks until q below its max size\n", "    \n", "# tell workers we're done\n", "for _ in range(NCORE):  \n", "    q.put(None)\n", "pool.close()\n", "pool.join()\n", "\n", "color_info = dict(color_info)"], "metadata": {"_cell_guid": "cb58a4ba-fcd5-4af7-9290-7683576a86c3", "_uuid": "6a3c34861ac79ede9a089aa0447195b686059a22", "collapsed": true}}, {"cell_type": "code", "execution_count": null, "outputs": [], "source": ["for n, col in enumerate(cols):\n", "    train[col] = train['fname'].apply(lambda x: color_info[x][n])\n", "    test[col] = test['fname'].apply(lambda x: color_info[x][n])"], "metadata": {"_cell_guid": "4014789a-4789-4543-9e9f-1cd7cdc0b914", "_uuid": "5e75d21c7550b513ff35d6d3908fa037affa45ae", "collapsed": true}}, {"cell_type": "code", "execution_count": null, "outputs": [], "source": ["train.sample(5)"], "metadata": {"_cell_guid": "a56e3882-df02-439d-82f4-025bcb7c11ff", "_uuid": "865b47cbf9959f160b808927c3cf16769923ea1b"}}, {"cell_type": "code", "execution_count": null, "outputs": [], "source": ["test.sample(5)"], "metadata": {"_cell_guid": "2aa72817-38f1-4a50-95da-0c5b0992e6a5", "_uuid": "b3649d5b210cb9a779bf70f8d9bc676bea23c9be"}}, {"cell_type": "code", "execution_count": null, "outputs": [], "source": ["y = train['camera'].values\n", "X_train = train[cols].values\n", "X_test = test[cols].values"], "metadata": {"_cell_guid": "53ba1145-9dac-4a47-a64a-140c637e25da", "_uuid": "aac1ba0a32fad86a992ba1c30aad45a888971398", "collapsed": true}}, {"cell_type": "code", "execution_count": null, "outputs": [], "source": ["clf = RandomForestClassifier(n_estimators=200)\n", "clf.fit(X_train, y)"], "metadata": {"_cell_guid": "683f1f9c-9980-453f-8056-4bb9ebd19208", "_uuid": "09804d7c6c035680b706e7d45788a76827f1268a"}}, {"cell_type": "code", "execution_count": null, "outputs": [], "source": ["y_pred = clf.predict(X_test)\n", "clueless = pd.read_csv(input_path / 'sample_submission.csv', index_col='fname')\n", "clueless['camera'] = y_pred\n", "clueless.to_csv('clueless.csv')"], "metadata": {"_cell_guid": "e0be27ae-9da7-4017-a187-411c476afd3c", "_uuid": "36a0fa3086766ea77f6b2b02dbf0280773343041", "collapsed": true}}], "nbformat_minor": 1, "nbformat": 4, "metadata": {"kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}, "language_info": {"nbconvert_exporter": "python", "pygments_lexer": "ipython3", "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "codemirror_mode": {"name": "ipython", "version": 3}, "version": "3.6.3"}}}