{"nbformat": 4, "cells": [{"cell_type": "markdown", "metadata": {"_uuid": "de87e4b1b18f48d3672221485a6577293ec10926", "_cell_guid": "077c0bd7-bc31-4528-a747-6b0c6e66ebe4"}, "source": ["This notebook is just a (very) small improvement over most common baseline.\n", "\n", "It loads a few images from train and resize it to 8x8 pixels to generate a 64 (8 x 8) feature vector.\n", "\n", "Then, it uses KNN to find the most similar image on test set.\n", "\n", "Unfortunatelly, due to limitations on Kernel, only a few test images are classified."]}, {"cell_type": "code", "outputs": [], "metadata": {"_uuid": "99b9a425db0c128d80fda1ecc9b42f952a77c93c", "collapsed": true, "_cell_guid": "7d7afe89-0c7e-43cb-9ba6-0e93db30b4bd"}, "execution_count": null, "source": ["import numpy as np\n", "import pandas as pd\n", "import io\n", "import bson\n", "import cv2\n", "import matplotlib.pyplot as plt\n", "from tqdm import tqdm_notebook\n", "import concurrent.futures\n", "from multiprocessing import cpu_count"]}, {"cell_type": "code", "outputs": [], "metadata": {"_uuid": "25a8edf7b1bf5b9d0726b3ede0c796717ab720ab", "collapsed": true, "_cell_guid": "ea4565d0-2922-4d0d-b7c5-f970cbdf1cc1"}, "execution_count": null, "source": ["num_images = 200000\n", "im_size = 16\n", "num_cpus = cpu_count()"]}, {"cell_type": "code", "outputs": [], "metadata": {"_uuid": "4a5081a65e4d22316b7306cddd80332712b21b0d", "_cell_guid": "82f5d889-b14d-4cb7-86b0-5bd726304d8b"}, "execution_count": null, "source": ["def imread(buf):\n", "    return cv2.imdecode(np.frombuffer(buf, np.uint8), cv2.IMREAD_ANYCOLOR)\n", "\n", "def img2feat(im):\n", "    x = cv2.resize(im, (im_size, im_size), interpolation=cv2.INTER_AREA)\n", "    return np.float32(x) / 255\n", "\n", "X = np.empty((num_images, im_size, im_size, 3), dtype=np.float32)\n", "y = []\n", "\n", "def load_image(pic, target, bar):\n", "    picture = imread(pic)\n", "    x = img2feat(picture)\n", "    bar.update()\n", "    \n", "    return x, target\n", "\n", "bar = tqdm_notebook(total=num_images)\n", "with open('../input/train.bson', 'rb') as f, \\\n", "        concurrent.futures.ThreadPoolExecutor(num_cpus) as executor:\n", "\n", "    data = bson.decode_file_iter(f)\n", "    delayed_load = []\n", "\n", "    i = 0\n", "    try:\n", "        for c, d in enumerate(data):\n", "            target = d['category_id']\n", "            for e, pic in enumerate(d['imgs']):\n", "                delayed_load.append(executor.submit(load_image, pic['picture'], target, bar))\n", "                \n", "                i = i + 1\n", "\n", "                if i >= num_images:\n", "                    raise IndexError()\n", "\n", "    except IndexError:\n", "        pass;\n", "    \n", "    for i, future in enumerate(concurrent.futures.as_completed(delayed_load)):\n", "        x, target = future.result()\n", "        \n", "        X[i] = x\n", "        y.append(target)"]}, {"cell_type": "code", "outputs": [], "metadata": {"_uuid": "39c439b021cfe966165d78d29486f97ea9b84c5b", "_cell_guid": "5cfda60d-d8c8-481c-833b-85f170564cc6"}, "execution_count": null, "source": ["X.shape, len(y)"]}, {"cell_type": "code", "outputs": [], "metadata": {"_uuid": "56969c620e60a3e55f0684b412fe1f6b7590ba01", "_cell_guid": "34c9bf08-9f89-461f-b74f-1f24291a5829"}, "execution_count": null, "source": ["y = pd.Series(y)\n", "\n", "num_classes = 500  # This will reduce the max accuracy to about 0.75\n", "\n", "# Now we must find the most `num_classes-1` frequent classes\n", "# (there will be an aditional 'other' class)\n", "valid_targets = set(y.value_counts().index[:num_classes-1].tolist())\n", "valid_y = y.isin(valid_targets)\n", "\n", "# Set other classes to -1\n", "y[~valid_y] = -1\n", "\n", "max_acc = valid_y.mean()\n", "print(max_acc)"]}, {"cell_type": "markdown", "metadata": {"_uuid": "f49981818f17dcfc127b0e4cb6b28f7bf3003c42", "_cell_guid": "2951efcb-a616-48c8-ad20-6c2bf2317a19"}, "source": ["Note that the max accuracy reported above is greater than ~0.75 reported [here](http://https://www.kaggle.com/bguberfain/naive-statistics) due to smaller train set."]}, {"cell_type": "code", "outputs": [], "metadata": {"_uuid": "1975ac67714c59998a9c8e587c7fd5554fd7597b", "_cell_guid": "3c137f38-d10d-49e7-a9e5-b1628d2e0567"}, "execution_count": null, "source": ["# Now we categorize the dataframe\n", "y, rev_labels = pd.factorize(y)"]}, {"cell_type": "code", "outputs": [], "metadata": {"_uuid": "96da9277c9c7fed8cb871a3c434be783c9eb10f1", "_cell_guid": "e6368858-f2ce-4426-85af-89219c651a06"}, "execution_count": null, "source": ["# Train a simple NN\n", "from keras.layers import Conv2D, MaxPooling2D, Dense, Flatten\n", "from keras.models import Sequential\n", "from keras.optimizers import Adam\n", "\n", "model = Sequential()\n", "model.add(Conv2D(16, 3, activation='relu', padding='same', input_shape=X.shape[1:]))\n", "model.add(Conv2D(16, 3, activation='relu', padding='same'))\n", "model.add(MaxPooling2D(2))\n", "model.add(Conv2D(32, 3, activation='relu', padding='same'))\n", "model.add(Conv2D(32, 3, activation='relu', padding='same'))\n", "model.add(MaxPooling2D(2))\n", "model.add(Flatten())\n", "model.add(Dense(num_classes, activation='relu'))\n", "model.add(Dense(num_classes, activation='softmax'))\n", "\n", "\n", "opt = Adam(lr=0.01)\n", "\n", "model.compile('adam', 'sparse_categorical_crossentropy', metrics=['accuracy'])\n", "\n", "model.summary()"]}, {"cell_type": "code", "outputs": [], "metadata": {"_uuid": "7683eaaa3211bdf9a17ca675bf6af5b1dd1ccd24", "_cell_guid": "663b6abe-c16e-445a-b55e-6895087dc321", "scrolled": true}, "execution_count": null, "source": ["model.fit(X, y, validation_split=0.1, epochs=2)"]}, {"cell_type": "code", "outputs": [], "metadata": {"_uuid": "290914eec155b2c14ae06309436235063c844339", "collapsed": true, "_cell_guid": "8980a025-38b1-4f1b-8860-ee413631c434"}, "execution_count": null, "source": ["model.save_weights('model.h5')  #You can download this model and run whole test localy"]}, {"cell_type": "markdown", "metadata": {"_uuid": "3dcc4c6c6261ac47970b5d0e33eaa0b9066a3cb9", "_cell_guid": "4b990f20-cb25-4be0-92bc-df033ada90ad"}, "source": ["Now we evaluate the test set using the previous trained model."]}, {"cell_type": "code", "outputs": [], "metadata": {"_uuid": "5811926ec436c8820ccaa468fc2ac6a09dfbd3bc", "_cell_guid": "4425bdc4-dd25-40c8-a26e-fe2eb11d3398"}, "execution_count": null, "source": ["submission = pd.read_csv('../input/sample_submission.csv', index_col='_id')\n", "\n", "most_frequent_guess = 1000018296\n", "submission['category_id'] = most_frequent_guess # Most frequent guess"]}, {"cell_type": "code", "outputs": [], "metadata": {"_uuid": "bc6ce0ae1620a9ac5a481d0a7e4a0f0e25510fd1", "_cell_guid": "f7465e07-cc74-4371-9cc1-3639641f4110"}, "execution_count": null, "source": ["num_images_test = 800000  # We only have time for a few test images..\n", "\n", "bar = tqdm_notebook(total=num_images_test * 2)\n", "with open('../input/test.bson', 'rb') as f, \\\n", "         concurrent.futures.ThreadPoolExecutor(num_cpus) as executor:\n", "\n", "    data = bson.decode_file_iter(f)\n", "\n", "    future_load = []\n", "    \n", "    for i,d in enumerate(data):\n", "        if i >= num_images_test:\n", "            break\n", "        future_load.append(executor.submit(load_image, d['imgs'][0]['picture'], d['_id'], bar))\n", "\n", "    print(\"Starting future processing\")\n", "    for future in concurrent.futures.as_completed(future_load):\n", "        x, _id = future.result()\n", "        \n", "        y_cat = rev_labels[np.argmax(model.predict(x[None])[0])]\n", "        if y_cat == -1:\n", "            y_cat = most_frequent_guess\n", "\n", "        bar.update()\n", "        submission.loc[_id, 'category_id'] = y_cat\n", "print('Finished')"]}, {"cell_type": "code", "outputs": [], "metadata": {"_uuid": "f806114fc478e841392589d4f7e2f0b828b20311", "_cell_guid": "dd34386c-f28a-42d7-a3be-f09126fd61c7"}, "execution_count": null, "source": ["submission.to_csv('new_submission.csv.gz', compression='gzip')"]}], "metadata": {"kernelspec": {"language": "python", "name": "python3", "display_name": "Python 3"}, "language_info": {"file_extension": ".py", "codemirror_mode": {"name": "ipython", "version": 3}, "pygments_lexer": "ipython3", "version": "3.6.1", "name": "python", "nbconvert_exporter": "python", "mimetype": "text/x-python"}}, "nbformat_minor": 1}