{"metadata":{"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","name":"python","mimetype":"text/x-python","version":"3.6.3","pygments_lexer":"ipython3","nbconvert_exporter":"python"},"kernelspec":{"display_name":"Python 3","name":"python3","language":"python"}},"cells":[{"outputs":[],"metadata":{"_uuid":"000ca54c5ae104339118737515a0c249e6a8b9e0","collapsed":true,"_cell_guid":"a7c119aa-bd53-4335-8b4b-fb52f3e87649"},"cell_type":"code","execution_count":null,"source":"import numpy as np\nimport pandas as pd\nimport io\nimport bson\nimport cv2\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm_notebook\nimport concurrent.futures\nfrom multiprocessing import cpu_count"},{"outputs":[],"metadata":{"_uuid":"939b5df9891321a696b5b7d7ad77c6c10f5feff8","collapsed":true,"_cell_guid":"be9ad405-6d7c-4c0c-a4d8-bdbedfe6b291"},"cell_type":"code","execution_count":null,"source":"num_images = 7000\nim_size = 90\nnum_cpus = cpu_count()"},{"outputs":[],"metadata":{"_uuid":"f287029a9bc0bf8349295e4451d8b14212fc3a45","_cell_guid":"adf7ff04-ce7a-4fcb-87e0-d71fb0fd4868"},"cell_type":"code","execution_count":null,"source":"def imread(buf):\n    return cv2.imdecode(np.frombuffer(buf, np.uint8), cv2.IMREAD_ANYCOLOR)\n\ndef img2feat(im):\n    x = cv2.resize(im, (im_size, im_size), interpolation=cv2.INTER_AREA)\n    return np.float32(x) / 255\n\nX = np.empty((num_images, im_size, im_size, 3), dtype=np.float32)\ny = []\n\ndef load_image(pic, target, bar):\n    picture = imread(pic)\n    x = img2feat(picture)\n    bar.update()\n    \n    return x, target\n\nbar = tqdm_notebook(total=num_images)\nwith open('../input/train.bson', 'rb') as f, \\\n        concurrent.futures.ThreadPoolExecutor(num_cpus) as executor:\n\n    data = bson.decode_file_iter(f)\n    delayed_load = []\n\n    i = 0\n    try:\n        for c, d in enumerate(data):\n            print(d)\n            target = d['category_id']\n            for e, pic in enumerate(d['imgs']):\n                delayed_load.append(executor.submit(load_image, pic['picture'], target, bar))\n                \n                i = i + 1\n\n                if i >= num_images:\n                    raise IndexError()\n\n    except IndexError:\n        pass;\n    \n    for i, future in enumerate(concurrent.futures.as_completed(delayed_load)):\n        x, target = future.result()\n        \n        X[i] = x\n        y.append(target)"},{"outputs":[],"metadata":{"_uuid":"d2a79afee31d2b8cf9940b407659ada232b6688d","collapsed":true,"_cell_guid":"e50158ec-afa7-4c2e-a39b-3d2a98c468aa"},"cell_type":"code","execution_count":null,"source":"X.shape, len(y)"},{"outputs":[],"metadata":{"_uuid":"f0a550c5ff4ab3463b2ea15a439a02ab0b3da6b8","collapsed":true,"_cell_guid":"f7ddaf41-1883-4f55-b58b-bf350229b3e8"},"cell_type":"code","execution_count":null,"source":"y = pd.Series(y)\n\nnum_classes =500 \nvalid_targets = set(y.value_counts().index[:num_classes-1].tolist())\nvalid_y = y.isin(valid_targets)\n\ny[~valid_y] = -1\n\nmax_acc = valid_y.mean()\nprint(max_acc)"},{"outputs":[],"metadata":{"_uuid":"27ce33e9cb77eb653663985f0cfe38fa7d34d9a7","collapsed":true,"_cell_guid":"63d0996b-fdbc-45c7-b102-7284ed51390f"},"cell_type":"code","execution_count":null,"source":"y, rev_labels = pd.factorize(y)"},{"outputs":[],"metadata":{"_uuid":"35ddb3061e3528bea2f5e00e36741d8d4251f89d","collapsed":true,"_cell_guid":"c7221e97-c757-4610-819e-000aa6d50793"},"cell_type":"code","execution_count":null,"source":"from keras.layers import Conv2D, MaxPooling2D, Dropout, Dense, Flatten\n#from keras.layers import Dense, Dropout, Activation, Flatten\nfrom keras.models import Sequential\nfrom keras.optimizers import Adam\n\nmodel = Sequential()\nmodel.add(Conv2D(16, 3 , activation='relu', padding='same', input_shape=X.shape[1:]))\nmodel.add(Conv2D(16, 3, activation='relu', padding='same'))\nmodel.add(MaxPooling2D(2))\nmodel.add(Dropout(0.25))\n\nmodel.add(Conv2D(32, 3, activation='relu', padding='same'))\nmodel.add(Conv2D(32, 3, activation='relu', padding='same'))\nmodel.add(MaxPooling2D(2))\nmodel.add(Dropout(0.25))\n\nmodel.add(Flatten())\nmodel.add(Dense(num_classes, activation='relu'))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(num_classes, activation='softmax'))\n\n\nopt = Adam(lr=0.001)\n\nmodel.compile('adam', 'sparse_categorical_crossentropy', metrics=['accuracy'])\n\nmodel.summary()\n\n\nmodel.fit(X, y, validation_split=0.1, epochs=3)\n\n\nmodel.save_weights('model.h5')"},{"outputs":[],"metadata":{"_uuid":"c5f309ae6044d5014534dea9436373c42fa8e289","collapsed":true,"_cell_guid":"ebb3d0a2-9ac3-439f-84b8-5c359f0e1692"},"cell_type":"code","execution_count":null,"source":"submission = pd.read_csv('../input/sample_submission.csv', index_col='_id')\n\nmost_frequent_guess =1000018296\nsubmission['category_id'] = most_frequent_guess \n\nnum_images_test = 840000\nwith open('../input/test.bson', 'rb') as f, \\\n         concurrent.futures.ThreadPoolExecutor(num_cpus) as executor:\n\n    data = bson.decode_file_iter(f)\n\n    future_load = []\n\n    for i,d in enumerate(data):\n        if i >= num_images_test:\n              break\n        future_load.append(executor.submit(load_image, d['imgs'][0]['picture'], d['_id'], bar))\n        \n        print(\"Starting future processing\")\n    for future in concurrent.futures.as_completed(future_load):\n        x, _id = future.result()\n        \n        y_cat = rev_labels[np.argmax(model.predict(x[None])[0])]\n        if y_cat == -1:\n            y_cat = most_frequent_guess\n\n        bar.update()\n        submission.loc[_id, 'category_id'] = y_cat\nprint('Finished')"},{"outputs":[],"metadata":{"_uuid":"0047a8d7b1b0f7667f299ac5c39d8e45862cfdbb","collapsed":true,"_cell_guid":"c328b383-2bbd-495d-920c-f8a06544036d"},"cell_type":"code","execution_count":null,"source":"submission.to_csv('new_submission.csv.gz', compression='gzip')"}],"nbformat_minor":1,"nbformat":4}