{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Problem description\n# --------------------\nWe pose the problem of the classification of UltraMNIST digits. UltraMNIST dataset comprises very large-scale images, each of 4000x4000 pixels with 3-5 digits per image. Each of these digits has been extracted from the original MNIST dataset. \n\n**Your task is to predict the sum of the digits per image, and this number can be anything from 0 to 27.**","metadata":{}},{"cell_type":"markdown","source":"## Install required packages","metadata":{}},{"cell_type":"code","source":"!pip install imutils -q\n","metadata":{"execution":{"iopub.status.busy":"2022-03-11T15:05:10.733199Z","iopub.execute_input":"2022-03-11T15:05:10.733616Z","iopub.status.idle":"2022-03-11T15:05:24.378342Z","shell.execute_reply.started":"2022-03-11T15:05:10.733518Z","shell.execute_reply":"2022-03-11T15:05:24.377483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import required packages","metadata":{}},{"cell_type":"code","source":"from skimage import measure\nimport numpy as np\nfrom PIL import Image as Img\nfrom IPython.display import Image\nimport cv2\nimport imutils\nimport matplotlib.pyplot as plt\nfrom imutils import contours\n\nimport tensorflow\nimport pandas as pd\nfrom tensorflow import keras\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Conv2D, MaxPool2D, Flatten\nimport matplotlib.pyplot as plt\n\nfrom keras.datasets import mnist\nfrom tensorflow.keras.utils import to_categorical\n","metadata":{"execution":{"iopub.status.busy":"2022-03-11T15:05:24.381925Z","iopub.execute_input":"2022-03-11T15:05:24.382238Z","iopub.status.idle":"2022-03-11T15:05:33.074099Z","shell.execute_reply.started":"2022-03-11T15:05:24.382204Z","shell.execute_reply":"2022-03-11T15:05:33.073151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Clean image","metadata":{}},{"cell_type":"code","source":"def cleaner(img):\n    img=img/255\n    for i in range(4):\n        for j in range(4):\n            border = img[1000*i:1000*(i+1),min(1000*j,3999)].sum() +img[1000*i:1000*(i+1),1000*(j+1)-1].sum() +img[min(1000*i,3999),1000*j:1000*(j+1)].sum()+img[1000*(i+1)-1,1000*j:1000*(j+1)].sum()\n            if border>=2000:\n                img[1000*i:1000*(i+1),1000*j:1000*(j+1)]=np.abs(img[1000*i:1000*(i+1),1000*j:1000*(j+1)]-1)\n    blurred = cv2.GaussianBlur(img, (9, 9), 0)\n    thresh = cv2.dilate(blurred, None, iterations=12)\n    thresh = cv2.erode(thresh, None, iterations=4)\n    return thresh","metadata":{"execution":{"iopub.status.busy":"2022-03-11T15:05:33.075563Z","iopub.execute_input":"2022-03-11T15:05:33.075814Z","iopub.status.idle":"2022-03-11T15:05:33.086894Z","shell.execute_reply.started":"2022-03-11T15:05:33.075786Z","shell.execute_reply":"2022-03-11T15:05:33.085773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Resize image","metadata":{}},{"cell_type":"code","source":"def resize_image(img, size=(28,28)):\n\n    h, w = img.shape[:2]\n    c = img.shape[2] if len(img.shape)>2 else 1\n\n    if h == w: \n        return cv2.resize(img, size, cv2.INTER_AREA)\n\n    dif = h if h > w else w\n\n    interpolation = cv2.INTER_AREA if dif > (size[0]+size[1])//2 else cv2.INTER_CUBIC\n\n    x_pos = (dif - w)//2\n    y_pos = (dif - h)//2\n\n    if len(img.shape) == 2:\n        mask = np.zeros((dif, dif), dtype=img.dtype)\n        mask[y_pos:y_pos+h, x_pos:x_pos+w] = img[:h, :w]\n    else:\n        mask = np.zeros((dif, dif, c), dtype=img.dtype)\n        mask[y_pos:y_pos+h, x_pos:x_pos+w, :] = img[:h, :w, :]\n\n    return cv2.resize(mask, size, interpolation)","metadata":{"execution":{"iopub.status.busy":"2022-03-11T15:05:33.090432Z","iopub.execute_input":"2022-03-11T15:05:33.090710Z","iopub.status.idle":"2022-03-11T15:05:33.115482Z","shell.execute_reply.started":"2022-03-11T15:05:33.090678Z","shell.execute_reply":"2022-03-11T15:05:33.114368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load a sample of image","metadata":{}},{"cell_type":"code","source":"img_original = cv2.imread(\"../input/ultra-mnist/train/aalsrckbqu.jpeg\", 0)\n#img_original = cv2.imread(\"../input/ultra-mnist/test/aaavulstaw.jpeg\", 0)\n\nimg_cleaned = cleaner(img_original)","metadata":{"execution":{"iopub.status.busy":"2022-03-11T15:12:41.582229Z","iopub.execute_input":"2022-03-11T15:12:41.582542Z","iopub.status.idle":"2022-03-11T15:12:42.311868Z","shell.execute_reply.started":"2022-03-11T15:12:41.582511Z","shell.execute_reply":"2022-03-11T15:12:42.311156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(img_original)","metadata":{"execution":{"iopub.status.busy":"2022-03-11T15:12:43.624257Z","iopub.execute_input":"2022-03-11T15:12:43.625377Z","iopub.status.idle":"2022-03-11T15:12:45.412729Z","shell.execute_reply.started":"2022-03-11T15:12:43.625335Z","shell.execute_reply":"2022-03-11T15:12:45.411818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(img_cleaned)","metadata":{"execution":{"iopub.status.busy":"2022-03-11T15:12:46.823221Z","iopub.execute_input":"2022-03-11T15:12:46.823531Z","iopub.status.idle":"2022-03-11T15:12:48.616120Z","shell.execute_reply.started":"2022-03-11T15:12:46.823494Z","shell.execute_reply":"2022-03-11T15:12:48.615215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = measure.label(img_cleaned,background = 0)\nprint (labels.shape)","metadata":{"execution":{"iopub.status.busy":"2022-03-11T15:12:51.025245Z","iopub.execute_input":"2022-03-11T15:12:51.025541Z","iopub.status.idle":"2022-03-11T15:12:51.185284Z","shell.execute_reply.started":"2022-03-11T15:12:51.025513Z","shell.execute_reply":"2022-03-11T15:12:51.183937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = measure.label(img_cleaned,background = 0)\nmask = np.zeros(img_cleaned.shape, dtype=\"uint8\")\n\nfor label in np.unique(labels):\n    if label == 0:\n        continue\n\n    labelMask = np.zeros(img_cleaned.shape, dtype=\"uint8\")\n    labelMask[labels == label] = 255\n    numPixels = cv2.countNonZero(labelMask)\n    if numPixels > 300:\n        mask = cv2.add(mask, labelMask)","metadata":{"execution":{"iopub.status.busy":"2022-03-11T15:13:10.320131Z","iopub.execute_input":"2022-03-11T15:13:10.320723Z","iopub.status.idle":"2022-03-11T15:13:10.940273Z","shell.execute_reply.started":"2022-03-11T15:13:10.320687Z","shell.execute_reply":"2022-03-11T15:13:10.939304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnts = cv2.findContours(mask.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\ncnts = imutils.grab_contours(cnts)\ncnts = contours.sort_contours(cnts)[0]\nprint(f'Found {len(cnts)} contours')\n","metadata":{"execution":{"iopub.status.busy":"2022-03-11T15:13:11.318595Z","iopub.execute_input":"2022-03-11T15:13:11.319443Z","iopub.status.idle":"2022-03-11T15:13:11.346798Z","shell.execute_reply.started":"2022-03-11T15:13:11.319399Z","shell.execute_reply":"2022-03-11T15:13:11.346158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"backtorgb = cv2.cvtColor(img_cleaned.astype('float32'), cv2.COLOR_GRAY2RGB)\nbbox_list = []\n\nfor (i, c) in enumerate(cnts):\n    (x, y, w, h) = cv2.boundingRect(c)\n    bbox_list.append([x, y, w, h])\n    cv2.rectangle(backtorgb, (x,y), (x+w, y+h), (255,0,0), 10)\n\nprint(f'BBoxes coordinates: {bbox_list}')","metadata":{"execution":{"iopub.status.busy":"2022-03-11T15:13:15.774404Z","iopub.execute_input":"2022-03-11T15:13:15.775011Z","iopub.status.idle":"2022-03-11T15:13:15.841994Z","shell.execute_reply.started":"2022-03-11T15:13:15.774930Z","shell.execute_reply":"2022-03-11T15:13:15.841150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(backtorgb)","metadata":{"execution":{"iopub.status.busy":"2022-03-11T15:13:19.665699Z","iopub.execute_input":"2022-03-11T15:13:19.666017Z","iopub.status.idle":"2022-03-11T15:13:23.826327Z","shell.execute_reply.started":"2022-03-11T15:13:19.665959Z","shell.execute_reply":"2022-03-11T15:13:23.825493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lst_exctracted_images=[]\nfor bbox in bbox_list:\n    bbox_img = img_cleaned[bbox[1]:bbox[1]+bbox[3], bbox[0]:bbox[0]+bbox[2]]\n    thresh = cv2.erode(bbox_img, None, iterations=12)\n    bbox_resized = resize_image(thresh,(28,28))\n    lst_exctracted_images.append(bbox_resized)\n    plt.figure()\n    plt.imshow(bbox_resized)","metadata":{"execution":{"iopub.status.busy":"2022-03-11T15:13:27.537282Z","iopub.execute_input":"2022-03-11T15:13:27.537610Z","iopub.status.idle":"2022-03-11T15:13:28.257546Z","shell.execute_reply.started":"2022-03-11T15:13:27.537562Z","shell.execute_reply":"2022-03-11T15:13:28.256374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load dataset\n(X_train, y_train),(X_test, y_test) = mnist.load_data()\n\ny_train = to_categorical(y_train)\ny_test = to_categorical(y_test)\n\n# X_train.max() is 255\nX_train = X_train/255\nX_test = X_test/255\n\nX_train = X_train.reshape(60000,28,28,1)\nX_test = X_test.reshape(10000,28,28,1)\n\n\nfrom keras.callbacks import EarlyStopping\nearly_stop = EarlyStopping(monitor='val_loss',patience=1)\n\n\nmodel = Sequential()\nmodel.add(Conv2D(filters=32,kernel_size=(4,4),input_shape=(28,28,1),activation='relu'))\nmodel.add(MaxPool2D(pool_size=(2,2)))\nmodel.add(Flatten())\nmodel.add(Dense(128,activation='relu'))\nmodel.add(Dense(10,activation='softmax'))\nmodel.compile(optimizer='adam', loss=keras.losses.categorical_crossentropy,metrics=['accuracy'])\nmodel.fit(x=X_train, y=y_train,epochs=100,validation_data=(X_test,y_test),callbacks=[early_stop])\n\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-03-11T15:06:08.420782Z","iopub.execute_input":"2022-03-11T15:06:08.421708Z","iopub.status.idle":"2022-03-11T15:07:31.530722Z","shell.execute_reply.started":"2022-03-11T15:06:08.421663Z","shell.execute_reply":"2022-03-11T15:07:31.528765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sum = 0\n\nfor i in range(len(lst_exctracted_images)):\n    predict_x=model.predict(lst_exctracted_images[i].reshape(1,28,28,1)) \n    classes_x=np.argmax(predict_x,axis=1)\n    print(classes_x)\n    sum+=int(classes_x)\n    plt.figure()\n    plt.imshow(lst_exctracted_images[i].reshape(28,28))\n    \n\nprint(\"The SUMMATION = \", sum)","metadata":{"execution":{"iopub.status.busy":"2022-03-11T15:14:33.142027Z","iopub.execute_input":"2022-03-11T15:14:33.142347Z","iopub.status.idle":"2022-03-11T15:14:33.980457Z","shell.execute_reply.started":"2022-03-11T15:14:33.142316Z","shell.execute_reply":"2022-03-11T15:14:33.979513Z"},"trusted":true},"execution_count":null,"outputs":[]}]}