{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import glob, cv2, tqdm\nimport os,numpy as np\nfrom PIL import Image","metadata":{"execution":{"iopub.status.busy":"2023-09-25T01:21:43.222540Z","iopub.execute_input":"2023-09-25T01:21:43.222990Z","iopub.status.idle":"2023-09-25T01:21:43.582860Z","shell.execute_reply.started":"2023-09-25T01:21:43.222867Z","shell.execute_reply":"2023-09-25T01:21:43.582017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_images = glob.glob(\"../input/ultra-mnist/train/*.jpeg\")\nprint(f\"Total Number of train Images: {len(train_images)}\")","metadata":{"execution":{"iopub.status.busy":"2023-09-25T01:21:43.585195Z","iopub.execute_input":"2023-09-25T01:21:43.585609Z","iopub.status.idle":"2023-09-25T01:21:43.963454Z","shell.execute_reply.started":"2023-09-25T01:21:43.585560Z","shell.execute_reply":"2023-09-25T01:21:43.962379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# images have 4*4 tile pattern. Lets remove them following this naive approach\ndef remove_tiles(image, stride= 128,size = 128):\n    window_area = size*size\n    hops = range(0,image.shape[0],stride)\n    for i,iv in enumerate(hops):\n        for j,jv in enumerate(hops):\n            window = image[iv:iv+size, jv:jv+size]  \n            window = window>0\n            white_pixel_counts = np.sum(window)\n            \n            if_white_window = white_pixel_counts == window_area \n            if_black_window = white_pixel_counts == 0\n            \n            if if_white_window: continue\n            elif if_black_window: window = 255  #make black tile white\n            else:\n                white_pixels_percentage = 100* white_pixel_counts/window_area\n                if white_pixels_percentage>50: continue                \n                window = 255 - image[iv:iv+size, jv:jv+size]     \n            image[iv:iv+size, jv:jv+size] = window\n                \n    return image","metadata":{"execution":{"iopub.status.busy":"2023-09-25T01:21:43.965863Z","iopub.execute_input":"2023-09-25T01:21:43.966123Z","iopub.status.idle":"2023-09-25T01:21:43.974495Z","shell.execute_reply.started":"2023-09-25T01:21:43.966090Z","shell.execute_reply":"2023-09-25T01:21:43.973589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Lets find out what this remove_tiles() do\nimage_with_tiles = cv2.imread(train_images[0],0)\nImage.fromarray(image_with_tiles)","metadata":{"execution":{"iopub.status.busy":"2023-09-25T01:21:43.976233Z","iopub.execute_input":"2023-09-25T01:21:43.977228Z","iopub.status.idle":"2023-09-25T01:21:44.582098Z","shell.execute_reply.started":"2023-09-25T01:21:43.977174Z","shell.execute_reply":"2023-09-25T01:21:44.581383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Image.fromarray(remove_tiles(image_with_tiles.copy(),1000,1000))","metadata":{"execution":{"iopub.status.busy":"2023-09-25T01:21:44.583483Z","iopub.execute_input":"2023-09-25T01:21:44.583753Z","iopub.status.idle":"2023-09-25T01:21:45.106493Z","shell.execute_reply.started":"2023-09-25T01:21:44.583721Z","shell.execute_reply":"2023-09-25T01:21:45.105656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# creates a binary image where pixel intensity is less than 255\ndef create_binary_mask(image, threshold=255):\n    image = image!=threshold\n    return image.astype('int')\n\n\n# finds cluster of axis where a digit exists and returns a list of tuple(start_of_a_cluster, end_of_a_cluster) \ndef get_cluster(array_):\n    clusters = []\n    start, end = 0,0\n    started = False\n    for idx, v in enumerate(array_):\n        if not started and v: \n            start = idx\n            started= True\n        if started and not v: \n            clusters.append((start, idx))\n            started = False      \n    if started:\n        clusters.append((start, len(array_)))      \n    return clusters","metadata":{"execution":{"iopub.status.busy":"2023-09-25T01:21:45.107821Z","iopub.execute_input":"2023-09-25T01:21:45.108633Z","iopub.status.idle":"2023-09-25T01:21:45.114971Z","shell.execute_reply.started":"2023-09-25T01:21:45.108590Z","shell.execute_reply":"2023-09-25T01:21:45.114347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(get_cluster([1,0,0,0,1,1,1,1,0,1,0,0,1,0,1])) ","metadata":{"execution":{"iopub.status.busy":"2023-09-25T01:21:45.116207Z","iopub.execute_input":"2023-09-25T01:21:45.116653Z","iopub.status.idle":"2023-09-25T01:21:45.130705Z","shell.execute_reply.started":"2023-09-25T01:21:45.116619Z","shell.execute_reply":"2023-09-25T01:21:45.130127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# take an binary_mask  and return bounding box of all digits\n# Splits image into disjoint segment using row-sum and column-sum recursively\n\ndef get_unique_segments(image):\n    unique_segments = []\n    \n    def make_unique_segment(current_segment,current_top_left_index,transpose = False,previous_cluster_len = 10, shift = 0):\n\n        selected_axis_sum = np.sum(current_segment,axis = int(transpose)) # 0-col, 1= row\n        selected_axis_index = selected_axis_sum>0\n        \n        if shift: selected_axis_index = np.bitwise_or(np.bitwise_or(selected_axis_index , np.roll(selected_axis_index,shift)),\n                                                        np.roll(selected_axis_index,-shift))\n                                            \n        clusters = get_cluster(selected_axis_index)\n        x,y = current_top_left_index \n        \n        if len(clusters)==1 and previous_cluster_len == 1:\n            start, end = clusters[0]\n            span = abs(end-start) \n            if transpose: \n                x2 = x + len(current_segment[0])\n                y  = y + start\n                y2 =  y + span \n            else:\n                x  = x + start\n                x2 = x + span \n                y2 = y + len(current_segment[:,0])\n                \n            unique_segments.append((x,y,x2,y2))\n            return\n        else:\n            for cluster in clusters:\n                start , end = cluster\n                if transpose:\n                    segment = current_segment[start:end,:]\n                    top_left_index = (current_top_left_index[0], current_top_left_index[1]+start)       \n                else:\n                    segment = current_segment[:,start:end]\n                    top_left_index = (current_top_left_index[0]+start, current_top_left_index[1])\n                make_unique_segment(segment, top_left_index, not transpose,len(clusters))\n                \n                \n    make_unique_segment(image,(0,0))\n    unique_segments = [i for i in unique_segments if i and (i[2]-i[0]>5 and i[3]-i[1]>5)]    #filter noisy segments\n    return  unique_segments","metadata":{"execution":{"iopub.status.busy":"2023-09-25T01:21:45.131811Z","iopub.execute_input":"2023-09-25T01:21:45.132478Z","iopub.status.idle":"2023-09-25T01:21:45.147404Z","shell.execute_reply.started":"2023-09-25T01:21:45.132445Z","shell.execute_reply":"2023-09-25T01:21:45.146294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for image in tqdm.tqdm(train_images[:1]): #lets visulize a image\n    try:\n        name = image.split('\\\\')[-1]    \n        image_ = cv2.imread(image,0)\n        image = remove_tiles(image_,1000,1000)\n        mask = create_binary_mask(image)\n        unique_segments = get_unique_segments(mask)\n        \n        image = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB)\n        image_ = cv2.cvtColor(image_, cv2.COLOR_GRAY2RGB)\n        for i in unique_segments:\n            image = cv2.rectangle(image, (i[0],i[1]), (i[2],i[3]), (255,0,0), 8) \n        #cv2.imwrite(f'F:\\S-Home\\projects\\mnist_ultra\\\\bbox\\{name}',np.concatenate((image_,image),axis=1))\n    except Exception as e:\n        print(e,name)\nImage.fromarray(image) ","metadata":{"execution":{"iopub.status.busy":"2023-09-25T01:21:45.148570Z","iopub.execute_input":"2023-09-25T01:21:45.148868Z","iopub.status.idle":"2023-09-25T01:21:46.807147Z","shell.execute_reply.started":"2023-09-25T01:21:45.148837Z","shell.execute_reply":"2023-09-25T01:21:46.806363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Directly adapted from keras.io\n\nimport numpy as np\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nimport tensorflow as tf\n\n\nnum_classes = 10\ninput_shape = (28, 28, 1)\n\n# the data, split between train and test sets\n(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()\n\n# Scale images to the [0, 1] range\nx_train = x_train.astype(\"float32\") / 255\nx_test = x_test.astype(\"float32\") / 255\n# Make sure images have shape (28, 28, 1)\nx_train = np.expand_dims(x_train, -1)\nx_test = np.expand_dims(x_test, -1)\nprint(\"x_train shape:\", x_train.shape)\nprint(x_train.shape[0], \"train samples\")\nprint(x_test.shape[0], \"test samples\")\n\n\n# convert class vectors to binary class matrices\ny_train = keras.utils.to_categorical(y_train, num_classes)\ny_test = keras.utils.to_categorical(y_test, num_classes)\n\n\nmodel = keras.Sequential(\n    [\n        keras.Input(shape=input_shape),\n        layers.Conv2D(32, kernel_size=(3, 3), activation=\"relu\"),\n        layers.MaxPooling2D(pool_size=(2, 2)),\n        layers.Conv2D(64, kernel_size=(3, 3), activation=\"relu\"),\n        layers.MaxPooling2D(pool_size=(2, 2)),\n        layers.Flatten(),\n        layers.Dropout(0.5),\n        layers.Dense(num_classes, activation=\"softmax\"),\n    ]\n)\n\nmodel.summary()\n\nmodel.compile(loss=\"categorical_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\n# model.load_weights(\"../input/mnist-model/model.02-0.99400.h5\")\nmodel.fit(x_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2023-09-25T01:23:43.438478Z","iopub.execute_input":"2023-09-25T01:23:43.439487Z","iopub.status.idle":"2023-09-25T01:24:04.912806Z","shell.execute_reply.started":"2023-09-25T01:23:43.439423Z","shell.execute_reply":"2023-09-25T01:24:04.911878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# lets load the labels\n\nimport csv\nwith open('../input/ultra-mnist/train.csv', 'r') as f:\n    reader = csv.reader(f)\n    data = list(reader)\nlabels = {i[0]:i[1] for i in data}","metadata":{"execution":{"iopub.status.busy":"2023-09-25T01:24:08.469674Z","iopub.execute_input":"2023-09-25T01:24:08.470024Z","iopub.status.idle":"2023-09-25T01:24:08.502379Z","shell.execute_reply.started":"2023-09-25T01:24:08.469981Z","shell.execute_reply":"2023-09-25T01:24:08.501296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#some preprocessing to prevent undesired transformation of image \n\ndef preprocess_image(image,bboxs, size = 28,padder = .2):\n    \n    x1,y1,x2,y2 = bboxs\n    \n    image = image[y1:y2,x1:x2]\n\n    x_ = int(padder*(x2-x1))\n    x1 = max(0,x1-x_)\n    x2 = min(image.shape[1],x2+x_)\n    y_ = int(padder*(y2-y1))\n    y1 = max(0,y1-y_)\n    y2 = min(image.shape[0],y2+y_)\n    \n    h, w  = image.shape\n    ratio = w/h\n    if ratio>1:\n        unit = w/size\n        h = int(h/unit)\n        image = cv2.resize(image, (size, h))\n        canvas = np.ones((size,size),dtype=np.uint8)*255\n        padding = int((size-h)/2)\n        canvas[padding:padding+h,:] = image\n        \n    else:\n        unit = h/size\n        w = int(w/unit)\n        image = cv2.resize(image, (w, size))\n        canvas = np.ones((size,size),dtype=np.uint8)*255\n        padding = int((size-w)/2)\n        canvas[:,padding:padding+w] = image\n    #Image.fromarray(image).show()  \n    #Image.fromarray(canvas).show()\n        \n    return canvas.reshape(size,size,1)/255","metadata":{"execution":{"iopub.status.busy":"2023-09-25T01:24:10.043951Z","iopub.execute_input":"2023-09-25T01:24:10.044257Z","iopub.status.idle":"2023-09-25T01:24:10.055540Z","shell.execute_reply.started":"2023-09-25T01:24:10.044219Z","shell.execute_reply":"2023-09-25T01:24:10.054237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_images[0]","metadata":{"execution":{"iopub.status.busy":"2023-09-25T01:24:12.273113Z","iopub.execute_input":"2023-09-25T01:24:12.273409Z","iopub.status.idle":"2023-09-25T01:24:12.279308Z","shell.execute_reply.started":"2023-09-25T01:24:12.273377Z","shell.execute_reply":"2023-09-25T01:24:12.278359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"correct = 0\nidx = 0\nresults = []\nfrom PIL import Image\nstart,end = 0, 1000\nfor image in tqdm.tqdm(train_images[start:end]):\n    try:\n        name = image.split('/')[-1][:-5]    \n        image_ = cv2.imread(image,0)\n        image = remove_tiles(image_,1000,1000)\n        mask = create_binary_mask(image.copy())\n        unique_segments = get_unique_segments(mask)\n        digits = []\n\n        for i in unique_segments:\n            try:\n                digit = preprocess_image(image.copy(),i)\n                digits.append(1-digit)\n            except:\n                 # mostly noisy segment with no digit and very small in size\n                pass\n            \n        digits = np.array(digits)\n        predictions = model.predict(digits)\n        predictions = np.argmax(predictions,axis=1)\n        total =  np.sum(predictions)\n        results.append((predictions,total,labels[name]))\n        \n        correct += int(total==int(labels[name]))\n    except Exception as e:\n        print(e,name)\n    idx +=1\n    #print('accuracy:',correct/idx)\nprint(f\"Accuracy on {end-start} examples is :\",correct/idx )    ","metadata":{"execution":{"iopub.status.busy":"2023-09-25T01:24:13.551985Z","iopub.execute_input":"2023-09-25T01:24:13.552303Z","iopub.status.idle":"2023-09-25T01:27:43.592214Z","shell.execute_reply.started":"2023-09-25T01:24:13.552253Z","shell.execute_reply":"2023-09-25T01:27:43.591213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}