{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import glob, cv2, tqdm\nimport os,numpy as np\nfrom PIL import Image","metadata":{"execution":{"iopub.status.busy":"2022-03-11T13:40:50.206717Z","iopub.execute_input":"2022-03-11T13:40:50.207077Z","iopub.status.idle":"2022-03-11T13:40:50.608353Z","shell.execute_reply.started":"2022-03-11T13:40:50.206980Z","shell.execute_reply":"2022-03-11T13:40:50.607253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_images = glob.glob(\"../input/ultra-mnist/train/*.jpeg\")\nprint(f\"Total Number of train Images: {len(train_images)}\")","metadata":{"execution":{"iopub.status.busy":"2022-03-11T13:40:54.314296Z","iopub.execute_input":"2022-03-11T13:40:54.314586Z","iopub.status.idle":"2022-03-11T13:40:54.933197Z","shell.execute_reply.started":"2022-03-11T13:40:54.314557Z","shell.execute_reply":"2022-03-11T13:40:54.931508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# images have 4*4 tile pattern. Lets remove them following this naive approach\ndef remove_tiles(image, stride= 128,size = 128):\n    window_area = size*size\n    hops = range(0,image.shape[0],stride)\n    for i,iv in enumerate(hops):\n        for j,jv in enumerate(hops):\n            window = image[iv:iv+size, jv:jv+size]  \n            window = window>0\n            white_pixel_counts = np.sum(window)\n            \n            if_white_window = white_pixel_counts == window_area \n            if_black_window = white_pixel_counts == 0\n            \n            if if_white_window: continue\n            elif if_black_window: window = 255  #make black tile white\n            else:\n                white_pixels_percentage = 100* white_pixel_counts/window_area\n                if white_pixels_percentage>50: continue                \n                window = 255 - image[iv:iv+size, jv:jv+size]     \n            image[iv:iv+size, jv:jv+size] = window\n                \n    return image","metadata":{"execution":{"iopub.status.busy":"2022-03-11T13:40:56.961479Z","iopub.execute_input":"2022-03-11T13:40:56.961781Z","iopub.status.idle":"2022-03-11T13:40:56.970615Z","shell.execute_reply.started":"2022-03-11T13:40:56.961747Z","shell.execute_reply":"2022-03-11T13:40:56.969948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Lets find out what this remove_tiles() do\nimage_with_tiles = cv2.imread(train_images[0],0)\nImage.fromarray(image_with_tiles)","metadata":{"execution":{"iopub.status.busy":"2022-03-11T13:41:10.582693Z","iopub.execute_input":"2022-03-11T13:41:10.583511Z","iopub.status.idle":"2022-03-11T13:41:10.888149Z","shell.execute_reply.started":"2022-03-11T13:41:10.583456Z","shell.execute_reply":"2022-03-11T13:41:10.887164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Image.fromarray(remove_tiles(image_with_tiles.copy(),1000,1000))","metadata":{"execution":{"iopub.status.busy":"2022-03-11T13:41:14.370381Z","iopub.execute_input":"2022-03-11T13:41:14.370722Z","iopub.status.idle":"2022-03-11T13:41:14.623097Z","shell.execute_reply.started":"2022-03-11T13:41:14.370690Z","shell.execute_reply":"2022-03-11T13:41:14.622107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# creates a binary image where pixel intensity is less than 255\ndef create_binary_mask(image, threshold=255):\n    image = image!=threshold\n    return image.astype('int')\n\n\n# finds cluster of axis where a digit exists and returns a list of tuple(start_of_a_cluster, end_of_a_cluster) \ndef get_cluster(array_):\n    clusters = []\n    start, end = 0,0\n    started = False\n    for idx, v in enumerate(array_):\n        if not started and v: \n            start = idx\n            started= True\n        if started and not v: \n            clusters.append((start, idx))\n            started = False      \n    if started:\n        clusters.append((start, len(array_)))      \n    return clusters","metadata":{"execution":{"iopub.status.busy":"2022-03-11T13:41:19.020294Z","iopub.execute_input":"2022-03-11T13:41:19.020924Z","iopub.status.idle":"2022-03-11T13:41:19.028916Z","shell.execute_reply.started":"2022-03-11T13:41:19.020887Z","shell.execute_reply":"2022-03-11T13:41:19.027619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(get_cluster([1,0,0,0,1,1,1,1,0,1,0,0,1,0,1])) ","metadata":{"execution":{"iopub.status.busy":"2022-03-11T13:41:22.020267Z","iopub.execute_input":"2022-03-11T13:41:22.021089Z","iopub.status.idle":"2022-03-11T13:41:22.026278Z","shell.execute_reply.started":"2022-03-11T13:41:22.021044Z","shell.execute_reply":"2022-03-11T13:41:22.025568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# take an binary_mask  and return bounding box of all digits\n# Splits image into disjoint segment using row-sum and column-sum recursively\n\ndef get_unique_segments(image):\n    unique_segments = []\n    \n    def make_unique_segment(current_segment,current_top_left_index,transpose = False,previous_cluster_len = 10, shift = 0):\n\n        selected_axis_sum = np.sum(current_segment,axis = int(transpose)) # 0-col, 1= row\n        selected_axis_index = selected_axis_sum>0\n        \n        if shift: selected_axis_index = np.bitwise_or(np.bitwise_or(selected_axis_index , np.roll(selected_axis_index,shift)),\n                                                        np.roll(selected_axis_index,-shift))\n                                            \n        clusters = get_cluster(selected_axis_index)\n        x,y = current_top_left_index \n        \n        if len(clusters)==1 and previous_cluster_len == 1:\n            start, end = clusters[0]\n            span = abs(end-start) \n            if transpose: \n                x2 = x + len(current_segment[0])\n                y  = y + start\n                y2 =  y + span \n            else:\n                x  = x + start\n                x2 = x + span \n                y2 = y + len(current_segment[:,0])\n                \n            unique_segments.append((x,y,x2,y2))\n            return\n        else:\n            for cluster in clusters:\n                start , end = cluster\n                if transpose:\n                    segment = current_segment[start:end,:]\n                    top_left_index = (current_top_left_index[0], current_top_left_index[1]+start)       \n                else:\n                    segment = current_segment[:,start:end]\n                    top_left_index = (current_top_left_index[0]+start, current_top_left_index[1])\n                make_unique_segment(segment, top_left_index, not transpose,len(clusters))\n                \n                \n    make_unique_segment(image,(0,0))\n    unique_segments = [i for i in unique_segments if i and (i[2]-i[0]>5 and i[3]-i[1]>5)]   # (i[2]-i[0])*(i[3]-i[1]>5)>50]  #filter noisy segments\n    return  unique_segments","metadata":{"execution":{"iopub.status.busy":"2022-03-11T13:41:39.024359Z","iopub.execute_input":"2022-03-11T13:41:39.024633Z","iopub.status.idle":"2022-03-11T13:41:39.039002Z","shell.execute_reply.started":"2022-03-11T13:41:39.024604Z","shell.execute_reply":"2022-03-11T13:41:39.037924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for image in tqdm.tqdm(train_images[:1]): #lets visulize a image\n    try:\n        name = image.split('\\\\')[-1]    \n        image_ = cv2.imread(image,0)\n        image = remove_tiles(image_,1000,1000)\n        mask = create_binary_mask(image)\n        unique_segments = get_unique_segments(mask)\n        \n        image = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB)\n        image_ = cv2.cvtColor(image_, cv2.COLOR_GRAY2RGB)\n        for i in unique_segments:\n            image = cv2.rectangle(image, (i[0],i[1]), (i[2],i[3]), (255,0,0), 8) \n        #cv2.imwrite(f'F:\\S-Home\\projects\\mnist_ultra\\\\bbox\\{name}',np.concatenate((image_,image),axis=1))\n    except Exception as e:\n        print(e,name)\nImage.fromarray(image) ","metadata":{"execution":{"iopub.status.busy":"2022-03-11T04:34:59.199346Z","iopub.execute_input":"2022-03-11T04:34:59.199851Z","iopub.status.idle":"2022-03-11T04:34:59.996038Z","shell.execute_reply.started":"2022-03-11T04:34:59.199817Z","shell.execute_reply":"2022-03-11T04:34:59.995373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Directly adapted from keras.io\n\nimport numpy as np\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nimport tensorflow as tf\n\n\nnum_classes = 10\ninput_shape = (28, 28, 1)\n\n'''# the data, split between train and test sets\n(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()\n\n# Scale images to the [0, 1] range\nx_train = x_train.astype(\"float32\") / 255\nx_test = x_test.astype(\"float32\") / 255\n# Make sure images have shape (28, 28, 1)\nx_train = np.expand_dims(x_train, -1)\nx_test = np.expand_dims(x_test, -1)\nprint(\"x_train shape:\", x_train.shape)\nprint(x_train.shape[0], \"train samples\")\nprint(x_test.shape[0], \"test samples\")\n\n\n# convert class vectors to binary class matrices\ny_train = keras.utils.to_categorical(y_train, num_classes)\ny_test = keras.utils.to_categorical(y_test, num_classes)\n'''\n\nmodel = keras.Sequential(\n    [\n        keras.Input(shape=input_shape),\n        layers.Conv2D(32, kernel_size=(3, 3), activation=\"relu\"),\n        layers.MaxPooling2D(pool_size=(2, 2)),\n        layers.Conv2D(64, kernel_size=(3, 3), activation=\"relu\"),\n        layers.MaxPooling2D(pool_size=(2, 2)),\n        layers.Flatten(),\n        layers.Dropout(0.5),\n        layers.Dense(num_classes, activation=\"softmax\"),\n    ]\n)\n\nmodel.summary()\n\nmodel.compile(loss=\"categorical_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\nmodel.load_weights(\"../input/mnist-model/model.02-0.99400.h5\")","metadata":{"execution":{"iopub.status.busy":"2022-03-11T13:44:05.502805Z","iopub.execute_input":"2022-03-11T13:44:05.503504Z","iopub.status.idle":"2022-03-11T13:44:12.250454Z","shell.execute_reply.started":"2022-03-11T13:44:05.503462Z","shell.execute_reply":"2022-03-11T13:44:12.249645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# lets load the labels\n\nimport csv\nwith open('../input/ultra-mnist/train.csv', 'r') as f:\n    reader = csv.reader(f)\n    data = list(reader)\nlabels = {i[0]:i[1] for i in data}","metadata":{"execution":{"iopub.status.busy":"2022-03-11T04:39:55.024593Z","iopub.execute_input":"2022-03-11T04:39:55.025347Z","iopub.status.idle":"2022-03-11T04:39:55.076192Z","shell.execute_reply.started":"2022-03-11T04:39:55.025283Z","shell.execute_reply":"2022-03-11T04:39:55.075162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#some preprocessing to prevent undesired transformation of image \n\ndef preprocess_image(image,bboxs, size = 28,padder = .2):\n    \n    x1,y1,x2,y2 = bboxs\n    \n    image = image[y1:y2,x1:x2]\n\n    x_ = int(padder*(x2-x1))\n    x1 = max(0,x1-x_)\n    x2 = min(image.shape[1],x2+x_)\n    y_ = int(padder*(y2-y1))\n    y1 = max(0,y1-y_)\n    y2 = min(image.shape[0],y2+y_)\n    \n    h, w  = image.shape\n    ratio = w/h\n    if ratio>1:\n        unit = w/size\n        h = int(h/unit)\n        image = cv2.resize(image, (size, h))\n        canvas = np.ones((size,size),dtype=np.uint8)*255\n        padding = int((size-h)/2)\n        canvas[padding:padding+h,:] = image\n        \n    else:\n        unit = h/size\n        w = int(w/unit)\n        image = cv2.resize(image, (w, size))\n        canvas = np.ones((size,size),dtype=np.uint8)*255\n        padding = int((size-w)/2)\n        canvas[:,padding:padding+w] = image\n    #Image.fromarray(image).show()  \n    #Image.fromarray(canvas).show()\n        \n    return canvas.reshape(size,size,1)/255","metadata":{"execution":{"iopub.status.busy":"2022-03-11T13:44:15.676349Z","iopub.execute_input":"2022-03-11T13:44:15.676769Z","iopub.status.idle":"2022-03-11T13:44:15.688155Z","shell.execute_reply.started":"2022-03-11T13:44:15.676735Z","shell.execute_reply":"2022-03-11T13:44:15.687039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#shows accuracy in train set\n\ncorrect = 0\nidx = 0\nresults = []\nfrom PIL import Image\nstart,end = 0, 1000\nfor image in tqdm.tqdm(train_images[start:end]):\n    try:\n        name = image.split('/')[-1][:-5]    \n        image_ = cv2.imread(image,0)\n        image = remove_tiles(image_,1000,1000)\n        mask = create_binary_mask(image.copy())\n        unique_segments = get_unique_segments(mask)\n        digits = []\n\n        for i in unique_segments:\n            try:\n                digit = preprocess_image(image.copy(),i)\n                digits.append(1-digit)\n            except:\n                 # mostly noisy segment with no digit and very small in size\n                pass\n            \n        digits = np.array(digits)\n        predictions = model.predict(digits)\n        predictions = np.argmax(predictions,axis=1)\n        total =  np.sum(predictions)\n        results.append((predictions,total,labels[name]))\n        \n        correct += int(total==int(labels[name]))\n    except Exception as e:\n        print(e,name)\n    idx +=1\n    #print('accuracy:',correct/idx)\nprint(f\"Accuracy on {end-start} examples is :\",correct/idx )    ","metadata":{"execution":{"iopub.status.busy":"2022-03-11T04:45:44.15384Z","iopub.execute_input":"2022-03-11T04:45:44.154173Z","iopub.status.idle":"2022-03-11T04:49:34.492487Z","shell.execute_reply.started":"2022-03-11T04:45:44.154139Z","shell.execute_reply":"2022-03-11T04:49:34.491686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = glob.glob(\"../input/ultra-mnist/test/*.jpeg\")\nstart,end = 0, len(test_data)\nresult_test = \"id,digit_sum\"\nfor image in tqdm.tqdm(test_data[start:end]):\n    \n    try:\n        name = image.split('/')[-1][:-5]    \n        image_ = cv2.imread(image,0)\n        image = remove_tiles(image_,1000,1000)\n        mask = create_binary_mask(image.copy())\n        unique_segments = get_unique_segments(mask)\n        digits = []\n\n        for i in unique_segments:\n            try:\n                digit = preprocess_image(image.copy(),i)\n                digits.append(1-digit)\n            except:\n                 # mostly noisy segment with no digit and very small in size\n                pass\n            \n        digits = np.array(digits)\n        predictions = model.predict(digits)\n        predictions = np.argmax(predictions,axis=1)\n        total =  np.sum(predictions)\n        total = min(total,27)\n        \n        result_test += f\"\\n{name},{str(total)}\"\n    except Exception as e:\n        result_test += f\"\\n{name},{str(np.random.choice(100)%27)}\"","metadata":{"execution":{"iopub.status.busy":"2022-03-11T13:48:07.519105Z","iopub.execute_input":"2022-03-11T13:48:07.519405Z","iopub.status.idle":"2022-03-11T15:26:22.315939Z","shell.execute_reply.started":"2022-03-11T13:48:07.519375Z","shell.execute_reply":"2022-03-11T15:26:22.314834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = '\\n'.join([i.replace(\"../input/ultra-mnist/test/\",'') for i in result_test.split('\\n')])\nopen('submission_naive.csv','w').write(temp)","metadata":{"execution":{"iopub.status.busy":"2022-03-11T15:30:32.428674Z","iopub.execute_input":"2022-03-11T15:30:32.429025Z","iopub.status.idle":"2022-03-11T15:30:32.451173Z","shell.execute_reply.started":"2022-03-11T15:30:32.428989Z","shell.execute_reply":"2022-03-11T15:30:32.450313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cp ./submission_naive.csv ../input/ultra-mnist/submission_naive.csv","metadata":{"execution":{"iopub.status.busy":"2022-03-11T15:33:59.987688Z","iopub.execute_input":"2022-03-11T15:33:59.988043Z","iopub.status.idle":"2022-03-11T15:34:00.827355Z","shell.execute_reply.started":"2022-03-11T15:33:59.988003Z","shell.execute_reply":"2022-03-11T15:34:00.826419Z"},"trusted":true},"execution_count":null,"outputs":[]}]}