{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":14897,"databundleVersionId":1020216,"sourceType":"competition"}],"dockerImageVersionId":29844,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import cv2\nfrom tqdm import tqdm_notebook as tqdm\nimport zipfile\nimport io\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"HEIGHT = 137\nWIDTH = 236\nSIZE = 128\n\nTRAIN = ['/kaggle/input/bengaliai-cv19/train_image_data_0.parquet',\n         '/kaggle/input/bengaliai-cv19/train_image_data_1.parquet',\n         '/kaggle/input/bengaliai-cv19/train_image_data_2.parquet',\n         '/kaggle/input/bengaliai-cv19/train_image_data_3.parquet']\n\nOUT_TRAIN = 'train.zip'","metadata":{"execution":{"iopub.status.busy":"2024-08-05T09:47:46.552419Z","iopub.execute_input":"2024-08-05T09:47:46.552769Z","iopub.status.idle":"2024-08-05T09:47:46.558557Z","shell.execute_reply.started":"2024-08-05T09:47:46.552704Z","shell.execute_reply":"2024-08-05T09:47:46.557381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def bbox(img):\n    rows = np.any(img, axis=1)\n    cols = np.any(img, axis=0)\n    rmin, rmax = np.where(rows)[0][[0, -1]]\n    cmin, cmax = np.where(cols)[0][[0, -1]]\n    return rmin, rmax, cmin, cmax\n\ndef crop_resize(img0, size=SIZE, pad=16):\n    #crop a box around pixels large than the threshold \n    #some images contain line at the sides\n    ymin,ymax,xmin,xmax = bbox(img0[5:-5,5:-5] > 80)\n    #cropping may cut too much, so we need to add it back\n    xmin = xmin - 13 if (xmin > 13) else 0\n    ymin = ymin - 10 if (ymin > 10) else 0\n    xmax = xmax + 13 if (xmax < WIDTH - 13) else WIDTH\n    ymax = ymax + 10 if (ymax < HEIGHT - 10) else HEIGHT\n    img = img0[ymin:ymax,xmin:xmax]\n    #remove lo intensity pixels as noise\n    img[img < 28] = 0\n    lx, ly = xmax-xmin,ymax-ymin\n    l = max(lx,ly) + pad\n    #make sure that the aspect ratio is kept in rescaling\n    img = np.pad(img, [((l-ly)//2,), ((l-lx)//2,)], mode='constant')\n    return cv2.resize(img,(size,size))","metadata":{"execution":{"iopub.status.busy":"2024-08-05T09:47:46.560432Z","iopub.execute_input":"2024-08-05T09:47:46.560875Z","iopub.status.idle":"2024-08-05T09:47:46.578134Z","shell.execute_reply.started":"2024-08-05T09:47:46.560798Z","shell.execute_reply":"2024-08-05T09:47:46.576893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_parquet(TRAIN[0])\nn_imgs = 8\nfig, axs = plt.subplots(n_imgs, 2, figsize=(10, 5*n_imgs))\n\nfor idx in range(n_imgs):\n    #somehow the original input is inverted\n    img0 = 255 - df.iloc[idx, 1:].values.reshape(HEIGHT, WIDTH).astype(np.uint8)\n    #normalize each image by its max val\n    img = (img0*(255.0/img0.max())).astype(np.uint8)\n    img = crop_resize(img)\n\n    axs[idx,0].imshow(img0)\n    axs[idx,0].set_title('Original image')\n    axs[idx,0].axis('off')\n    axs[idx,1].imshow(img)\n    axs[idx,1].set_title('Crop & resize')\n    axs[idx,1].axis('off')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-05T09:47:46.580014Z","iopub.execute_input":"2024-08-05T09:47:46.580373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_tot,x2_tot = [],[]\nwith zipfile.ZipFile(OUT_TRAIN, 'w') as img_out:\n    for fname in TRAIN:\n        df = pd.read_parquet(fname)\n        #the input is inverted\n        data = 255 - df.iloc[:, 1:].values.reshape(-1, HEIGHT, WIDTH).astype(np.uint8)\n        for idx in tqdm(range(len(df))):\n            name = df.iloc[idx,0]\n            #normalize each image by its max val\n            img = (data[idx]*(255.0/data[idx].max())).astype(np.uint8)\n            img = crop_resize(img)\n        \n            x_tot.append((img/255.0).mean())\n            x2_tot.append(((img/255.0)**2).mean()) \n            img = cv2.imencode('.png',img)[1]\n            img_out.writestr(name + '.png', img)","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#image stats\nimg_avr =  np.array(x_tot).mean()\nimg_std =  np.sqrt(np.array(x2_tot).mean() - img_avr**2)\nprint('mean:',img_avr, ', std:', img_std)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}