{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# EDA on the  Vesuvius Challenge - Ink Detection Data","metadata":{}},{"cell_type":"code","source":"# Import all import libraries\nimport cv2 \nimport matplotlib.pyplot as plt \nimport numpy as np \nimport seaborn as sns\n%matplotlib inline","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-18T09:07:37.632659Z","iopub.execute_input":"2023-03-18T09:07:37.633095Z","iopub.status.idle":"2023-03-18T09:07:37.641084Z","shell.execute_reply.started":"2023-03-18T09:07:37.633057Z","shell.execute_reply":"2023-03-18T09:07:37.639569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Defining the paths for the data\ndata_path = \"/kaggle/input/vesuvius-challenge-ink-detection/\"\ntrain_path = data_path + \"train/\"\ntest_path = data_path + \"test/\"","metadata":{"execution":{"iopub.status.busy":"2023-03-18T08:57:57.961093Z","iopub.execute_input":"2023-03-18T08:57:57.961602Z","iopub.status.idle":"2023-03-18T08:57:57.970468Z","shell.execute_reply.started":"2023-03-18T08:57:57.961549Z","shell.execute_reply":"2023-03-18T08:57:57.969382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plotting sample. Read image --> convert to grayscale --> plot\ntrain_1 = cv2.imread(f\"{train_path}/1/inklabels.png\", 0)\nplt.imshow(train_1, cmap = \"gray\")","metadata":{"execution":{"iopub.status.busy":"2023-03-18T08:57:57.972403Z","iopub.execute_input":"2023-03-18T08:57:57.973225Z","iopub.status.idle":"2023-03-18T08:58:01.158397Z","shell.execute_reply.started":"2023-03-18T08:57:57.973175Z","shell.execute_reply":"2023-03-18T08:58:01.157148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Check shape of all the inklabels","metadata":{}},{"cell_type":"code","source":"# Loop through the files and print the shape\nfor i in range(1, 4):\n    exec(f\"train_img_{i}  = cv2.imread(f'{train_path}/{i}/inklabels.png', 0) \")\n    print(f\"Train :: Set {i} :: Inklabel shape:\", eval(f\"train_img_{i}\").shape)","metadata":{"execution":{"iopub.status.busy":"2023-03-18T08:58:01.161384Z","iopub.execute_input":"2023-03-18T08:58:01.161745Z","iopub.status.idle":"2023-03-18T08:58:02.670710Z","shell.execute_reply.started":"2023-03-18T08:58:01.161710Z","shell.execute_reply":"2023-03-18T08:58:02.669419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Okay so we notice that all the Inklabel sizes are NOT same!\nThis has to be kept in mind while preparing our Data for Training","metadata":{}},{"cell_type":"markdown","source":"## One thing that can be done is that we can create our Dataset class which does not care about which \"set\" it belongs to and takes into consideration wheather the pixel is \"0\" or \"1\"","metadata":{}},{"cell_type":"code","source":"# Finding the Unique values\nnp.unique(train_img_1)","metadata":{"execution":{"iopub.status.busy":"2023-03-18T08:58:02.672025Z","iopub.execute_input":"2023-03-18T08:58:02.672358Z","iopub.status.idle":"2023-03-18T08:58:03.690688Z","shell.execute_reply.started":"2023-03-18T08:58:02.672326Z","shell.execute_reply":"2023-03-18T08:58:03.689507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(train_img_1.flatten()))\nprint((train_img_1.flatten()/255).sum())","metadata":{"execution":{"iopub.status.busy":"2023-03-18T08:58:03.691992Z","iopub.execute_input":"2023-03-18T08:58:03.692349Z","iopub.status.idle":"2023-03-18T08:58:03.916200Z","shell.execute_reply.started":"2023-03-18T08:58:03.692314Z","shell.execute_reply":"2023-03-18T08:58:03.914750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_pos = 0\nall_neg = 0 \n\nfor i in range(1, 4):\n    exec(f\"white = (train_img_{i}.flatten()/255).sum()\")\n    exec(f\"black = len(train_img_{i}.flatten()) - white\")\n    all_pos += int(white)\n    all_neg += int(black)\nprint(\"Positive (White Pixels):\", all_pos)\nprint(\"Negative (Black Pixels):\", all_neg)\n\nplt.bar([\"Black\", \"White\"], [all_neg, all_pos])","metadata":{"execution":{"iopub.status.busy":"2023-03-18T08:58:03.917698Z","iopub.execute_input":"2023-03-18T08:58:03.918156Z","iopub.status.idle":"2023-03-18T08:58:05.041094Z","shell.execute_reply.started":"2023-03-18T08:58:03.918120Z","shell.execute_reply":"2023-03-18T08:58:05.039649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## We can see there is a huge imbalance between Black and White Pixels","metadata":{}},{"cell_type":"code","source":"BOX_SIZE = 100\ndef find_important_box(img):\n    w, h = img.shape\n    print(w, h)\n    all_pos = 0\n    all_neg = 0\n    for i in range(0, w, BOX_SIZE):\n        for j in range(0, h, BOX_SIZE):\n            if len(np.unique(img[i:i + BOX_SIZE, j:j + BOX_SIZE])) == 2:\n                p = (img[i:i + BOX_SIZE, j:j + BOX_SIZE].flatten()/225).sum()\n                n = len(img[i:i + BOX_SIZE, j:j + BOX_SIZE].flatten()) - p\n                all_pos += p \n                all_neg += n\n#                 print((i, j), (i + BOX_SIZE, j + BOX_SIZE), len(np.unique(img[i:i + BOX_SIZE, j:j + BOX_SIZE])))\n#                 img = cv2.rectangle(img, (i, j), (i + BOX_SIZE, j + BOX_SIZE), (255,  255 , 255), 7)\n    return int(all_pos), int(all_neg), img","metadata":{"execution":{"iopub.status.busy":"2023-03-18T09:16:33.446944Z","iopub.execute_input":"2023-03-18T09:16:33.447427Z","iopub.status.idle":"2023-03-18T09:16:33.457251Z","shell.execute_reply.started":"2023-03-18T09:16:33.447383Z","shell.execute_reply":"2023-03-18T09:16:33.455844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Now we will slide a BOX across the image and take only those boxes into consideration which has any White pixel","metadata":{}},{"cell_type":"code","source":"white = 0 \nblack = 0\n\nfor i in range(1, 4):\n    exec(f\"x = find_important_box(train_img_{i})\")\n    white+= x[0]\n    black += x[1]\n#     plt.subplot(1, 3, i)\n#     plt.imshow(x[2], cmap = \"gray\")\n    ","metadata":{"execution":{"iopub.status.busy":"2023-03-18T09:16:49.788755Z","iopub.execute_input":"2023-03-18T09:16:49.789639Z","iopub.status.idle":"2023-03-18T09:16:52.902906Z","shell.execute_reply.started":"2023-03-18T09:16:49.789594Z","shell.execute_reply":"2023-03-18T09:16:52.901845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.bar([\"Black\", \"White\"], [white, black])","metadata":{"execution":{"iopub.status.busy":"2023-03-18T09:16:52.904915Z","iopub.execute_input":"2023-03-18T09:16:52.905405Z","iopub.status.idle":"2023-03-18T09:16:53.070045Z","shell.execute_reply.started":"2023-03-18T09:16:52.905353Z","shell.execute_reply":"2023-03-18T09:16:53.068888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Okay now it look pretty good..... \n\nAgain ..... How to use it while training.... \nAddressing it soon \n\n# This is a work in progress.... I hope you found it helpful 🤜👩‍🔬","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}