{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"It might be helpful to understand correlation between pixel value of given data with pixel manually labelled as ink, beyond the simple ink/not-ink histogram.\nHopefully, visualizations below will give you an insight.","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport glob\nimport PIL.Image as Image\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nfrom tqdm import tqdm\nfrom io import StringIO\nfrom sklearn.metrics import fbeta_score\nfrom skimage.util import view_as_windows\nfrom scipy.ndimage import distance_transform_edt\nfrom numba import jit","metadata":{"execution":{"iopub.status.busy":"2023-05-17T05:26:09.785956Z","iopub.execute_input":"2023-05-17T05:26:09.786365Z","iopub.status.idle":"2023-05-17T05:26:09.793263Z","shell.execute_reply.started":"2023-05-17T05:26:09.786334Z","shell.execute_reply":"2023-05-17T05:26:09.792032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Constants\nPREFIX = '/kaggle/input/vesuvius-challenge-ink-detection/train/3/'\n\n# Load mask image\nmask = np.array(Image.open(PREFIX+\"mask.png\").convert('1'))\n\n# Load label image\nlabel = (np.array(Image.open(PREFIX+\"inklabels.png\")) > 0).astype(np.float32)\n\n# Load infrared image\nir = np.array(Image.open(PREFIX+\"ir.png\"))","metadata":{"execution":{"iopub.status.busy":"2023-05-17T05:26:09.795197Z","iopub.execute_input":"2023-05-17T05:26:09.795524Z","iopub.status.idle":"2023-05-17T05:26:10.872690Z","shell.execute_reply.started":"2023-05-17T05:26:09.795495Z","shell.execute_reply":"2023-05-17T05:26:10.871404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@jit(nopython=True)\ndef get_value_ink_ratio(value_count_ink, value_count_all, a, label):\n    for v, l in zip(a.ravel(), label.ravel()):\n        value_count_all[v] += 1\n        if l:\n            value_count_ink[v] += 1","metadata":{"execution":{"iopub.status.busy":"2023-05-17T05:26:10.874304Z","iopub.execute_input":"2023-05-17T05:26:10.874671Z","iopub.status.idle":"2023-05-17T05:26:10.880761Z","shell.execute_reply.started":"2023-05-17T05:26:10.874640Z","shell.execute_reply":"2023-05-17T05:26:10.879910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_ink_ratio(value_count_ink, value_count_all, img):\n    value_ink_ratio = np.where(value_count_all == 0, 0, value_count_ink/value_count_all)\n    x = np.arange(len(value_ink_ratio))\n    \n    # plot ink ratio distribution\n    fig, ax = plt.subplots(1, 1, figsize=(14,1))\n    ax.plot(x, value_ink_ratio, linestyle='', marker='.')\n    ax.set_title('Ink ratio by value')\n    plt.show()\n    \n    # select \n    sorted_by_ink = np.argsort(value_ink_ratio)\n    sorted_ink_ratio = value_ink_ratio[sorted_by_ink]\n    truth = value_count_all.sum()\n    f05 = np.zeros(101)\n    best_f05 = 0\n    best_th = 0\n    for th in range(101):\n        high_ink_ratio = sorted_by_ink[sorted_ink_ratio > th / 100]\n        tp = value_count_ink[high_ink_ratio].sum()\n        fp = value_count_all[high_ink_ratio].sum() - tp\n        fn = truth - tp\n        f05[th] = 1.25 * tp / (1.25 * tp + fp + 0.25 * fn)\n        if best_f05 < f05[th]:\n            best_th = th / 100\n            best_f05 = f05[th]\n            \n    fig, ax = plt.subplots(1, 1, figsize=(14,1))\n    ax.plot(np.arange(101) / 100, f05, linestyle='', marker='.')\n    ax.set_title('F05 score by threshold')\n    plt.show()\n            \n    high_ink_ratio = sorted_by_ink[sorted_ink_ratio > best_th]\n    high_ink = np.isin(img, high_ink_ratio)\n        \n    print('Best scoring threshold:', best_th)\n    print('Best score:', best_f05)\n    print('% of values:', len(high_ink_ratio)/len(value_ink_ratio))\n\n    fig, ax = plt.subplots(1, 4, figsize=(14,5))\n    ax[0].imshow(img, cmap='gray')\n    ax[1].imshow(value_ink_ratio[img], cmap='gray')\n    ax[2].imshow(high_ink, cmap='gray')\n    ax[3].imshow(label, cmap='gray')\n    ax[0].set_title('Source image')\n    ax[1].set_title('Ink ratio by pixel')\n    ax[2].set_title('Best prediction')\n    ax[3].set_title('Ink labels')\n    plt.show()\n    \n    return high_ink, best_f05\n\nNUM_VALUES = 256    \nvalue_count_ink = np.zeros(NUM_VALUES, dtype=int)\nvalue_count_all = np.zeros(NUM_VALUES, dtype=int)\nget_value_ink_ratio(value_count_ink, value_count_all, ir, label)\nplot_ink_ratio(value_count_ink, value_count_all, ir)","metadata":{"execution":{"iopub.status.busy":"2023-05-17T05:41:36.066337Z","iopub.execute_input":"2023-05-17T05:41:36.066750Z","iopub.status.idle":"2023-05-17T05:41:43.549677Z","shell.execute_reply.started":"2023-05-17T05:41:36.066719Z","shell.execute_reply":"2023-05-17T05:41:43.548628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There is clear correlation between pixel values of Infrared image and labels","metadata":{}},{"cell_type":"code","source":"NUM_VALUES = 65536\n\nbest_f05 = 0\nbest_ink = 0\nbest_z = 0\nfor z, filename in tqdm(enumerate(sorted(glob.glob(PREFIX+\"surface_volume/*.tif\")))):\n    value_count_ink = np.zeros(NUM_VALUES, dtype=int)\n    value_count_all = np.zeros(NUM_VALUES, dtype=int)\n    img = np.array(Image.open(filename), dtype=int)\n    get_value_ink_ratio(value_count_ink, value_count_all, img, label)\n    ink, f05 = plot_ink_ratio(value_count_ink, value_count_all, img)\n    if best_f05 < f05:\n        best_ink = ink\n        best_f05 = f05\n        best_z = z\n    \nfig, ax = plt.subplots(1, 2, figsize=(14,5))\nax[0].imshow(best_ink, cmap='gray')\nax[1].imshow(label, cmap='gray')\nax[0].set_title('Best prediction')\nax[1].set_title('Label')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-17T05:41:55.674484Z","iopub.execute_input":"2023-05-17T05:41:55.674918Z","iopub.status.idle":"2023-05-17T05:42:32.728690Z","shell.execute_reply.started":"2023-05-17T05:41:55.674881Z","shell.execute_reply":"2023-05-17T05:42:32.726964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There seems to be small correlation in layers 27-33","metadata":{}},{"cell_type":"code","source":"# Function to generate run-length encoding (RLE) for the binary mask\ndef rle(img):\n    pixels = img.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    f = StringIO()\n    np.savetxt(f, runs.reshape(1, -1), delimiter=\" \", fmt=\"%d\")\n    predicted = f.getvalue().strip()\n    return predicted\n\n# Generate RLE for the binary output\nrle_output = rle(best_ink)\n\n# Save the RLE to a CSV file for submission\nwith open('submission.csv', 'w') as f:\n    f.write(\"Id,Predicted\\n\")\n    f.write(\"a,\" + rle_output + \"\\n\")\n    f.write(\"b,\" + rle_output + \"\\n\")\n\nprint(\"Submission file 'submission.csv' has been generated.\")","metadata":{"execution":{"iopub.status.busy":"2023-05-17T05:26:34.407695Z","iopub.status.idle":"2023-05-17T05:26:34.408326Z","shell.execute_reply.started":"2023-05-17T05:26:34.408033Z","shell.execute_reply":"2023-05-17T05:26:34.408061Z"},"trusted":true},"execution_count":null,"outputs":[]}]}