{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":117682,"databundleVersionId":14443416,"sourceType":"competition"}],"dockerImageVersionId":31192,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Vesuvius EDA\n\nA simple and brief notebook looking into the Vesuvius Challenge data just to get a handle on what the data looks like at a glass\n\n## Installs","metadata":{}},{"cell_type":"code","source":"!pip install imagecodecs","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-24T13:26:35.652136Z","iopub.execute_input":"2025-11-24T13:26:35.652424Z","iopub.status.idle":"2025-11-24T13:26:46.591556Z","shell.execute_reply.started":"2025-11-24T13:26:35.652395Z","shell.execute_reply":"2025-11-24T13:26:46.589864Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# imports\nimport os\nimport numpy as np\nimport pandas as pd \nimport imagecodecs\nimport tifffile \n\nimport matplotlib.pyplot as plt\n\nfrom pathlib import Path\nfrom tqdm import tqdm\n\ninput_path = Path(\"/kaggle/input/vesuvius-challenge-surface-detection\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-11-24T14:27:18.156319Z","iopub.execute_input":"2025-11-24T14:27:18.156859Z","iopub.status.idle":"2025-11-24T14:27:18.166311Z","shell.execute_reply.started":"2025-11-24T14:27:18.156826Z","shell.execute_reply":"2025-11-24T14:27:18.164950Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# setting up paths\ntrain_image_path = input_path / \"train_images\"\ntrain_labels_path = input_path / \"train_labels\"\ntrain_df_path = input_path / \"train.csv\"\n\ntrain_image_files = sorted([os.path.join(train_image_path, f) for f in os.listdir(train_image_path) if f.endswith('.tif')])\ntrain_labels = sorted([os.path.join(train_labels_path, f) for f in os.listdir(train_labels_path) if f.endswith('.tif')])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-24T13:26:47.329863Z","iopub.execute_input":"2025-11-24T13:26:47.330429Z","iopub.status.idle":"2025-11-24T13:26:47.378168Z","shell.execute_reply.started":"2025-11-24T13:26:47.330395Z","shell.execute_reply":"2025-11-24T13:26:47.376850Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_image_files[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-24T14:22:29.902487Z","iopub.execute_input":"2025-11-24T14:22:29.903205Z","iopub.status.idle":"2025-11-24T14:22:29.918036Z","shell.execute_reply.started":"2025-11-24T14:22:29.903169Z","shell.execute_reply":"2025-11-24T14:22:29.916424Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_labels[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-24T14:22:36.132090Z","iopub.execute_input":"2025-11-24T14:22:36.132480Z","iopub.status.idle":"2025-11-24T14:22:36.140279Z","shell.execute_reply.started":"2025-11-24T14:22:36.132456Z","shell.execute_reply":"2025-11-24T14:22:36.139061Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Using tiff files for reading and writing data\nexample_image = tifffile.imread(train_image_files[0])\nexample_label = tifffile.imread(train_labels[0])\ntrain_df = pd.read_csv(train_df_path)\ntrain_df.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-24T14:27:54.797104Z","iopub.execute_input":"2025-11-24T14:27:54.798381Z","iopub.status.idle":"2025-11-24T14:27:55.395773Z","shell.execute_reply.started":"2025-11-24T14:27:54.798345Z","shell.execute_reply":"2025-11-24T14:27:55.394683Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Looking at the scroll ids\ntrain_df[\"scroll_id\"].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-24T14:28:14.197040Z","iopub.execute_input":"2025-11-24T14:28:14.197441Z","iopub.status.idle":"2025-11-24T14:28:14.208753Z","shell.execute_reply.started":"2025-11-24T14:28:14.197416Z","shell.execute_reply":"2025-11-24T14:28:14.207818Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Notes\n\nWe can see here that there are 6 different scrolls to look for. Therefore, it would be best to train and validate on different scroll id's to not have any data leakage\n\n## Looking at tiff dimensions","metadata":{}},{"cell_type":"code","source":"def get_tiff_dimensions(file_paths):\n    records = []\n    \n    for path in tqdm(file_paths, desc=\"Reading TIFF metadata\"):\n        try:\n            with tifffile.TiffFile(path) as tif:\n                shape = tif.series[0].shape\n                axes = tif.series[0].axes\n                records.append({\n                    \"file\": path,\n                    \"shape\": shape,\n                    \"axes\": axes\n                })\n        except Exception as e:\n            records.append({\n                \"file\": path,\n                \"shape\": None,\n                \"axes\": None,\n                \"error\": str(e)\n            })\n    \n    return pd.DataFrame(records)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-24T13:26:48.472304Z","iopub.execute_input":"2025-11-24T13:26:48.472879Z","iopub.status.idle":"2025-11-24T13:26:48.493154Z","shell.execute_reply.started":"2025-11-24T13:26:48.472842Z","shell.execute_reply":"2025-11-24T13:26:48.492155Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# this takes a few minutes to run\ndf = get_tiff_dimensions(train_image_files)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-24T13:27:20.614141Z","iopub.execute_input":"2025-11-24T13:27:20.614493Z","iopub.status.idle":"2025-11-24T13:31:01.984696Z","shell.execute_reply.started":"2025-11-24T13:27:20.614467Z","shell.execute_reply":"2025-11-24T13:31:01.983422Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# extract from file path\ndef extract_id(x):\n    return int(x.split(\"/\")[-1].split(\".\")[0])\n\ndf[\"id\"] = df[\"file\"].apply(lambda x: extract_id(x))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-24T13:37:34.391164Z","iopub.execute_input":"2025-11-24T13:37:34.392398Z","iopub.status.idle":"2025-11-24T13:37:34.398101Z","shell.execute_reply.started":"2025-11-24T13:37:34.392345Z","shell.execute_reply":"2025-11-24T13:37:34.396482Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-24T14:31:36.448394Z","iopub.execute_input":"2025-11-24T14:31:36.448715Z","iopub.status.idle":"2025-11-24T14:31:36.461864Z","shell.execute_reply.started":"2025-11-24T14:31:36.448693Z","shell.execute_reply":"2025-11-24T14:31:36.460594Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df[\"shape\"].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-24T14:31:16.980325Z","iopub.execute_input":"2025-11-24T14:31:16.980796Z","iopub.status.idle":"2025-11-24T14:31:16.998228Z","shell.execute_reply.started":"2025-11-24T14:31:16.980765Z","shell.execute_reply":"2025-11-24T14:31:16.997301Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Thoughts\n\n- tiffile is useful, but I understand now why people just use PIL image and NumPy as there is next to none documentation on how it works (if you have found some, message me!)\n- Most are 320x320x320, but there are differences so they either needed to be resized or the model to use needs to be able to handle dynamic input sizes\n\n## Basic visualization","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\ndef show_slices(volume, slice_indices=None):\n    \"\"\"\n    Display orthogonal slices of a 3D volume\n    \"\"\"\n    if slice_indices is None:\n        slice_indices = [volume.shape[0]//2, volume.shape[1]//2, volume.shape[2]//2]\n    \n    fig, axes = plt.subplots(1, 3, figsize=(15, 5))\n    \n    axes[0].imshow(volume[slice_indices[0], :, :], cmap='gray')\n    axes[0].set_title('Axial slice')\n    \n    axes[1].imshow(volume[:, slice_indices[1], :], cmap='gray')\n    axes[1].set_title('Coronal slice')\n    \n    axes[2].imshow(volume[:, :, slice_indices[2]], cmap='gray')\n    axes[2].set_title('Sagittal slice')\n    \n    plt.tight_layout()\n    plt.show()\n\n# Show slices\nshow_slices(example_image)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-24T13:38:42.113359Z","iopub.execute_input":"2025-11-24T13:38:42.113701Z","iopub.status.idle":"2025-11-24T13:38:43.203439Z","shell.execute_reply.started":"2025-11-24T13:38:42.113677Z","shell.execute_reply":"2025-11-24T13:38:43.201781Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"show_slices(example_label)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-24T13:39:06.151614Z","iopub.execute_input":"2025-11-24T13:39:06.152489Z","iopub.status.idle":"2025-11-24T13:39:06.875799Z","shell.execute_reply.started":"2025-11-24T13:39:06.152440Z","shell.execute_reply":"2025-11-24T13:39:06.874465Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Thoughts\n\n- 3d EDA is hard! Will be worth looking into a 3D visualization tool (ITK-SNAP or Volume Cartographer) to be able to better understand the labels and images","metadata":{}}]}