{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84969,"databundleVersionId":10033515,"sourceType":"competition"},{"sourceId":10445208,"sourceType":"datasetVersion","datasetId":6465432},{"sourceId":10451318,"sourceType":"datasetVersion","datasetId":6469474}],"dockerImageVersionId":30787,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!ls /kaggle/input/sdewgg","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T14:13:15.680444Z","iopub.execute_input":"2025-01-12T14:13:15.68088Z","iopub.status.idle":"2025-01-12T14:13:16.774691Z","shell.execute_reply.started":"2025-01-12T14:13:15.680841Z","shell.execute_reply":"2025-01-12T14:13:16.773073Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install --no-index --find-links /kaggle/input/sdewgg -r /kaggle/input/sdewgg/requirements.txt\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T14:13:18.864927Z","iopub.execute_input":"2025-01-12T14:13:18.865401Z","iopub.status.idle":"2025-01-12T14:13:30.045098Z","shell.execute_reply.started":"2025-01-12T14:13:18.865359Z","shell.execute_reply":"2025-01-12T14:13:30.043616Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install /kaggle/working/asciitree-0.3.3/\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T14:13:33.020153Z","iopub.execute_input":"2025-01-12T14:13:33.020539Z","iopub.status.idle":"2025-01-12T14:14:17.313954Z","shell.execute_reply.started":"2025-01-12T14:13:33.020504Z","shell.execute_reply":"2025-01-12T14:14:17.31253Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip show asciitree\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T14:14:20.883413Z","iopub.execute_input":"2025-01-12T14:14:20.883794Z","iopub.status.idle":"2025-01-12T14:14:30.452279Z","shell.execute_reply.started":"2025-01-12T14:14:20.883761Z","shell.execute_reply":"2025-01-12T14:14:30.450823Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport zarr\nimport json\nimport matplotlib.pyplot as plt\nfrom mpl_toolkits.mplot3d import Axes3D\n\nBASE_PATH = '/kaggle/input/czii-cryo-et-object-identification'\nTRAIN_PATH = os.path.join(BASE_PATH, 'train')\nTEST_PATH = os.path.join(BASE_PATH, 'test')\n\n\ndef load_zarr(file_path):\n    z = zarr.open(file_path, mode='r')\n    return np.array(z[0])  \n\ndef load_json(file_path):\n    with open(file_path, 'r') as f:\n        return json.load(f)\n\n# Visual 2D slice of 3D tomogram\ndef visualize_slice(tomogram, slice_idx):\n    plt.figure(figsize=(10, 10))\n    plt.imshow(tomogram[slice_idx], cmap='gray')\n    plt.title(f'Slice {slice_idx}')\n    plt.colorbar()\n    plt.show()\n\n# Ex\nexperiment = 'TS_5_4'\nzarr_file = os.path.join(TRAIN_PATH, f'static/ExperimentRuns/{experiment}/VoxelSpacing10.000/denoised.zarr')\njson_file = os.path.join(TRAIN_PATH, f'overlay/ExperimentRuns/{experiment}/Picks/apo-ferritin.json')\n\n\ntomogram = load_zarr(zarr_file)\nannotations = load_json(json_file)\n\nprint(f'Tomogram shape: {tomogram.shape}')\nprint(f'Annotations: {annotations}')\n\n# Visual\nvisualize_slice(tomogram, slice_idx=tomogram.shape[0] // 2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T14:14:34.070752Z","iopub.execute_input":"2025-01-12T14:14:34.071718Z","iopub.status.idle":"2025-01-12T14:14:37.694789Z","shell.execute_reply.started":"2025-01-12T14:14:34.07167Z","shell.execute_reply":"2025-01-12T14:14:37.693659Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from scipy.ndimage import gaussian_filter\nimport numpy as np\n\n# Normalize \ndef normalize_tomogram(tomogram):\n    tomogram = (tomogram - np.mean(tomogram)) / np.std(tomogram)\n    return tomogram\n\n# Convert angstrom coordi to voxel indices\ndef angstrom_to_voxel(point, voxel_spacing):\n    return {\n        'x': int(point['x'] / voxel_spacing[0]),\n        'y': int(point['y'] / voxel_spacing[1]),\n        'z': int(point['z'] / voxel_spacing[2])\n    }\n\n# Adjusting out-of-bounds coordi \ndef scale_coordinates(x, y, z, tomogram_shape):\n    x = np.clip(x, 0, tomogram_shape[0] - 1)\n    y = np.clip(y, 0, tomogram_shape[1] - 1)\n    z = np.clip(z, 0, tomogram_shape[2] - 1)\n    return x, y, z\n\n# Gaussian mask gen\ndef create_gaussian_mask(tomogram_shape, points, voxel_spacing, sigma=3):\n    mask = np.zeros(tomogram_shape, dtype=np.float32)\n    for point in points:\n        # Convert angstrom coord to voxel indices\n        voxel_coords = angstrom_to_voxel(point['location'], voxel_spacing)\n\n        # Check if coord are out of bound\n        if not (0 <= voxel_coords['x'] < tomogram_shape[0] and \\\n                0 <= voxel_coords['y'] < tomogram_shape[1] and \\\n                0 <= voxel_coords['z'] < tomogram_shape[2]):\n            print(f\"Warning: Point {voxel_coords} out of bounds, skipping.\")\n            continue\n\n        # Adjust \n        x, y, z = scale_coordinates(voxel_coords['x'], voxel_coords['y'], voxel_coords['z'], tomogram_shape)\n\n        mask[x, y, z] = 1\n    \n    # Gaussian filter\n    mask = gaussian_filter(mask, sigma=sigma)\n\n    mask /= np.max(mask)\n    return mask\n\n# Processing annotations\ndef process_annotations(json_file, tomogram_shape, voxel_spacing):\n    data = load_json(json_file)\n    points = data.get('points', [])\n    for point in points:\n        angstrom_coords = point['location']\n        voxel_coords = angstrom_to_voxel(angstrom_coords, voxel_spacing)\n        print(f\"Angstrom Coordinate: {angstrom_coords}\")\n        print(f\"Voxel Coordinate: x={voxel_coords['x']}, y={voxel_coords['y']}, z={voxel_coords['z']}\")\n        if not (0 <= voxel_coords['x'] < tomogram_shape[0] and \\\n                0 <= voxel_coords['y'] < tomogram_shape[1] and \\\n                0 <= voxel_coords['z'] < tomogram_shape[2]):\n            print(f\"Warning: Point out of tomogram bounds!\")\n\n    return create_gaussian_mask(tomogram_shape, points, voxel_spacing)\n\nvoxel_spacing = (10.012444196428572, 10.012444196428572, 10.012444537618887)\n\nnormalized_tomogram = normalize_tomogram(tomogram)\ngaussian_mask = process_annotations(json_file, tomogram.shape, voxel_spacing)\n\nvisualize_slice(normalized_tomogram, slice_idx=tomogram.shape[0] // 2)\nvisualize_slice(gaussian_mask, slice_idx=tomogram.shape[0] // 2)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T14:14:43.666156Z","iopub.execute_input":"2025-01-12T14:14:43.666749Z","iopub.status.idle":"2025-01-12T14:14:48.758484Z","shell.execute_reply.started":"2025-01-12T14:14:43.666712Z","shell.execute_reply":"2025-01-12T14:14:48.75736Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def visualize_annotations_on_slice(tomogram, mask, slice_idx, points):\n    fig = plt.figure(figsize=(10, 10))\n    ax = fig.add_subplot(111)\n    im = ax.imshow(tomogram[slice_idx], cmap='gray', alpha=0.7)\n    mask_slice = mask[slice_idx]\n    ax.imshow(mask_slice, cmap='jet', alpha=0.5)  # Overlay the Gaussian mask\n    for point in points:\n        angstrom_coords = point['location']\n        voxel_coords = angstrom_to_voxel(angstrom_coords, voxel_spacing)\n        if 0 <= voxel_coords['x'] < tomogram.shape[0] and 0 <= voxel_coords['y'] < tomogram.shape[1]:\n            ax.scatter(voxel_coords['y'], voxel_coords['x'], color='red', s=50)\n    \n    plt.title(f'Annotations on Slice {slice_idx}')\n    \n    # Adding colorbar using the imshow return object\n    plt.colorbar(im)\n    \n    plt.show()\n\n\nvisualize_annotations_on_slice(tomogram, gaussian_mask, slice_idx=tomogram.shape[0] // 2, points=annotations['points'])\n\n# Improved Gaussian mask generation with dynamic sigma based on point density\ndef create_gaussian_mask_dynamic_sigma(tomogram_shape, points, voxel_spacing, base_sigma=3):\n    mask = np.zeros(tomogram_shape, dtype=np.float32)\n    for point in points:\n        voxel_coords = angstrom_to_voxel(point['location'], voxel_spacing)\n\n        x, y, z = scale_coordinates(voxel_coords['x'], voxel_coords['y'], voxel_coords['z'], tomogram_shape)\n\n        # Placing the point in the mask with dynamic sigma based on proximity\n        distances = np.linalg.norm([point['location']['x'], point['location']['y'], point['location']['z']])\n        sigma = max(base_sigma / (distances + 1), 1)  \n        mask[x, y, z] = 1\n    \n    mask = gaussian_filter(mask, sigma=sigma)\n\n    mask /= np.max(mask)\n    return mask\n\ndynamic_gaussian_mask = create_gaussian_mask_dynamic_sigma(tomogram.shape, annotations['points'], voxel_spacing)\n\nvisualize_slice(dynamic_gaussian_mask, slice_idx=tomogram.shape[0] // 2)\n\n# Save (after scaling)\nadjusted_annotations = []\nfor point in annotations['points']:\n    angstrom_coords = point['location']\n    voxel_coords = angstrom_to_voxel(angstrom_coords, voxel_spacing)\n    adjusted_annotations.append(voxel_coords)\n\n# Export to a JSON file\nwith open('adjusted_annotations.json', 'w') as f:\n    json.dump(adjusted_annotations, f)\n\n# Save \nnp.save('gaussian_mask.npy', dynamic_gaussian_mask)\n\n# Implement chunking to optimize memory\ndef create_gaussian_mask_in_chunks(tomogram_shape, points, voxel_spacing, chunk_size=50, sigma=3):\n    mask = np.zeros(tomogram_shape, dtype=np.float32)\n    for i in range(0, len(points), chunk_size):\n        chunk = points[i:i+chunk_size]\n        mask = create_gaussian_mask(tomogram_shape, chunk, voxel_spacing, sigma)\n    return mask\n\nchunked_gaussian_mask = create_gaussian_mask_in_chunks(tomogram.shape, annotations['points'], voxel_spacing)\nvisualize_slice(chunked_gaussian_mask, slice_idx=tomogram.shape[0] // 2)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T14:14:53.391462Z","iopub.execute_input":"2025-01-12T14:14:53.392562Z","iopub.status.idle":"2025-01-12T14:15:00.833424Z","shell.execute_reply.started":"2025-01-12T14:14:53.392515Z","shell.execute_reply":"2025-01-12T14:15:00.832055Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_gaussian_mask_dynamic_sigma(tomogram_shape, points, voxel_spacing, base_sigma=3):\n    \n    mask = np.zeros(tomogram_shape, dtype=np.float32)\n    for point in points:\n        # Convert angstrom coordinates to voxel indices\n        voxel_coords = angstrom_to_voxel(point['location'], voxel_spacing)\n        \n        if not (0 <= voxel_coords['x'] < tomogram_shape[0] and \\\n                0 <= voxel_coords['y'] < tomogram_shape[1] and \\\n                0 <= voxel_coords['z'] < tomogram_shape[2]):\n            print(f\"Warning: Point {voxel_coords} out of bounds, skipping.\")\n            continue\n\n        x, y, z = scale_coordinates(voxel_coords['x'], voxel_coords['y'], voxel_coords['z'], tomogram_shape)\n\n        sigma = base_sigma\n        if len(points) > 500:  \n            sigma = base_sigma / 2  \n\n        mask[x, y, z] = 1\n    \n    mask = gaussian_filter(mask, sigma=sigma)\n\n    mask /= np.max(mask)\n    return mask\n\ndef visualize_dynamic_gaussian_mask(tomogram, json_file, voxel_spacing, slice_idx):\n    # Loading annotations and process mask\n    annotations = load_json(json_file)\n    dynamic_gaussian_mask = create_gaussian_mask_dynamic_sigma(tomogram.shape, annotations['points'], voxel_spacing)\n    \n    # Visualizing the tomogram slice and overlay the mask\n    plt.figure(figsize=(10, 10))\n    plt.imshow(tomogram[slice_idx], cmap='gray', alpha=0.7)\n    plt.imshow(dynamic_gaussian_mask[slice_idx], cmap='jet', alpha=0.5)  # Overlay the Gaussian mask\n    plt.title(f'Annotations and Dynamic Mask on Slice {slice_idx}')\n    plt.colorbar()\n    plt.show()\n\nvisualize_dynamic_gaussian_mask(tomogram, json_file, voxel_spacing, slice_idx=tomogram.shape[0] // 2)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T14:15:14.88194Z","iopub.execute_input":"2025-01-12T14:15:14.882373Z","iopub.status.idle":"2025-01-12T14:15:19.132346Z","shell.execute_reply.started":"2025-01-12T14:15:14.882334Z","shell.execute_reply":"2025-01-12T14:15:19.131082Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from mpl_toolkits.mplot3d import Axes3D\n\n# Function to visualize the annotated points as a 3D scatter plot\ndef visualize_3d_points(tomogram, json_file, voxel_spacing):\n    annotations = load_json(json_file)\n    points = annotations.get('points', [])\n\n    voxel_coords = [angstrom_to_voxel(point['location'], voxel_spacing) for point in points]\n\n    fig = plt.figure(figsize=(10, 10))\n    ax = fig.add_subplot(111, projection='3d')\n\n    # Extracting x, y, z coordinates\n    x = [v['x'] for v in voxel_coords]\n    y = [v['y'] for v in voxel_coords]\n    z = [v['z'] for v in voxel_coords]\n\n    # Scatter plot\n    ax.scatter(x, y, z, c='red', s=50, alpha=0.5)\n    ax.set_xlabel('X Voxel')\n    ax.set_ylabel('Y Voxel')\n    ax.set_zlabel('Z Voxel')\n    ax.set_title('3D Scatter Plot of Annotations')\n    plt.show()\n\nvisualize_3d_points(tomogram, json_file, voxel_spacing)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T14:15:26.977941Z","iopub.execute_input":"2025-01-12T14:15:26.978385Z","iopub.status.idle":"2025-01-12T14:15:27.253917Z","shell.execute_reply.started":"2025-01-12T14:15:26.97835Z","shell.execute_reply":"2025-01-12T14:15:27.252538Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from scipy.ndimage import gaussian_filter, sobel\nimport numpy as np\n\n# High-pass filter\ndef high_pass_filter(tomogram, sigma=3):\n    smooth_tomogram = gaussian_filter(tomogram, sigma=sigma)\n    high_passed = tomogram - smooth_tomogram\n    return high_passed\n\n# Low-pass filter\ndef low_pass_filter(tomogram, sigma=3):\n    low_passed = gaussian_filter(tomogram, sigma=sigma)\n    return low_passed\n\nhigh_passed_tomogram = high_pass_filter(tomogram, sigma=5)\nlow_passed_tomogram = low_pass_filter(tomogram, sigma=5)\n\nplt.figure(figsize=(10, 10))\nplt.subplot(1, 2, 1)\nplt.imshow(high_passed_tomogram[tomogram.shape[0] // 2], cmap='gray')\nplt.title('High-Passed Tomogram')\nplt.colorbar()\n\nplt.subplot(1, 2, 2)\nplt.imshow(low_passed_tomogram[tomogram.shape[0] // 2], cmap='gray')\nplt.title('Low-Passed Tomogram')\nplt.colorbar()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T14:15:31.122214Z","iopub.execute_input":"2025-01-12T14:15:31.122613Z","iopub.status.idle":"2025-01-12T14:15:42.903932Z","shell.execute_reply.started":"2025-01-12T14:15:31.122577Z","shell.execute_reply":"2025-01-12T14:15:42.902463Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Sobel filter for edge detection\ndef sobel_filter(tomogram):\n    sobel_x = sobel(tomogram, axis=0)\n    sobel_y = sobel(tomogram, axis=1)\n    sobel_z = sobel(tomogram, axis=2)\n\n    sobel_magnitude = np.sqrt(sobel_x**2 + sobel_y**2 + sobel_z**2)\n    return sobel_magnitude\n\nsobel_magnitude = sobel_filter(tomogram)\n\nplt.figure(figsize=(10, 10))\nplt.imshow(sobel_magnitude[tomogram.shape[0] // 2], cmap='hot')\nplt.title('Sobel Filter Magnitude')\nplt.colorbar()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T14:16:03.156654Z","iopub.execute_input":"2025-01-12T14:16:03.157072Z","iopub.status.idle":"2025-01-12T14:16:08.471379Z","shell.execute_reply.started":"2025-01-12T14:16:03.157033Z","shell.execute_reply":"2025-01-12T14:16:08.469879Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\n\n# Function save tomogram or mask as .npy file\ndef save_as_npy(data, filename):\n    np.save(filename, data)\n    print(f\"Saved {filename}\")\n\nsave_as_npy(high_passed_tomogram, \"/kaggle/working/high_passed_tomogram.npy\")\nsave_as_npy(low_passed_tomogram, \"/kaggle/working/low_passed_tomogram.npy\")\nsave_as_npy(gaussian_mask, \"/kaggle/working/gaussian_mask.npy\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T14:16:25.344168Z","iopub.execute_input":"2025-01-12T14:16:25.344574Z","iopub.status.idle":"2025-01-12T14:16:25.908842Z","shell.execute_reply.started":"2025-01-12T14:16:25.344535Z","shell.execute_reply":"2025-01-12T14:16:25.907773Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Function to save visualizations\ndef save_visualization(tomogram_slice, filename):\n    plt.figure(figsize=(10, 10))\n    plt.imshow(tomogram_slice, cmap='gray')\n    plt.colorbar()\n    plt.title(\"Processed Tomogram Slice\")\n    plt.savefig(filename)\n    print(f\"Saved visualization to {filename}\")\n    plt.close()\n\nsave_visualization(high_passed_tomogram[tomogram.shape[0] // 2], \"/kaggle/working/high_passed_slice.png\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T14:16:30.399506Z","iopub.execute_input":"2025-01-12T14:16:30.399878Z","iopub.status.idle":"2025-01-12T14:16:31.052618Z","shell.execute_reply.started":"2025-01-12T14:16:30.399847Z","shell.execute_reply":"2025-01-12T14:16:31.051368Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Function to save annotations as JSON\ndef save_annotations_to_json(annotations, filename):\n    with open(filename, 'w') as f:\n        json.dump(annotations, f)\n    print(f\"Annotations saved to {filename}\")\n\nsave_annotations_to_json(annotations, \"/kaggle/working/processed_annotations.json\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T14:16:35.305932Z","iopub.execute_input":"2025-01-12T14:16:35.306389Z","iopub.status.idle":"2025-01-12T14:16:35.316501Z","shell.execute_reply.started":"2025-01-12T14:16:35.306352Z","shell.execute_reply":"2025-01-12T14:16:35.315352Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import csv\n\n# Function to save annotation points to a CSV\ndef save_annotations_to_csv(annotations, filename):\n    points = annotations.get('points', [])\n    with open(filename, mode='w', newline='') as f:\n        writer = csv.writer(f)\n        writer.writerow(['X', 'Y', 'Z'])\n        for point in points:\n            voxel_coords = angstrom_to_voxel(point['location'], voxel_spacing)\n            writer.writerow([voxel_coords['x'], voxel_coords['y'], voxel_coords['z']])\n    print(f\"Annotations saved to {filename}\")\n\nsave_annotations_to_csv(annotations, \"/kaggle/working/annotations.csv\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T14:16:41.559848Z","iopub.execute_input":"2025-01-12T14:16:41.56093Z","iopub.status.idle":"2025-01-12T14:16:41.571271Z","shell.execute_reply.started":"2025-01-12T14:16:41.560871Z","shell.execute_reply":"2025-01-12T14:16:41.569763Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Function to plot and compare filtered tomograms and masks\ndef compare_filters(tomogram, high_passed, low_passed, sobel_magnitude):\n    plt.figure(figsize=(15, 10))\n    \n    # Original Tomogram Slice\n    plt.subplot(2, 2, 1)\n    plt.imshow(tomogram[tomogram.shape[0] // 2], cmap='gray')\n    plt.title('Original Tomogram')\n    plt.colorbar()\n    \n    # High-passed Tomogram Slice\n    plt.subplot(2, 2, 2)\n    plt.imshow(high_passed[tomogram.shape[0] // 2], cmap='hot')\n    plt.title('High-Passed Tomogram')\n    plt.colorbar()\n    \n    # Low-passed Tomogram Slice\n    plt.subplot(2, 2, 3)\n    plt.imshow(low_passed[tomogram.shape[0] // 2], cmap='gray')\n    plt.title('Low-Passed Tomogram')\n    plt.colorbar()\n    \n    # Sobel Filtered Magnitude Slice\n    plt.subplot(2, 2, 4)\n    plt.imshow(sobel_magnitude[tomogram.shape[0] // 2], cmap='hot')\n    plt.title('Sobel Filtered Magnitude')\n    plt.colorbar()\n    \n    plt.tight_layout()\n    plt.show()\n\ncompare_filters(tomogram, high_passed_tomogram, low_passed_tomogram, sobel_magnitude)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T14:16:46.26142Z","iopub.execute_input":"2025-01-12T14:16:46.261816Z","iopub.status.idle":"2025-01-12T14:16:47.966464Z","shell.execute_reply.started":"2025-01-12T14:16:46.261782Z","shell.execute_reply":"2025-01-12T14:16:47.965328Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def count_particles(mask, threshold=0.5):\n    return np.sum(mask >= threshold)\n\nparticle_count = count_particles(gaussian_mask)\nprint(f\"Number of particles detected: {particle_count}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T14:16:54.955841Z","iopub.execute_input":"2025-01-12T14:16:54.956279Z","iopub.status.idle":"2025-01-12T14:16:55.061398Z","shell.execute_reply.started":"2025-01-12T14:16:54.95624Z","shell.execute_reply":"2025-01-12T14:16:55.060162Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Machine learning part #","metadata":{}},{"cell_type":"markdown","source":"## main work ##","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Input, Conv3D, MaxPooling3D, Flatten, Dense\n\ndef create_3d_cnn(input_shape=(64, 64, 64, 1)):  # (depth, height, width, channels)\n    inputs = Input(shape=input_shape)\n\n    # 3D convolutional layers\n    x = Conv3D(32, (3, 3, 3), activation='relu')(inputs)\n    x = MaxPooling3D(pool_size=(2, 2, 2))(x)\n    x = Conv3D(64, (3, 3, 3), activation='relu')(x)\n    x = MaxPooling3D(pool_size=(2, 2, 2))(x)\n    x = Conv3D(128, (3, 3, 3), activation='relu')(x)\n\n    # Flatten and dense layers for final output\n    x = Flatten()(x)\n    x = Dense(256, activation='relu')(x)\n    outputs = Dense(5, activation='softmax')(x)  # 5 particle types to classify\n\n    model = Model(inputs, outputs)\n    model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n    return model\n\ninput_shape = (64, 64, 64, 1)  \nmodel = create_3d_cnn(input_shape)\nmodel.summary()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T14:17:58.636942Z","iopub.execute_input":"2025-01-12T14:17:58.637409Z","iopub.status.idle":"2025-01-12T14:17:59.093949Z","shell.execute_reply.started":"2025-01-12T14:17:58.637369Z","shell.execute_reply":"2025-01-12T14:17:59.092815Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport zarr\nimport numpy as np\n\ndef load_data():\n    data_path = \"/kaggle/input/czii-cryo-et-object-identification/train/static/ExperimentRuns\"  \n    annotations_path = \"/kaggle/working/adjusted_annotations.json\"  \n    \n    tomograms = [] \n    annotations = []  \n\n    if not os.path.exists(data_path):\n        print(f\"Data path {data_path} does not exist.\")\n        return tomograms, annotations\n    \n    print(f\"Listing files in data path: {data_path}\")\n    for experiment_folder in sorted(os.listdir(data_path)):\n        experiment_folder_path = os.path.join(data_path, experiment_folder)\n        print(f\"Checking folder: {experiment_folder_path}\")\n        \n        if os.path.isdir(experiment_folder_path):\n            for subfolder in sorted(os.listdir(experiment_folder_path)):\n                subfolder_path = os.path.join(experiment_folder_path, subfolder)\n                print(f\"Checking subfolder: {subfolder_path}\")\n                \n                if os.path.isdir(subfolder_path):\n                    for tomogram_file in sorted(os.listdir(subfolder_path)):\n                        print(f\"Found file: {tomogram_file}\")\n                        \n                        if tomogram_file.endswith(\".zarr\"):\n                            zarr_file_path = os.path.join(subfolder_path, tomogram_file)\n                            print(f\"Inspecting Zarr file: {zarr_file_path}\")\n                            zarr_group = zarr.open(zarr_file_path, mode='r')\n                            \n                            available_keys = list(zarr_group.keys())\n                            print(f\"Available keys in Zarr file: {available_keys}\")\n                            \n                            if available_keys:\n                                key = available_keys[0]  \n                                tomogram_data = zarr_group[key]\n                                tomograms.append(tomogram_data)\n                            else:\n                                print(f\"No data found in {tomogram_file}\")\n    \n    print(f\"Total tomograms loaded: {len(tomograms)}\")\n    print(f\"Total annotations loaded: {len(annotations)}\")\n    \n    tomograms = np.array(tomograms)\n    annotations = np.array(annotations)\n    \n    return tomograms, annotations\n\ntomograms, annotations = load_data()\n\nprint(f\"Loaded {len(tomograms)} tomograms and {len(annotations)} annotations.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-12T14:18:12.997871Z","iopub.execute_input":"2025-01-12T14:18:12.998309Z","iopub.status.idle":"2025-01-12T14:19:40.976606Z","shell.execute_reply.started":"2025-01-12T14:18:12.998264Z","shell.execute_reply":"2025-01-12T14:19:40.974641Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import json\n\nannotations_path = \"/kaggle/working/adjusted_annotations.json\"\n\ntry:\n    with open(annotations_path, \"r\") as file:\n        annotations = json.load(file)\n    \n    if isinstance(annotations, list):\n        print(f\"The file contains a list with {len(annotations)} entries.\")\n        \n        print(\"\\nFirst 5 entries in the list:\")\n        for i, entry in enumerate(annotations[:5]):\n            print(f\"Entry {i+1}: {entry}\")\n    else:\n        print(f\"The file contains an unexpected data type: {type(annotations)}\")\nexcept FileNotFoundError:\n    print(f\"File {annotations_path} not found.\")\nexcept json.JSONDecodeError as e:\n    print(f\"Error decoding JSON: {e}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-10T18:20:42.785939Z","iopub.execute_input":"2025-01-10T18:20:42.786354Z","iopub.status.idle":"2025-01-10T18:20:42.797675Z","shell.execute_reply.started":"2025-01-10T18:20:42.786318Z","shell.execute_reply":"2025-01-10T18:20:42.796345Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nx_coords = [entry['x'] for entry in annotations]\ny_coords = [entry['y'] for entry in annotations]\nz_coords = [entry['z'] for entry in annotations]\n\nplt.figure(figsize=(8, 6))\nplt.scatter(x_coords, y_coords, c='red', alpha=0.6)\nplt.xlabel(\"X Coordinate\")\nplt.ylabel(\"Y Coordinate\")\nplt.title(\"2D Projection of Annotations (X, Y)\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-10T18:20:42.799331Z","iopub.execute_input":"2025-01-10T18:20:42.800088Z","iopub.status.idle":"2025-01-10T18:20:43.09289Z","shell.execute_reply.started":"2025-01-10T18:20:42.800033Z","shell.execute_reply":"2025-01-10T18:20:43.091413Z"}},"outputs":[],"execution_count":null}]}