{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":84969,"databundleVersionId":10033515,"sourceType":"competition"}],"dockerImageVersionId":31153,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import subprocess\nimport sys\n\n# Install zarr\nsubprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"zarr\"])\nprint(\"Packages installed!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T14:50:36.971419Z","iopub.execute_input":"2025-10-30T14:50:36.971611Z","iopub.status.idle":"2025-10-30T14:50:38.459097Z","shell.execute_reply.started":"2025-10-30T14:50:36.971596Z","shell.execute_reply":"2025-10-30T14:50:38.458172Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import zarr\nimport numpy as np\nimport pandas as pd\nimport json\nimport os\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nfrom tensorflow.keras import layers, models\nfrom tensorflow.keras.utils import to_categorical\nfrom sklearn.preprocessing import LabelEncoder\nimport pickle\nimport gc\n\nprint(\"All imports successful!\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"home = \"/kaggle/input/czii-cryo-et-object-identification\"\n\n# Example: Explore one tomogram\ncurrent_file = \"/kaggle/input/czii-cryo-et-object-identification/train/static/ExperimentRuns/TS_86_3/VoxelSpacing10.000/denoised.zarr\"\n\nz = zarr.open(current_file, mode='r')\n\n# See what's inside\nprint(\"Contents:\", list(z.keys()))\nprint(\"Tree structure:\")\nprint(z.tree())\n\n# Examine the zarr data at level 2 resolution\narr = z['2']\n\nprint(\"Shape:\", arr.shape)\nprint(\"Data type:\", arr.dtype)\nprint(\"Data sample:\")\nprint(arr[:2])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T14:50:38.459834Z","iopub.execute_input":"2025-10-30T14:50:38.460109Z","iopub.status.idle":"2025-10-30T15:09:46.056057Z","shell.execute_reply.started":"2025-10-30T14:50:38.460080Z","shell.execute_reply":"2025-10-30T15:09:46.055128Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_overlay = \"/kaggle/input/czii-cryo-et-object-identification/train/overlay/ExperimentRuns\"\n\n# Get all experiments\nexperiments = os.listdir(train_overlay)\n\nall_particles = []\nparticle_id = 0\n\nfor experiment in experiments:\n    picks_path = os.path.join(train_overlay, experiment, \"Picks\")\n    \n    if not os.path.exists(picks_path):\n        continue\n    \n    # Get all particle type JSON files\n    json_files = [f for f in os.listdir(picks_path) if f.endswith('.json')]\n    \n    for json_file in json_files:\n        # Particle type is the filename without .json\n        particle_type = json_file.replace('.json', '')\n        \n        json_path = os.path.join(picks_path, json_file)\n        \n        with open(json_path, 'r') as f:\n            data = json.load(f)\n        \n        # Extract coordinates from each point\n        for point in data['points']:\n            loc = point['location']\n            all_particles.append({\n                'id': particle_id,\n                'experiment': experiment,\n                'particle_type': particle_type,\n                'x': loc['x'],\n                'y': loc['y'],\n                'z': loc['z']\n            })\n            particle_id += 1\n\n# Create DataFrame\ntrain_labels = pd.DataFrame(all_particles)\n\nprint(train_labels.head(20))\nprint(f\"\\nTotal particles: {len(train_labels)}\")\nprint(f\"\\nParticle type counts:\")\nprint(train_labels['particle_type'].value_counts())\nprint(f\"\\nExperiments:\")\nprint(train_labels['experiment'].unique())\n\n# Save to working directory\ntrain_labels.to_csv('/kaggle/working/train_labels.csv', index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\nExtracting training patches from ALL experiments...\")\n\npatch_size = 32\nX_train_all = []\ny_train_all = []\nvoxel_spacing = 10.0\n\ntrain_static = \"/kaggle/input/czii-cryo-et-object-identification/train/static/ExperimentRuns\"\nall_train_experiments = os.listdir(train_static)\n\nfor experiment in tqdm(all_train_experiments, desc=\"Processing experiments\"):\n    zarr_path = os.path.join(train_static, experiment, \"VoxelSpacing10.000\", \"denoised.zarr\")\n    \n    if not os.path.exists(zarr_path):\n        print(f\"Skipping {experiment} - zarr not found\")\n        continue\n    \n    # Load tomogram\n    z_exp = zarr.open(zarr_path, mode='r')\n    arr_exp = z_exp['2']  # Use level 2 for memory efficiency\n    scale_factor = 4  # Level 2 is 4x downsampled\n    \n    # Load particles for this experiment\n    particles = train_labels[train_labels['experiment'] == experiment].reset_index(drop=True)\n    \n    if len(particles) == 0:\n        continue\n    \n    # Convert Angstrom to pixel coordinates (accounting for scale)\n    particles['x_pixel'] = (particles['x'] / voxel_spacing / scale_factor).astype(int)\n    particles['y_pixel'] = (particles['y'] / voxel_spacing / scale_factor).astype(int)\n    particles['z_pixel'] = (particles['z'] / voxel_spacing / scale_factor).astype(int)\n    \n    # Extract positive examples\n    for idx, particle in particles.iterrows():\n        x, y, z_coord = particle['x_pixel'], particle['y_pixel'], particle['z_pixel']\n        particle_type = particle['particle_type']\n        \n        # Check bounds\n        if (z_coord - patch_size//2 >= 0 and z_coord + patch_size//2 <= arr_exp.shape[0] and\n            y - patch_size//2 >= 0 and y + patch_size//2 <= arr_exp.shape[1] and\n            x - patch_size//2 >= 0 and x + patch_size//2 <= arr_exp.shape[2]):\n            \n            patch = arr_exp[\n                z_coord - patch_size//2 : z_coord + patch_size//2,\n                y - patch_size//2 : y + patch_size//2,\n                x - patch_size//2 : x + patch_size//2\n            ]\n            \n            if patch.shape == (patch_size, patch_size, patch_size):\n                X_train_all.append(patch)\n                y_train_all.append(particle_type)\n    \n    # Extract negative examples (limit to avoid too many)\n    num_negatives = min(len(particles), 30)\n    negative_count = 0\n    attempts = 0\n    max_attempts = num_negatives * 10\n    \n    while negative_count < num_negatives and attempts < max_attempts:\n        attempts += 1\n        z_rand = np.random.randint(patch_size//2, arr_exp.shape[0] - patch_size//2)\n        y_rand = np.random.randint(patch_size//2, arr_exp.shape[1] - patch_size//2)\n        x_rand = np.random.randint(patch_size//2, arr_exp.shape[2] - patch_size//2)\n        \n        # Check if far from particles\n        min_dist = 20\n        is_far = True\n        for _, p in particles.iterrows():\n            dist = np.sqrt((x_rand - p['x_pixel'])**2 + (y_rand - p['y_pixel'])**2 + (z_rand - p['z_pixel'])**2)\n            if dist < min_dist:\n                is_far = False\n                break\n        \n        if is_far:\n            patch = arr_exp[\n                z_rand - patch_size//2 : z_rand + patch_size//2,\n                y_rand - patch_size//2 : y_rand + patch_size//2,\n                x_rand - patch_size//2 : x_rand + patch_size//2\n            ]\n            if patch.shape == (patch_size, patch_size, patch_size):\n                X_train_all.append(patch)\n                y_train_all.append('background')\n                negative_count += 1\n    \n    # Clean up\n    del z_exp, arr_exp\n    gc.collect()\n\nprint(f\"\\nTotal patches collected: {len(X_train_all)}\")\n\nX_train_all = np.array(X_train_all)\ny_train_all = np.array(y_train_all)\n\nprint(f\"Training data shape: {X_train_all.shape}\")\nprint(f\"Unique labels: {np.unique(y_train_all)}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Encode labels\nlabel_encoder = LabelEncoder()\ny_train_encoded = label_encoder.fit_transform(y_train_all)\ny_train_categorical = to_categorical(y_train_encoded)\n\nnum_classes = len(label_encoder.classes_)\nprint(f\"Number of classes: {num_classes}\")\nprint(f\"Classes: {label_encoder.classes_}\")\n\n# Normalize\nX_train_all = X_train_all.astype('float32')\nmean = X_train_all.mean()\nstd = X_train_all.std()\nX_train_all = (X_train_all - mean) / std\n\n# Add channel dimension\nX_train_all = X_train_all[..., np.newaxis]\n\nprint(f\"Final X_train shape: {X_train_all.shape}\")\nprint(f\"Final y_train shape: {y_train_categorical.shape}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\nBuilding model...\")\nmodel = models.Sequential([\n    layers.Input(shape=(32, 32, 32, 1)),\n    \n    layers.Conv3D(32, (3, 3, 3), activation='relu', padding='same'),\n    layers.MaxPooling3D((2, 2, 2)),\n    \n    layers.Conv3D(64, (3, 3, 3), activation='relu', padding='same'),\n    layers.MaxPooling3D((2, 2, 2)),\n    \n    layers.Conv3D(128, (3, 3, 3), activation='relu', padding='same'),\n    layers.MaxPooling3D((2, 2, 2)),\n    \n    layers.Flatten(),\n    layers.Dense(128, activation='relu'),\n    layers.Dropout(0.5),\n    layers.Dense(num_classes, activation='softmax')\n])\n\nmodel.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\nmodel.summary()\n\nprint(\"\\nTraining model...\")\nhistory = model.fit(\n    X_train_all, \n    y_train_categorical, \n    epochs=10,  # Reduced from 20 for faster submission\n    batch_size=16,\n    validation_split=0.2,\n    verbose=1\n)\n\n# Save model and parameters\nmodel.save('/kaggle/working/particle_detector_3d.h5')\nprint(\"\\nModel saved!\")\n\n# Save label encoder and normalization params\nwith open('/kaggle/working/label_encoder.pkl', 'wb') as f:\n    pickle.dump(label_encoder, f)\n\nwith open('/kaggle/working/normalization_params.pkl', 'wb') as f:\n    pickle.dump({'mean': mean, 'std': std}, f)\n\nprint(\"Label encoder and normalization params saved!\")\n\n# Clean up training data to free memory\ndel X_train_all, y_train_all, y_train_categorical\ngc.collect()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n\\n=== MAKING PREDICTIONS ===\\n\")\n\nclass_names = label_encoder.classes_\nprint(f\"Class names: {class_names}\")\n\n# Path to test data\ntest_path = \"/kaggle/input/czii-cryo-et-object-identification/test/static/ExperimentRuns\"\n\n# Get all test experiments\ntest_experiments = os.listdir(test_path)\nprint(f\"Found {len(test_experiments)} test experiments\")\n\nall_predictions = []\nprediction_id = 0\n\n# Parameters\npatch_size = 32\nstride = 48\nvoxel_spacing = 10.0\nconfidence_threshold = 0.6\nbatch_size = 32\n\n# Process each test experiment with error handling\nfor experiment in test_experiments:\n    print(f\"\\nProcessing experiment: {experiment}\")\n    \n    try:\n        test_zarr_path = os.path.join(test_path, experiment, \"VoxelSpacing10.000\", \"denoised.zarr\")\n        \n        if not os.path.exists(test_zarr_path):\n            print(f\"  Skipping {experiment} - zarr file not found\")\n            continue\n        \n        z_test = zarr.open(test_zarr_path, mode='r')\n        arr_test = z_test['2']\n        scale_factor = 4\n        \n        print(f\"  Tomogram shape (level 2): {arr_test.shape}\")\n        \n        patches = []\n        patch_coords = []\n        \n        z_range = range(0, max(1, arr_test.shape[0] - patch_size), stride)\n        y_range = range(0, max(1, arr_test.shape[1] - patch_size), stride)\n        x_range = range(0, max(1, arr_test.shape[2] - patch_size), stride)\n        \n        total_windows = len(z_range) * len(y_range) * len(x_range)\n        print(f\"  Total windows: {total_windows}\")\n        \n        processed = 0\n        for z in z_range:\n            for y in y_range:\n                for x in x_range:\n                    z_end = min(z + patch_size, arr_test.shape[0])\n                    y_end = min(y + patch_size, arr_test.shape[1])\n                    x_end = min(x + patch_size, arr_test.shape[2])\n                    \n                    patch = arr_test[z:z_end, y:y_end, x:x_end]\n                    \n                    if patch.shape != (patch_size, patch_size, patch_size):\n                        continue\n                    \n                    patch_norm = (patch - mean) / std\n                    patches.append(patch_norm)\n                    patch_coords.append((z, y, x))\n                    \n                    if len(patches) >= batch_size:\n                        patches_array = np.array(patches)[..., np.newaxis]\n                        preds = model.predict(patches_array, verbose=0)\n                        \n                        for i, pred in enumerate(preds):\n                            if pred.max() > confidence_threshold:\n                                particle_type = class_names[pred.argmax()]\n                                \n                                if particle_type != 'background':\n                                    z_p, y_p, x_p = patch_coords[i]\n                                    \n                                    x_center = (x_p + patch_size // 2) * scale_factor\n                                    y_center = (y_p + patch_size // 2) * scale_factor\n                                    z_center = (z_p + patch_size // 2) * scale_factor\n                                    \n                                    x_angstrom = x_center * voxel_spacing\n                                    y_angstrom = y_center * voxel_spacing\n                                    z_angstrom = z_center * voxel_spacing\n                                    \n                                    all_predictions.append({\n                                        'id': prediction_id,\n                                        'experiment': experiment,\n                                        'particle_type': particle_type,\n                                        'x': x_angstrom,\n                                        'y': y_angstrom,\n                                        'z': z_angstrom\n                                    })\n                                    prediction_id += 1\n                        \n                        patches = []\n                        patch_coords = []\n                        processed += batch_size\n                        \n                        if processed % 500 == 0:\n                            print(f\"  Processed {processed}/{total_windows} windows\")\n        \n        # Process remaining patches\n        if len(patches) > 0:\n            patches_array = np.array(patches)[..., np.newaxis]\n            preds = model.predict(patches_array, verbose=0)\n            \n            for i, pred in enumerate(preds):\n                if pred.max() > confidence_threshold:\n                    particle_type = class_names[pred.argmax()]\n                    \n                    if particle_type != 'background':\n                        z_p, y_p, x_p = patch_coords[i]\n                        \n                        x_center = (x_p + patch_size // 2) * scale_factor\n                        y_center = (y_p + patch_size // 2) * scale_factor\n                        z_center = (z_p + patch_size // 2) * scale_factor\n                        \n                        x_angstrom = x_center * voxel_spacing\n                        y_angstrom = y_center * voxel_spacing\n                        z_angstrom = z_center * voxel_spacing\n                        \n                        all_predictions.append({\n                            'id': prediction_id,\n                            'experiment': experiment,\n                            'particle_type': particle_type,\n                            'x': x_angstrom,\n                            'y': y_angstrom,\n                            'z': z_angstrom\n                        })\n                        prediction_id += 1\n        \n        print(f\"  Completed {experiment}\")\n        \n        del z_test, arr_test, patches, patch_coords\n        gc.collect()\n        \n    except Exception as e:\n        print(f\"  ERROR processing {experiment}: {str(e)}\")\n        continue\n\nprint(f\"\\n=== PREDICTION SUMMARY ===\")\nprint(f\"Total predictions: {len(all_predictions)}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = pd.DataFrame(all_predictions)\n\nif len(submission) > 0:\n    print(f\"\\nPredictions by type:\")\n    print(submission['particle_type'].value_counts())\n    \n    print(f\"\\nPredictions by experiment:\")\n    print(submission['experiment'].value_counts())\nelse:\n    print(\"\\nWARNING: No predictions made!\")\n    submission = pd.DataFrame(columns=['id', 'experiment', 'particle_type', 'x', 'y', 'z'])\n\nsubmission.to_csv('/kaggle/working/submission.csv', index=False)\nprint(\"\\nSubmission saved to /kaggle/working/submission.csv\")\nprint(\"Ready to submit!\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}