I am a beginner on Kaggle. Even after training for 300 epochs, the loss does not decrease(about0.7). What steps can I take to address this issue? Below is the code I am currently using.
\nimport os
\nimport shutil
config_blob = \"\"\"{
\n \"name\": \"czii_cryoet_mlchallenge_2024\",
\n \"description\": \"2024 CZII CryoET ML Challenge training data.\",
\n \"version\": \"1.0.0\",
\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n\n \n\n \n \n\n\n \n}\"\"\"
\ncopick_config_path = \"/kaggle/working/copick.config\"
\noutput_overlay = \"/kaggle/working/overlay\"
with open(copick_config_path, \"w\") as f:
\n f.write(config_blob)
source_dir = '/kaggle/input/czii-cryo-et-object-identification/train/overlay'
\ndestination_dir = '/kaggle/working/overlay'
for root, dirs, files in os.walk(source_dir):
\n # Create corresponding subdirectories in the destination
\n relative_path = os.path.relpath(root, source_dir)
\n target_dir = os.path.join(destination_dir, relative_path)
\n os.makedirs(target_dir, exist_ok=True)
# and rename \n in files:\n .startswith():\n new_filename = \n :\n new_filename = f\n\n\n # Define full paths the and destination files\n source_file = os.path.(root, )\n destination_file = os.path.(target_dir, new_filename)\n\n # the with the name\n shutil.copy2(source_file, destination_file)\n (f)\nimport os
\nimport numpy as np
\nfrom pathlib import Path
\nimport torch
\nimport torchinfo
\nimport zarr, copick
\nfrom tqdm import tqdm
\nfrom monai.data import DataLoader, Dataset, CacheDataset, decollate_batch
\nfrom monai.transforms import (
\n Compose,
\n EnsureChannelFirstd,
\n Orientationd,
\n AsDiscrete,
\n RandFlipd,
\n RandRotate90d,
\n NormalizeIntensityd,
\n RandCropByLabelClassesd,
\n)
\nfrom monai.networks.nets import UNet
\nfrom monai.losses import DiceLoss, FocalLoss, TverskyLoss
\nfrom monai.metrics import DiceMetric, ConfusionMatrixMetric
\nimport mlflow
\nimport mlflow.pytorch
root = copick.from_file(copick_config_path)
\ncopick_user_name = \"copickUtils\"
\ncopick_segmentation_name = \"paintedPicks\"
\nvoxel_size = 10
\ntomo_type = \"denoised\"
from copick_utils.segmentation import segmentation_from_picks
\nimport copick_utils.writers.write as write
\nfrom collections import defaultdict
generate_masks = True
\nif generate_masks:
\n target_objects = defaultdict(dict)
\n for object in root.pickable_objects:
\n if object.is_particle:
\n target_objects[object.name]['label'] = object.label
\n target_objects[object.name]['radius'] = object.radius
tqdm(root.runs):\n tomo = run.get_voxel_spacing(10)\n tomo = tomo.get_tomogram(tomo_type).numpy()\n target = np.zeros(tomo.shape, =np.uint8)\n pickable_object root.pickable_objects:\n pick = run.get_picks(=pickable_object.name, =)\n len(pick): \n target = segmentation_from_picks.from_picks(pick[0], \n target, \n target_objects[pickable_object.name][] * 0.8,\n target_objects[pickable_object.name][]\n )\n write.segmentation(run, target, copick_user_name, =copick_segmentation_name)\ndata_dicts = []
\nfor run in tqdm(root.runs):
\n tomogram = run.get_voxel_spacing(voxel_size).get_tomogram(tomo_type).numpy()
\n segmentation = run.get_segmentations(name=copick_segmentation_name, user_id=copick_user_name, voxel_size=voxel_size, is_multilabel=True)[0].numpy()
\n data_dicts.append({\"image\": tomogram, \"label\": segmentation})
print(np.unique(data_dicts[0]['label']))
\nimport matplotlib.pyplot as plt
\nplt.figure(figsize=(15, 5))
\nplt.subplot(1, 2, 1)
\nplt.title('Tomogram')
\nplt.imshow(data_dicts[0]['image'][100],cmap='gray')
\nplt.axis('off')
plt.subplot(1, 2, 2)
\nplt.title('Painted Segmentation from Picks')
\nplt.imshow(data_dicts[0]['label'][100], cmap='viridis')
\nplt.axis('off')
plt.tight_layout()
\nplt.show()
my_num_samples = 16
\ntrain_batch_size = 1
\nval_batch_size = 1
train_files, val_files = data_dicts[:5], data_dicts[5:7]
\nprint(f\"Number of training samples: {len(train_files)}\")
\nprint(f\"Number of validation samples: {len(val_files)}\")
non_random_transforms = Compose([
\n EnsureChannelFirstd(keys=[\"image\", \"label\"], channel_dim=\"no_channel\"),
\n NormalizeIntensityd(keys=\"image\"),
\n Orientationd(keys=[\"image\", \"label\"], axcodes=\"RAS\")
\n])
random_transforms = Compose([
\n RandCropByLabelClassesd(
\n keys=[\"image\", \"label\"],
\n label_key=\"label\",
\n spatial_size=[96, 96, 96],
\n num_classes=8,
\n num_samples=my_num_samples
\n ),
\n RandRotate90d(keys=[\"image\", \"label\"], prob=0.5, spatial_axes=[0, 2]),
\n RandFlipd(keys=[\"image\", \"label\"], prob=0.5, spatial_axis=0),
\n])
train_ds = CacheDataset(data=train_files, transform=non_random_transforms, cache_rate=1.0)
\ntrain_ds = Dataset(data=train_ds, transform=random_transforms)
\ntrain_loader = DataLoader(
\n train_ds,
\n batch_size=train_batch_size,
\n shuffle=True,
\n num_workers=4,
\n pin_memory=torch.cuda.is_available()
\n)
val_transforms = Compose([
\n EnsureChannelFirstd(keys=[\"image\", \"label\"], channel_dim=\"no_channel\"),
\n NormalizeIntensityd(keys=\"image\"),
\n RandCropByLabelClassesd(
\n keys=[\"image\", \"label\"],
\n label_key=\"label\",
\n spatial_size=[96, 96, 96],
\n num_classes=8,
\n num_samples=my_num_samples, # Use 1 to get a single, consistent crop per image
\n ),
\n])
val_ds = CacheDataset(data=val_files, transform=non_random_transforms, cache_rate=1.0)
\nval_ds = Dataset(data=val_ds, transform=random_transforms)
\nval_loader = DataLoader(
\n val_ds,
\n batch_size=val_batch_size,
\n num_workers=4,
\n pin_memory=torch.cuda.is_available(),
\n shuffle=False, # Ensure the data order remains consistent
\n)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
\nprint(device)
model = UNet(
\n spatial_dims=3,
\n in_channels=1,
\n out_channels=len(root.pickable_objects)+1,
\n channels=(48, 64, 80, 80),
\n strides=(2, 2, 1),
\n num_res_units=1,
\n).to(device)
lr = 1e-3
\noptimizer = torch.optim.Adam(model.parameters(), lr)
loss_function = TverskyLoss(include_background=True, to_onehot_y=True, softmax=True) # softmax=True for multiclass
\ndice_metric = DiceMetric(include_background=False, reduction=\"mean\", ignore_empty=True) # must use onehot for multiclass
\nrecall_metric = ConfusionMatrixMetric(include_background=False, metric_name=\"recall\", reduction=\"None\")
post_pred = AsDiscrete(argmax=True, to_onehot=len(root.pickable_objects)+1)
\npost_label = AsDiscrete(to_onehot=len(root.pickable_objects)+1)
def train(train_loader, model, loss_function, metrics_function, optimizer, max_epochs=100):
\n val_interval = 2
\n best_metric = -1
\n best_metric_epoch = -1
\n epoch_loss_values = []
\n metric_values = []
\n for epoch in range(max_epochs):
\n print(\"-\" * 10)
\n print(f\"epoch {epoch + 1}/{max_epochs}\")
\n model.train()
\n epoch_loss = 0
\n step = 0
\n for batch_data in train_loader:
\n step += 1
\n inputs = batch_data[\"image\"].to(device)
\n labels = batch_data[\"label\"].to(device)
\n optimizer.zero_grad()
\n outputs = model(inputs)
\n loss = loss_function(outputs, labels)
\n loss.backward()
\n optimizer.step()
\n epoch_loss += loss.item()
\n print(f\"batch {step}/{len(train_ds) // train_loader.batch_size}, \" f\"train_loss: {loss.item():.4f}\")
\n epoch_loss /= step
\n epoch_loss_values.append(epoch_loss)
\n print(f\"epoch {epoch + 1} average loss: {epoch_loss:.4f}\")
\n mlflow.log_metric(\"train_loss\", epoch_loss, step=epoch+1)
(epoch + 1) % val_interval == 0:\n model.eval()\n with torch.no_grad():\n val_data val_loader:\n val_inputs = val_data[].(device)\n val_labels = val_data[].(device)\n val_outputs = model(val_inputs)\n metric_val_outputs = [post_pred(i) i decollate_batch(val_outputs)]\n metric_val_labels = [post_label(i) i decollate_batch(val_labels)]\n\n\n # compute metric current iteration\n metrics_function(=metric_val_outputs, =metric_val_labels)\n\n metrics = metrics_function.aggregate(=)\n metric_per_class = [.format(x) x metrics]\n metric = torch.mean(metrics).numpy(=)\n mlflow.log_metric(, metric, =epoch+1)\n i,m enumerate(metrics):\n mlflow.log_metric(f, m, =epoch+1)\n metrics_function.reset()\n\n metric_values.append(metric)\n metric > best_metric:\n best_metric = metric\n best_metric_epoch = epoch + 1\n torch.save(model.state_dict(), os.path.join(, ))\n\n ()\n (\n f\n f\n f\n )\nfrom torchinfo import summary
\nmlflow.end_run()
\nmlflow.set_experiment('training 3D U-Net model for the cryoET ML Challenge')
\nepochs = 300
\nwith mlflow.start_run():
\n params = {
\n \"epochs\": epochs,
\n \"learning_rate\": lr,
\n \"loss_function\": loss_function.class.name,
\n \"metric_function\": recall_metric.class.name,
\n \"optimizer\": \"Adam\",
\n }
\n # Log training parameters.
\n mlflow.log_params(params)
\n (, ) f:\n f.(str(summary(model)))\nmlflow.log_artifact()\n\ntrain(train_loader, model, loss_function, dice_metric, optimizer, max_epochs=epochs)\n\n\nmlflow.pytorch.log_model(model, )\nI am a beginner on Kaggle. Even after training for 300 epochs, the loss does not decrease(about0.7). What steps can I take to address this issue? Below is the code I am currently using.
\nimport os
\nimport shutil
config_blob = \"\"\"{
\n \"name\": \"czii_cryoet_mlchallenge_2024\",
\n \"description\": \"2024 CZII CryoET ML Challenge training data.\",
\n \"version\": \"1.0.0\",
\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n\n \n\n \n \n\n\n \n}\"\"\"
\ncopick_config_path = \"/kaggle/working/copick.config\"
\noutput_overlay = \"/kaggle/working/overlay\"
with open(copick_config_path, \"w\") as f:
\n f.write(config_blob)
source_dir = '/kaggle/input/czii-cryo-et-object-identification/train/overlay'
\ndestination_dir = '/kaggle/working/overlay'
for root, dirs, files in os.walk(source_dir):
\n # Create corresponding subdirectories in the destination
\n relative_path = os.path.relpath(root, source_dir)
\n target_dir = os.path.join(destination_dir, relative_path)
\n os.makedirs(target_dir, exist_ok=True)
# and rename \n in files:\n .startswith():\n new_filename = \n :\n new_filename = f\n\n\n # Define full paths the and destination files\n source_file = os.path.(root, )\n destination_file = os.path.(target_dir, new_filename)\n\n # the with the name\n shutil.copy2(source_file, destination_file)\n (f)\nimport os
\nimport numpy as np
\nfrom pathlib import Path
\nimport torch
\nimport torchinfo
\nimport zarr, copick
\nfrom tqdm import tqdm
\nfrom monai.data import DataLoader, Dataset, CacheDataset, decollate_batch
\nfrom monai.transforms import (
\n Compose,
\n EnsureChannelFirstd,
\n Orientationd,
\n AsDiscrete,
\n RandFlipd,
\n RandRotate90d,
\n NormalizeIntensityd,
\n RandCropByLabelClassesd,
\n)
\nfrom monai.networks.nets import UNet
\nfrom monai.losses import DiceLoss, FocalLoss, TverskyLoss
\nfrom monai.metrics import DiceMetric, ConfusionMatrixMetric
\nimport mlflow
\nimport mlflow.pytorch
root = copick.from_file(copick_config_path)
\ncopick_user_name = \"copickUtils\"
\ncopick_segmentation_name = \"paintedPicks\"
\nvoxel_size = 10
\ntomo_type = \"denoised\"
from copick_utils.segmentation import segmentation_from_picks
\nimport copick_utils.writers.write as write
\nfrom collections import defaultdict
generate_masks = True
\nif generate_masks:
\n target_objects = defaultdict(dict)
\n for object in root.pickable_objects:
\n if object.is_particle:
\n target_objects[object.name]['label'] = object.label
\n target_objects[object.name]['radius'] = object.radius
tqdm(root.runs):\n tomo = run.get_voxel_spacing(10)\n tomo = tomo.get_tomogram(tomo_type).numpy()\n target = np.zeros(tomo.shape, =np.uint8)\n pickable_object root.pickable_objects:\n pick = run.get_picks(=pickable_object.name, =)\n len(pick): \n target = segmentation_from_picks.from_picks(pick[0], \n target, \n target_objects[pickable_object.name][] * 0.8,\n target_objects[pickable_object.name][]\n )\n write.segmentation(run, target, copick_user_name, =copick_segmentation_name)\ndata_dicts = []
\nfor run in tqdm(root.runs):
\n tomogram = run.get_voxel_spacing(voxel_size).get_tomogram(tomo_type).numpy()
\n segmentation = run.get_segmentations(name=copick_segmentation_name, user_id=copick_user_name, voxel_size=voxel_size, is_multilabel=True)[0].numpy()
\n data_dicts.append({\"image\": tomogram, \"label\": segmentation})
print(np.unique(data_dicts[0]['label']))
\nimport matplotlib.pyplot as plt
\nplt.figure(figsize=(15, 5))
\nplt.subplot(1, 2, 1)
\nplt.title('Tomogram')
\nplt.imshow(data_dicts[0]['image'][100],cmap='gray')
\nplt.axis('off')
plt.subplot(1, 2, 2)
\nplt.title('Painted Segmentation from Picks')
\nplt.imshow(data_dicts[0]['label'][100], cmap='viridis')
\nplt.axis('off')
plt.tight_layout()
\nplt.show()
my_num_samples = 16
\ntrain_batch_size = 1
\nval_batch_size = 1
train_files, val_files = data_dicts[:5], data_dicts[5:7]
\nprint(f\"Number of training samples: {len(train_files)}\")
\nprint(f\"Number of validation samples: {len(val_files)}\")
non_random_transforms = Compose([
\n EnsureChannelFirstd(keys=[\"image\", \"label\"], channel_dim=\"no_channel\"),
\n NormalizeIntensityd(keys=\"image\"),
\n Orientationd(keys=[\"image\", \"label\"], axcodes=\"RAS\")
\n])
random_transforms = Compose([
\n RandCropByLabelClassesd(
\n keys=[\"image\", \"label\"],
\n label_key=\"label\",
\n spatial_size=[96, 96, 96],
\n num_classes=8,
\n num_samples=my_num_samples
\n ),
\n RandRotate90d(keys=[\"image\", \"label\"], prob=0.5, spatial_axes=[0, 2]),
\n RandFlipd(keys=[\"image\", \"label\"], prob=0.5, spatial_axis=0),
\n])
train_ds = CacheDataset(data=train_files, transform=non_random_transforms, cache_rate=1.0)
\ntrain_ds = Dataset(data=train_ds, transform=random_transforms)
\ntrain_loader = DataLoader(
\n train_ds,
\n batch_size=train_batch_size,
\n shuffle=True,
\n num_workers=4,
\n pin_memory=torch.cuda.is_available()
\n)
val_transforms = Compose([
\n EnsureChannelFirstd(keys=[\"image\", \"label\"], channel_dim=\"no_channel\"),
\n NormalizeIntensityd(keys=\"image\"),
\n RandCropByLabelClassesd(
\n keys=[\"image\", \"label\"],
\n label_key=\"label\",
\n spatial_size=[96, 96, 96],
\n num_classes=8,
\n num_samples=my_num_samples, # Use 1 to get a single, consistent crop per image
\n ),
\n])
val_ds = CacheDataset(data=val_files, transform=non_random_transforms, cache_rate=1.0)
\nval_ds = Dataset(data=val_ds, transform=random_transforms)
\nval_loader = DataLoader(
\n val_ds,
\n batch_size=val_batch_size,
\n num_workers=4,
\n pin_memory=torch.cuda.is_available(),
\n shuffle=False, # Ensure the data order remains consistent
\n)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
\nprint(device)
model = UNet(
\n spatial_dims=3,
\n in_channels=1,
\n out_channels=len(root.pickable_objects)+1,
\n channels=(48, 64, 80, 80),
\n strides=(2, 2, 1),
\n num_res_units=1,
\n).to(device)
lr = 1e-3
\noptimizer = torch.optim.Adam(model.parameters(), lr)
loss_function = TverskyLoss(include_background=True, to_onehot_y=True, softmax=True) # softmax=True for multiclass
\ndice_metric = DiceMetric(include_background=False, reduction=\"mean\", ignore_empty=True) # must use onehot for multiclass
\nrecall_metric = ConfusionMatrixMetric(include_background=False, metric_name=\"recall\", reduction=\"None\")
post_pred = AsDiscrete(argmax=True, to_onehot=len(root.pickable_objects)+1)
\npost_label = AsDiscrete(to_onehot=len(root.pickable_objects)+1)
def train(train_loader, model, loss_function, metrics_function, optimizer, max_epochs=100):
\n val_interval = 2
\n best_metric = -1
\n best_metric_epoch = -1
\n epoch_loss_values = []
\n metric_values = []
\n for epoch in range(max_epochs):
\n print(\"-\" * 10)
\n print(f\"epoch {epoch + 1}/{max_epochs}\")
\n model.train()
\n epoch_loss = 0
\n step = 0
\n for batch_data in train_loader:
\n step += 1
\n inputs = batch_data[\"image\"].to(device)
\n labels = batch_data[\"label\"].to(device)
\n optimizer.zero_grad()
\n outputs = model(inputs)
\n loss = loss_function(outputs, labels)
\n loss.backward()
\n optimizer.step()
\n epoch_loss += loss.item()
\n print(f\"batch {step}/{len(train_ds) // train_loader.batch_size}, \" f\"train_loss: {loss.item():.4f}\")
\n epoch_loss /= step
\n epoch_loss_values.append(epoch_loss)
\n print(f\"epoch {epoch + 1} average loss: {epoch_loss:.4f}\")
\n mlflow.log_metric(\"train_loss\", epoch_loss, step=epoch+1)
(epoch + 1) % val_interval == 0:\n model.eval()\n with torch.no_grad():\n val_data val_loader:\n val_inputs = val_data[].(device)\n val_labels = val_data[].(device)\n val_outputs = model(val_inputs)\n metric_val_outputs = [post_pred(i) i decollate_batch(val_outputs)]\n metric_val_labels = [post_label(i) i decollate_batch(val_labels)]\n\n\n # compute metric current iteration\n metrics_function(=metric_val_outputs, =metric_val_labels)\n\n metrics = metrics_function.aggregate(=)\n metric_per_class = [.format(x) x metrics]\n metric = torch.mean(metrics).numpy(=)\n mlflow.log_metric(, metric, =epoch+1)\n i,m enumerate(metrics):\n mlflow.log_metric(f, m, =epoch+1)\n metrics_function.reset()\n\n metric_values.append(metric)\n metric > best_metric:\n best_metric = metric\n best_metric_epoch = epoch + 1\n torch.save(model.state_dict(), os.path.join(, ))\n\n ()\n (\n f\n f\n f\n )\nfrom torchinfo import summary
\nmlflow.end_run()
\nmlflow.set_experiment('training 3D U-Net model for the cryoET ML Challenge')
\nepochs = 300
\nwith mlflow.start_run():
\n params = {
\n \"epochs\": epochs,
\n \"learning_rate\": lr,
\n \"loss_function\": loss_function.class.name,
\n \"metric_function\": recall_metric.class.name,
\n \"optimizer\": \"Adam\",
\n }
\n # Log training parameters.
\n mlflow.log_params(params)
\n (, ) f:\n f.(str(summary(model)))\nmlflow.log_artifact()\n\ntrain(train_loader, model, loss_function, dice_metric, optimizer, max_epochs=epochs)\n\n\nmlflow.pytorch.log_model(model, )\nIf you're new, I would start here:
\nhttps://www.kaggle.com/competitions/czii-cryo-et-object-identification/discussion/549715
Thank you! I'll read it.
", "rawMarkdown": "Thank you! I'll read it.", "votes": null } ], "comments": [ { "id": 3095497, "author_name": "davidlist", "author_url": "", "post_date": "01/13/2025 12:04:14", "content": "If you're new, I would start here:
\nhttps://www.kaggle.com/competitions/czii-cryo-et-object-identification/discussion/549715
Thank you! I'll read it.
", "votes": null, "replies": [] } ] } ], "raw_markdown_by_id": { "3095475": "I am a beginner on Kaggle. Even after training for 300 epochs, the loss does not decrease(about0.7). What steps can I take to address this issue? Below is the code I am currently using.\n\n\n# Make a copick project\nimport os\nimport shutil\n\nconfig_blob = \"\"\"{\n \"name\": \"czii_cryoet_mlchallenge_2024\",\n \"description\": \"2024 CZII CryoET ML Challenge training data.\",\n \"version\": \"1.0.0\",\n\n \"pickable_objects\": [\n {\n \"name\": \"apo-ferritin\",\n \"is_particle\": true,\n \"pdb_id\": \"4V1W\",\n \"label\": 1,\n \"color\": [ 0, 117, 220, 128],\n \"radius\": 60,\n \"map_threshold\": 0.0418\n },\n {\n \"name\": \"beta-galactosidase\",\n \"is_particle\": true,\n \"pdb_id\": \"6X1Q\",\n \"label\": 3,\n \"color\": [ 76, 0, 92, 128],\n \"radius\": 90,\n \"map_threshold\": 0.0578\n },\n {\n \"name\": \"ribosome\",\n \"is_particle\": true,\n \"pdb_id\": \"6EK0\",\n \"label\": 4,\n \"color\": [ 0, 92, 49, 128],\n \"radius\": 150,\n \"map_threshold\": 0.0374\n },\n {\n \"name\": \"thyroglobulin\",\n \"is_particle\": true,\n \"pdb_id\": \"6SCJ\",\n \"label\": 5,\n \"color\": [ 43, 206, 72, 128],\n \"radius\": 130,\n \"map_threshold\": 0.0278\n },\n {\n \"name\": \"virus-like-particle\",\n \"is_particle\": true,\n \"label\": 6,\n \"color\": [255, 204, 153, 128],\n \"radius\": 135,\n \"map_threshold\": 0.201\n },\n {\n \"name\": \"membrane\",\n \"is_particle\": false,\n \"label\": 8,\n \"color\": [100, 100, 100, 128]\n },\n {\n \"name\": \"background\",\n \"is_particle\": false,\n \"label\": 9,\n \"color\": [10, 150, 200, 128]\n }\n ],\n\n \"overlay_root\": \"/kaggle/working/overlay\",\n\n \"overlay_fs_args\": {\n \"auto_mkdir\": true\n },\n\n \"static_root\": \"/kaggle/input/czii-cryo-et-object-identification/train/static\"\n}\"\"\"\n\ncopick_config_path = \"/kaggle/working/copick.config\"\noutput_overlay = \"/kaggle/working/overlay\"\n\nwith open(copick_config_path, \"w\") as f:\n f.write(config_blob)\n \n# Update the overlay\n# Define source and destination directories\nsource_dir = '/kaggle/input/czii-cryo-et-object-identification/train/overlay'\ndestination_dir = '/kaggle/working/overlay'\n\n# Walk through the source directory\nfor root, dirs, files in os.walk(source_dir):\n # Create corresponding subdirectories in the destination\n relative_path = os.path.relpath(root, source_dir)\n target_dir = os.path.join(destination_dir, relative_path)\n os.makedirs(target_dir, exist_ok=True)\n \n # Copy and rename each file\n for file in files:\n if file.startswith(\"curation_0_\"):\n new_filename = file\n else:\n new_filename = f\"curation_0_{file}\"\n \n \n # Define full paths for the source and destination files\n source_file = os.path.join(root, file)\n destination_file = os.path.join(target_dir, new_filename)\n \n # Copy the file with the new name\n shutil.copy2(source_file, destination_file)\n print(f\"Copied {source_file} to {destination_file}\")\n\nimport os\nimport numpy as np\nfrom pathlib import Path\nimport torch\nimport torchinfo\nimport zarr, copick\nfrom tqdm import tqdm\nfrom monai.data import DataLoader, Dataset, CacheDataset, decollate_batch\nfrom monai.transforms import (\n Compose, \n EnsureChannelFirstd, \n Orientationd, \n AsDiscrete, \n RandFlipd, \n RandRotate90d, \n NormalizeIntensityd,\n RandCropByLabelClassesd,\n)\nfrom monai.networks.nets import UNet\nfrom monai.losses import DiceLoss, FocalLoss, TverskyLoss\nfrom monai.metrics import DiceMetric, ConfusionMatrixMetric\nimport mlflow\nimport mlflow.pytorch\n\nroot = copick.from_file(copick_config_path)\n\ncopick_user_name = \"copickUtils\"\ncopick_segmentation_name = \"paintedPicks\"\nvoxel_size = 10\ntomo_type = \"denoised\"\n\nfrom copick_utils.segmentation import segmentation_from_picks\nimport copick_utils.writers.write as write\nfrom collections import defaultdict\n\n# Just do this once\ngenerate_masks = True\n\nif generate_masks:\n target_objects = defaultdict(dict)\n for object in root.pickable_objects:\n if object.is_particle:\n target_objects[object.name]['label'] = object.label\n target_objects[object.name]['radius'] = object.radius\n\n\n for run in tqdm(root.runs):\n tomo = run.get_voxel_spacing(10)\n tomo = tomo.get_tomogram(tomo_type).numpy()\n target = np.zeros(tomo.shape, dtype=np.uint8)\n for pickable_object in root.pickable_objects:\n pick = run.get_picks(object_name=pickable_object.name, user_id=\"curation\")\n if len(pick): \n target = segmentation_from_picks.from_picks(pick[0], \n target, \n target_objects[pickable_object.name]['radius'] * 0.8,\n target_objects[pickable_object.name]['label']\n )\n write.segmentation(run, target, copick_user_name, name=copick_segmentation_name)\n\ndata_dicts = []\nfor run in tqdm(root.runs):\n tomogram = run.get_voxel_spacing(voxel_size).get_tomogram(tomo_type).numpy()\n segmentation = run.get_segmentations(name=copick_segmentation_name, user_id=copick_user_name, voxel_size=voxel_size, is_multilabel=True)[0].numpy()\n data_dicts.append({\"image\": tomogram, \"label\": segmentation})\n \nprint(np.unique(data_dicts[0]['label']))\n\nimport matplotlib.pyplot as plt\n\n# Plot the images\nplt.figure(figsize=(15, 5))\n\nplt.subplot(1, 2, 1)\nplt.title('Tomogram')\nplt.imshow(data_dicts[0]['image'][100],cmap='gray')\nplt.axis('off')\n\nplt.subplot(1, 2, 2)\nplt.title('Painted Segmentation from Picks')\nplt.imshow(data_dicts[0]['label'][100], cmap='viridis')\nplt.axis('off')\n\nplt.tight_layout()\nplt.show()\n\nmy_num_samples = 16\ntrain_batch_size = 1\nval_batch_size = 1\n\ntrain_files, val_files = data_dicts[:5], data_dicts[5:7]\nprint(f\"Number of training samples: {len(train_files)}\")\nprint(f\"Number of validation samples: {len(val_files)}\")\n\n# Non-random transforms to be cached\nnon_random_transforms = Compose([\n EnsureChannelFirstd(keys=[\"image\", \"label\"], channel_dim=\"no_channel\"),\n NormalizeIntensityd(keys=\"image\"),\n Orientationd(keys=[\"image\", \"label\"], axcodes=\"RAS\")\n])\n\n# Random transforms to be applied during training\nrandom_transforms = Compose([\n RandCropByLabelClassesd(\n keys=[\"image\", \"label\"],\n label_key=\"label\",\n spatial_size=[96, 96, 96],\n num_classes=8,\n num_samples=my_num_samples\n ),\n RandRotate90d(keys=[\"image\", \"label\"], prob=0.5, spatial_axes=[0, 2]),\n RandFlipd(keys=[\"image\", \"label\"], prob=0.5, spatial_axis=0), \n])\n\n# Create the cached dataset with non-random transforms\ntrain_ds = CacheDataset(data=train_files, transform=non_random_transforms, cache_rate=1.0)\n\n# Wrap the cached dataset to apply random transforms during iteration\ntrain_ds = Dataset(data=train_ds, transform=random_transforms)\n\n# DataLoader remains the same\ntrain_loader = DataLoader(\n train_ds,\n batch_size=train_batch_size,\n shuffle=True,\n num_workers=4,\n pin_memory=torch.cuda.is_available()\n)\n\n# Validation transforms\nval_transforms = Compose([\n EnsureChannelFirstd(keys=[\"image\", \"label\"], channel_dim=\"no_channel\"),\n NormalizeIntensityd(keys=\"image\"),\n RandCropByLabelClassesd(\n keys=[\"image\", \"label\"],\n label_key=\"label\",\n spatial_size=[96, 96, 96],\n num_classes=8,\n num_samples=my_num_samples, # Use 1 to get a single, consistent crop per image\n ),\n])\n\n# Create validation dataset\nval_ds = CacheDataset(data=val_files, transform=non_random_transforms, cache_rate=1.0)\n\n# Wrap the cached dataset to apply random transforms during iteration\nval_ds = Dataset(data=val_ds, transform=random_transforms)\n\n# Create validation DataLoader\nval_loader = DataLoader(\n val_ds,\n batch_size=val_batch_size,\n num_workers=4,\n pin_memory=torch.cuda.is_available(),\n shuffle=False, # Ensure the data order remains consistent\n)\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(device)\n# Create UNet, DiceLoss and Adam optimizer\nmodel = UNet(\n spatial_dims=3,\n in_channels=1,\n out_channels=len(root.pickable_objects)+1,\n channels=(48, 64, 80, 80),\n strides=(2, 2, 1),\n num_res_units=1,\n).to(device)\n\nlr = 1e-3\noptimizer = torch.optim.Adam(model.parameters(), lr)\n#loss_function = DiceLoss(include_background=True, to_onehot_y=True, softmax=True) # softmax=True for multiclass\nloss_function = TverskyLoss(include_background=True, to_onehot_y=True, softmax=True) # softmax=True for multiclass\ndice_metric = DiceMetric(include_background=False, reduction=\"mean\", ignore_empty=True) # must use onehot for multiclass\nrecall_metric = ConfusionMatrixMetric(include_background=False, metric_name=\"recall\", reduction=\"None\")\n\npost_pred = AsDiscrete(argmax=True, to_onehot=len(root.pickable_objects)+1)\npost_label = AsDiscrete(to_onehot=len(root.pickable_objects)+1)\n\ndef train(train_loader, model, loss_function, metrics_function, optimizer, max_epochs=100):\n val_interval = 2\n best_metric = -1\n best_metric_epoch = -1\n epoch_loss_values = []\n metric_values = []\n for epoch in range(max_epochs):\n print(\"-\" * 10)\n print(f\"epoch {epoch + 1}/{max_epochs}\")\n model.train()\n epoch_loss = 0\n step = 0\n for batch_data in train_loader:\n step += 1\n inputs = batch_data[\"image\"].to(device)\n labels = batch_data[\"label\"].to(device)\n optimizer.zero_grad()\n outputs = model(inputs)\n loss = loss_function(outputs, labels)\n loss.backward()\n optimizer.step()\n epoch_loss += loss.item()\n print(f\"batch {step}/{len(train_ds) // train_loader.batch_size}, \" f\"train_loss: {loss.item():.4f}\")\n epoch_loss /= step\n epoch_loss_values.append(epoch_loss)\n print(f\"epoch {epoch + 1} average loss: {epoch_loss:.4f}\")\n mlflow.log_metric(\"train_loss\", epoch_loss, step=epoch+1)\n\n if (epoch + 1) % val_interval == 0:\n model.eval()\n with torch.no_grad():\n for val_data in val_loader:\n val_inputs = val_data[\"image\"].to(device)\n val_labels = val_data[\"label\"].to(device)\n val_outputs = model(val_inputs)\n metric_val_outputs = [post_pred(i) for i in decollate_batch(val_outputs)]\n metric_val_labels = [post_label(i) for i in decollate_batch(val_labels)]\n \n \n # compute metric for current iteration\n metrics_function(y_pred=metric_val_outputs, y=metric_val_labels)\n\n metrics = metrics_function.aggregate(reduction=\"mean_batch\")\n metric_per_class = [\"{:.4g}\".format(x) for x in metrics]\n metric = torch.mean(metrics).numpy(force=True)\n mlflow.log_metric(\"validation metric\", metric, step=epoch+1)\n for i,m in enumerate(metrics):\n mlflow.log_metric(f\"validation metric class {i+1}\", m, step=epoch+1)\n metrics_function.reset()\n\n metric_values.append(metric)\n if metric > best_metric:\n best_metric = metric\n best_metric_epoch = epoch + 1\n torch.save(model.state_dict(), os.path.join('./', \"best_metric_model.pth\"))\n \n print(\"saved new best metric model\")\n print(\n f\"current epoch: {epoch + 1} current mean recall per class: {', '.join(metric_per_class)}\"\n f\"\\nbest mean recall: {best_metric:.4f} \"\n f\"at epoch: {best_metric_epoch}\"\n )\n\nfrom torchinfo import summary\n\nmlflow.end_run()\nmlflow.set_experiment('training 3D U-Net model for the cryoET ML Challenge')\nepochs = 300\nwith mlflow.start_run():\n params = {\n \"epochs\": epochs,\n \"learning_rate\": lr,\n \"loss_function\": loss_function.__class__.__name__,\n \"metric_function\": recall_metric.__class__.__name__,\n \"optimizer\": \"Adam\",\n }\n # Log training parameters.\n mlflow.log_params(params)\n\n # Log model summary.\n with open(\"model_summary.txt\", \"w\") as f:\n f.write(str(summary(model)))\n mlflow.log_artifact(\"model_summary.txt\")\n\n train(train_loader, model, loss_function, dice_metric, optimizer, max_epochs=epochs)\n\n # Save the trained model to MLflow.\n mlflow.pytorch.log_model(model, \"model\")", "3095497": "If you're new, I would start here:\n[https://www.kaggle.com/competitions/czii-cryo-et-object-identification/discussion/549715](https://www.kaggle.com/competitions/czii-cryo-et-object-identification/discussion/549715)", "3095592": "Thank you! I'll read it." }, "source": "meta" }