numpy np\n pandas pd\n os\n torch\n torch.nn nn\n torch.optim optim\n torch.utils.data Dataset, DataLoader\n torch.cuda.amp autocast, GradScaler\n tqdm tqdm\n cv2\n glob\n warnings\n\n\nwarnings.filterwarnings()\n\n\nworking_dir = \ndataset_dir = \nTRAIN_LABELS_PATH = os.path.join(dataset_dir, )\nTRAIN_DIR = os.path.join(dataset_dir, )\nTEST_DIR = os.path.join(dataset_dir, )\nOUTPUT_MODEL = os.path.join(working_dir, )\n\n\nDEVICE = torch.device( torch.cuda.is_available() )\nPATCH_SIZE = \nBATCH_SIZE = \nEPOCHS = \nNUM_WORKERS = \nSEED = \nLEARNING_RATE = \nWEIGHT_DECAY = \n\n\n ():\n torch.manual_seed(seed)\n np.random.seed(seed)\n torch.cuda.is_available():\n torch.cuda.manual_seed(seed)\n torch.backends.cudnn.deterministic = \n torch.backends.cudnn.benchmark = \n\nset_seed(SEED)\n\n\n (nn.Module):\n ():\n ().__init__()\n .conv_layers = nn.Sequential(\n nn.Conv3d(, , kernel_size=, padding=),\n nn.BatchNorm3d(),\n nn.ReLU(inplace=),\n nn.MaxPool3d(),\n\n nn.Conv3d(, , kernel_size=, padding=),\n nn.BatchNorm3d(),\n nn.ReLU(inplace=),\n nn.MaxPool3d(),\n\n nn.Conv3d(, , kernel_size=, padding=),\n nn.BatchNorm3d(),\n nn.ReLU(inplace=),\n nn.AdaptiveAvgPool3d()\n )\n\n .regressor = nn.Sequential(\n nn.Flatten(),\n nn.Linear(, ),\n nn.Sigmoid()\n )\n\n ():\n features = .conv_layers(x)\n coords = .regressor(features)\n coords\n\n\n ():\n slice_paths = (glob.glob(os.path.join(base_path, tomo_id, )))\n slices = []\n i (PATCH_SIZE):\n i < (slice_paths):\n img = cv2.imread(slice_paths[i], )\n img :\n img = cv2.resize(img, (PATCH_SIZE, PATCH_SIZE))\n slices.append(img)\n :\n slices.append(np.zeros((PATCH_SIZE, PATCH_SIZE)))\n :\n slices.append(np.zeros((PATCH_SIZE, PATCH_SIZE)))\n\n volume = np.stack(slices) \n volume\n\n ():\n ():\n .ids = ids\n .base_path = base_path\n .labels = labels.set_index() labels \n\n ():\n (.ids)\n\n ():\n tomo_id = .ids[idx]\n :\n vol = load_tomogram(tomo_id, .base_path)\n vol = (vol - vol.mean()) / (vol.std() + ) \n vol_tensor = torch.tensor(vol, dtype=torch.float32).unsqueeze() \n\n .labels tomo_id .labels.index:\n label = .labels.loc[tomo_id]\n \n (label, pd.Series):\n coords = label[[, , ]].values\n : \n coords = label[[, , ]].iloc[].values\n coords = torch.tensor(coords, dtype=torch.float32) / PATCH_SIZE\n vol_tensor, coords\n \n vol_tensor, torch.zeros(, dtype=torch.float32)\n Exception e:\n ()\n vol_tensor = torch.zeros((, PATCH_SIZE, PATCH_SIZE, PATCH_SIZE), dtype=torch.float32)\n vol_tensor, torch.zeros(, dtype=torch.float32)\n\n\n ():\n volumes = torch.stack([item[] item batch])\n coords = torch.stack([item[] item batch])\n volumes, coords\n\n\n (nn.Module):\n ():\n ().__init__()\n .coord_loss = nn.MSELoss()\n\n ():\n loss = .coord_loss(outputs, targets)\n loss\n\n\n ():\n model.train()\n optimizer = optim.AdamW(model.parameters(), lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)\n scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=EPOCHS)\n scaler = GradScaler()\n\n epoch (EPOCHS):\n epoch_loss = \n inputs, targets tqdm(train_loader, desc=):\n inputs = inputs.to(DEVICE)\n targets = targets.to(DEVICE)\n\n optimizer.zero_grad()\n autocast():\n outputs = model(inputs)\n loss = criterion(outputs, targets)\n\n scaler.scale(loss).backward()\n scaler.step(optimizer)\n scaler.update()\n epoch_loss += loss.item()\n\n scheduler.step()\n ()\n\n\n __name__ == :\n \n train_labels = pd.read_csv(TRAIN_LABELS_PATH).rename(columns={: })\n train_ids = (os.listdir(TRAIN_DIR))\n\n \n dataset = TomogramDataset(train_ids, TRAIN_DIR, train_labels)\n train_loader = DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=, \n num_workers=NUM_WORKERS, pin_memory=,\n collate_fn=custom_collate)\n\n \n model = FastFlagellumDetector().to(DEVICE)\n criterion = FlagellumLoss()\n\n \n train_fast(model, train_loader, criterion)\n\n \n torch.save(model.state_dict(), OUTPUT_MODEL)\n ()**\n numpy np\n pandas pd\n os\n torch\n torch.nn nn\n torch.optim optim\n torch.utils.data Dataset, DataLoader\n torch.cuda.amp autocast, GradScaler\n tqdm tqdm\n cv2\n glob\n warnings\n\n\nwarnings.filterwarnings()\n\n\nworking_dir = \ndataset_dir = \nTRAIN_LABELS_PATH = os.path.join(dataset_dir, )\nTRAIN_DIR = os.path.join(dataset_dir, )\nTEST_DIR = os.path.join(dataset_dir, )\nOUTPUT_MODEL = os.path.join(working_dir, )\n\n\nDEVICE = torch.device( torch.cuda.is_available() )\nPATCH_SIZE = \nBATCH_SIZE = \nEPOCHS = \nNUM_WORKERS = \nSEED = \nLEARNING_RATE = \nWEIGHT_DECAY = \n\n\n ():\n torch.manual_seed(seed)\n np.random.seed(seed)\n torch.cuda.is_available():\n torch.cuda.manual_seed(seed)\n torch.backends.cudnn.deterministic = \n torch.backends.cudnn.benchmark = \n\nset_seed(SEED)\n\n\n (nn.Module):\n ():\n ().__init__()\n .conv_layers = nn.Sequential(\n nn.Conv3d(, , kernel_size=, padding=),\n nn.BatchNorm3d(),\n nn.ReLU(inplace=),\n nn.MaxPool3d(),\n\n nn.Conv3d(, , kernel_size=, padding=),\n nn.BatchNorm3d(),\n nn.ReLU(inplace=),\n nn.MaxPool3d(),\n\n nn.Conv3d(, , kernel_size=, padding=),\n nn.BatchNorm3d(),\n nn.ReLU(inplace=),\n nn.AdaptiveAvgPool3d()\n )\n\n .regressor = nn.Sequential(\n nn.Flatten(),\n nn.Linear(, ),\n nn.Sigmoid()\n )\n\n ():\n features = .conv_layers(x)\n coords = .regressor(features)\n coords\n\n\n ():\n slice_paths = (glob.glob(os.path.join(base_path, tomo_id, )))\n slices = []\n i (PATCH_SIZE):\n i < (slice_paths):\n img = cv2.imread(slice_paths[i], )\n img :\n img = cv2.resize(img, (PATCH_SIZE, PATCH_SIZE))\n slices.append(img)\n :\n slices.append(np.zeros((PATCH_SIZE, PATCH_SIZE)))\n :\n slices.append(np.zeros((PATCH_SIZE, PATCH_SIZE)))\n\n volume = np.stack(slices) \n volume\n\n ():\n ():\n .ids = ids\n .base_path = base_path\n .labels = labels.set_index() labels \n\n ():\n (.ids)\n\n ():\n tomo_id = .ids[idx]\n :\n vol = load_tomogram(tomo_id, .base_path)\n vol = (vol - vol.mean()) / (vol.std() + ) \n vol_tensor = torch.tensor(vol, dtype=torch.float32).unsqueeze() \n\n .labels tomo_id .labels.index:\n label = .labels.loc[tomo_id]\n \n (label, pd.Series):\n coords = label[[, , ]].values\n : \n coords = label[[, , ]].iloc[].values\n coords = torch.tensor(coords, dtype=torch.float32) / PATCH_SIZE\n vol_tensor, coords\n \n vol_tensor, torch.zeros(, dtype=torch.float32)\n Exception e:\n ()\n vol_tensor = torch.zeros((, PATCH_SIZE, PATCH_SIZE, PATCH_SIZE), dtype=torch.float32)\n vol_tensor, torch.zeros(, dtype=torch.float32)\n\n\n ():\n volumes = torch.stack([item[] item batch])\n coords = torch.stack([item[] item batch])\n volumes, coords\n\n\n (nn.Module):\n ():\n ().__init__()\n .coord_loss = nn.MSELoss()\n\n ():\n loss = .coord_loss(outputs, targets)\n loss\n\n\n ():\n model.train()\n optimizer = optim.AdamW(model.parameters(), lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)\n scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=EPOCHS)\n scaler = GradScaler()\n\n epoch (EPOCHS):\n epoch_loss = \n inputs, targets tqdm(train_loader, desc=):\n inputs = inputs.to(DEVICE)\n targets = targets.to(DEVICE)\n\n optimizer.zero_grad()\n autocast():\n outputs = model(inputs)\n loss = criterion(outputs, targets)\n\n scaler.scale(loss).backward()\n scaler.step(optimizer)\n scaler.update()\n epoch_loss += loss.item()\n\n scheduler.step()\n ()\n\n\n __name__ == :\n \n train_labels = pd.read_csv(TRAIN_LABELS_PATH).rename(columns={: })\n train_ids = (os.listdir(TRAIN_DIR))\n\n \n dataset = TomogramDataset(train_ids, TRAIN_DIR, train_labels)\n train_loader = DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=, \n num_workers=NUM_WORKERS, pin_memory=,\n collate_fn=custom_collate)\n\n \n model = FastFlagellumDetector().to(DEVICE)\n criterion = FlagellumLoss()\n\n \n train_fast(model, train_loader, criterion)\n\n \n torch.save(model.state_dict(), OUTPUT_MODEL)\n ()**\nI think model is to simple to learn something useful, and how you will determine if motor not presented in tomogram too?
", "rawMarkdown": "I think model is to simple to learn something useful, and how you will determine if motor not presented in tomogram too?", "votes": null }, { "id": "3160487", "postDate": "03/26/2025 20:37:24", "content": "I mean…run an experiment and see
", "rawMarkdown": "I mean…run an experiment and see", "votes": null }, { "id": "3160498", "postDate": "03/26/2025 21:08:26", "content": "Like you were told, run an experiment and you will find out.
\nAn educated guess is that this won't be a good model. If we had 100,000+ images, maybe it would be possible to learn the patterns from scratch, like you are trying to do. Given a relatively small dataset size, most likely it is necessary that we resort to transfer learning. That means using an existing model that can recognize objects and fine-tuning it to our specific problem.
", "rawMarkdown": "Like you were told, run an experiment and you will find out.\n\nAn educated guess is that this won't be a good model. If we had 100,000+ images, maybe it would be possible to learn the patterns from scratch, like you are trying to do. Given a relatively small dataset size, most likely it is necessary that we resort to [**transfer learning**](https://www.google.com/search?q=transfer+learning). That means using an existing model that can recognize objects and fine-tuning it to our specific problem.", "votes": null }, { "id": "3160674", "postDate": "03/27/2025 03:18:40", "content": "I've tried this. Not working in this comp. My current method just uses KNN.
", "rawMarkdown": "I've tried this. Not working in this comp. My current method just uses KNN.", "votes": null } ], "comments": [ { "id": 3160369, "author_name": "fautei", "author_url": "", "post_date": "03/26/2025 17:22:59", "content": "I think model is to simple to learn something useful, and how you will determine if motor not presented in tomogram too?
", "votes": null, "replies": [] }, { "id": 3160487, "author_name": "connorjd", "author_url": "", "post_date": "03/26/2025 20:37:24", "content": "I mean…run an experiment and see
", "votes": null, "replies": [] }, { "id": 3160498, "author_name": "tilii7", "author_url": "", "post_date": "03/26/2025 21:08:26", "content": "Like you were told, run an experiment and you will find out.
\nAn educated guess is that this won't be a good model. If we had 100,000+ images, maybe it would be possible to learn the patterns from scratch, like you are trying to do. Given a relatively small dataset size, most likely it is necessary that we resort to transfer learning. That means using an existing model that can recognize objects and fine-tuning it to our specific problem.
", "votes": null, "replies": [] }, { "id": 3160674, "author_name": "tom99763", "author_url": "", "post_date": "03/27/2025 03:18:40", "content": "I've tried this. Not working in this comp. My current method just uses KNN.
", "votes": null, "replies": [] } ], "raw_markdown_by_id": { "3160014": "```python\nimport numpy as np\nimport pandas as pd\nimport os\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.cuda.amp import autocast, GradScaler\nfrom tqdm import tqdm\nimport cv2\nimport glob\nimport warnings\n\n# Suppress warnings\nwarnings.filterwarnings('ignore')\n\n# Paths from read file code\nworking_dir = '/kaggle/working/'\ndataset_dir = '/kaggle/input/byu-locating-bacterial-flagellar-motors-2025/'\nTRAIN_LABELS_PATH = os.path.join(dataset_dir, 'train_labels.csv')\nTRAIN_DIR = os.path.join(dataset_dir, 'train')\nTEST_DIR = os.path.join(dataset_dir, 'test')\nOUTPUT_MODEL = os.path.join(working_dir, 'fast_flagellum_model.pth')\n\n# Configuration\nDEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nPATCH_SIZE = 48 # Fixed patch size for all dimensions\nBATCH_SIZE = 32\nEPOCHS = 10\nNUM_WORKERS = 8\nSEED = 42\nLEARNING_RATE = 3e-4\nWEIGHT_DECAY = 1e-5\n\n# Set random seed for reproducibility\ndef set_seed(seed):\n torch.manual_seed(seed)\n np.random.seed(seed)\n if torch.cuda.is_available():\n torch.cuda.manual_seed(seed)\n torch.backends.cudnn.deterministic = True\n torch.backends.cudnn.benchmark = False\n\nset_seed(SEED)\n\n# Simplified Model Architecture - only coordinate regression\nclass FastFlagellumDetector(nn.Module):\n def __init__(self):\n super().__init__()\n self.conv_layers = nn.Sequential(\n nn.Conv3d(1, 32, kernel_size=3, padding=1),\n nn.BatchNorm3d(32),\n nn.ReLU(inplace=True),\n nn.MaxPool3d(2),\n \n nn.Conv3d(32, 64, kernel_size=3, padding=1),\n nn.BatchNorm3d(64),\n nn.ReLU(inplace=True),\n nn.MaxPool3d(2),\n \n nn.Conv3d(64, 128, kernel_size=3, padding=1),\n nn.BatchNorm3d(128),\n nn.ReLU(inplace=True),\n nn.AdaptiveAvgPool3d(1)\n )\n \n self.regressor = nn.Sequential(\n nn.Flatten(),\n nn.Linear(128, 3),\n nn.Sigmoid()\n )\n\n def forward(self, x):\n features = self.conv_layers(x)\n coords = self.regressor(features)\n return coords\n\n# Optimized Data Loading with fixed dimensions\ndef load_tomogram(tomo_id, base_path):\n slice_paths = sorted(glob.glob(os.path.join(base_path, tomo_id, '*.jpg')))\n slices = []\n for i in range(PATCH_SIZE):\n if i < len(slice_paths):\n img = cv2.imread(slice_paths[i], 0)\n if img is not None:\n img = cv2.resize(img, (PATCH_SIZE, PATCH_SIZE))\n slices.append(img)\n else:\n slices.append(np.zeros((PATCH_SIZE, PATCH_SIZE)))\n else:\n slices.append(np.zeros((PATCH_SIZE, PATCH_SIZE)))\n \n volume = np.stack(slices) # Shape: (PATCH_SIZE, PATCH_SIZE, PATCH_SIZE)\n return volume\n\nclass TomogramDataset(Dataset):\n def __init__(self, ids, base_path, labels=None):\n self.ids = ids\n self.base_path = base_path\n self.labels = labels.set_index('tomo_id') if labels is not None else None\n\n def __len__(self):\n return len(self.ids)\n\n def __getitem__(self, idx):\n tomo_id = self.ids[idx]\n try:\n vol = load_tomogram(tomo_id, self.base_path)\n vol = (vol - vol.mean()) / (vol.std() + 1e-6) # Normalize\n vol_tensor = torch.tensor(vol, dtype=torch.float32).unsqueeze(0) # Shape: (1, D, H, W)\n \n if self.labels is not None and tomo_id in self.labels.index:\n label = self.labels.loc[tomo_id]\n # Ensure we get a single row and convert to 1D array\n if isinstance(label, pd.Series):\n coords = label[['Motor axis 0', 'Motor axis 1', 'Motor axis 2']].values\n else: # If we got a DataFrame (multiple rows)\n coords = label[['Motor axis 0', 'Motor axis 1', 'Motor axis 2']].iloc[0].values\n coords = torch.tensor(coords, dtype=torch.float32) / PATCH_SIZE\n return vol_tensor, coords\n # Return dummy coords if no labels or tomo_id not found\n return vol_tensor, torch.zeros(3, dtype=torch.float32)\n except Exception as e:\n print(f\"Error processing {tomo_id}: {str(e)}\")\n vol_tensor = torch.zeros((1, PATCH_SIZE, PATCH_SIZE, PATCH_SIZE), dtype=torch.float32)\n return vol_tensor, torch.zeros(3, dtype=torch.float32)\n\n# Custom collate function\ndef custom_collate(batch):\n volumes = torch.stack([item[0] for item in batch])\n coords = torch.stack([item[1] for item in batch])\n return volumes, coords\n\n# Loss Function - only coordinate regression\nclass FlagellumLoss(nn.Module):\n def __init__(self):\n super().__init__()\n self.coord_loss = nn.MSELoss()\n\n def forward(self, outputs, targets):\n loss = self.coord_loss(outputs, targets)\n return loss\n\n# Training Function\ndef train_fast(model, train_loader, criterion):\n model.train()\n optimizer = optim.AdamW(model.parameters(), lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)\n scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=EPOCHS)\n scaler = GradScaler()\n\n for epoch in range(EPOCHS):\n epoch_loss = 0\n for inputs, targets in tqdm(train_loader, desc=f\"Epoch {epoch+1}/{EPOCHS}\"):\n inputs = inputs.to(DEVICE)\n targets = targets.to(DEVICE)\n \n optimizer.zero_grad()\n with autocast():\n outputs = model(inputs)\n loss = criterion(outputs, targets)\n \n scaler.scale(loss).backward()\n scaler.step(optimizer)\n scaler.update()\n epoch_loss += loss.item()\n \n scheduler.step()\n print(f\"Epoch {epoch+1} Loss: {epoch_loss/len(train_loader):.4f}\")\n\n# Main Execution\nif __name__ == '__main__':\n # Load data\n train_labels = pd.read_csv(TRAIN_LABELS_PATH).rename(columns={'id': 'tomo_id'})\n train_ids = sorted(os.listdir(TRAIN_DIR))\n \n # Create dataset and dataloader with custom collate\n dataset = TomogramDataset(train_ids, TRAIN_DIR, train_labels)\n train_loader = DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=True, \n num_workers=NUM_WORKERS, pin_memory=True,\n collate_fn=custom_collate)\n \n # Initialize model and loss\n model = FastFlagellumDetector().to(DEVICE)\n criterion = FlagellumLoss()\n \n # Train model\n train_fast(model, train_loader, criterion)\n \n # Save model\n torch.save(model.state_dict(), OUTPUT_MODEL)\n print(f\"Model saved successfully to {OUTPUT_MODEL}\")**\n```", "3160369": "I think model is to simple to learn something useful, and how you will determine if motor not presented in tomogram too?", "3160487": "I mean…run an experiment and see", "3160498": "Like you were told, run an experiment and you will find out.\n\nAn educated guess is that this won't be a good model. If we had 100,000+ images, maybe it would be possible to learn the patterns from scratch, like you are trying to do. Given a relatively small dataset size, most likely it is necessary that we resort to [**transfer learning**](https://www.google.com/search?q=transfer+learning). That means using an existing model that can recognize objects and fine-tuning it to our specific problem.", "3160674": "I've tried this. Not working in this comp. My current method just uses KNN." }, "source": "meta" }