{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":94689,"databundleVersionId":11605086,"sourceType":"competition"}],"dockerImageVersionId":31012,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"\n# **Forams Classification 2025: Improved Semi-Supervised Model**\n\nThis notebook implements an improved semi-supervised classification model for the Forams Classification 2025 competition. The model uses enhanced feature engineering, an autoencoder for unknown class detection, and pseudo-labeling to leverage unlabeled data. The code extracts features from 3D volumetric scans and 2D visualizations, trains an XGBoost classifier, and generates predictions for submission.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom glob import glob\nimport tifffile\nfrom tqdm import tqdm\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import cross_val_score, cross_val_predict\nfrom sklearn.metrics import f1_score, confusion_matrix, log_loss, accuracy_score\nimport xgboost as xgb\nimport torch\nimport torchvision.models as models\nimport torchvision.transforms as transforms\nfrom PIL import Image\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom skimage import feature, measure\nimport cv2\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Input, Conv3D, MaxPooling3D, UpSampling3D\nimport os\nimport re\n\n# Set random seed for reproducibility\nnp.random.seed(42)\ntorch.manual_seed(42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-07T16:42:30.272743Z","iopub.execute_input":"2025-05-07T16:42:30.272990Z","iopub.status.idle":"2025-05-07T16:42:37.045391Z","shell.execute_reply.started":"2025-05-07T16:42:30.272971Z","shell.execute_reply":"2025-05-07T16:42:37.044720Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Enhanced Feature Engineering\n\nFunctions to extract 3D features from volumetric scans and 2D CNN features from visualization images.","metadata":{}},{"cell_type":"code","source":"\ndef compute_3d_features(volume):\n    \"\"\"Extract advanced 3D features from a volume.\"\"\"\n    # Basic statistical features\n    features = [\n        volume.mean(),  # Mean intensity\n        volume[volume > 0].mean() if np.any(volume > 0) else 0,  # Mean of non-zero voxels\n        volume.max(),  # Maximum intensity\n        volume.std(),  # Standard deviation\n    ]\n\n    # Shape features (Hu moments on central slice)\n    slice_z = volume[:, :, 64].astype(np.uint8)\n    moments = cv2.HuMoments(cv2.moments(slice_z)).flatten()\n    features.extend(moments)\n\n    # Texture features (3D local binary patterns - simplified)\n    lbp = np.zeros_like(volume)\n    for i in range(1, volume.shape[0]-1):\n        for j in range(1, volume.shape[1]-1):\n            for k in range(1, volume.shape[2]-1):\n                center = volume[i, j, k]\n                neighbors = volume[i-1:i+2, j-1:j+2, k-1:k+2].flatten()\n                lbp[i, j, k] = np.sum(neighbors > center)\n    features.append(lbp.mean())\n\n    # Gradient features\n    grad_x = np.abs(np.diff(volume, axis=0)).mean()\n    grad_y = np.abs(np.diff(volume, axis=1)).mean()\n    grad_z = np.abs(np.diff(volume, axis=2)).mean()\n    features.extend([grad_x, grad_y, grad_z])\n\n    return np.array(features)\n\ndef extract_cnn_features(image_path):\n    \"\"\"Extract features from JPG visualizations using pre-trained ResNet18.\"\"\"\n    # Load pre-trained ResNet18\n    resnet = models.resnet18(pretrained=True)\n    resnet.eval()\n    # Remove the final classification layer\n    feature_extractor = torch.nn.Sequential(*list(resnet.children())[:-1])\n\n    # Load and preprocess image\n    img = Image.open(image_path).convert('RGB')\n    transform = transforms.Compose([\n        transforms.Resize((224, 224)),\n        transforms.ToTensor(),\n        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n    ])\n    img_tensor = transform(img).unsqueeze(0)\n\n    # Extract features\n    with torch.no_grad():\n        features = feature_extractor(img_tensor).flatten().numpy()\n    return features\n\ndef make_feature_array(vol_list, vis_list=None):\n    \"\"\"Compute features from volumes and optionally visualizations.\"\"\"\n    n_features = 14 + 512  # 14 from 3D + 512 from ResNet18\n    X = np.full((len(vol_list), n_features), np.nan)\n    for i, vol_path in enumerate(tqdm(vol_list)):\n        # Load volume\n        volume = tifffile.imread(vol_path)\n        # Extract 3D features\n        X[i, :14] = compute_3d_features(volume)\n        # Extract CNN features from visualization if provided\n        if vis_list:\n            vis_path = vol_path.replace('volumes/volumes', 'visualizations/visualizations').replace('.tif', '.jpg')\n            X[i, 14:] = extract_cnn_features(vis_path)\n    return X","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-07T16:42:37.046087Z","iopub.execute_input":"2025-05-07T16:42:37.046559Z","iopub.status.idle":"2025-05-07T16:42:37.058357Z","shell.execute_reply.started":"2025-05-07T16:42:37.046523Z","shell.execute_reply":"2025-05-07T16:42:37.057572Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Autoencoder for Unknown Class Detection\n\nBuild and use a 3D convolutional autoencoder to detect anomalies (unknown class) based on reconstruction error.","metadata":{}},{"cell_type":"code","source":"\ndef build_autoencoder():\n    \"\"\"Build a 3D convolutional autoencoder for anomaly detection.\"\"\"\n    input_shape = (64, 64, 64, 1)  # Downsample to save memory\n    input_vol = Input(shape=input_shape)\n    x = Conv3D(16, (3, 3, 3), activation='relu', padding='same')(input_vol)\n    x = MaxPooling3D((2, 2, 2), padding='same')(x)\n    x = Conv3D(8, (3, 3, 3), activation='relu', padding='same')(x)\n    encoded = MaxPooling3D((2, 2, 2), padding='same')(x)\n    x = Conv3D(8, (3, 3, 3), activation='relu', padding='same')(encoded)\n    x = UpSampling3D((2, 2, 2))(x)\n    x = Conv3D(16, (3, 3, 3), activation='relu', padding='same')(x)\n    x = UpSampling3D((2, 2, 2))(x)\n    decoded = Conv3D(1, (3, 3, 3), activation='sigmoid', padding='same')(x)\n    autoencoder = Model(input_vol, decoded)\n    autoencoder.compile(optimizer='adam', loss='mse')\n    return autoencoder\n\ndef detect_anomalies(volumes, autoencoder, threshold=0.05):\n    \"\"\"Detect anomalies (unknown class) using autoencoder reconstruction error.\"\"\"\n    anomalies = []\n    for vol in tqdm(volumes):\n        # Resize 3D volume to 64x64x64\n        vol_resized = np.zeros((64, 64, 64))\n        for z in range(64):\n            vol_resized[:, :, z] = cv2.resize(vol[:, :, z], (64, 64))\n        vol_resized = vol_resized.reshape(1, 64, 64, 64, 1) / 255.0\n        recon = autoencoder.predict(vol_resized, verbose=0)\n        mse = np.mean((recon - vol_resized) ** 2)\n        anomalies.append(mse > threshold)\n    return np.array(anomalies)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-07T16:42:37.059212Z","iopub.execute_input":"2025-05-07T16:42:37.059437Z","iopub.status.idle":"2025-05-07T16:42:37.075034Z","shell.execute_reply.started":"2025-05-07T16:42:37.059420Z","shell.execute_reply":"2025-05-07T16:42:37.074350Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load Data\n\nLoad labeled and unlabeled data from the competition dataset.","metadata":{}},{"cell_type":"code","source":"\n# Read labeled data\nlabels = pd.read_csv('/kaggle/input/forams-classification-2025/labelled.csv', index_col='id')['label']\nvol_list_labeled = sorted(glob('/kaggle/input/forams-classification-2025/volumes/volumes/labelled/*.tif'))\nvis_list_labeled = sorted(glob('/kaggle/input/forams-classification-2025/visualizations/visualizations/labelled/*.jpg'))\n\n# Read unlabeled data\nvol_list_unlabeled = sorted(glob('/kaggle/input/forams-classification-2025/volumes/volumes/unlabelled/*.tif'))\nvis_list_unlabeled = sorted(glob('/kaggle/input/forams-classification-2025/visualizations/visualizations/unlabelled/*.jpg'))\n\n# Extract features\nprint(\"Extracting features for labeled data...\")\nX_labeled = make_feature_array(vol_list_labeled, vis_list_labeled)\nscaler = StandardScaler()\nX_labeled = scaler.fit_transform(X_labeled)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-07T16:42:37.075577Z","iopub.execute_input":"2025-05-07T16:42:37.075743Z","iopub.status.idle":"2025-05-07T17:28:54.070218Z","shell.execute_reply.started":"2025-05-07T16:42:37.075730Z","shell.execute_reply":"2025-05-07T17:28:54.069392Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Train Autoencoder for Unknown Class\n\nTrain the autoencoder on labeled volumes to detect anomalies.","metadata":{}},{"cell_type":"code","source":"print(\"Training autoencoder for anomaly detection...\")\nvolumes_labeled = [tifffile.imread(vol) for vol in tqdm(vol_list_labeled)]\nvolumes_labeled_resized = []\nfor vol in volumes_labeled:\n    vol_resized = np.zeros((64, 64, 64))\n    for z in range(64):\n        # Resize each 2D slice along the z-axis\n        slice_idx = int(z * vol.shape[2] / 64)  # Sample slices evenly\n        vol_resized[:, :, z] = cv2.resize(vol[:, :, slice_idx], (64, 64))\n    vol_resized = vol_resized.reshape(64, 64, 64, 1) / 255.0\n    volumes_labeled_resized.append(vol_resized)\nautoencoder = build_autoencoder()\nautoencoder.fit(np.array(volumes_labeled_resized), np.array(volumes_labeled_resized), epochs=10, batch_size=8, verbose=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-07T17:32:48.592494Z","iopub.execute_input":"2025-05-07T17:32:48.593259Z","iopub.status.idle":"2025-05-07T17:33:23.765490Z","shell.execute_reply.started":"2025-05-07T17:32:48.593230Z","shell.execute_reply":"2025-05-07T17:33:23.764911Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Semi-Supervised Learning\n\nTrain an XGBoost classifier with pseudo-labeling to leverage unlabeled data.","metadata":{}},{"cell_type":"code","source":"# Initialize model (XGBoost)\nmodel = xgb.XGBClassifier(objective='multi:softprob', num_class=14, eval_metric='mlogloss', random_state=42)\n\n# Initial training on labeled data\nprint(\"Training initial model on labeled data...\")\nmodel.fit(X_labeled, labels)\n\n# Cross-validation\noof = cross_val_predict(model, X_labeled, labels, cv=5)\nprint(f\"Initial F1: {f1_score(labels, oof, average='macro'):.3f}\")\nprint(f\"Initial Accuracy: {accuracy_score(labels, oof):.3f}\")\n\n# Pseudo-labeling\nprint(\"Extracting features for unlabeled data...\")\nX_unlabeled = make_feature_array(vol_list_unlabeled, vis_list_unlabeled)\nX_unlabeled = scaler.transform(X_unlabeled)\n\n# Detect anomalies (unknown class)\nprint(\"Detecting anomalies...\")\nvolumes_unlabeled = [tifffile.imread(vol) for vol in tqdm(vol_list_unlabeled[:2000])]  # Subset for speed\nanomalies = detect_anomalies(volumes_unlabeled, autoencoder)\n\n# Pseudo-labeling loop\nn_iterations = 3\nconfidence_threshold = 0.9\nfor iter in range(n_iterations):\n    print(f\"Pseudo-labeling iteration {iter+1}/{n_iterations}...\")\n    probs = model.predict_proba(X_unlabeled)\n    max_probs = np.max(probs, axis=1)\n    pseudo_labels = np.argmax(probs, axis=1)\n    \n    # Assign unknown class to anomalies or low-confidence predictions\n    pseudo_labels[max_probs < confidence_threshold] = 14\n    if iter == 0:  # Apply anomalies in first iteration\n        pseudo_labels[:len(anomalies)][anomalies] = 14\n    \n    # Select high-confidence pseudo-labels\n    confident = max_probs >= confidence_threshold\n    X_combined = np.vstack([X_labeled, X_unlabeled[confident]])\n    y_combined = np.concatenate([labels, pseudo_labels[confident]])\n    \n    # Retrain model\n    model.fit(X_combined, y_combined)\n    \n    # Evaluate on labeled data\n    oof = cross_val_predict(model, X_labeled, labels, cv=5)\n    print(f\"Iter {iter+1} F1: {f1_score(labels, oof, average='macro'):.3f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-07T17:33:35.843994Z","iopub.execute_input":"2025-05-07T17:33:35.850754Z","execution_failed":"2025-05-07T17:35:36.125Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Generate Submission\n\nGenerate predictions for the unlabeled data and save to submission.csv.","metadata":{}},{"cell_type":"code","source":"print(\"Generating submission...\")\ny_pred = model.predict(X_unlabeled)\n# Re-apply anomaly detection for final predictions\ny_pred[:len(anomalies)][anomalies] = 14\n\nunlabelled_index = pd.read_csv('/kaggle/input/forams-classification-2025/unlabelled.csv', index_col='id').index\nsubmission = pd.Series(y_pred, index=unlabelled_index[:len(y_pred)], name='label')\nsubmission.to_csv('submission.csv')\n\nprint(\"Submission saved. First few rows:\")\nprint(submission.head())\n\n# Visualize confusion matrix\nsns.heatmap(confusion_matrix(labels, cross_val_predict(model, X_labeled, labels, cv=5)), annot=True)\nplt.title(\"Confusion Matrix (Labeled Data)\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-07T17:28:55.147482Z","iopub.status.idle":"2025-05-07T17:28:55.147782Z","shell.execute_reply.started":"2025-05-07T17:28:55.147609Z","shell.execute_reply":"2025-05-07T17:28:55.147625Z"}},"outputs":[],"execution_count":null}]}