{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.13"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":108394,"databundleVersionId":14167939,"sourceType":"competition"}],"dockerImageVersionId":31193,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true},"papermill":{"default_parameters":{},"duration":1746.908006,"end_time":"2025-11-20T12:02:21.913095","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-11-20T11:33:15.005089","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"71189a54","cell_type":"markdown","source":"# Pottery Sherd Reconstruction Solution\n\n1.  **Feature Extraction**:\n    *   **Deep Features**: Use a pre-trained **ResNet50** model to extract high-level texture and shape features (2048 dimensions).\n    *   **Color Features**: Extract **HSV Color Histograms** to capture the clay color profile, which is crucial for matching fragments from the same vessel.\n3.  **Constrained Matching**: \n    *   We strictly constrain matching to sherds within the same archaeological **`unit`**. Sherds found in different units are assumed to be unrelated.\n4.  **Clustering**:\n    *   We use **Agglomerative Clustering** with Cosine Distance on the combined feature vectors to form \"Assembly Groups\". This allows us to find natural groupings without pre-defining the number of clusters.","metadata":{"papermill":{"duration":0.0023,"end_time":"2025-11-20T11:33:18.488075","exception":false,"start_time":"2025-11-20T11:33:18.485775","status":"completed"},"tags":[]}},{"id":"5f878316","cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nfrom sklearn.cluster import AgglomerativeClustering\nfrom sklearn.metrics.pairwise import cosine_similarity\nfrom sklearn.preprocessing import normalize\nimport torch\nimport torchvision.models as models\nimport torchvision.transforms as transforms\nfrom PIL import Image\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"Using device: {device}\")","metadata":{"execution":{"iopub.execute_input":"2025-11-20T11:33:18.492633Z","iopub.status.busy":"2025-11-20T11:33:18.492432Z","iopub.status.idle":"2025-11-20T11:33:30.550259Z","shell.execute_reply":"2025-11-20T11:33:30.549331Z"},"papermill":{"duration":12.0614,"end_time":"2025-11-20T11:33:30.551475","exception":false,"start_time":"2025-11-20T11:33:18.490075","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"016616ce","cell_type":"code","source":"BASE_DIR = '/kaggle/input/h690/h690/h690'\nCSV_PATH = os.path.join(BASE_DIR, 'jd_sherds_info.csv')\nIMG_DIR = os.path.join(BASE_DIR, 'sherd_images')\n\nif os.path.exists(CSV_PATH):\n    df = pd.read_csv(CSV_PATH)\n    print(f\"Successfully loaded metadata. Total sherds: {len(df)}\")\n    display(df.head())\nelse:\n    print(f\"ERROR: CSV file not found at {CSV_PATH}. Please check the path.\")","metadata":{"execution":{"iopub.execute_input":"2025-11-20T11:33:30.556372Z","iopub.status.busy":"2025-11-20T11:33:30.556037Z","iopub.status.idle":"2025-11-20T11:33:30.682315Z","shell.execute_reply":"2025-11-20T11:33:30.681558Z"},"papermill":{"duration":0.130001,"end_time":"2025-11-20T11:33:30.683436","exception":false,"start_time":"2025-11-20T11:33:30.553435","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"5663e6de","cell_type":"markdown","source":"## 1. Feature Extraction Class\nWe define a class to handle image loading and feature extraction.","metadata":{"papermill":{"duration":0.002022,"end_time":"2025-11-20T11:33:30.687589","exception":false,"start_time":"2025-11-20T11:33:30.685567","status":"completed"},"tags":[]}},{"id":"f89dbf69","cell_type":"code","source":"class FeatureExtractor:\n    def __init__(self):\n        # We use the model up to the average pooling layer to get a 2048-d vector\n        print(\"Loading ResNet50 model...\")\n        self.model = models.resnet50(pretrained=True)\n        self.model = torch.nn.Sequential(*(list(self.model.children())[:-1]))\n        self.model.to(device)\n        self.model.eval()\n        \n        self.preprocess = transforms.Compose([\n            transforms.Resize(256),\n            transforms.CenterCrop(224),\n            transforms.ToTensor(),\n            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n        ])\n\n    def get_deep_features(self, img_path):\n        \"\"\"Extracts 2048-d deep features from the image.\"\"\"\n        try:\n            img = Image.open(img_path).convert('RGB')\n            img_t = self.preprocess(img).unsqueeze(0).to(device)\n            with torch.no_grad():\n                emb = self.model(img_t)\n            return emb.cpu().numpy().flatten()\n        except Exception as e:\n            return np.zeros(2048)\n\n    def get_color_features(self, img_path):\n        \"\"\"Extracts HSV color histogram.\"\"\"\n        try:\n            img = cv2.imread(img_path)\n            if img is None: return np.zeros(48)\n            \n            img_hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)\n            \n            # We focus on Hue (Color) and Value (Lightness)\n            hist = cv2.calcHist([img_hsv], [0, 1, 2], None, [8, 2, 3], [0, 180, 0, 256, 0, 256])\n            cv2.normalize(hist, hist)\n            return hist.flatten()\n        except Exception as e:\n            return np.zeros(48)\n\nextractor = FeatureExtractor()","metadata":{"execution":{"iopub.execute_input":"2025-11-20T11:33:30.692531Z","iopub.status.busy":"2025-11-20T11:33:30.692081Z","iopub.status.idle":"2025-11-20T11:33:31.967487Z","shell.execute_reply":"2025-11-20T11:33:31.966881Z"},"papermill":{"duration":1.279361,"end_time":"2025-11-20T11:33:31.968841","exception":false,"start_time":"2025-11-20T11:33:30.689480","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"139709d2","cell_type":"markdown","source":"## 2. Extract Features for All Images\nThis step may take some time depending on the number of images and GPU availability.","metadata":{"papermill":{"duration":0.002403,"end_time":"2025-11-20T11:33:31.973889","exception":false,"start_time":"2025-11-20T11:33:31.971486","status":"completed"},"tags":[]}},{"id":"8b816816","cell_type":"code","source":"features = []\nimage_ids = df['image_id'].tolist()\nvalid_indices = []\n\nprint(\"Starting feature extraction...\")\nfor idx, img_id in enumerate(tqdm(image_ids)):\n    img_filename = f\"{img_id}.jpg\"\n    img_path = os.path.join(IMG_DIR, img_filename)\n    \n    if not os.path.exists(img_path):\n        continue\n        \n    deep_feat = extractor.get_deep_features(img_path)\n    color_feat = extractor.get_color_features(img_path)\n\n    norm_deep = np.linalg.norm(deep_feat)\n    if norm_deep > 0:\n        deep_feat = deep_feat / norm_deep\n        \n    combined = np.concatenate([deep_feat, color_feat * 0.5])\n    \n    features.append(combined)\n    valid_indices.append(idx)\n\nfeatures_matrix = np.array(features)\nprint(f\"Feature Extraction Complete. Matrix Shape: {features_matrix.shape}\")","metadata":{"execution":{"iopub.execute_input":"2025-11-20T11:33:31.979503Z","iopub.status.busy":"2025-11-20T11:33:31.979306Z","iopub.status.idle":"2025-11-20T12:01:55.160498Z","shell.execute_reply":"2025-11-20T12:01:55.159692Z"},"papermill":{"duration":1703.185578,"end_time":"2025-11-20T12:01:55.161770","exception":false,"start_time":"2025-11-20T11:33:31.976192","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"2ff83f93","cell_type":"markdown","source":"## 3. Clustering and Grouping\nWe perform clustering **within each unit** to respect the archaeological context.","metadata":{"papermill":{"duration":0.494571,"end_time":"2025-11-20T12:01:56.159229","exception":false,"start_time":"2025-11-20T12:01:55.664658","status":"completed"},"tags":[]}},{"id":"0bb6f4af","cell_type":"code","source":"df['Assembly Group'] = -1\ngroup_id_counter = 1\n\nunits = df['unit'].unique()\n\nprint(f\"Clustering across {len(units)} units...\")\n\nfor unit in tqdm(units):\n    unit_mask = df['unit'] == unit\n    unit_indices = df.index[unit_mask]\n    \n    local_feats = []\n    local_df_indices = []\n    \n    for i in unit_indices:\n        if i in valid_indices:\n            feat_idx = valid_indices.index(i)\n            local_feats.append(features_matrix[feat_idx])\n            local_df_indices.append(i)\n            \n    if len(local_feats) == 0:\n        continue\n        \n    local_feats = np.array(local_feats)\n\n    if len(local_feats) == 1:\n        df.loc[local_df_indices[0], 'Assembly Group'] = f\"AssemblyGroup{group_id_counter}\"\n        group_id_counter += 1\n        continue\n        \n    sim_matrix = cosine_similarity(local_feats)\n    dist_matrix = 1 - sim_matrix\n    dist_matrix[dist_matrix < 0] = 0\n    \n    clustering = AgglomerativeClustering(\n        n_clusters=None, \n        distance_threshold=0.25, \n        metric='precomputed',\n        linkage='average'\n    )\n    \n    labels = clustering.fit_predict(dist_matrix)\n    \n    for lbl in np.unique(labels):\n        members_mask = (labels == lbl)\n        member_indices = [local_df_indices[j] for j in range(len(members_mask)) if members_mask[j]]\n        \n        df.loc[member_indices, 'Assembly Group'] = f\"AssemblyGroup{group_id_counter}\"\n        group_id_counter += 1\n\nprint(f\"Clustering Complete. Total Assembly Groups created: {group_id_counter - 1}\")","metadata":{"execution":{"iopub.execute_input":"2025-11-20T12:01:57.225260Z","iopub.status.busy":"2025-11-20T12:01:57.224990Z","iopub.status.idle":"2025-11-20T12:02:18.501138Z","shell.execute_reply":"2025-11-20T12:02:18.500358Z"},"papermill":{"duration":21.851151,"end_time":"2025-11-20T12:02:18.502329","exception":false,"start_time":"2025-11-20T12:01:56.651178","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"eeafd20f","cell_type":"code","source":"missed_mask = df['Assembly Group'] == -1\nif missed_mask.any():\n    print(f\"Warning: {missed_mask.sum()} images were not clustered. Assigning unique groups.\")\n    for idx in df[missed_mask].index:\n        df.loc[idx, 'Assembly Group'] = f\"AssemblyGroup{group_id_counter}\"\n        group_id_counter += 1\n\nvalid_df = df.iloc[valid_indices].copy()\n\nsubmission = valid_df[['image_id', 'Assembly Group']].copy()\nprint(f\"Before removing duplicates: {len(submission)} rows\")\nsubmission = submission.drop_duplicates(subset=['image_id'], keep='first')\nprint(f\"After removing duplicates: {len(submission)} rows\")\n\nsubmission.columns = ['image_id', 'assembly_id']\nsubmission = submission.sort_values('image_id').reset_index(drop=True)\n\nduplicates = submission['image_id'].duplicated().sum()\nif duplicates > 0:\n    print(f\"⚠️ WARNING: Found {duplicates} duplicate image_ids!\")\n    submission = submission.drop_duplicates(subset=['image_id'], keep='first')\nelse:\n    print(\"✅ No duplicate image_ids found.\")\n\nsubmission.to_csv('submission.csv', index=False)\nprint(f\"\\n✅ Submission saved to 'submission.csv' with {len(submission)} rows.\")\nprint(f\"Expected: 35159 rows\")\nprint(f\"Difference: {35159 - len(submission)} rows\")\n\ndisplay(submission.head(10))\nprint(f\"\\nTotal images in submission: {len(submission)}\")\nprint(f\"Unique image_ids: {submission['image_id'].nunique()}\")\nprint(f\"Unique assembly_ids: {submission['assembly_id'].nunique()}\")","metadata":{"execution":{"iopub.execute_input":"2025-11-20T12:02:19.499988Z","iopub.status.busy":"2025-11-20T12:02:19.499519Z","iopub.status.idle":"2025-11-20T12:02:19.589781Z","shell.execute_reply":"2025-11-20T12:02:19.588908Z"},"papermill":{"duration":0.589003,"end_time":"2025-11-20T12:02:19.590816","exception":false,"start_time":"2025-11-20T12:02:19.001813","status":"completed"},"tags":[]},"outputs":[],"execution_count":null}]}