{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.13"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":22962,"databundleVersionId":3171193,"sourceType":"competition"},{"sourceId":3398941,"sourceType":"datasetVersion","datasetId":2040833}],"dockerImageVersionId":31090,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":41.006696,"end_time":"2025-08-03T05:59:48.024918","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-08-03T05:59:07.018222","version":"2.5.0"},"widgets":{"application/vnd.jupyter.widget-state+json":{"state":{"0242f0ca0559484daa5f53e208b4940d":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"0ad43bed980645c0847df078c41bfa0e":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_8668b357f7ab4259bdd3f60f9d8cea89","max":18,"min":0,"orientation":"horizontal","style":"IPY_MODEL_8933328305d4487da91eca0b59f929d9","value":18}},"0b53c5206eb34f6689cc32900b345a31":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"0d8b8e8413f049c7b1f71e9d63016f2f":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"14a5c260b4e142b79661775e9ab2f2e3":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_0b53c5206eb34f6689cc32900b345a31","placeholder":"​","style":"IPY_MODEL_8f4503703d6c48d095e387d9c761bebc","value":"Epoch 2: 100%"}},"2be52247723d4bfc92b0113b03262625":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":"inline-flex","flex":null,"flex_flow":"row wrap","grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":"100%"}},"3f1e58790231494ea54ca1c1dbae1c22":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_e61efd293cd84ff6bc853b34f8e9860c","placeholder":"​","style":"IPY_MODEL_0242f0ca0559484daa5f53e208b4940d","value":"Testing DataLoader 0: 100%"}},"4776a80a5d3c4f1dae8c032fd298aed9":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":"2","flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"4ddfb0468dc84f36b157b401dab0b040":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"7a9d7263f1f1439ba8edb9ea26923257":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_3f1e58790231494ea54ca1c1dbae1c22","IPY_MODEL_0ad43bed980645c0847df078c41bfa0e","IPY_MODEL_f9d0430c993e44598bc43694a8dae3ce"],"layout":"IPY_MODEL_2be52247723d4bfc92b0113b03262625"}},"7cb0b283f32a46529d5d345211178ca4":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":"inline-flex","flex":null,"flex_flow":"row wrap","grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":"100%"}},"8668b357f7ab4259bdd3f60f9d8cea89":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":"2","flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"8933328305d4487da91eca0b59f929d9":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"8f4503703d6c48d095e387d9c761bebc":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"a9b18ee47ae84012bf67a69167983661":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_f39e84f54d614d64b8c7529b009af830","placeholder":"​","style":"IPY_MODEL_4ddfb0468dc84f36b157b401dab0b040","value":" 70/70 [00:40<00:00,  1.72it/s, v_num=0]"}},"d0d8a7dee29b439084621db10efc20e1":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_14a5c260b4e142b79661775e9ab2f2e3","IPY_MODEL_f693cca8123c4f17830fe2b749632ad6","IPY_MODEL_a9b18ee47ae84012bf67a69167983661"],"layout":"IPY_MODEL_7cb0b283f32a46529d5d345211178ca4"}},"d5eecd7849aa4202b2c5717233b5dd81":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"e61efd293cd84ff6bc853b34f8e9860c":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"f39e84f54d614d64b8c7529b009af830":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"f693cca8123c4f17830fe2b749632ad6":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_4776a80a5d3c4f1dae8c032fd298aed9","max":70,"min":0,"orientation":"horizontal","style":"IPY_MODEL_f6eb65a6c7aa4cf8882db6dc8fa3a295","value":70}},"f6eb65a6c7aa4cf8882db6dc8fa3a295":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"f9d0430c993e44598bc43694a8dae3ce":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_d5eecd7849aa4202b2c5717233b5dd81","placeholder":"​","style":"IPY_MODEL_0d8b8e8413f049c7b1f71e9d63016f2f","value":" 18/18 [00:07<00:00,  2.44it/s]"}}},"version_major":2,"version_minor":0}}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.007357,"end_time":"2025-08-03T05:59:11.600526","exception":false,"start_time":"2025-08-03T05:59:11.593169","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **Whale Identification Pipeline: Debug & Validation**\nhttps://www.kaggle.com/competitions/happy-whale-and-dolphin (2022)","metadata":{"papermill":{"duration":0.005626,"end_time":"2025-08-03T05:59:11.612345","exception":false,"start_time":"2025-08-03T05:59:11.606719","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"\n## **Introduction**\nThis notebook serves as a **debugging and validation prototype** for a whale and dolphin re-identification system. Originally designed to test core functionality before scaling to the full dataset (15,000+ images), it demonstrates key components of the pipeline using a small subset of data for rapid iteration.\n\n## **Pipeline Overview**\n\n### **1. Controlled Data Sampling**\nThe system begins by intentionally limiting the dataset to:\n- Only the first 10 unique individual whales/dolphins\n- Maximum 10 images per individual\n\nThis creates a manageable test environment while preserving real-world data characteristics. The balanced sampling further ensures each individual is equally represented by selecting 3 random images per creature (when available).\n\n### **2. Deep Feature Extraction**\nUsing EfficientNetB0 (pretrained on ImageNet) as a feature extractor:\n- Images are resized to 224x224 pixels\n- Converted to normalized tensor format\n- Processed through the neural network\n- Final 1280-dimensional feature vectors are extracted from the global average pooling layer\n\nThis step transforms visual data into compact numerical representations suitable for similarity comparisons.\n\n### **3. Feature Space Analysis**\nTwo key analytical methods are applied:\n\n**Distance Distribution Analysis:**\n- Calculates Euclidean distances between all feature pairs\n- Separates distances into:\n  - Intra-class (same individual)\n  - Inter-class (different individuals)\n- Visualizes the distributions to assess feature discriminability\n\n**PCA Visualization:**\n- Projects high-dimensional features into 2D space\n- Color-codes points by individual identity\n- Helps visually verify clustering of same-individual samples\n\n### **4. Similarity Search Demonstration**\nThe system implements a query-by-example search:\n1. Randomly selects a query image\n2. Computes cosine similarity against all other images\n3. Returns top 5 most similar matches\n4. Displays results with similarity scores\n\nThis provides intuitive validation of whether the features capture meaningful visual similarities.\n\n","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nfrom sklearn.decomposition import PCA\nfrom tensorflow.keras.preprocessing import image\nfrom tensorflow.keras.applications.efficientnet import preprocess_input\n\n# 1. Data Preparation (with limits for testing)\ntrain_csv = pd.read_csv(\"/kaggle/input/happy-whale-and-dolphin/train.csv\")\ntrain_dir = \"/kaggle/input/happy-whale-and-dolphin/train_images\"\n\n# Limit number of individuals and images per individual (for testing)\nunique_individuals = train_csv['individual_id'].unique()[:10]  # Only first 10 individuals\nlimited_df = train_csv[train_csv['individual_id'].isin(unique_individuals)].copy()\n\n# Limit to max 10 images per individual\nlimited_df = limited_df.groupby('individual_id').head(10).reset_index(drop=True)\n\nprint(f\"Limited sample count: {len(limited_df)}\")\nprint(f\"Limited unique individuals: {limited_df['individual_id'].nunique()}\")\n\n# 2. Balanced Sampling (max 5 samples per individual)\ndef balanced_sample(df, samples_per_class=5):\n    return df.groupby('individual_id').apply(\n        lambda x: x.sample(min(len(x), samples_per_class))\n    ).reset_index(drop=True)\n\nsample_df = balanced_sample(limited_df, 3)  # Limit to 3 samples\nprint(f\"Sample count: {len(sample_df)}\")\n\n# 3. Feature Extraction Model\ndef create_feature_extractor():\n    from tensorflow.keras.applications import EfficientNetB0\n    from tensorflow.keras.models import Model\n    from tensorflow.keras.layers import GlobalAveragePooling2D\n    \n    base_model = EfficientNetB0(weights='imagenet', include_top=False)\n    x = GlobalAveragePooling2D()(base_model.output)\n    return Model(inputs=base_model.input, outputs=x)\n\nfeature_model = create_feature_extractor()\n\n# 4. Feature Extraction\ndef extract_features(img_path, model):\n    try:\n        img = image.load_img(os.path.join(train_dir, img_path), \n                            target_size=(224, 224), color_mode='rgb')\n        x = image.img_to_array(img)\n        x = np.expand_dims(x, axis=0)\n        x = preprocess_input(x)\n        return model.predict(x, verbose=0).flatten()\n    except Exception as e:\n        print(f\"Error: {img_path} - {str(e)}\")\n        return None\n\n# Extract features from sample data\nfeatures = []\nvalid_indices = []\nfor i, row in tqdm(sample_df.iterrows(), total=len(sample_df)):\n    feat = extract_features(row['image'], feature_model)\n    if feat is not None:\n        features.append(feat)\n        valid_indices.append(i)\n\nfeatures = np.array(features)\nsample_df = sample_df.iloc[valid_indices]\n\nprint(f\"\\nValid feature count: {len(features)}\")\nprint(f\"Feature shape: {features.shape}\")\n\n# 5. Feature Analysis\ndef analyze_features(features, labels):\n    # Calculate distance distributions\n    def calculate_distances(feats, lbls):\n        intra_dist, inter_dist = [], []\n        for i in range(len(feats)):\n            for j in range(i+1, len(feats)):\n                dist = np.linalg.norm(feats[i] - feats[j])\n                if lbls[i] == lbls[j]:\n                    intra_dist.append(dist)\n                else:\n                    inter_dist.append(dist)\n        return intra_dist, inter_dist\n    \n    intra, inter = calculate_distances(features, sample_df['individual_id'].values)\n    \n    # Visualization\n    plt.figure(figsize=(12, 5))\n    \n    plt.subplot(1, 2, 1)\n    plt.hist(intra, bins=30, alpha=0.7, label='Same individual')\n    plt.hist(inter, bins=30, alpha=0.7, label='Different individuals')\n    plt.xlabel('Feature distance')\n    plt.ylabel('Frequency')\n    plt.legend()\n    \n    plt.subplot(1, 2, 2)\n    pca = PCA(n_components=2)\n    feat_pca = pca.fit_transform(features)\n    for uid in np.unique(sample_df['individual_id'])[:5]:  # Show first 5 individuals\n        idx = sample_df['individual_id'] == uid\n        plt.scatter(feat_pca[idx, 0], feat_pca[idx, 1], label=uid)\n    plt.xlabel('PCA1')\n    plt.ylabel('PCA2')\n    plt.legend()\n    \n    plt.tight_layout()\n    plt.show()\n    \n    print(f\"Same individual mean distance: {np.mean(intra):.3f} ± {np.std(intra):.3f}\")\n    print(f\"Different individuals mean distance: {np.mean(inter):.3f} ± {np.std(inter):.3f}\")\n\nanalyze_features(features, sample_df['individual_id'])\n\n# 6. Similarity Search Demo\ndef similarity_demo(query_idx, features, df, top_k=5):\n    query_feat = features[query_idx]\n    query_img = df.iloc[query_idx]['image']\n    query_id = df.iloc[query_idx]['individual_id']\n    \n    # Calculate cosine similarity\n    norms = np.linalg.norm(features, axis=1, keepdims=True)\n    sims = np.dot(features/norms, query_feat/np.linalg.norm(query_feat))\n    \n    # Get results (excluding query itself)\n    results = pd.DataFrame({\n        'image': df['image'],\n        'individual_id': df['individual_id'],\n        'similarity': sims\n    }).sort_values('similarity', ascending=False)\n    results = results[results['image'] != query_img].head(top_k)\n    \n    # Visualization\n    plt.figure(figsize=(15, 5))\n    plt.subplot(1, top_k+1, 1)\n    plt.imshow(image.load_img(os.path.join(train_dir, query_img)))\n    plt.title(f\"Query\\n{query_id}\")\n    \n    for i, (_, row) in enumerate(results.iterrows(), 1):\n        plt.subplot(1, top_k+1, i+1)\n        plt.imshow(image.load_img(os.path.join(train_dir, row['image'])))\n        plt.title(f\"Top {i}\\n{row['individual_id']}\\n{row['similarity']:.3f}\")\n    \n    plt.tight_layout()\n    plt.show()\n\n# Test with 3 random queries\nfor _ in range(3):\n    idx = np.random.choice(len(sample_df))\n    similarity_demo(idx, features, sample_df)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **Key Insights from Output**\nThe current results show:\n- Same-individual distance (10.41 ± 2.73) is smaller than different-individual distance (12.90 ± 2.72)\n- Visual clusters in PCA space correspond to individual identities\n- Similarity search returns logically related matches\n\nThis confirms the pipeline is fundamentally functional, though the modest distance separation suggests potential areas for improvement in feature discrimination.\n\n## **Development Notes**\nOriginally created as a debugging tool, this notebook has evolved into:\n- A validation framework for architectural changes\n- A safe environment for testing preprocessing modifications\n- A visual demonstration of the core re-identification concept\n\nThe limited scale (27 samples) enables rapid experimentation while maintaining biological relevance through careful individual-balanced sampling.","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}