{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":11848,"databundleVersionId":862157,"sourceType":"competition"}],"dockerImageVersionId":31011,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\nsecret_value_0 = user_secrets.get_secret(\"SD3.5_token\")\n\nfrom huggingface_hub import login\nlogin(token=secret_value_0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-30T16:22:39.502709Z","iopub.execute_input":"2025-04-30T16:22:39.502974Z","iopub.status.idle":"2025-04-30T16:22:40.463883Z","shell.execute_reply.started":"2025-04-30T16:22:39.502954Z","shell.execute_reply":"2025-04-30T16:22:40.463099Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## import","metadata":{}},{"cell_type":"code","source":"# !pip install -q tf-nightly[and-cuda]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-30T16:22:41.979351Z","iopub.execute_input":"2025-04-30T16:22:41.980025Z","iopub.status.idle":"2025-04-30T16:24:32.880820Z","shell.execute_reply.started":"2025-04-30T16:22:41.979994Z","shell.execute_reply":"2025-04-30T16:24:32.880104Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport glob\nimport time\nimport h5py\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nfrom tqdm import tqdm\nimport tensorflow as tf\nfrom huggingface_hub import from_pretrained_keras\n\n# Define paths\nDATA_PATH = \"/kaggle/input/histopathologic-cancer-detection/train/\"\nOUTPUT_PATH = \"/kaggle/working/embeddings/\"\nMETADATA_PATH = \"/kaggle/working/metadata/\"\n\n# Create directories if they don't exist\nos.makedirs(OUTPUT_PATH, exist_ok=True)\nos.makedirs(METADATA_PATH, exist_ok=True)\n\ndef load_model():\n    \"\"\"Load PathFoundation model from Hugging Face\"\"\"\n    print(\"Loading PathFoundation model...\")\n    model = from_pretrained_keras(\"google/path-foundation\")\n    infer = model.signatures[\"serving_default\"]\n    print(\"Model loaded!\")\n    return infer\n\ndef process_image(image_path, infer_function):\n    \"\"\"Process a single image and get embedding\"\"\"\n    try:\n        # Open and prepare image\n        img = Image.open(image_path).convert('RGB')\n        \n        # Resize to 224x224 if needed\n        if img.size != (224, 224):\n            img = img.resize((224, 224))\n        \n        # Convert to tensor and normalize\n        tensor = tf.cast(tf.expand_dims(np.array(img), axis=0), tf.float32) / 255.0\n        \n        # Get embedding\n        embeddings = infer_function(tf.constant(tensor))\n        embedding_vector = embeddings['output_0'].numpy().flatten()\n        \n        return embedding_vector\n    except Exception as e:\n        print(f\"Error processing {image_path}: {e}\")\n        return None\n\ndef process_in_batches(file_list, batch_size=1000, model_infer=None, labels_df=None):\n    \"\"\"Process files in batches and save embeddings to HDF5\"\"\"\n    if model_infer is None:\n        model_infer = load_model()\n    \n    # Create labels dictionary for faster lookup\n    labels_dict = {}\n    if labels_df is not None:\n        labels_dict = dict(zip(labels_df['id'], labels_df['label']))\n    \n    total_batches = (len(file_list) + batch_size - 1) // batch_size\n    \n    for batch_idx in range(total_batches):\n        start_idx = batch_idx * batch_size\n        end_idx = min((batch_idx + 1) * batch_size, len(file_list))\n        batch_files = file_list[start_idx:end_idx]\n        \n        batch_name = f\"batch_{batch_idx:05d}\"\n        h5_path = os.path.join(OUTPUT_PATH, f\"{batch_name}.h5\")\n        \n        # Create metadata for this batch\n        metadata = {\n            'file_id': [],\n            'file_path': [],\n            'embedding_batch': [],\n            'embedding_index': [],\n            'label': []\n        }\n        \n        # Process batch\n        with h5py.File(h5_path, 'w') as h5f:\n            embeddings_dataset = h5f.create_dataset(\n                'embeddings', \n                shape=(len(batch_files), 384),\n                dtype='float32'\n            )\n            \n            for i, file_path in enumerate(tqdm(batch_files, desc=f\"Processing batch {batch_idx+1}/{total_batches}\")):\n                file_id = os.path.basename(file_path).split('.')[0]\n                embedding = process_image(file_path, model_infer)\n                \n                if embedding is not None:\n                    # Save embedding to HDF5\n                    embeddings_dataset[i] = embedding\n                    \n                    # Store metadata\n                    metadata['file_id'].append(file_id)\n                    metadata['file_path'].append(file_path)\n                    metadata['embedding_batch'].append(batch_name)\n                    metadata['embedding_index'].append(i)\n                    # Add label if available\n                    if file_id in labels_dict:\n                        metadata['label'].append(labels_dict[file_id])\n                    else:\n                        metadata['label'].append(None)\n        \n        # Save metadata for this batch\n        metadata_df = pd.DataFrame(metadata)\n        metadata_df.to_parquet(os.path.join(METADATA_PATH, f\"{batch_name}_metadata.parquet\"))\n        \n        print(f\"Completed batch {batch_idx+1}/{total_batches}\")\n\ndef main():\n    # Get list of all .tif files\n    print(\"Finding all .tif files...\")\n    tif_files = glob.glob(os.path.join(DATA_PATH, \"*.tif\"))\n    print(f\"Found {len(tif_files)} .tif files\")\n    \n    # Load labels if available\n    labels_df = None\n    labels_path = \"/kaggle/input/histopathologic-cancer-detection/train_labels.csv\"\n    if os.path.exists(labels_path):\n        print(\"Loading cancer labels...\")\n        labels_df = pd.read_csv(labels_path)\n        print(f\"Loaded {len(labels_df)} labels\")\n    \n    # Load model once\n    model_infer = load_model()\n    \n    # Process in batches\n    start_time = time.time()\n    process_in_batches(tif_files, batch_size=1000, model_infer=model_infer, labels_df=labels_df)\n    end_time = time.time()\n    \n    print(f\"Processing completed in {(end_time - start_time)/60:.2f} minutes\")\n    \n    # Create a master metadata file linking all batches\n    print(\"Creating master metadata file...\")\n    metadata_files = glob.glob(os.path.join(METADATA_PATH, \"*_metadata.parquet\"))\n    if metadata_files:\n        master_metadata = pd.concat([pd.read_parquet(f) for f in metadata_files])\n        master_metadata.to_parquet(os.path.join(METADATA_PATH, \"master_metadata.parquet\"))\n        print(f\"Master metadata saved with {len(master_metadata)} entries\")\n    \n    print(\"Done!\")\n\nif __name__ == \"__main__\":\n    main()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-30T16:24:32.882286Z","iopub.execute_input":"2025-04-30T16:24:32.882527Z","execution_failed":"2025-04-30T16:26:07.163Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}