{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":14774,"databundleVersionId":875431}],"dockerImageVersionId":31287,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"e3ca33e2","cell_type":"markdown","source":"# APTOS 2019 — FULL HYBRID TRANSFORMER KAGGLE NOTEBOOK\n\n\n","metadata":{}},{"id":"98af0b1e-e15c-47fe-8d4a-a57e09609a10","cell_type":"markdown","source":"# 1️⃣ Install","metadata":{}},{"id":"d5d60771-0657-4621-9379-41b9564cfd3a","cell_type":"code","source":"import os\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import cohen_kappa_score\nfrom tensorflow.keras import layers, models\nfrom tensorflow.keras.applications import EfficientNetB3\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T17:10:57.501613Z","iopub.execute_input":"2026-02-23T17:10:57.502284Z","iopub.status.idle":"2026-02-23T17:10:57.506938Z","shell.execute_reply.started":"2026-02-23T17:10:57.502246Z","shell.execute_reply":"2026-02-23T17:10:57.506171Z"}},"outputs":[],"execution_count":null},{"id":"f1922631-934a-4795-81c1-cd6eb50a0e4d","cell_type":"markdown","source":"# 2️⃣ Load Dataset (Your Required Format)","metadata":{}},{"id":"1a3e6ff0-8004-4026-b2d1-36e4871e048f","cell_type":"code","source":"import os\nprint(os.listdir(\"/kaggle/input\"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T17:10:57.508764Z","iopub.execute_input":"2026-02-23T17:10:57.509083Z","iopub.status.idle":"2026-02-23T17:10:57.52526Z","shell.execute_reply.started":"2026-02-23T17:10:57.509061Z","shell.execute_reply":"2026-02-23T17:10:57.524588Z"}},"outputs":[],"execution_count":null},{"id":"06b4ec22-f1f4-4410-9f48-7803897db92f","cell_type":"code","source":"DATA_DIR = \"/kaggle/input/aptos2019-blindness-detection\"\n\ndf = pd.read_csv(f\"{DATA_DIR}/train.csv\")\n\ndf[\"image_path\"] = df[\"id_code\"].apply(\n    lambda x: f\"{DATA_DIR}/train_images/{x}.png\"\n)\n\ntrain_df, val_df = train_test_split(\n    df,\n    test_size=0.2,\n    stratify=df[\"diagnosis\"],\n    random_state=42\n)\n\nprint(df.shape, train_df.shape, val_df.shape)\nprint(df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T17:10:57.526167Z","iopub.execute_input":"2026-02-23T17:10:57.526406Z","iopub.status.idle":"2026-02-23T17:10:57.55586Z","shell.execute_reply.started":"2026-02-23T17:10:57.526385Z","shell.execute_reply":"2026-02-23T17:10:57.555092Z"}},"outputs":[],"execution_count":null},{"id":"a431d69d-43ba-4aca-aac5-67a35438cc15","cell_type":"markdown","source":"# 3️⃣ Retina Preprocessing","metadata":{}},{"id":"5da681f4-e32b-4229-b69a-712e157211a0","cell_type":"code","source":"IMG_SIZE = 300\n\ndef preprocess_image(path):\n    img = cv2.imread(path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    img = cv2.resize(img, (IMG_SIZE, IMG_SIZE))\n\n    img = cv2.addWeighted(\n        img, 4,\n        cv2.GaussianBlur(img, (0,0), IMG_SIZE/10),\n        -4, 128\n    )\n\n    return img / 255.0\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T17:10:57.556844Z","iopub.execute_input":"2026-02-23T17:10:57.557107Z","iopub.status.idle":"2026-02-23T17:10:57.561677Z","shell.execute_reply.started":"2026-02-23T17:10:57.557074Z","shell.execute_reply":"2026-02-23T17:10:57.560903Z"}},"outputs":[],"execution_count":null},{"id":"3b0716fe-f14e-4706-a553-e941d466ee21","cell_type":"markdown","source":"# 4️⃣ Dataset Pipeline","metadata":{}},{"id":"24432ae2-76f9-4175-9560-a5bc6a9a443e","cell_type":"code","source":"def build_dataset(df, training=True):\n    paths = df[\"image_path\"].values\n    labels = df[\"diagnosis\"].values\n\n    def generator():\n        for p, l in zip(paths, labels):\n            yield preprocess_image(p), l\n\n    ds = tf.data.Dataset.from_generator(\n        generator,\n        output_signature=(\n            tf.TensorSpec(shape=(IMG_SIZE, IMG_SIZE, 3), dtype=tf.float32),\n            tf.TensorSpec(shape=(), dtype=tf.int32)\n        )\n    )\n\n    if training:\n        ds = ds.shuffle(512)\n\n    return ds.batch(8).prefetch(tf.data.AUTOTUNE)\n\ntrain_ds = build_dataset(train_df)\nval_ds = build_dataset(val_df, training=False)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T17:10:57.563756Z","iopub.execute_input":"2026-02-23T17:10:57.564174Z","iopub.status.idle":"2026-02-23T17:10:59.082579Z","shell.execute_reply.started":"2026-02-23T17:10:57.564139Z","shell.execute_reply":"2026-02-23T17:10:59.08179Z"}},"outputs":[],"execution_count":null},{"id":"cc421859-c73d-4c79-8abd-f9deb210dd4d","cell_type":"markdown","source":"# 5️⃣ Lightweight Vision Transformer (Manual Implementation)","metadata":{}},{"id":"39e082b5-40b1-4602-8cbe-df2093040ad4","cell_type":"markdown","source":"We implement patch embedding + transformer encoder.","metadata":{}},{"id":"a8a3fc97-2aae-4bab-a22a-d2858353b7df","cell_type":"markdown","source":"# Patch Embedding","metadata":{}},{"id":"029c2685-e03d-4c0b-bad4-77a501f1c27e","cell_type":"code","source":"def patch_embedding(x, patch_size=16, embed_dim=256):\n    patches = layers.Conv2D(\n        embed_dim,\n        kernel_size=patch_size,\n        strides=patch_size,\n        padding=\"valid\"\n    )(x)\n\n    patches = layers.Reshape((-1, embed_dim))(patches)\n    return patches\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T17:10:59.083474Z","iopub.execute_input":"2026-02-23T17:10:59.084194Z","iopub.status.idle":"2026-02-23T17:10:59.088271Z","shell.execute_reply.started":"2026-02-23T17:10:59.08416Z","shell.execute_reply":"2026-02-23T17:10:59.087492Z"}},"outputs":[],"execution_count":null},{"id":"dda0fafb-44af-4524-b49a-fe0d3af03d9c","cell_type":"markdown","source":"# Transformer Encoder Block","metadata":{}},{"id":"00a91787-15eb-4b9f-a803-44a805aefac0","cell_type":"code","source":"def transformer_block(x, num_heads=4, key_dim=64, mlp_dim=256):\n\n    attn = layers.MultiHeadAttention(\n        num_heads=num_heads,\n        key_dim=key_dim\n    )(x, x)\n\n    x = layers.Add()([x, attn])\n    x = layers.LayerNormalization()(x)\n\n    mlp = layers.Dense(mlp_dim, activation=\"gelu\")(x)\n    mlp = layers.Dense(x.shape[-1])(mlp)\n\n    x = layers.Add()([x, mlp])\n    x = layers.LayerNormalization()(x)\n\n    return x\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T17:10:59.089256Z","iopub.execute_input":"2026-02-23T17:10:59.08955Z","iopub.status.idle":"2026-02-23T17:10:59.104929Z","shell.execute_reply.started":"2026-02-23T17:10:59.089521Z","shell.execute_reply":"2026-02-23T17:10:59.104258Z"}},"outputs":[],"execution_count":null},{"id":"9c98c641-c59e-4393-a133-f21b6528e33b","cell_type":"markdown","source":"# 6️⃣ Lesion Attention Gate","metadata":{}},{"id":"78b5186a-e4cc-4ad5-a400-63045cf7852f","cell_type":"code","source":"def lesion_attention_block(x):\n    mask = layers.Conv2D(64, 3, padding=\"same\", activation=\"relu\")(x)\n    mask = layers.Conv2D(1, 1, activation=\"sigmoid\")(mask)\n    return layers.multiply([x, mask])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T17:10:59.105896Z","iopub.execute_input":"2026-02-23T17:10:59.106171Z","iopub.status.idle":"2026-02-23T17:10:59.118303Z","shell.execute_reply.started":"2026-02-23T17:10:59.106143Z","shell.execute_reply":"2026-02-23T17:10:59.11773Z"}},"outputs":[],"execution_count":null},{"id":"32930aed-9722-4a1c-92f8-03caff7895b2","cell_type":"markdown","source":"# 7️⃣ Build Hybrid Model","metadata":{}},{"id":"b6232d01-ce6c-4392-92a4-77492f27948d","cell_type":"code","source":"def build_hybrid_model():\n\n    inputs = layers.Input(shape=(IMG_SIZE, IMG_SIZE, 3))\n\n    # ----- CNN Branch -----\n    cnn_base = EfficientNetB3(\n        include_top=False,\n        weights=\"imagenet\",\n        input_tensor=inputs\n    )\n\n    cnn_feat = cnn_base.output\n    cnn_feat = lesion_attention_block(cnn_feat)\n    cnn_feat = layers.GlobalAveragePooling2D()(cnn_feat)\n\n    # ----- Transformer Branch -----\n    patches = patch_embedding(inputs)\n    x = transformer_block(patches)\n    x = transformer_block(x)\n    vit_feat = layers.GlobalAveragePooling1D()(x)\n\n    # ----- Fusion -----\n    fusion = layers.concatenate([cnn_feat, vit_feat])\n    fusion = layers.BatchNormalization()(fusion)\n    fusion = layers.Dropout(0.5)(fusion)\n    fusion = layers.Dense(256, activation=\"relu\")(fusion)\n    fusion = layers.Dropout(0.4)(fusion)\n\n    output = layers.Dense(5, activation=\"softmax\")(fusion)\n\n    model = models.Model(inputs, output)\n\n    return model\n\nmodel = build_hybrid_model()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T17:10:59.119215Z","iopub.execute_input":"2026-02-23T17:10:59.119552Z","iopub.status.idle":"2026-02-23T17:11:05.669991Z","shell.execute_reply.started":"2026-02-23T17:10:59.11952Z","shell.execute_reply":"2026-02-23T17:11:05.669154Z"}},"outputs":[],"execution_count":null},{"id":"ddc1e1a7-c883-45c1-9032-6a0613f8dba0","cell_type":"markdown","source":"# 8️⃣ Focal Loss (No Addons)","metadata":{}},{"id":"3c18470b-2866-4774-ba75-3f72f09a0fa0","cell_type":"code","source":"def focal_loss(gamma=2., alpha=0.25):\n    def loss(y_true, y_pred):\n\n        y_true = tf.cast(y_true, tf.int32)   # <-- FIX\n        y_true = tf.one_hot(y_true, depth=5)\n\n        ce = tf.keras.losses.categorical_crossentropy(y_true, y_pred)\n        pt = tf.exp(-ce)\n        focal = alpha * tf.pow(1 - pt, gamma) * ce\n\n        return tf.reduce_mean(focal)\n\n    return loss\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T17:11:05.671038Z","iopub.execute_input":"2026-02-23T17:11:05.671473Z","iopub.status.idle":"2026-02-23T17:11:05.676553Z","shell.execute_reply.started":"2026-02-23T17:11:05.671439Z","shell.execute_reply":"2026-02-23T17:11:05.675785Z"}},"outputs":[],"execution_count":null},{"id":"2673d05c-5754-4b77-82b6-c07ad7031f3b","cell_type":"markdown","source":"# 9️⃣ Custom QWK Metric","metadata":{}},{"id":"8ec97671-e507-4a62-91f3-72da0007503a","cell_type":"code","source":"def qwk_metric(y_true, y_pred):\n    y_pred = tf.argmax(y_pred, axis=1)\n    return tf.py_function(\n        lambda t,p: cohen_kappa_score(t, p, weights=\"quadratic\"),\n        [y_true, y_pred],\n        tf.float32\n    )\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T17:11:05.677402Z","iopub.execute_input":"2026-02-23T17:11:05.677672Z","iopub.status.idle":"2026-02-23T17:11:05.692885Z","shell.execute_reply.started":"2026-02-23T17:11:05.677645Z","shell.execute_reply":"2026-02-23T17:11:05.691983Z"}},"outputs":[],"execution_count":null},{"id":"c83df895-4f0e-4cc4-bda3-645823b106c1","cell_type":"markdown","source":"# 🔟 Compile","metadata":{}},{"id":"b46b4a4c-8648-411e-a98f-36327aaa8ab5","cell_type":"code","source":"model.compile(\n    optimizer=tf.keras.optimizers.Adam(1e-4),\n    loss=focal_loss(),\n    metrics=[\"accuracy\"]\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T17:11:05.693985Z","iopub.execute_input":"2026-02-23T17:11:05.694247Z","iopub.status.idle":"2026-02-23T17:11:05.71699Z","shell.execute_reply.started":"2026-02-23T17:11:05.694217Z","shell.execute_reply":"2026-02-23T17:11:05.716391Z"}},"outputs":[],"execution_count":null},{"id":"23e84b9d-0b3b-47b6-83ac-39e65c83b87c","cell_type":"markdown","source":"# 11️⃣ Train","metadata":{}},{"id":"70f16341-5d49-4104-9c59-532c8ff12048","cell_type":"code","source":"callbacks = [\n    tf.keras.callbacks.ReduceLROnPlateau(patience=3, factor=0.5),\n    tf.keras.callbacks.EarlyStopping(patience=6, restore_best_weights=True)\n]\n\nmodel.fit(\n    train_ds,\n    validation_data=val_ds,\n    epochs=25,\n    callbacks=callbacks\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T17:11:05.718031Z","iopub.execute_input":"2026-02-23T17:11:05.718319Z","iopub.status.idle":"2026-02-23T20:15:24.850094Z","shell.execute_reply.started":"2026-02-23T17:11:05.718288Z","shell.execute_reply":"2026-02-23T20:15:24.849273Z"}},"outputs":[],"execution_count":null},{"id":"87d5a3fb-d997-4bed-a513-e8f7c12a7fdc","cell_type":"markdown","source":"انا","metadata":{}},{"id":"26947682-7896-4ab8-9cfb-a3e96493f310","cell_type":"code","source":"history = model.fit(\n    train_ds,\n    validation_data=val_ds,\n    epochs=25,\n    callbacks=callbacks\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T20:54:46.454552Z","iopub.execute_input":"2026-02-23T20:54:46.455281Z","iopub.status.idle":"2026-02-23T21:05:28.800811Z","shell.execute_reply.started":"2026-02-23T20:54:46.455252Z","shell.execute_reply":"2026-02-23T21:05:28.799523Z"}},"outputs":[],"execution_count":null},{"id":"9184d99a-a120-4221-96d9-0cc1c8471513","cell_type":"code","source":"import matplotlib.pyplot as plt\n\nplt.figure(figsize=(12,5))\n\n# Loss\nplt.subplot(1,2,1)\nplt.plot(history.history['loss'], label='Train Loss')\nplt.plot(history.history['val_loss'], label='Val Loss')\nplt.legend()\nplt.title(\"Loss Curve\")\n\n# Accuracy\nplt.subplot(1,2,2)\nplt.plot(history.history['accuracy'], label='Train Acc')\nplt.plot(history.history['val_accuracy'], label='Val Acc')\nplt.legend()\nplt.title(\"Accuracy Curve\")\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T20:54:35.060161Z","iopub.execute_input":"2026-02-23T20:54:35.060906Z","iopub.status.idle":"2026-02-23T20:54:35.252156Z","shell.execute_reply.started":"2026-02-23T20:54:35.060874Z","shell.execute_reply":"2026-02-23T20:54:35.251142Z"}},"outputs":[],"execution_count":null},{"id":"fa1addf7-c7c1-4442-bcac-1014165f536f","cell_type":"code","source":"from sklearn.metrics import classification_report, cohen_kappa_score\nimport numpy as np\n\ny_true = []\ny_pred = []\n\nfor x, y in val_ds:\n    preds = model.predict(x, verbose=0)\n    y_pred.extend(np.argmax(preds, axis=1))\n    y_true.extend(np.argmax(y.numpy(), axis=1))\n\nqwk = cohen_kappa_score(y_true, y_pred, weights=\"quadratic\")\n\nprint(\"QWK =\", qwk)\nprint(classification_report(y_true, y_pred))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T20:54:27.075969Z","iopub.status.idle":"2026-02-23T20:54:27.077841Z"}},"outputs":[],"execution_count":null},{"id":"770befad-a9cc-4896-bdae-44ce5b530fcb","cell_type":"code","source":"from sklearn.metrics import confusion_matrix\nimport seaborn as sns\n\ncm = confusion_matrix(y_true, y_pred)\n\nplt.figure(figsize=(6,5))\nsns.heatmap(cm, annot=True, fmt=\"d\", cmap=\"Blues\")\nplt.title(\"Confusion Matrix\")\nplt.xlabel(\"Predicted\")\nplt.ylabel(\"True\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T20:54:27.081724Z","iopub.status.idle":"2026-02-23T20:54:27.082028Z"}},"outputs":[],"execution_count":null},{"id":"a38f47d0-1909-4d45-a4d3-7b50e8d71896","cell_type":"code","source":"tf.keras.callbacks.ModelCheckpoint(\n    \"best_model.keras\",\n    monitor=\"val_loss\",\n    save_best_only=True\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-23T20:54:27.087062Z","iopub.status.idle":"2026-02-23T20:54:27.087463Z"}},"outputs":[],"execution_count":null}]}