{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":10338,"databundleVersionId":862042,"sourceType":"competition"},{"sourceId":104953,"sourceType":"datasetVersion","datasetId":54936}],"dockerImageVersionId":31090,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# =================================================================\n# ■ ステップ1：ライブラリと環境の準備\n# =================================================================\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport os\nfrom pathlib import Path\nimport pydicom # DICOMファイルの読み込みに必要\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n\nimport tensorflow as tf\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.applications import ResNet50\nfrom tensorflow.keras.layers import Dense, GlobalAveragePooling2D, Dropout\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau\nfrom tensorflow.keras.models import load_model\nfrom tensorflow import keras\n\nfrom tqdm.notebook import tqdm\n\n# --- 環境判定とパス設定 ---\n# Kaggle環境かColab環境かを自動で判定し、パスを切り替えます\nif 'KAGGLE_KERNEL_RUN_TYPE' in os.environ:\n    print(\"Kaggle環境を検出しました。\")\n    # Kaggleのデータセットパス\n    BASE_DIR = Path('/kaggle/input/rsna-pneumonia-detection-challenge')\n    DICOM_DIR = BASE_DIR / 'stage_2_train_images'\n    PNG_DIR = Path('/kaggle/input/rsna-pneu-train-png/orig')\n    # モデルの保存/読み込みパス\n    NOTEBOOK_SLUG = os.environ.get('KAGGLE_URL_BASE', 'default-notebook').split('/')[-1]\n    MODEL_LOAD_PATH = Path(f'/kaggle/input/{NOTEBOOK_SLUG}/best_pneumonia_classifier.keras')\n    MODEL_SAVE_PATH = Path('/kaggle/working/best_pneumonia_classifier.keras')\nelse:\n    print(\"Colab環境またはローカル環境を検出しました。\")\n    from google.colab import drive\n    drive.mount('/content/drive')\n    # Google Driveのパス\n    BASE_DIR = Path('/content/drive/MyDrive/RSNA_Pneumonia/')\n    PNG_DIR = BASE_DIR / 'stage_2_train_images_png'\n    # モデルの保存/読み込みパス\n    MODEL_SAVE_PATH = BASE_DIR / 'best_pneumonia_classifier.keras'\n    MODEL_LOAD_PATH = MODEL_SAVE_PATH\n\nprint(f\"画像ディレクトリ: {PNG_DIR}\")\nprint(f\"モデル保存先: {MODEL_SAVE_PATH}\")\nprint(f\"モデル読み込み元 (次のセッション用): {MODEL_LOAD_PATH}\")\n\nprint(\"\\nTensorFlow Version:\", tf.__version__)\nprint(\"▶ ライブラリの準備完了\")\n\n# グラフのスタイルを設定\nsns.set_style(\"whitegrid\")\n\n\n# =================================================================\n# ■ ステップ2：データの読み込みと準備\n# =================================================================\n# --- 1. ラベルデータの読み込み ---\ndf_labels = pd.read_csv(BASE_DIR / 'stage_2_train_labels.csv')\n\n# --- 2. 分類タスク用のデータを作成 ---\ndf_class = df_labels.drop_duplicates('patientId')[['patientId', 'Target']].copy()\ndf_class['filename'] = df_class['patientId'].apply(lambda x: f\"{x}.png\")\ndf_class['Target'] = df_class['Target'].astype(str)\n\n# 実際に存在する画像ファイルのみを対象にする\nprint(\"\\n存在する画像ファイルを確認中...\")\ndf_class['filepath'] = df_class['filename'].apply(lambda f: PNG_DIR / f)\ndf_class['file_exists'] = df_class['filepath'].apply(lambda p: p.exists())\nmissing_files_count = len(df_class) - df_class['file_exists'].sum()\nif missing_files_count > 0:\n    print(f\"注意: {missing_files_count}件の画像ファイルが見つかりませんでした。これらはデータセットから除外されます。\")\ndf_class = df_class[df_class['file_exists']].copy().reset_index(drop=True)\n\n# 公平性分析のためにメタ情報を自動で追加\n# この処理はKaggle環境でのみ実行されます（Colabでは前処理済みと想定）\nif 'KAGGLE_KERNEL_RUN_TYPE' in os.environ:\n    ages = []\n    sexes = []\n    view_positions = []\n    print(\"\\nDICOMファイルからメタ情報を抽出中...\")\n    for patient_id in tqdm(df_class['patientId']):\n        dcm_path = DICOM_DIR / f\"{patient_id}.dcm\"\n        dcm_data = pydicom.dcmread(dcm_path, stop_before_pixels=True)\n        ages.append(dcm_data.PatientAge)\n        sexes.append(dcm_data.PatientSex)\n        view_positions.append(dcm_data.ViewPosition)\n    df_class['Age'] = ages\n    df_class['Sex'] = sexes\n    df_class['ViewPosition'] = view_positions\n    df_class['Age'] = df_class['Age'].astype(int)\n    print(\"メタ情報の抽出完了。\")\n\n# --- 4. データの分割 ---\ndf_train, df_val = train_test_split(\n    df_class,\n    test_size=0.2,\n    random_state=42,\n    stratify=df_class['Target']\n)\n\nprint(f\"\\n利用可能な画像総数: {len(df_class)}件\")\nprint(f\"学習用データ数: {len(df_train)}件\")\nprint(f\"検証用データ数: {len(df_val)}件\")\nprint(\"\\n▶ データの準備完了\")\n\n\n# =================================================================\n# ■ ステップ3：学習・評価用の関数を定義\n# =================================================================\n# (このセクションは変更ありません)\ndef train_and_finetune_stable_model(train_df, val_df, image_dir, image_size=224, batch_size=32, initial_epochs=8, fine_tune_epochs=25):\n    print(f\"\\n--- 画像サイズ: {image_size}x{image_size}, バッチサイズ: {batch_size} ---\")\n    train_datagen = ImageDataGenerator(\n        rescale=1./255, rotation_range=15, width_shift_range=0.1, height_shift_range=0.1,\n        shear_range=0.1, zoom_range=0.1, horizontal_flip=True, fill_mode='nearest'\n    )\n    val_datagen = ImageDataGenerator(rescale=1./255)\n    train_generator = train_datagen.flow_from_dataframe(\n        dataframe=train_df, directory=image_dir, x_col='filename', y_col='Target',\n        target_size=(image_size, image_size), batch_size=batch_size, class_mode='binary',\n        validate_filenames=False\n    )\n    validation_generator = val_datagen.flow_from_dataframe(\n        dataframe=val_df, directory=image_dir, x_col='filename', y_col='Target',\n        target_size=(image_size, image_size), batch_size=batch_size, class_mode='binary', shuffle=False,\n        validate_filenames=False\n    )\n    if len(train_generator.classes) == 0:\n        print(\"学習データが見つかりません。\")\n        return None, None\n    class_weights = compute_class_weight('balanced', classes=np.unique(train_generator.classes), y=train_generator.classes)\n    class_weight_dict = dict(enumerate(class_weights))\n    print(f\"▶ 計算されたクラス重み: {class_weight_dict}\")\n    base_model = ResNet50(weights='imagenet', include_top=False, input_shape=(image_size, image_size, 3))\n    x = base_model.output\n    x = GlobalAveragePooling2D()(x)\n    x = Dense(256, activation='relu')(x)\n    x = Dropout(0.5)(x)\n    predictions = Dense(1, activation='sigmoid')(x)\n    model = Model(inputs=base_model.input, outputs=predictions)\n    METRICS = [keras.metrics.BinaryAccuracy(name='accuracy'), keras.metrics.Precision(name='precision'), keras.metrics.Recall(name='recall')]\n    base_model.trainable = False\n    model.compile(optimizer=Adam(learning_rate=1e-4), loss='binary_crossentropy', metrics=METRICS)\n    print(\"\\n--- Phase 1: ヘッド層の学習開始 ---\")\n    model.fit(train_generator, epochs=initial_epochs, validation_data=validation_generator, class_weight=class_weight_dict, verbose=2)\n    base_model.trainable = True\n    fine_tune_at = int(len(base_model.layers) * 0.8)\n    for layer in base_model.layers[:fine_tune_at]:\n        layer.trainable = False\n    model.compile(optimizer=Adam(learning_rate=1e-5), loss='binary_crossentropy', metrics=METRICS)\n    callbacks = [\n        ReduceLROnPlateau(monitor='val_loss', mode='min', factor=0.2, patience=3, verbose=1),\n        EarlyStopping(monitor='val_loss', mode='min', patience=5, restore_best_weights=True, verbose=1)\n    ]\n    print(\"\\n--- Phase 2: ファインチューニング開始 ---\")\n    model.fit(\n        train_generator, epochs=initial_epochs + fine_tune_epochs, initial_epoch=initial_epochs,\n        validation_data=validation_generator, class_weight=class_weight_dict, callbacks=callbacks, verbose=2\n    )\n    print(\"\\n--- 検証データで評価中 ---\")\n    y_true = validation_generator.classes\n    y_pred_proba = model.predict(validation_generator)\n    y_pred = (y_pred_proba > 0.5).astype(int).flatten()\n    metrics = {\n        'accuracy': accuracy_score(y_true, y_pred),\n        'precision': precision_score(y_true, y_pred, zero_division=0),\n        'recall': recall_score(y_true, y_pred, zero_division=0),\n        'f1_score': f1_score(y_true, y_pred, zero_division=0)\n    }\n    print(f\"Accuracy: {metrics['accuracy']:.4f}, Precision: {metrics['precision']:.4f}, Recall: {metrics['recall']:.4f}, F1-Score: {metrics['f1_score']:.4f}\")\n    return metrics, model\n\nprint(\"▶ 関数定義完了\")\n\n\n# =================================================================\n# ■ ステップ4：実験Aの実行\n# =================================================================\n# (このセクションは変更ありません)\ndata_sizes = [500, 1000, 2000, 4000, 8000]\nresults_A = []\nfinal_model = None\nfor size in data_sizes:\n    print(f\"\\n{'='*60}\\n▶▶▶ データ量 {size}件 での処理を開始... \\n{'='*60}\")\n    _, df_train_subset = train_test_split(df_train, train_size=size, random_state=42, stratify=df_train['Target'])\n    metrics, trained_model = train_and_finetune_stable_model(\n        train_df=df_train_subset, val_df=df_val, image_dir=PNG_DIR,\n        image_size=224, batch_size=16, initial_epochs=8, fine_tune_epochs=22\n    )\n    if metrics:\n        metrics['size'] = size\n        results_A.append(metrics)\n    if size == max(data_sizes) and trained_model:\n        final_model = trained_model\nif final_model:\n    final_model.save(MODEL_SAVE_PATH)\n    print(f\"\\n▶▶▶ 最強の分類モデルを '{MODEL_SAVE_PATH}' として保存しました。\")\nprint(\"\\n▶ 実験Aが完了しました。\")\ndf_results = pd.DataFrame(results_A)\n\n\n# =================================================================\n# ■ ステップ5：実験Aの結果を可視化\n# =================================================================\n# (このセクションは変更ありません)\nprint(\"\\n--- Experiment A: Results Summary ---\")\ndisplay(df_results)\nmetrics_to_plot = ['accuracy', 'precision', 'recall', 'f1_score']\ntitles = ['Accuracy', 'Precision', 'Recall', 'F1-Score']\nfig, axes = plt.subplots(2, 2, figsize=(16, 12))\nfig.suptitle('Relationship between Data Size and Performance Metrics', fontsize=20, y=1.02)\naxes = axes.flatten()\nfor i, (metric, title) in enumerate(zip(metrics_to_plot, titles)):\n    sns.lineplot(x='size', y=metric, data=df_results, ax=axes[i], marker='o', color='royalblue')\n    axes[i].set_title(title, fontsize=14)\n    axes[i].set_xlabel('Training Data Size', fontsize=12)\n    axes[i].set_ylabel('Score', fontsize=12)\n    axes[i].set_ylim(0.0, 1.0)\n    axes[i].grid(True, which='both', linestyle='--', linewidth=0.5)\n    for index, row in df_results.iterrows():\n        axes[i].text(row['size'], row[metric], f\"{row[metric]:.3f}\", ha='center', va='bottom', fontsize=9)\nplt.tight_layout(rect=[0, 0, 1, 0.98])\nplt.show()\n\n\n# =================================================================\n# ■ ステップ6：実験Bの準備と予測\n# =================================================================\nprint(\"\\n--- 実験B：公平性分析を開始します ---\")\nbest_model = None\n# まず現在のセッションで保存したモデルを探す\nif MODEL_SAVE_PATH.exists():\n    print(f\"現在のセッションで保存したモデル '{MODEL_SAVE_PATH}' を読み込みます。\")\n    best_model = load_model(MODEL_SAVE_PATH)\n# 見つからなければ、前のバージョンの出力（input）を探す\nelif MODEL_LOAD_PATH.exists():\n    print(f\"以前のバージョンで保存されたモデル '{MODEL_LOAD_PATH}' を読み込みます。\")\n    best_model = load_model(MODEL_LOAD_PATH)\nelse:\n    print(f\"エラー: モデルファイルが見つかりません。\")\n    print(f\"確認したパス: {MODEL_SAVE_PATH}, {MODEL_LOAD_PATH}\")\n\nif best_model:\n    validation_generator = ImageDataGenerator(rescale=1./255).flow_from_dataframe(\n        dataframe=df_val, directory=PNG_DIR, x_col='filename', y_col='Target',\n        target_size=(224, 224), batch_size=32, class_mode='binary', shuffle=False,\n        validate_filenames=False\n    )\n    print(\"\\n▶ 検証データ全体の予測を実行中...\")\n    y_true = validation_generator.classes\n    y_pred_proba = best_model.predict(validation_generator)\n    y_pred = (y_pred_proba > 0.5).astype(int).flatten()\n    df_val_results = df_val.copy()\n    df_val_results['y_true'] = y_true\n    df_val_results['y_pred'] = y_pred\n    print(\"▶ 予測完了。\")\n\n# =================================================================\n# ■ ステップ7：実験Bのグループ評価\n# =================================================================\ndef evaluate_on_groups(df_results, group_col):\n    results = []\n    for name, group_df in df_results.groupby(group_col):\n        if len(group_df) == 0: continue\n        y_true_group, y_pred_group = group_df['y_true'], group_df['y_pred']\n        metrics = {\n            'group_name': name, 'group_col': group_col, 'count': len(group_df),\n            'accuracy': accuracy_score(y_true_group, y_pred_group),\n            'precision': precision_score(y_true_group, y_pred_group, zero_division=0),\n            'recall': recall_score(y_true_group, y_pred_group, zero_division=0),\n            'f1_score': f1_score(y_true_group, y_pred_group, zero_division=0)\n        }\n        results.append(metrics)\n    return pd.DataFrame(results)\n\nif 'df_val_results' in locals() and all(c in df_val_results for c in ['Age', 'Sex', 'ViewPosition']):\n    df_val_results['AgeGroup'] = pd.cut(df_val_results['Age'], bins=[0, 59, 150], labels=['Under 60', '60 and Over'])\n    results_sex = evaluate_on_groups(df_val_results, 'Sex')\n    results_age = evaluate_on_groups(df_val_results, 'AgeGroup')\n    results_view = evaluate_on_groups(df_val_results, 'ViewPosition')\n    overall_metrics = {\n        'group_name': 'Overall', 'group_col': 'Overall', 'count': len(df_val_results),\n        'accuracy': accuracy_score(y_true, y_pred), 'precision': precision_score(y_true, y_pred, zero_division=0),\n        'recall': recall_score(y_true, y_pred, zero_division=0), 'f1_score': f1_score(y_true, y_pred, zero_division=0)\n    }\n    df_overall = pd.DataFrame([overall_metrics])\n    df_fairness_results = pd.concat([df_overall, results_sex, results_age, results_view], ignore_index=True)\n    print(\"\\n--- Fairness Analysis Results Summary ---\")\n    display(df_fairness_results)\nelse:\n    print(\"\\n--- 公平性分析に必要なメタ情報（Age, Sexなど）がないため、実験Bをスキップします ---\")\n\n\n# =================================================================\n# ■ ステップ8：実験Bの結果を可視化\n# =================================================================\nif 'df_fairness_results' in locals():\n    metrics_to_plot = ['accuracy', 'precision', 'recall', 'f1_score']\n    titles = ['Accuracy', 'Precision', 'Recall', 'F1-Score']\n    fig, axes = plt.subplots(len(metrics_to_plot), 1, figsize=(12, 20), sharex=False)\n    fig.suptitle('Experiment B: Model Performance Comparison by Attribute Group', fontsize=20, y=1.0)\n    for i, metric in enumerate(metrics_to_plot):\n        data_to_plot = df_fairness_results[df_fairness_results['group_col'].isin(['Overall', 'Sex', 'AgeGroup', 'ViewPosition'])]\n        sns.barplot(x='group_name', y=metric, data=data_to_plot, ax=axes[i], palette='viridis')\n        axes[i].set_title(f'Comparison of {titles[i]}', fontsize=14)\n        axes[i].set_xlabel('Attribute Group', fontsize=12)\n        axes[i].set_ylabel('Score', fontsize=12)\n        axes[i].set_ylim(0.0, 1.0)\n        axes[i].tick_params(axis='x', rotation=10)\n        for p in axes[i].patches:\n            axes[i].annotate(f\"{p.get_height():.3f}\",\n                             (p.get_x() + p.get_width() / 2., p.get_height()),\n                             ha='center', va='center', fontsize=11, color='black', xytext=(0, 5),\n                             textcoords='offset points')\n    plt.tight_layout(rect=[0, 0, 1, 0.97])\n    plt.show()\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null}]}