{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":10338,"databundleVersionId":862042,"sourceType":"competition"},{"sourceId":104953,"sourceType":"datasetVersion","datasetId":54936}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# =================================================================\n# ■ ステップ1：ライブラリの準備\n# =================================================================\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport os\nimport pydicom # DICOMファイルを読むために追加\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n\nimport tensorflow as tf\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.applications import ResNet50\nfrom tensorflow.keras.layers import Dense, GlobalAveragePooling2D, Dropout\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau\nfrom tensorflow.keras.models import load_model\n\nfrom tensorflow import keras\n\nfrom tqdm.notebook import tqdm\n\nprint(\"TensorFlow Version:\", tf.__version__)\nprint(\"▶ ライブラリの準備完了\")\n\n# グラフのスタイルを設定\nsns.set_style(\"whitegrid\")\nplt.rcParams['font.family'] = 'sans-serif'\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-07-16T01:50:30.256928Z","iopub.execute_input":"2025-07-16T01:50:30.257286Z","iopub.status.idle":"2025-07-16T01:50:30.265837Z","shell.execute_reply.started":"2025-07-16T01:50:30.257251Z","shell.execute_reply":"2025-07-16T01:50:30.264669Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =================================================================\n# ■ ステップ2：データの読み込みと準備\n# =================================================================\n# Kaggleのデータセットパスを定義\nBASE_DIR = '/kaggle/input/rsna-pneumonia-detection-challenge'\nPNG_DIR = '/kaggle/input/rsna-pneu-train-png/orig'\nDICOM_DIR = '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_images'\n\n# --- 1. ラベルデータの読み込み ---\ndf_labels = pd.read_csv(os.path.join(BASE_DIR, 'stage_2_train_labels.csv'))\n\n# --- 2. 分類タスク用のデータを作成 ---\n# patientIdで重複を削除し、必要なカラムのみに絞る\ndf_class = df_labels.drop_duplicates('patientId')[['patientId', 'Target']].copy()\n\n# ファイル名カラムを追加\ndf_class['filename'] = df_class['patientId'].apply(lambda x: f\"{x}.png\")\ndf_class['Target'] = df_class['Target'].astype(str)\n\n# --- 3. DICOMからメタ情報(年齢、性別、撮影方向)を抽出 ---\n# この処理は少し時間がかかります\nages = []\nsexes = []\nview_positions = []\nprint(\"DICOMファイルからメタ情報を抽出中...\")\nfor patient_id in tqdm(df_class['patientId']):\n    dcm_path = os.path.join(DICOM_DIR, f\"{patient_id}.dcm\")\n    dcm_data = pydicom.dcmread(dcm_path, stop_before_pixels=True)\n    ages.append(dcm_data.PatientAge)\n    sexes.append(dcm_data.PatientSex)\n    view_positions.append(dcm_data.ViewPosition)\n\n# 抽出した情報をデータフレームに追加\ndf_class['Age'] = ages\ndf_class['Sex'] = sexes\ndf_class['ViewPosition'] = view_positions\ndf_class['Age'] = df_class['Age'].astype(int) # 年齢を数値型に変換\n\n# --- 4. データの分割 ---\ndf_train, df_val = train_test_split(\n    df_class,\n    test_size=0.2,\n    random_state=42,\n    stratify=df_class['Target']\n)\n\nprint(f\"\\nデータ総数: {len(df_class)}件\")\nprint(f\"学習用データ数: {len(df_train)}件\")\nprint(f\"検証用データ数: {len(df_val)}件\")\nprint(\"\\n--- 検証データの属性情報サマリー ---\")\n# この行でエラーが出ていましたが、修正により解決するはずです\nprint(df_val[['Age', 'Sex', 'ViewPosition']].info())\nprint(\"\\n▶ データの準備完了\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-16T01:50:30.26722Z","iopub.execute_input":"2025-07-16T01:50:30.267469Z","iopub.status.idle":"2025-07-16T01:51:07.890582Z","shell.execute_reply.started":"2025-07-16T01:50:30.267445Z","shell.execute_reply":"2025-07-16T01:51:07.889667Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =================================================================\n# ■ ステップ3：【安定化版】学習・評価用の関数を定義\n# =================================================================\n\ndef train_and_finetune_stable_model(train_df, val_df, image_dir, image_size=224, batch_size=32, initial_epochs=8, fine_tune_epochs=25):\n    \"\"\"\n    学習プロセスを安定させ、Recallを重視するよう修正した関数\n    \"\"\"\n    print(f\"\\n--- 画像サイズ: {image_size}x{image_size}, バッチサイズ: {batch_size} ---\")\n\n    # --- 1. ImageDataGeneratorの準備 ---\n    train_datagen = ImageDataGenerator(\n        rescale=1./255, rotation_range=15, width_shift_range=0.1, height_shift_range=0.1,\n        shear_range=0.1, zoom_range=0.1, horizontal_flip=True, fill_mode='nearest'\n    )\n    val_datagen = ImageDataGenerator(rescale=1./255)\n    \n    train_generator = train_datagen.flow_from_dataframe(\n        dataframe=train_df, directory=image_dir, x_col='filename', y_col='Target',\n        target_size=(image_size, image_size), batch_size=batch_size, class_mode='binary'\n    )\n    validation_generator = val_datagen.flow_from_dataframe(\n        dataframe=val_df, directory=image_dir, x_col='filename', y_col='Target',\n        target_size=(image_size, image_size), batch_size=batch_size, class_mode='binary', shuffle=False\n    )\n    \n    # --- 2. クラス重みの計算 ---\n    if len(train_generator.classes) == 0:\n        print(\"学習データが見つかりません。処理を中断します。\")\n        return None, None\n    class_weights = compute_class_weight(\n        'balanced', classes=np.unique(train_generator.classes), y=train_generator.classes\n    )\n    class_weight_dict = dict(enumerate(class_weights))\n    print(f\"▶ 計算されたクラス重み: {class_weight_dict}\")\n\n    # --- 3. モデルの構築 ---\n    base_model = ResNet50(weights='imagenet', include_top=False, input_shape=(image_size, image_size, 3))\n    x = base_model.output\n    x = GlobalAveragePooling2D()(x)\n    x = Dense(256, activation='relu')(x)\n    x = Dropout(0.5)(x)\n    predictions = Dense(1, activation='sigmoid')(x)\n    model = Model(inputs=base_model.input, outputs=predictions)\n    \n    # Kerasに標準で含まれていないRecallをメトリクスとして追加\n    METRICS = [\n        keras.metrics.TruePositives(name='tp'),\n        keras.metrics.FalsePositives(name='fp'),\n        keras.metrics.TrueNegatives(name='tn'),\n        keras.metrics.FalseNegatives(name='fn'), \n        keras.metrics.BinaryAccuracy(name='accuracy'),\n        keras.metrics.Precision(name='precision'),\n        keras.metrics.Recall(name='recall'),\n    ]\n\n    # =================================================================\n    # ★★★ Phase 1: ヘッド層の学習 ★★★\n    # =================================================================\n    base_model.trainable = False\n    \n    # ★ 変更点: 初期学習率を慎重な値に設定\n    model.compile(optimizer=Adam(learning_rate=1e-4),\n                  loss='binary_crossentropy',\n                  metrics=METRICS)\n    \n    print(\"\\n--- Phase 1: ヘッド層の学習開始 ---\")\n    model.fit(\n        train_generator,\n        epochs=initial_epochs,\n        validation_data=validation_generator,\n        class_weight=class_weight_dict,\n        verbose=2\n    )\n\n    # =================================================================\n    # ★★★ Phase 2: ファインチューニング ★★★\n    # =================================================================\n    base_model.trainable = True\n    fine_tune_at = int(len(base_model.layers) * 0.8)\n    for layer in base_model.layers[:fine_tune_at]:\n        layer.trainable = False\n\n    model.compile(optimizer=Adam(learning_rate=1e-5),\n                  loss='binary_crossentropy',\n                  metrics=METRICS)\n                  \n    # ★ 変更点: Recallを監視し、patienceを増やす\n    callbacks = [\n        ReduceLROnPlateau(monitor='val_recall', mode='max', factor=0.2, patience=5, min_lr=1e-7, verbose=1),\n        EarlyStopping(monitor='val_recall', mode='max', patience=8, restore_best_weights=True, verbose=1)\n    ]\n\n    print(\"\\n--- Phase 2: ファインチューニング開始 ---\")\n    total_epochs = initial_epochs + fine_tune_epochs\n    model.fit(\n        train_generator,\n        epochs=total_epochs,\n        initial_epoch=initial_epochs,\n        validation_data=validation_generator,\n        class_weight=class_weight_dict,\n        callbacks=callbacks,\n        verbose=2\n    )\n\n    # --- 6. 最終評価 ---\n    print(\"\\n--- 検証データで評価中 ---\")\n    y_true = validation_generator.classes\n    y_pred_proba = model.predict(validation_generator)\n    y_pred = (y_pred_proba > 0.5).astype(int).flatten()\n    \n    metrics = {\n        'accuracy': accuracy_score(y_true, y_pred),\n        'precision': precision_score(y_true, y_pred, zero_division=0),\n        'recall': recall_score(y_true, y_pred, zero_division=0),\n        'f1_score': f1_score(y_true, y_pred, zero_division=0)\n    }\n    \n    print(f\"Accuracy: {metrics['accuracy']:.4f}, Precision: {metrics['precision']:.4f}, Recall: {metrics['recall']:.4f}, F1-Score: {metrics['f1_score']:.4f}\")\n    \n    return metrics, model\n\nprint(\"▶ 関数定義完了\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-16T01:51:07.892155Z","iopub.execute_input":"2025-07-16T01:51:07.892731Z","iopub.status.idle":"2025-07-16T01:51:07.910403Z","shell.execute_reply.started":"2025-07-16T01:51:07.892706Z","shell.execute_reply":"2025-07-16T01:51:07.909308Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =================================================================\n# ■ ステップ4：【安定化版】実験の実行\n# =================================================================\n# 検証するデータ量を定義\ndata_sizes = [500, 1000, 2000, 4000, 8000, len(df_train)]\nresults_A = []\nfinal_model = None \n\nfor size in data_sizes:\n    # ★ 変更点: ループの最後はサンプリングしない\n    if size == len(df_train):\n        df_train_subset = df_train.copy()\n    else:\n        # クラス比率を保ったままサンプリング\n        n_class_0 = int(size * 0.775)\n        n_class_1 = size - n_class_0\n        df_train_0 = df_train[df_train['Target'] == '0'].sample(n=n_class_0, random_state=42)\n        df_train_1 = df_train[df_train['Target'] == '1'].sample(n=n_class_1, random_state=42)\n        df_train_subset = pd.concat([df_train_0, df_train_1])\n\n    print(f\"\\n{'='*60}\\n▶▶▶ データ量 {size}件 での処理を開始... \\n{'='*60}\")\n\n    metrics, trained_model = train_and_finetune_stable_model(\n        train_df=df_train_subset,\n        val_df=df_val,\n        image_dir=PNG_DIR,\n        image_size=224,\n        batch_size=16, # バッチサイズを小さくして学習をより安定させる\n        initial_epochs=8,\n        fine_tune_epochs=22\n    )\n    \n    if metrics:\n        metrics['size'] = size\n        results_A.append(metrics)\n    \n    if size == len(df_train) and trained_model:\n        final_model = trained_model\n        \nif final_model:\n    final_model.save('best_pneumonia_classifier.keras')\n    print(\"\\n▶▶▶ 最強の分類モデルを 'best_pneumonia_classifier.keras' として保存しました。\")\n\nprint(\"\\n▶ 実験Aが完了しました。\")\n\ndf_results_A = pd.DataFrame(results_A)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-16T01:51:07.911319Z","iopub.execute_input":"2025-07-16T01:51:07.91164Z","execution_failed":"2025-07-16T06:22:31.106Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =================================================================\n# ■ ステップ5：実験結果の可視化\n# =================================================================\nprint(\"\\n--- 実験結果サマリー ---\")\ndisplay(df_results)\n\n# --- 描画 ---\nmetrics_to_plot = ['accuracy', 'precision', 'recall', 'f1_score']\ntitles = ['正解率 (Accuracy)', '適合率 (Precision)', '再現率 (Recall)', 'F1スコア (F1-Score)']\n\n# 2x2のグリッドでグラフを作成\nfig, axes = plt.subplots(2, 2, figsize=(16, 12))\nfig.suptitle('データ量と各評価指標の関係', fontsize=20, y=1.02)\naxes = axes.flatten() # 2x2のaxesを1次元配列に変換してループしやすくする\n\nfor i, (metric, title) in enumerate(zip(metrics_to_plot, titles)):\n    sns.lineplot(x='size', y=metric, data=df_results, ax=axes[i], marker='o', color='royalblue')\n    axes[i].set_title(title, fontsize=14)\n    axes[i].set_xlabel('学習データ数', fontsize=12)\n    axes[i].set_ylabel('スコア', fontsize=12)\n    axes[i].set_ylim(0.5, 1.0) # y軸の範囲を0.5〜1.0に調整\n    axes[i].grid(True, which='both', linestyle='--', linewidth=0.5)\n    \n    # 各点に数値を表示\n    for index, row in df_results.iterrows():\n        axes[i].text(row['size'], row[metric], f\"{row[metric]:.3f}\", ha='center', va='bottom', fontsize=9)\n\nplt.tight_layout(rect=[0, 0, 1, 0.98])\nplt.show()\n","metadata":{"trusted":true,"execution":{"execution_failed":"2025-07-16T06:22:31.107Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =================================================================\n# ■ ステップ6：実験Bの準備と予測\n# =================================================================\nprint(\"\\n--- 実験B：公平性分析を開始します ---\")\n\n# --- 1. モデルの読み込み ---\ntry:\n    best_model = load_model('/kaggle/working/best_pneumonia_classifier.keras')\n    print(\"▶ 保存されたモデルの読み込みに成功しました。\")\nexcept Exception as e:\n    print(f\"モデルの読み込みに失敗しました: {e}\")\n    best_model = None\n\n# --- 2. 検証データ全体の予測を実行 ---\nif best_model:\n    # 検証データ用のジェネレータを再作成\n    val_datagen = ImageDataGenerator(rescale=1./255)\n    validation_generator = val_datagen.flow_from_dataframe(\n        dataframe=df_val, directory=PNG_DIR, x_col='filename', y_col='Target',\n        target_size=(224, 224), batch_size=32, class_mode='binary', shuffle=False\n    )\n    \n    print(\"\\n▶ 検証データ全体の予測を実行中...\")\n    y_true = validation_generator.classes\n    y_pred_proba = best_model.predict(validation_generator)\n    y_pred = (y_pred_proba > 0.5).astype(int).flatten()\n    \n    # 予測結果を検証データフレームに追加\n    df_val_results = df_val.copy()\n    df_val_results['y_true'] = y_true\n    df_val_results['y_pred'] = y_pred\n    \n    print(\"▶ 予測完了。\")\n","metadata":{"trusted":true,"execution":{"execution_failed":"2025-07-16T06:22:31.108Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =================================================================\n# ■ ステップ7：実験Bのグループ評価\n# =================================================================\n\ndef evaluate_on_groups(df_results, group_col):\n    \"\"\"指定されたカラムでグループ分けし、各グループの評価指標を計算する関数\"\"\"\n    results = []\n    for name, group_df in df_results.groupby(group_col):\n        if len(group_df) == 0:\n            continue\n        \n        y_true_group = group_df['y_true']\n        y_pred_group = group_df['y_pred']\n        \n        metrics = {\n            'group_name': name,\n            'group_col': group_col,\n            'count': len(group_df),\n            'accuracy': accuracy_score(y_true_group, y_pred_group),\n            'precision': precision_score(y_true_group, y_pred_group, zero_division=0),\n            'recall': recall_score(y_true_group, y_pred_group, zero_division=0),\n            'f1_score': f1_score(y_true_group, y_pred_group, zero_division=0)\n        }\n        results.append(metrics)\n    return pd.DataFrame(results)\n\nif 'df_val_results' in locals():\n    # --- 1. 年齢グループの作成 ---\n    # 60歳を閾値としてグループ分け\n    df_val_results['AgeGroup'] = pd.cut(df_val_results['Age'], \n                                        bins=[0, 59, 150], \n                                        labels=['59歳以下', '60歳以上'])\n\n    # --- 2. 各切り口で評価を実行 ---\n    results_sex = evaluate_on_groups(df_val_results, 'Sex')\n    results_age = evaluate_on_groups(df_val_results, 'AgeGroup')\n    results_view = evaluate_on_groups(df_val_results, 'ViewPosition')\n    \n    # --- 3. 全体評価も追加 ---\n    overall_metrics = {\n        'group_name': '全体', 'group_col': 'Overall', 'count': len(df_val_results),\n        'accuracy': accuracy_score(y_true, y_pred), 'precision': precision_score(y_true, y_pred, zero_division=0),\n        'recall': recall_score(y_true, y_pred, zero_division=0), 'f1_score': f1_score(y_true, y_pred, zero_division=0)\n    }\n    df_overall = pd.DataFrame([overall_metrics])\n    \n    # 全ての結果を結合\n    df_fairness_results = pd.concat([df_overall, results_sex, results_age, results_view], ignore_index=True)\n    \n    print(\"\\n--- 公平性分析 結果サマリー ---\")\n    display(df_fairness_results)\n","metadata":{"trusted":true,"execution":{"execution_failed":"2025-07-16T06:22:31.108Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =================================================================\n# ■ ステップ8：実験Bの結果を可視化\n# =================================================================\nif 'df_fairness_results' in locals():\n    # --- 描画 ---\n    group_cols_to_plot = ['Overall', 'Sex', 'AgeGroup', 'ViewPosition']\n    plot_titles = ['全体 vs 性別', '全体 vs 年齢層', '全体 vs 撮影方向']\n    \n    fig, axes = plt.subplots(len(metrics_to_plot), 1, figsize=(12, 20), sharex=False)\n    fig.suptitle('実験B: 属性グループごとのモデル性能比較', fontsize=20, y=1.0)\n\n    for i, metric in enumerate(metrics_to_plot):\n        data_to_plot = df_fairness_results[df_fairness_results['group_col'].isin(['Overall', 'Sex', 'AgeGroup', 'ViewPosition'])]\n        sns.barplot(x='group_name', y=metric, data=data_to_plot, ax=axes[i], palette='viridis')\n        axes[i].set_title(f'{titles[i]} の比較', fontsize=14)\n        axes[i].set_xlabel('属性グループ', fontsize=12)\n        axes[i].set_ylabel('スコア', fontsize=12)\n        axes[i].set_ylim(0.5, 1.0)\n        axes[i].tick_params(axis='x', rotation=10)\n        \n        # 棒グラフの上に数値を表示\n        for p in axes[i].patches:\n            axes[i].annotate(f\"{p.get_height():.3f}\",\n                             (p.get_x() + p.get_width() / 2., p.get_height()),\n                             ha='center', va='center', fontsize=11, color='black', xytext=(0, 5),\n                             textcoords='offset points')\n\n    plt.tight_layout(rect=[0, 0, 1, 0.97])\n    plt.show()\n","metadata":{"trusted":true,"execution":{"execution_failed":"2025-07-16T06:22:31.108Z"}},"outputs":[],"execution_count":null}]}