{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":10338,"databundleVersionId":862042,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":104953,"sourceType":"datasetVersion","datasetId":54936}],"dockerImageVersionId":31090,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# # =================================================================\n# # ■ ステップ1：ライブラリの準備\n# # =================================================================\n# import pandas as pd\n# import numpy as np\n# import matplotlib.pyplot as plt\n# import seaborn as sns\n# import os\n# import pydicom # DICOMファイルを読むために追加\n# import japanize_matplotlib # ★★★ インポートして有効化 ★★★\n\n# from sklearn.model_selection import train_test_split\n# from sklearn.utils.class_weight import compute_class_weight\n# from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n\n# import tensorflow as tf\n# from tensorflow.keras.preprocessing.image import ImageDataGenerator\n# from tensorflow.keras.applications import ResNet50\n# from tensorflow.keras.layers import Dense, GlobalAveragePooling2D, Dropout\n# from tensorflow.keras.models import Model\n# from tensorflow.keras.optimizers import Adam\n# from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau\n# from tensorflow.keras.models import load_model\n\n# from tensorflow import keras\n\n# from tqdm.notebook import tqdm\n\n# print(\"TensorFlow Version:\", tf.__version__)\n# print(\"▶ ライブラリの準備完了\")\n\n# # グラフのスタイルを設定\n# sns.set_style(\"whitegrid\")\n# plt.rcParams['font.family'] = 'sans-serif'","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-07-18T15:46:35.782540Z","iopub.execute_input":"2025-07-18T15:46:35.782759Z","iopub.status.idle":"2025-07-18T15:46:39.594198Z","shell.execute_reply.started":"2025-07-18T15:46:35.782735Z","shell.execute_reply":"2025-07-18T15:46:39.592931Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # =================================================================\n# # ■ ステップ2：データの読み込みと準備\n# # =================================================================\n# # Kaggleのデータセットパスを定義\n# BASE_DIR = '/kaggle/input/rsna-pneumonia-detection-challenge'\n# PNG_DIR = '/kaggle/input/rsna-pneu-train-png/orig'\n# DICOM_DIR = '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_images'\n\n# # --- 1. ラベルデータの読み込み ---\n# df_labels = pd.read_csv(os.path.join(BASE_DIR, 'stage_2_train_labels.csv'))\n\n# # --- 2. 分類タスク用のデータを作成 ---\n# # patientIdで重複を削除し、必要なカラムのみに絞る\n# df_class = df_labels.drop_duplicates('patientId')[['patientId', 'Target']].copy()\n\n# # ファイル名カラムを追加\n# df_class['filename'] = df_class['patientId'].apply(lambda x: f\"{x}.png\")\n# df_class['Target'] = df_class['Target'].astype(str)\n\n# # --- 3. DICOMからメタ情報(年齢、性別、撮影方向)を抽出 ---\n# # この処理は少し時間がかかります\n# ages = []\n# sexes = []\n# view_positions = []\n# print(\"DICOMファイルからメタ情報を抽出中...\")\n# for patient_id in tqdm(df_class['patientId']):\n#     dcm_path = os.path.join(DICOM_DIR, f\"{patient_id}.dcm\")\n#     dcm_data = pydicom.dcmread(dcm_path, stop_before_pixels=True)\n#     ages.append(dcm_data.PatientAge)\n#     sexes.append(dcm_data.PatientSex)\n#     view_positions.append(dcm_data.ViewPosition)\n\n# # 抽出した情報をデータフレームに追加\n# df_class['Age'] = ages\n# df_class['Sex'] = sexes\n# df_class['ViewPosition'] = view_positions\n# df_class['Age'] = df_class['Age'].astype(int) # 年齢を数値型に変換\n\n# # --- 4. データの分割 ---\n# df_train, df_val = train_test_split(\n#     df_class,\n#     test_size=0.2,\n#     random_state=42,\n#     stratify=df_class['Target']\n# )\n\n# print(f\"\\nデータ総数: {len(df_class)}件\")\n# print(f\"学習用データ数: {len(df_train)}件\")\n# print(f\"検証用データ数: {len(df_val)}件\")\n# print(\"\\n--- 検証データの属性情報サマリー ---\")\n# # この行でエラーが出ていましたが、修正により解決するはずです\n# print(df_val[['Age', 'Sex', 'ViewPosition']].info())\n# print(\"\\n▶ データの準備完了\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-18T15:46:39.594718Z","iopub.status.idle":"2025-07-18T15:46:39.594955Z","shell.execute_reply.started":"2025-07-18T15:46:39.594849Z","shell.execute_reply":"2025-07-18T15:46:39.594860Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # =================================================================\n# # ■ ステップ3：【安定化版】学習・評価用の関数を定義\n# # =================================================================\n\n# def train_and_finetune_stable_model(train_df, val_df, image_dir, image_size=224, batch_size=32, initial_epochs=8, fine_tune_epochs=25):\n#     \"\"\"\n#     学習プロセスを安定させ、Recallを重視するよう修正した関数\n#     \"\"\"\n#     print(f\"\\n--- 画像サイズ: {image_size}x{image_size}, バッチサイズ: {batch_size} ---\")\n\n#     # --- 1. ImageDataGeneratorの準備 ---\n#     train_datagen = ImageDataGenerator(\n#         rescale=1./255, rotation_range=15, width_shift_range=0.1, height_shift_range=0.1,\n#         shear_range=0.1, zoom_range=0.1, horizontal_flip=True, fill_mode='nearest'\n#     )\n#     val_datagen = ImageDataGenerator(rescale=1./255)\n    \n#     train_generator = train_datagen.flow_from_dataframe(\n#         dataframe=train_df, directory=image_dir, x_col='filename', y_col='Target',\n#         target_size=(image_size, image_size), batch_size=batch_size, class_mode='binary'\n#     )\n#     validation_generator = val_datagen.flow_from_dataframe(\n#         dataframe=val_df, directory=image_dir, x_col='filename', y_col='Target',\n#         target_size=(image_size, image_size), batch_size=batch_size, class_mode='binary', shuffle=False\n#     )\n    \n#     # --- 2. クラス重みの計算 ---\n#     if len(train_generator.classes) == 0:\n#         print(\"学習データが見つかりません。処理を中断します。\")\n#         return None, None\n#     class_weights = compute_class_weight(\n#         'balanced', classes=np.unique(train_generator.classes), y=train_generator.classes\n#     )\n#     class_weight_dict = dict(enumerate(class_weights))\n#     print(f\"▶ 計算されたクラス重み: {class_weight_dict}\")\n\n#     # --- 3. モデルの構築 ---\n#     base_model = ResNet50(weights='imagenet', include_top=False, input_shape=(image_size, image_size, 3))\n#     x = base_model.output\n#     x = GlobalAveragePooling2D()(x)\n#     x = Dense(256, activation='relu')(x)\n#     x = Dropout(0.5)(x)\n#     predictions = Dense(1, activation='sigmoid')(x)\n#     model = Model(inputs=base_model.input, outputs=predictions)\n    \n#     # Kerasに標準で含まれていないRecallをメトリクスとして追加\n#     METRICS = [\n#         keras.metrics.TruePositives(name='tp'),\n#         keras.metrics.FalsePositives(name='fp'),\n#         keras.metrics.TrueNegatives(name='tn'),\n#         keras.metrics.FalseNegatives(name='fn'), \n#         keras.metrics.BinaryAccuracy(name='accuracy'),\n#         keras.metrics.Precision(name='precision'),\n#         keras.metrics.Recall(name='recall'),\n#     ]\n\n#     # =================================================================\n#     # ★★★ Phase 1: ヘッド層の学習 ★★★\n#     # =================================================================\n#     base_model.trainable = False\n    \n#     # ★ 変更点: 初期学習率を慎重な値に設定\n#     model.compile(optimizer=Adam(learning_rate=1e-4),\n#                   loss='binary_crossentropy',\n#                   metrics=METRICS)\n    \n#     print(\"\\n--- Phase 1: ヘッド層の学習開始 ---\")\n#     model.fit(\n#         train_generator,\n#         epochs=initial_epochs,\n#         validation_data=validation_generator,\n#         class_weight=class_weight_dict,\n#         verbose=2\n#     )\n\n#     # =================================================================\n#     # ★★★ Phase 2: ファインチューニング ★★★\n#     # =================================================================\n#     base_model.trainable = True\n#     fine_tune_at = int(len(base_model.layers) * 0.8)\n#     for layer in base_model.layers[:fine_tune_at]:\n#         layer.trainable = False\n\n#     model.compile(optimizer=Adam(learning_rate=1e-5),\n#                   loss='binary_crossentropy',\n#                   metrics=METRICS)\n                  \n#     # ★ 変更点: Recallを監視し、patienceを増やす\n#     callbacks = [\n#         ReduceLROnPlateau(monitor='val_f1_score', mode='max', factor=0.2, patience=5, min_lr=1e-7, verbose=1),\n#         EarlyStopping(monitor='val_f1_score', mode='max', patience=8, restore_best_weights=True, verbose=1)\n#     ]\n\n#     print(\"\\n--- Phase 2: ファインチューニング開始 ---\")\n#     total_epochs = initial_epochs + fine_tune_epochs\n#     model.fit(\n#         train_generator,\n#         epochs=total_epochs,\n#         initial_epoch=initial_epochs,\n#         validation_data=validation_generator,\n#         class_weight=class_weight_dict,\n#         callbacks=callbacks,\n#         verbose=2\n#     )\n\n#     # --- 6. 最終評価 ---\n#     print(\"\\n--- 検証データで評価中 ---\")\n#     y_true = validation_generator.classes\n#     y_pred_proba = model.predict(validation_generator)\n#     y_pred = (y_pred_proba > 0.5).astype(int).flatten()\n    \n#     metrics = {\n#         'accuracy': accuracy_score(y_true, y_pred),\n#         'precision': precision_score(y_true, y_pred, zero_division=0),\n#         'recall': recall_score(y_true, y_pred, zero_division=0),\n#         'f1_score': f1_score(y_true, y_pred, zero_division=0)\n#     }\n    \n#     print(f\"Accuracy: {metrics['accuracy']:.4f}, Precision: {metrics['precision']:.4f}, Recall: {metrics['recall']:.4f}, F1-Score: {metrics['f1_score']:.4f}\")\n    \n#     return metrics, model\n\n# print(\"▶ 関数定義完了\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-18T15:46:39.596410Z","iopub.status.idle":"2025-07-18T15:46:39.596769Z","shell.execute_reply.started":"2025-07-18T15:46:39.596600Z","shell.execute_reply":"2025-07-18T15:46:39.596614Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # =================================================================\n# # ■ ステップ4：【安定化版】実験の実行\n# # =================================================================\n# # 検証するデータ量を定義\n# data_sizes = [500,1000,2000,4000,8000]\n# results_A = []\n# final_model = None \n\n# for size in data_sizes:\n#     # ★ 変更点: ループの最後はサンプリングしない\n#     if size == len(df_train):\n#         df_train_subset = df_train.copy()\n#     else:\n#         # クラス比率を保ったままサンプリング\n#         n_class_0 = int(size * 0.775)\n#         n_class_1 = size - n_class_0\n#         df_train_0 = df_train[df_train['Target'] == '0'].sample(n=n_class_0, random_state=42)\n#         df_train_1 = df_train[df_train['Target'] == '1'].sample(n=n_class_1, random_state=42)\n#         df_train_subset = pd.concat([df_train_0, df_train_1])\n\n#     print(f\"\\n{'='*60}\\n▶▶▶ データ量 {size}件 での処理を開始... \\n{'='*60}\")\n\n#     metrics, trained_model = train_and_finetune_stable_model(\n#         train_df=df_train_subset,\n#         val_df=df_val,\n#         image_dir=PNG_DIR,\n#         image_size=224,\n#         batch_size=16, # バッチサイズを小さくして学習をより安定させる\n#         initial_epochs=8,\n#         fine_tune_epochs=22\n#     )\n    \n#     if metrics:\n#         metrics['size'] = size\n#         results_A.append(metrics)\n    \n#     if size == 8000 and trained_model:\n#         final_model = trained_model\n        \n# if final_model:\n#     final_model.save('best_pneumonia_classifier.keras')\n#     print(\"\\n▶▶▶ 最強の分類モデルを 'best_pneumonia_classifier.keras' として保存しました。\")\n\n# print(\"\\n▶ 実験Aが完了しました。\")\n\n# df_results = pd.DataFrame(results_A)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-18T15:46:39.597979Z","iopub.status.idle":"2025-07-18T15:46:39.598272Z","shell.execute_reply.started":"2025-07-18T15:46:39.598136Z","shell.execute_reply":"2025-07-18T15:46:39.598153Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # =================================================================\n# # ■ ステップ5：実験結果の可視化\n# # =================================================================\n# print(\"\\n--- 実験結果サマリー ---\")\n# display(df_results)\n\n# # --- 描画 ---\n# metrics_to_plot = ['accuracy', 'precision', 'recall', 'f1_score']\n# titles = ['正解率 (Accuracy)', '適合率 (Precision)', '再現率 (Recall)', 'F1スコア (F1-Score)']\n\n# # 2x2のグリッドでグラフを作成\n# fig, axes = plt.subplots(2, 2, figsize=(16, 12))\n# fig.suptitle('データ量と各評価指標の関係', fontsize=20, y=1.02)\n# axes = axes.flatten() # 2x2のaxesを1次元配列に変換してループしやすくする\n\n# for i, (metric, title) in enumerate(zip(metrics_to_plot, titles)):\n#     sns.lineplot(x='size', y=metric, data=df_results, ax=axes[i], marker='o', color='royalblue')\n#     axes[i].set_title(title, fontsize=14)\n#     axes[i].set_xlabel('学習データ数', fontsize=12)\n#     axes[i].set_ylabel('スコア', fontsize=12)\n#     axes[i].set_ylim(0.0, 1.0) # ★y軸の範囲を0.0〜1.0に調整\n#     axes[i].grid(True, which='both', linestyle='--', linewidth=0.5)\n    \n#     # 各点に数値を表示\n#     for index, row in df_results.iterrows():\n#         axes[i].text(row['size'], row[metric], f\"{row[metric]:.3f}\", ha='center', va='bottom', fontsize=9)\n\n# plt.tight_layout(rect=[0, 0, 1, 0.98])\n# plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-18T15:46:39.599071Z","iopub.status.idle":"2025-07-18T15:46:39.599522Z","shell.execute_reply.started":"2025-07-18T15:46:39.599316Z","shell.execute_reply":"2025-07-18T15:46:39.599337Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # =================================================================\n# # ■ ステップ6以降の修正版コード\n# # =================================================================\n# # このセルを実行する前に、ステップ1とステップ2が実行済みであることを確認してください。\n\n# # --- モデルの読み込み ---\n# MODEL_PATH = '/kaggle/working/best_pneumonia_classifier.keras'\n# PNG_DIR = '/kaggle/input/rsna-pneu-train-png/orig' # 画像パスを再定義\n\n# try:\n#     best_model = load_model(MODEL_PATH)\n#     print(f\"▶ 保存されたモデル {MODEL_PATH} の読み込みに成功しました。\")\n# except Exception as e:\n#     print(f\"モデルの読み込みに失敗しました: {e}\")\n#     best_model = None\n\n# # --- 検証データ全体の予測を実行 ---\n# if best_model:\n#     # 検証データ用のジェネレータを再作成\n#     val_datagen = ImageDataGenerator(rescale=1./255)\n#     validation_generator = val_datagen.flow_from_dataframe(\n#         dataframe=df_val, \n#         directory=PNG_DIR, \n#         x_col='filename', \n#         y_col='Target',\n#         target_size=(224, 224), \n#         batch_size=32, \n#         class_mode='binary', \n#         shuffle=False\n#     )\n    \n#     print(\"\\n▶ 検証データ全体の予測を実行中...\")\n#     y_true = validation_generator.classes\n#     y_pred_proba = best_model.predict(validation_generator)\n#     y_pred = (y_pred_proba > 0.5).astype(int).flatten()\n    \n#     # ジェネレータが実際に読み込んだファイル名を取得\n#     valid_filenames = validation_generator.filenames\n    \n#     # 実際に読み込まれたファイルの結果だけで新しいDataFrameを作成\n#     df_pred_results = pd.DataFrame({\n#         'filename': valid_filenames,\n#         'y_true': y_true,\n#         'y_pred': y_pred\n#     })\n    \n#     # 元の検証データフレーム(df_val)と、予測結果のフレームを'filename'をキーにして結合する\n#     df_val_results = pd.merge(\n#         df_val,\n#         df_pred_results,\n#         on='filename'\n#     )\n    \n#     print(f\"▶ 予測完了。検証に成功した {len(df_val_results)} 件のデータで分析を続行します。\")\n\n#     # --- グループ評価の実行 ---\n#     df_val_results['AgeGroup'] = pd.cut(df_val_results['Age'], \n#                                         bins=[0, 59, 150], \n#                                         labels=['59歳以下', '60歳以上'])\n\n#     results_sex = evaluate_on_groups(df_val_results, 'Sex')\n#     results_age = evaluate_on_groups(df_val_results, 'AgeGroup')\n#     results_view = evaluate_on_groups(df_val_results, 'ViewPosition')\n    \n#     overall_metrics = {\n#         'group_name': '全体', 'group_col': 'Overall', 'count': len(df_val_results),\n#         'accuracy': accuracy_score(df_val_results['y_true'], df_val_results['y_pred']), \n#         'precision': precision_score(df_val_results['y_true'], df_val_results['y_pred'], zero_division=0),\n#         'recall': recall_score(df_val_results['y_true'], df_val_results['y_pred'], zero_division=0), \n#         'f1_score': f1_score(df_val_results['y_true'], df_val_results['y_pred'], zero_division=0)\n#     }\n#     df_overall = pd.DataFrame([overall_metrics])\n    \n#     df_fairness_results = pd.concat([df_overall, results_sex, results_age, results_view], ignore_index=True)\n    \n#     print(\"\\n--- 公平性分析 結果サマリー ---\")\n#     display(df_fairness_results)\n\n#     # --- 結果の可視化 ---\n#     metrics_to_plot = ['accuracy', 'precision', 'recall', 'f1_score']\n#     metric_titles = ['正解率 (Accuracy)', '適合率 (Precision)', '再現率 (Recall)', 'F1スコア (F1-Score)']\n    \n#     fig, axes = plt.subplots(len(metrics_to_plot), 1, figsize=(12, 20), sharex=False)\n#     fig.suptitle('実験B: 属性グループごとのモデル性能比較', fontsize=20, y=1.0)\n\n#     for i, metric in enumerate(metrics_to_plot):\n#         data_to_plot = df_fairness_results[df_fairness_results['group_col'].isin(['Overall', 'Sex', 'AgeGroup', 'ViewPosition'])]\n#         sns.barplot(x='group_name', y=metric, data=data_to_plot, ax=axes[i], palette='viridis')\n#         axes[i].set_title(f'{metric_titles[i]} の比較', fontsize=14)\n#         axes[i].set_xlabel('属性グループ', fontsize=12)\n#         axes[i].set_ylabel('スコア', fontsize=12)\n#         axes[i].set_ylim(0.0, 1.0)\n#         axes[i].tick_params(axis='x', rotation=10)\n        \n#         for p in axes[i].patches:\n#             axes[i].annotate(f\"{p.get_height():.3f}\",\n#                              (p.get_x() + p.get_width() / 2., p.get_height()),\n#                              ha='center', va='center', fontsize=11, color='black', xytext=(0, 5),\n#                              textcoords='offset points')\n\n#     plt.tight_layout(rect=[0, 0, 1, 0.97])\n#     plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-18T15:46:39.600753Z","iopub.status.idle":"2025-07-18T15:46:39.600988Z","shell.execute_reply.started":"2025-07-18T15:46:39.600875Z","shell.execute_reply":"2025-07-18T15:46:39.600887Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =================================================================\n# ■【最終実行用】学習・分析・保存 全機能版コード\n# =================================================================\n# このセルを一度だけ「Save Version」で実行してください。\n# 2回目以降の実行では、保存された結果を読み込み、学習をスキップします。\n\n# --- 1. 必要なライブラリの準備 ---\nprint(\"■ ステップ1：ライブラリの準備\")\n!pip install -q japanize-matplotlib\nimport japanize_matplotlib\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport os\nimport pydicom\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom tensorflow.keras.models import load_model\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\nfrom IPython.display import display\nfrom tqdm.notebook import tqdm\nfrom tensorflow.keras.applications import ResNet50\nfrom tensorflow.keras.layers import Dense, GlobalAveragePooling2D, Dropout\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau\nfrom tensorflow import keras\nprint(\"▶ ライブラリの準備完了\")\n\n# --- 2. データの準備 ---\nprint(\"\\n■ ステップ2：データの準備\")\nBASE_DIR = '/kaggle/input/rsna-pneumonia-detection-challenge'\nDICOM_DIR = '/kaggle/input/rsna-pneumonia-detection-challenge/stage_2_train_images'\nPNG_DIR = '/kaggle/input/rsna-pneu-train-png/orig'\ndf_labels = pd.read_csv(os.path.join(BASE_DIR, 'stage_2_train_labels.csv'))\ndf_class = df_labels.drop_duplicates('patientId')[['patientId', 'Target']].copy()\ndf_class['filename'] = df_class['patientId'].apply(lambda x: f\"{x}.png\")\ndf_class['Target'] = df_class['Target'].astype(str)\n\n# メタ情報抽出\nages, sexes, view_positions = [], [], []\nprint(\"DICOMファイルからメタ情報を抽出中...\")\nfor patient_id in tqdm(df_class['patientId']):\n    dcm_path = os.path.join(DICOM_DIR, f\"{patient_id}.dcm\")\n    dcm_data = pydicom.dcmread(dcm_path, stop_before_pixels=True)\n    ages.append(dcm_data.PatientAge)\n    sexes.append(dcm_data.PatientSex)\n    view_positions.append(dcm_data.ViewPosition)\n\ndf_class['Age'] = ages\ndf_class['Sex'] = sexes\ndf_class['ViewPosition'] = view_positions\ndf_class['Age'] = df_class['Age'].astype(int)\n\n# データの分割\ndf_train, df_val = train_test_split(df_class, test_size=0.2, random_state=42, stratify=df_class['Target'])\nprint(f\"▶ 検証用データ {len(df_val)} 件、学習用データ {len(df_train)} 件の準備完了\")\n\n# --- 3. グループ評価用の関数を定義 ---\ndef evaluate_on_groups(df_results, group_col):\n    results = []\n    for name, group_df in df_results.groupby(group_col):\n        if len(group_df) == 0: continue\n        y_true_group = group_df['y_true']\n        y_pred_group = group_df['y_pred']\n        metrics = {\n            'group_name': name, 'group_col': group_col, 'count': len(group_df),\n            'accuracy': accuracy_score(y_true_group, y_pred_group),\n            'precision': precision_score(y_true_group, y_pred_group, zero_division=0),\n            'recall': recall_score(y_true_group, y_pred_group, zero_division=0),\n            'f1_score': f1_score(y_true_group, y_pred_group, zero_division=0)\n        }\n        results.append(metrics)\n    return pd.DataFrame(results)\nprint(\"▶ 評価用関数の準備完了\")\n\n# --- 4. モデルの読み込み、または学習の実行 ---\nprint(\"\\n■ ステップ4：モデルの読み込み、または学習の実行\")\nMODEL_PATH = '/kaggle/working/best_pneumonia_classifier.keras'\nRESULTS_CSV_PATH = '/kaggle/working/experiment_A_results.csv'\nbest_model = None\ndf_results = None\n\nif os.path.exists(MODEL_PATH) and os.path.exists(RESULTS_CSV_PATH):\n    print(f\"▶▶▶ 発見済みのモデル {MODEL_PATH} を読み込みます。\")\n    try:\n        best_model = load_model(MODEL_PATH)\n        df_results = pd.read_csv(RESULTS_CSV_PATH)\n        print(\"▶ モデルと実験結果の読み込みに成功しました。学習はスキップされます。\")\n    except Exception as e:\n        print(f\"▶ 読み込みに失敗しました: {e}。学習を再実行します。\")\n        best_model = None\n\nif best_model is None:\n    print(\"\\n▶▶▶ モデルが存在しないため、実験Aを最初から実行します...\")\n    \n    # --- 学習用の関数定義 ---\n    def train_and_finetune_stable_model(train_df, val_df, image_dir, image_size=224, batch_size=32, initial_epochs=8, fine_tune_epochs=25):\n        print(f\"\\n--- 画像サイズ: {image_size}x{image_size}, バッチサイズ: {batch_size} ---\")\n        train_datagen = ImageDataGenerator(rescale=1./255, rotation_range=15, width_shift_range=0.1, height_shift_range=0.1, shear_range=0.1, zoom_range=0.1, horizontal_flip=True, fill_mode='nearest')\n        val_datagen = ImageDataGenerator(rescale=1./255)\n        train_generator = train_datagen.flow_from_dataframe(dataframe=train_df, directory=image_dir, x_col='filename', y_col='Target', target_size=(image_size, image_size), batch_size=batch_size, class_mode='binary')\n        validation_generator = val_datagen.flow_from_dataframe(dataframe=val_df, directory=image_dir, x_col='filename', y_col='Target', target_size=(image_size, image_size), batch_size=batch_size, class_mode='binary', shuffle=False)\n        \n        if len(train_generator.classes) == 0: return None, None\n        class_weights = compute_class_weight('balanced', classes=np.unique(train_generator.classes), y=train_generator.classes)\n        class_weight_dict = dict(enumerate(class_weights))\n        print(f\"▶ 計算されたクラス重み: {class_weight_dict}\")\n\n        base_model = ResNet50(weights='imagenet', include_top=False, input_shape=(image_size, image_size, 3))\n        x = GlobalAveragePooling2D()(base_model.output)\n        x = Dense(256, activation='relu')(x)\n        x = Dropout(0.5)(x)\n        predictions = Dense(1, activation='sigmoid')(x)\n        model = Model(inputs=base_model.input, outputs=predictions)\n        \n        METRICS = [keras.metrics.BinaryAccuracy(name='accuracy'), keras.metrics.Precision(name='precision'), keras.metrics.Recall(name='recall')]\n        \n        base_model.trainable = False\n        model.compile(optimizer=Adam(learning_rate=1e-4), loss='binary_crossentropy', metrics=METRICS)\n        print(\"\\n--- Phase 1: ヘッド層の学習開始 ---\")\n        model.fit(train_generator, epochs=initial_epochs, validation_data=validation_generator, class_weight=class_weight_dict, verbose=2)\n\n        base_model.trainable = True\n        fine_tune_at = int(len(base_model.layers) * 0.8)\n        for layer in base_model.layers[:fine_tune_at]: layer.trainable = False\n        model.compile(optimizer=Adam(learning_rate=1e-5), loss='binary_crossentropy', metrics=METRICS)\n        callbacks = [ReduceLROnPlateau(monitor='val_recall', mode='max', factor=0.2, patience=5, min_lr=1e-7, verbose=1), EarlyStopping(monitor='val_recall', mode='max', patience=8, restore_best_weights=True, verbose=1)]\n        print(\"\\n--- Phase 2: ファインチューニング開始 ---\")\n        model.fit(train_generator, epochs=initial_epochs + fine_tune_epochs, initial_epoch=initial_epochs, validation_data=validation_generator, class_weight=class_weight_dict, callbacks=callbacks, verbose=2)\n        \n        y_true = validation_generator.classes\n        y_pred = (model.predict(validation_generator) > 0.5).astype(int).flatten()\n        metrics = {'accuracy': accuracy_score(y_true, y_pred), 'precision': precision_score(y_true, y_pred, zero_division=0), 'recall': recall_score(y_true, y_pred, zero_division=0), 'f1_score': f1_score(y_true, y_pred, zero_division=0)}\n        return metrics, model\n\n    # --- 学習ループ ---\n    data_sizes = [500, 1000, 2000, 4000, 8000]\n    results_A = []\n    \n    for size in data_sizes:\n        n_class_0 = int(size * 0.775)\n        n_class_1 = size - n_class_0\n        df_train_0 = df_train[df_train['Target'] == '0'].sample(n=n_class_0, random_state=42)\n        df_train_1 = df_train[df_train['Target'] == '1'].sample(n=n_class_1, random_state=42)\n        df_train_subset = pd.concat([df_train_0, df_train_1])\n        print(f\"\\n{'='*60}\\n▶▶▶ データ量 {size}件 での処理を開始... \\n{'='*60}\")\n        metrics, trained_model = train_and_finetune_stable_model(train_df=df_train_subset, val_df=df_val, image_dir=PNG_DIR, image_size=224, batch_size=16, initial_epochs=8, fine_tune_epochs=22)\n        if metrics:\n            metrics['size'] = size\n            results_A.append(metrics)\n        if size == 8000 and trained_model:\n            best_model = trained_model\n            \n    if best_model:\n        best_model.save(MODEL_PATH)\n        print(f\"\\n▶▶▶ 8000件で学習したモデルを '{MODEL_PATH}' として保存しました。\")\n    df_results = pd.DataFrame(results_A)\n    df_results.to_csv(RESULTS_CSV_PATH, index=False)\n    print(f\"▶ 実験Aの結果を '{RESULTS_CSV_PATH}' に保存しました。\")\n\n# --- 5. 実験Aの結果を可視化 ---\nif df_results is not None:\n    print(\"\\n■ ステップ5：実験Aの結果を可視化\")\n    display(df_results)\n    metrics_to_plot = ['accuracy', 'precision', 'recall', 'f1_score']\n    titles = ['正解率 (Accuracy)', '適合率 (Precision)', '再現率 (Recall)', 'F1スコア (F1-Score)']\n    fig, axes = plt.subplots(2, 2, figsize=(16, 12))\n    fig.suptitle('データ量と各評価指標の関係', fontsize=20, y=1.02)\n    axes = axes.flatten()\n    for i, (metric, title) in enumerate(zip(metrics_to_plot, titles)):\n        sns.lineplot(x='size', y=metric, data=df_results, ax=axes[i], marker='o', color='royalblue')\n        axes[i].set_title(title, fontsize=14)\n        axes[i].set_xlabel('学習データ数', fontsize=12)\n        axes[i].set_ylabel('スコア', fontsize=12)\n        axes[i].set_ylim(0.0, 1.0)\n        axes[i].grid(True, which='both', linestyle='--', linewidth=0.5)\n        for index, row in df_results.iterrows():\n            axes[i].text(row['size'], max(row[metric], 0.01), f\"{row[metric]:.3f}\", ha='center', va='bottom', fontsize=9)\n    plt.tight_layout(rect=[0, 0, 1, 0.98])\n    plt.show()\n\n# --- 6. 予測と公平性分析の実行 ---\nif best_model:\n    print(\"\\n■ ステップ6：予測と公平性分析の実行\")\n    val_datagen = ImageDataGenerator(rescale=1./255)\n    validation_generator = val_datagen.flow_from_dataframe(dataframe=df_val, directory=PNG_DIR, x_col='filename', y_col='Target', target_size=(224, 244), batch_size=32, class_mode='binary', shuffle=False)\n    \n    print(\"▶ 検証データ全体の予測を実行中...\")\n    y_pred_proba = best_model.predict(validation_generator)\n    y_pred = (y_pred_proba > 0.5).astype(int).flatten()\n    \n    valid_filenames = validation_generator.filenames\n    df_pred_results = pd.DataFrame({'filename': valid_filenames, 'y_true': validation_generator.classes, 'y_pred': y_pred})\n    df_val_results = pd.merge(df_val, df_pred_results, on='filename')\n    print(f\"▶ 予測完了。検証に成功した {len(df_val_results)} 件のデータで分析を続行します。\")\n\n    df_val_results['AgeGroup'] = pd.cut(df_val_results['Age'], bins=[0, 59, 150], labels=['59歳以下', '60歳以上'])\n    results_sex = evaluate_on_groups(df_val_results, 'Sex')\n    results_age = evaluate_on_groups(df_val_results, 'AgeGroup')\n    results_view = evaluate_on_groups(df_val_results, 'ViewPosition')\n    \n    overall_metrics = {'group_name': '全体', 'group_col': 'Overall', 'count': len(df_val_results), 'accuracy': accuracy_score(df_val_results['y_true'], df_val_results['y_pred']), 'precision': precision_score(df_val_results['y_true'], df_val_results['y_pred'], zero_division=0), 'recall': recall_score(df_val_results['y_true'], df_val_results['y_pred'], zero_division=0), 'f1_score': f1_score(df_val_results['y_true'], df_val_results['y_pred'], zero_division=0)}\n    df_overall = pd.DataFrame([overall_metrics])\n    df_fairness_results = pd.concat([df_overall, results_sex, results_age, results_view], ignore_index=True)\n    \n    print(\"\\n--- 公平性分析 結果サマリー ---\")\n    display(df_fairness_results)\n\n    # --- 7. 公平性分析の結果を可視化 ---\n    print(\"\\n■ ステップ7：公平性分析の結果を可視化\")\n    metrics_to_plot = ['accuracy', 'precision', 'recall', 'f1_score']\n    metric_titles = ['正解率 (Accuracy)', '適合率 (Precision)', '再現率 (Recall)', 'F1スコア (F1-Score)']\n    fig, axes = plt.subplots(len(metrics_to_plot), 1, figsize=(12, 20), sharex=False)\n    fig.suptitle('実験B: 属性グループごとのモデル性能比較', fontsize=20, y=1.0)\n    for i, metric in enumerate(metrics_to_plot):\n        data_to_plot = df_fairness_results[df_fairness_results['group_col'].isin(['Overall', 'Sex', 'AgeGroup', 'ViewPosition'])]\n        sns.barplot(x='group_name', y=metric, data=data_to_plot, ax=axes[i], palette='viridis')\n        axes[i].set_title(f'{metric_titles[i]} の比較', fontsize=14)\n        axes[i].set_xlabel('属性グループ', fontsize=12)\n        axes[i].set_ylabel('スコア', fontsize=12)\n        axes[i].set_ylim(0.0, 1.0)\n        axes[i].tick_params(axis='x', rotation=10)\n        for p in axes[i].patches:\n            axes[i].annotate(f\"{p.get_height():.3f}\", (p.get_x() + p.get_width() / 2., p.get_height()), ha='center', va='center', fontsize=11, color='black', xytext=(0, 5), textcoords='offset points')\n    plt.tight_layout(rect=[0, 0, 1, 0.97])\n    plt.show()\n\nelse:\n    print(\"エラー: モデルが利用できないため、分析を中断しました。\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-18T15:50:49.900362Z","iopub.execute_input":"2025-07-18T15:50:49.900694Z","iopub.status.idle":"2025-07-18T15:55:28.471290Z","shell.execute_reply.started":"2025-07-18T15:50:49.900670Z","shell.execute_reply":"2025-07-18T15:55:28.470595Z"}},"outputs":[],"execution_count":null}]}