{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.17","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":21154,"databundleVersionId":1243559,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-07-04T06:46:28.242077Z","iopub.execute_input":"2025-07-04T06:46:28.242545Z","iopub.status.idle":"2025-07-04T06:46:28.291379Z","shell.execute_reply.started":"2025-07-04T06:46:28.242511Z","shell.execute_reply":"2025-07-04T06:46:28.288193Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#math,re,osは数学の計算を行うライブラリ,numpyも画像データや行列演算につかうやつ。\n#tensorflow：Googleが開発した深層学習ライブラリ。ニューラルネットの定義、訓練、推論などを行う。\nimport math, re, os\nimport numpy as np\nimport tensorflow as tf\n\n#設定したバージョンの表示\nprint(\"Tensorflow version \" + tf.__version__)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-04T06:46:28.294366Z","iopub.execute_input":"2025-07-04T06:46:28.295048Z","iopub.status.idle":"2025-07-04T06:46:28.305302Z","shell.execute_reply.started":"2025-07-04T06:46:28.295021Z","shell.execute_reply":"2025-07-04T06:46:28.299559Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# TPUが使えるかどうかを確認\tKaggleやColabはTPUが使えることがある\n#適切な学習戦略（strategy）を決定\tTPUなら TPUStrategy、なければ get_strategy()\n#分散学習の準備\t後のモデル訓練時に with strategy.scope(): を使うことで効率よく学習できる\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver() \n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() \n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-04T06:46:45.028737Z","iopub.execute_input":"2025-07-04T06:46:45.029054Z","iopub.status.idle":"2025-07-04T06:46:45.040131Z","shell.execute_reply.started":"2025-07-04T06:46:45.029029Z","shell.execute_reply":"2025-07-04T06:46:45.035392Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#バージョン入れる\n!pip install protobuf==3.20.*","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-04T06:46:28.320047Z","iopub.execute_input":"2025-07-04T06:46:28.320265Z","iopub.status.idle":"2025-07-04T06:46:32.001947Z","shell.execute_reply.started":"2025-07-04T06:46:28.320246Z","shell.execute_reply":"2025-07-04T06:46:31.997193Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow_datasets as tfds\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom tensorflow import keras\nfrom tensorflow.keras import layers, models\nimport math\nimport os\nimport re\nimport pandas as pd\nimport sklearn","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-04T06:46:32.002907Z","iopub.execute_input":"2025-07-04T06:46:32.003164Z","iopub.status.idle":"2025-07-04T06:46:32.014896Z","shell.execute_reply.started":"2025-07-04T06:46:32.003137Z","shell.execute_reply":"2025-07-04T06:46:32.010371Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"①環境構築：作業内容\n\nTensorFlow がインストールできている\n\nTPU / GPU が正しく検出できる\n\n必要な Python ライブラリが使える\n\n状態になっているはず。","metadata":{}},{"cell_type":"markdown","source":"【手順②】データセット読み込み・前処理\n◉ 手順\nTFRecord ファイルの読み込み\n\n特徴量とラベルのデコード\n\n画像サイズ・データ型変換\n\nバッチ化・シャッフル","metadata":{}},{"cell_type":"code","source":"# TFRecordの読み込み関数\ndef parse_tfrecord_fn(example):\n    feature_description = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"class\": tf.io.FixedLenFeature([], tf.int64),\n    }\n    example = tf.io.parse_single_example(example, feature_description)\n    \n    image = tf.io.decode_jpeg(example[\"image\"], channels=3)\n    image = tf.image.resize(image, [224, 224])  # 必要に応じてサイズ調整\n    image = tf.cast(image, tf.float32) / 255.0  # 0-1に正規化\n\n    label = tf.cast(example[\"class\"], tf.int32)\n\n    return image, label\n\n# TFRecordのデータセット読み込み\ndef load_dataset(tfrecord_paths, batch_size=32, shuffle_buffer=1000):\n    raw_dataset = tf.data.TFRecordDataset(tfrecord_paths)\n    dataset = raw_dataset.map(parse_tfrecord_fn, num_parallel_calls=tf.data.AUTOTUNE)\n    dataset = dataset.shuffle(shuffle_buffer).batch(batch_size).prefetch(tf.data.AUTOTUNE)\n    return dataset\n\n# 例：trainとvalidのファイルパス\ntrain_tfrecords = [\"./train-00000-of-00002.tfrecord\", \"./train-00001-of-00002.tfrecord\"]\nvalid_tfrecords = [\"./valid-00000-of-00001.tfrecord\"]\n\n# データセット作成\ntrain_dataset = load_dataset(train_tfrecords, batch_size=32)\nvalid_dataset = load_dataset(valid_tfrecords, batch_size=32)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-04T06:46:32.016079Z","iopub.execute_input":"2025-07-04T06:46:32.016279Z","iopub.status.idle":"2025-07-04T06:46:32.098914Z","shell.execute_reply.started":"2025-07-04T06:46:32.016260Z","shell.execute_reply":"2025-07-04T06:46:32.093494Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import tensorflow as tf\n\n# TFRecordファイルのパス（trainフォルダの224x224データの例）\ntrain_tfrecord_paths = tf.io.gfile.glob('/kaggle/input/tpu-getting-started/tfrecords-jpeg-224x224/train/*.tfrec')\n\n# TFRecordの特徴を解析するための関数（例）\ndef decode_tfrecord(record):\n    feature_description = {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'id': tf.io.FixedLenFeature([], tf.string),\n        'class': tf.io.FixedLenFeature([], tf.int64),\n    }\n    example = tf.io.parse_single_example(record, feature_description)\n    image = tf.io.decode_jpeg(example['image'], channels=3)\n    image = tf.image.convert_image_dtype(image, tf.float32)\n    label = tf.cast(example['class'], tf.int32)\n    return image, label\n\n# データセットを作成\ntrain_dataset = tf.data.TFRecordDataset(train_tfrecord_paths)\ntrain_dataset = train_dataset.map(decode_tfrecord)\ntrain_dataset = train_dataset.batch(32).prefetch(tf.data.AUTOTUNE)\n\n# 動作確認（最初のバッチの画像とラベルの形状を表示）\nfor images, labels in train_dataset.take(1):\n    print(images.shape)  # (バッチサイズ, 224, 224, 3)\n    print(labels.shape)  # (バッチサイズ,)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-04T06:46:32.101059Z","iopub.execute_input":"2025-07-04T06:46:32.101295Z","iopub.status.idle":"2025-07-04T06:46:32.260265Z","shell.execute_reply.started":"2025-07-04T06:46:32.101274Z","shell.execute_reply":"2025-07-04T06:46:32.255006Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"上で分かったこと\n(32, 224, 224, 3) はバッチサイズ32の224x224のカラー画像データ、\n(32,) はそのラベルが32個分ある形です。\n\nこれでデータセットの読み込み・前処理が正しくできています。","metadata":{}},{"cell_type":"markdown","source":"手順３モデルの定義、作成（転移学習）\n今回のデータは画像サイズ224×224、3チャンネルなので、\nシンプルなCNNモデルを作る","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras import layers, models\n\nmodel = models.Sequential([\n    layers.InputLayer(input_shape=(224, 224, 3)),\n    layers.Conv2D(32, (3,3), activation='relu'),\n    layers.MaxPooling2D((2,2)),\n    layers.Conv2D(64, (3,3), activation='relu'),\n    layers.MaxPooling2D((2,2)),\n    layers.Conv2D(128, (3,3), activation='relu'),\n    layers.Flatten(),\n    layers.Dense(256, activation='relu'),\n    layers.Dense(100, activation='softmax')  # クラス数100なら\n])\n\nmodel.compile(optimizer='adam',\n              loss='sparse_categorical_crossentropy',\n              metrics=['accuracy'])\n\nmodel.summary()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-04T06:46:32.261258Z","iopub.execute_input":"2025-07-04T06:46:32.261536Z","iopub.status.idle":"2025-07-04T06:46:32.511763Z","shell.execute_reply.started":"2025-07-04T06:46:32.261509Z","shell.execute_reply":"2025-07-04T06:46:32.506397Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tensorflow.keras import layers, models\n\nmodel = models.Sequential([\n    layers.Input(shape=(224, 224, 3)),\n    layers.Conv2D(32, (3,3), activation='relu'),\n    layers.MaxPooling2D((2,2)),\n    layers.Conv2D(64, (3,3), activation='relu'),\n    layers.MaxPooling2D((2,2)),\n    layers.Conv2D(128, (3,3), activation='relu'),\n    layers.GlobalAveragePooling2D(),  # ここを変更\n    layers.Dense(256, activation='relu'),\n    layers.Dense(100, activation='softmax')\n])\n\nmodel.compile(optimizer='adam',\n              loss='sparse_categorical_crossentropy',\n              metrics=['accuracy'])\n\nmodel.summary()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-04T06:46:32.513735Z","iopub.execute_input":"2025-07-04T06:46:32.514025Z","iopub.status.idle":"2025-07-04T06:46:32.591861Z","shell.execute_reply.started":"2025-07-04T06:46:32.514000Z","shell.execute_reply":"2025-07-04T06:46:32.586624Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"分かったこと。\n\n画像を扱うCNNモデルを作った。\n\n最初は特徴マップを平らにするFlattenを使ってたけど、パラメータが多すぎた。\n\n次に、GlobalAveragePooling2Dを使って特徴を平均化し、パラメータを大幅に減らした。\n\n得られた内容\nFlattenだとパラメータが多すぎて重いモデルになる。\n\nGlobalAveragePooling2Dを使うとパラメータが少なくなり、軽くて効率の良いモデルになる。\n\nモデルは軽いほうが学習が速くて安定する。","metadata":{}},{"cell_type":"markdown","source":"  次、「5. 学習率調整設定」のコード例を示します。","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\n\ndef count_tfrecords(filenames):\n    count = 0\n    for fn in filenames:\n        for _ in tf.data.TFRecordDataset(fn):\n            count += 1\n    return count\n\n# 例えばtrain_filesはTFRecordファイルのリスト\ntrain_files = [\n    '/kaggle/input/tpu-getting-started/tfrecords-jpeg-224x224/train/00-224x224-798.tfrec',\n    '/kaggle/input/tpu-getting-started/tfrecords-jpeg-224x224/train/01-224x224-798.tfrec',\n    # 他のファイルもリストに追加\n]\n\ntrain_data_size = count_tfrecords(train_files)\nprint(f\"Train data size: {train_data_size}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-04T06:46:32.592950Z","iopub.execute_input":"2025-07-04T06:46:32.593179Z","iopub.status.idle":"2025-07-04T06:46:32.790552Z","shell.execute_reply.started":"2025-07-04T06:46:32.593156Z","shell.execute_reply":"2025-07-04T06:46:32.783947Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data_size = 1596\nbatch_size = 32\nsteps_per_epoch = train_data_size // batch_size  # 1596 // 32 = 49\nepochs = 10\ntotal_steps = steps_per_epoch * epochs  # 49 * 10 = 490\n\nprint(f\"steps_per_epoch: {steps_per_epoch}\")\nprint(f\"total_steps: {total_steps}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-04T06:46:32.792458Z","iopub.execute_input":"2025-07-04T06:46:32.792713Z","iopub.status.idle":"2025-07-04T06:46:32.802176Z","shell.execute_reply.started":"2025-07-04T06:46:32.792687Z","shell.execute_reply":"2025-07-04T06:46:32.797254Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"できたこと\n学習率スケジューラの設定ができた。","metadata":{}},{"cell_type":"markdown","source":"次はモデル学習 をする。\ntrain_dataset：学習用データセット（前処理済み）\n\nepochs：何回データ全体を学習するか\n\nsteps_per_epoch：1エポックあたりのバッチ数","metadata":{}},{"cell_type":"code","source":"history = model.fit(\n    train_dataset,\n    epochs=epochs,\n    steps_per_epoch=steps_per_epoch,\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-04T06:46:32.804044Z","iopub.execute_input":"2025-07-04T06:46:32.804279Z","iopub.status.idle":"2025-07-04T06:46:34.096810Z","shell.execute_reply.started":"2025-07-04T06:46:32.804255Z","shell.execute_reply":"2025-07-04T06:46:34.092161Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"with strategy.scope():\n    model = build_model(num_classes=100)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-04T06:46:34.099411Z","iopub.status.idle":"2025-07-04T06:46:34.100241Z","shell.execute_reply.started":"2025-07-04T06:46:34.099600Z","shell.execute_reply":"2025-07-04T06:46:34.099615Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import tensorflow as tf\n\ntry:\n    resolver = tf.distribute.cluster_resolver.TPUClusterResolver()\n    tf.config.experimental_connect_to_cluster(resolver)\n    tf.tpu.experimental.initialize_tpu_system(resolver)\n    strategy = tf.distribute.TPUStrategy(resolver)\n    print(\"All devices:\", tf.config.list_logical_devices())\n    print(\"TPU initialized successfully\")\nexcept ValueError:\n    strategy = tf.distribute.get_strategy()  # CPU or GPU\n    print(\"TPU not found, using default strategy\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-04T06:46:59.813027Z","iopub.execute_input":"2025-07-04T06:46:59.813317Z","iopub.status.idle":"2025-07-04T06:46:59.825906Z","shell.execute_reply.started":"2025-07-04T06:46:59.813293Z","shell.execute_reply":"2025-07-04T06:46:59.820417Z"}},"outputs":[],"execution_count":null}]}