{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nfrom sklearn.preprocessing import LabelEncoder\n\n%config Completer.use_jedi = False","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-21T11:15:05.062696Z","iopub.execute_input":"2022-03-21T11:15:05.063052Z","iopub.status.idle":"2022-03-21T11:15:05.082443Z","shell.execute_reply.started":"2022-03-21T11:15:05.063003Z","shell.execute_reply":"2022-03-21T11:15:05.081436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ライブラリのバージョンチェック\nprint(np.__version__)\nprint(pd.__version__)\nprint(tf.__version__)","metadata":{"execution":{"iopub.status.busy":"2022-03-21T11:15:05.696979Z","iopub.execute_input":"2022-03-21T11:15:05.699215Z","iopub.status.idle":"2022-03-21T11:15:05.708803Z","shell.execute_reply.started":"2022-03-21T11:15:05.699166Z","shell.execute_reply":"2022-03-21T11:15:05.707106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# オリジナルデータセットの保存パスを定義\nINPUT_BASE = '../input/'\nINPUT_DATA_PATH = f'{INPUT_BASE}happy-whale-and-dolphin/'\nTRAIN_IMAGES_PATH = f'{INPUT_DATA_PATH}train_images/'\nTEST_IMAGES_PATH = f'{INPUT_DATA_PATH}test_images/'\n\n# トリミング用のboxが格納されたデータセットの保存パスを定義\nCROPPED_DATA_PATH = f'{INPUT_BASE}cropped-dataset/'\n\n# 出力用のデータセット\nOUTPUT_DIR_PATH = '/kaggle/working/'\nOUTPUT_TRAIN_IMAGES_PATH = f'{OUTPUT_DIR_PATH}train_images/'\nOUTPUT_TEST_IMAGES_PATH = f'{OUTPUT_DIR_PATH}test_images/'\n\n# 出力用のファイル\n#COPY_SAMPLE_CSV = 'train_sample.csv'\nTRAIN_LABEL_CSV = 'train_label_box.csv'\nTEST_LABEL_CSV = 'test_label_box.csv'\nLABEL_MASTER_CSV = 'label_master.csv'\n\nTRAIN_IMAGES_TFREC = 'train_images.tfrec'\nTEST_IMAGES_TFREC = 'test_images.tfrec'\n\n# 作業に使用する画像数を定義\nIMAGE_COUNT = 10","metadata":{"execution":{"iopub.status.busy":"2022-03-21T11:15:06.221359Z","iopub.execute_input":"2022-03-21T11:15:06.221988Z","iopub.status.idle":"2022-03-21T11:15:06.229032Z","shell.execute_reply.started":"2022-03-21T11:15:06.221954Z","shell.execute_reply":"2022-03-21T11:15:06.227552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 正解ラベルをエンコーディングしたデータが記載された訓練データCSVを出力\ndef write_label_encode_train_csv():\n    # 正解ラベルのエンコーディング(説明変数として利用するわけではないため、ラベルエンコーディングで実施)\n    df_train = pd.read_csv(f'{INPUT_DATA_PATH}train.csv')\n    le = LabelEncoder()\n\n    # individual_idのエンコード\n    encoded_individual_id = pd.Series(le.fit_transform(df_train['individual_id']))\n    df_train['encoded_individual_id'] = encoded_individual_id\n\n    # speciesのエンコード\n    encoded_species = pd.Series(le.fit_transform(df_train['species']))\n    df_train['encoded_species'] = encoded_species\n    \n    # トリミング用のbox情報をセットを読み込んで、訓練データCSVに追加\n    df_train_cropped_info = pd.read_csv(f'{CROPPED_DATA_PATH}train-cropped-dataset.csv')\n    df_train['box'] = df_train_cropped_info['box']\n    \n    # エンコードした正解ラベルが記載されたCSVファイルを出力\n    df_train.to_csv(f'{OUTPUT_DIR_PATH}{TRAIN_LABEL_CSV}')\n\nwrite_label_encode_train_csv()","metadata":{"execution":{"iopub.status.busy":"2022-03-21T11:15:06.917002Z","iopub.execute_input":"2022-03-21T11:15:06.917693Z","iopub.status.idle":"2022-03-21T11:15:07.489907Z","shell.execute_reply.started":"2022-03-21T11:15:06.917658Z","shell.execute_reply":"2022-03-21T11:15:07.488555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# テストデータのトリミングしたbox情報を格納したcsvを出力\ndef write_test_box_csv():\n    # 提出用サンプルにテスト画像が登録されているため、これをベースとする。\n    df_test = pd.read_csv(f'{INPUT_DATA_PATH}sample_submission.csv')\n    # \"predictions\"項目を削除する。\n    df_test = df_test.drop('predictions', axis=1)\n\n    # トリミング用のbox情報をセットを読み込んで、訓練データCSVに追加\n    df_test_cropped_info = pd.read_csv(f'{CROPPED_DATA_PATH}test-cropped-dataset.csv')\n    df_test['box'] = df_test_cropped_info['box']\n    \n    print(df_test.head())\n    \n    # エンコードした正解ラベルが記載されたCSVファイルを出力\n    df_test.to_csv(f'{OUTPUT_DIR_PATH}{TEST_LABEL_CSV}')\n\nwrite_test_box_csv()","metadata":{"execution":{"iopub.status.busy":"2022-03-21T11:15:07.491922Z","iopub.execute_input":"2022-03-21T11:15:07.492505Z","iopub.status.idle":"2022-03-21T11:15:07.728718Z","shell.execute_reply.started":"2022-03-21T11:15:07.492453Z","shell.execute_reply":"2022-03-21T11:15:07.72766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 正解ラベルのマスタcsv(label_master.csv)を作成\ndef create_label_master():\n    # 正解ラベルのエンコーディング(説明変数として利用するわけではないため、ラベルエンコーディングで実施)\n    df_train = pd.read_csv(f'{OUTPUT_DIR_PATH}{TRAIN_LABEL_CSV}')\n    df_train = df_train.groupby(['encoded_individual_id', 'individual_id'],as_index=False)\n    df_train = df_train.size()\n    \n    # \"size\"項目を削除する。\n    df_train = df_train.drop('size', axis=1)\n    df_train = df_train.rename(columns={'encoded_individual_id': 'index'})\n    df_train.to_csv(f'{OUTPUT_DIR_PATH}{LABEL_MASTER_CSV}', index=False)\n    \ncreate_label_master()","metadata":{"execution":{"iopub.status.busy":"2022-03-21T11:15:07.997215Z","iopub.execute_input":"2022-03-21T11:15:07.997518Z","iopub.status.idle":"2022-03-21T11:15:08.190314Z","shell.execute_reply.started":"2022-03-21T11:15:07.997486Z","shell.execute_reply":"2022-03-21T11:15:08.18931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## (暫定)とりあえず学習データを10件までに絞る(お試し用ロジック)\ndf_train_label = pd.read_csv(f'{OUTPUT_DIR_PATH}{TRAIN_LABEL_CSV}')\ndf_train_label[:IMAGE_COUNT].to_csv(f'{OUTPUT_DIR_PATH}{TRAIN_LABEL_CSV}')","metadata":{"execution":{"iopub.status.busy":"2022-03-21T11:15:08.99642Z","iopub.execute_input":"2022-03-21T11:15:08.996722Z","iopub.status.idle":"2022-03-21T11:15:09.101424Z","shell.execute_reply.started":"2022-03-21T11:15:08.99669Z","shell.execute_reply":"2022-03-21T11:15:09.100606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## 訓練画像のいくつかを出力\ndef copy_train_images():\n    import glob\n    import shutil\n    import os\n\n    # お試し用の数枚の画像用の作業ディレクトリを作成\n    if os.path.exists(OUTPUT_TRAIN_IMAGES_PATH):\n        shutil.rmtree(OUTPUT_TRAIN_IMAGES_PATH)\n    os.makedirs(OUTPUT_TRAIN_IMAGES_PATH)\n\n    # 訓練データcsvから画像名を取得\n    train_df = pd.read_csv(f'{OUTPUT_DIR_PATH}{TRAIN_LABEL_CSV}')\n    # train_df = pd.read_csv(f'{INPUT_TRAIN_SAMPLE_PATH}train_sample.csv')\n    # # print(train_df.head(10))\n\n    # 訓練画像を作業ディレクトリにコピー\n    for image_name in train_df['image'].to_list():\n        shutil.copy(f'{TRAIN_IMAGES_PATH}{image_name}', f'{OUTPUT_TRAIN_IMAGES_PATH}{image_name}')\n\ncopy_train_images()","metadata":{"execution":{"iopub.status.busy":"2022-03-21T11:15:09.676335Z","iopub.execute_input":"2022-03-21T11:15:09.676607Z","iopub.status.idle":"2022-03-21T11:15:10.081648Z","shell.execute_reply.started":"2022-03-21T11:15:09.676577Z","shell.execute_reply":"2022-03-21T11:15:10.080555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 前処理用の変数を定義\nAUTO = tf.data.experimental.AUTOTUNE\nIMAGE_SIZE = 512","metadata":{"execution":{"iopub.status.busy":"2022-03-21T11:15:10.33657Z","iopub.execute_input":"2022-03-21T11:15:10.336864Z","iopub.status.idle":"2022-03-21T11:15:10.343034Z","shell.execute_reply.started":"2022-03-21T11:15:10.336832Z","shell.execute_reply":"2022-03-21T11:15:10.341233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # https://www.kaggle.com/lextoumbourou/happywhale-tfrecords-with-bounding-boxes\n# def read_bbox(bbox):\n#     return np.array([int(i) for i in bbox.split()])\n\ndef _bytes_feature(value):\n    \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n    if isinstance(value, type(tf.constant(0))):\n        # BytesList won't unpack a string from an EagerTensor.\n        value = value.numpy()\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\n# def _float_feature(value):\n#   \"\"\"Returns a float_list from a float / double.\"\"\"\n#   return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\ndef _int64_feature(value):\n    \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\ndef _bb_feature(bb):\n    \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=bb))\n\n#def serialize_example(image,image_name,target,species,yolov5_bb,detic_bb):\n# 画像のシリアライズ(イメージ、正解ラベルを付与したデータセットを作成)\ndef serialize_image(image, image_name, target, species, box):\n    feature = {\n        'image': _bytes_feature(image),\n        'image_name': _bytes_feature(image_name),\n        'target': _int64_feature(target),\n        'species': _int64_feature(species),\n        'box': _bb_feature(box)\n        #'yolov5_box': _bb_feature(yolov5_bb),\n        #'detic_box': _bb_feature(detic_bb)\n    }\n    serialize_data = tf.train.Example(features=tf.train.Features(feature=feature))\n    return serialize_data.SerializeToString()","metadata":{"execution":{"iopub.status.busy":"2022-03-21T11:15:19.863292Z","iopub.execute_input":"2022-03-21T11:15:19.863563Z","iopub.status.idle":"2022-03-21T11:15:19.874158Z","shell.execute_reply.started":"2022-03-21T11:15:19.863533Z","shell.execute_reply":"2022-03-21T11:15:19.872886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 訓練画像をtfrecファイルに変換\ndef train_image_to_tfrec():\n    tfr_filename = f'{OUTPUT_DIR_PATH}{TRAIN_IMAGES_TFREC}'\n    train_df = pd.read_csv(f'{OUTPUT_DIR_PATH}{TRAIN_LABEL_CSV}')\n    \n    with tf.io.TFRecordWriter(tfr_filename) as writer:\n        for index, row in train_df.iterrows():\n            image_id = row['image']\n            # 正解ラベル(エンコーディング済み)\n            target = row['encoded_individual_id']\n            # イルカ・クジラの分類(エンコーディング済み)\n            species = row['encoded_species']\n            \n            # 画像の読み込み\n            image_path = f\"{OUTPUT_TRAIN_IMAGES_PATH}{image_id}\"\n            image_encoded = tf.io.read_file(image_path)\n            image_name = str.encode(image_id)\n            \n            if type(row['box']) is float:\n                box = [-1, -1, -1, -1]\n            else:\n                #box = list(read_bbox(row['box']))\n                box = [int(b) for b in row['box'].split()]\n\n            #example = serialize_example(image_encoded, image_name, target, species, yolov5_bb, detic_bb)\n            #serialize_image_data = serialize_image(image_encoded, image_name, target, species, detic_bb)\n            serialize_image_data = serialize_image(image_encoded, image_name, target, species, box)\n\n            writer.write(serialize_image_data)\n\ntrain_image_to_tfrec()","metadata":{"execution":{"iopub.status.busy":"2022-03-21T11:16:48.272285Z","iopub.execute_input":"2022-03-21T11:16:48.273098Z","iopub.status.idle":"2022-03-21T11:16:48.338248Z","shell.execute_reply.started":"2022-03-21T11:16:48.273021Z","shell.execute_reply":"2022-03-21T11:16:48.337116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# テスト画像をtfrecファイルに変換\ndef test_image_to_tfrec():\n    tfr_filename = f'{OUTPUT_DIR_PATH}{TEST_IMAGES_TFREC}'\n    test_df = pd.read_csv(f'{OUTPUT_DIR_PATH}{TEST_LABEL_CSV}')\n    test_df = test_df.head(10)\n\n    with tf.io.TFRecordWriter(tfr_filename) as writer:\n        for index, row in test_df.iterrows():\n            image_id = row['image']\n            # 正解ラベル(正解が分からないので固定値)\n            target = -1\n            # イルカ・クジラの分類(正解が分からないので固定値)\n            species = -1\n            \n            # 画像の読み込み\n            image_path = f\"{TEST_IMAGES_PATH}{image_id}\"\n            image_encoded = tf.io.read_file(image_path)\n            image_name = str.encode(image_id)\n            \n            if type(row['box']) is float:\n                box = [-1, -1, -1, -1]\n            else:\n                #box = list(read_bbox(row['box']))\n                box = [int(b) for b in row['box'].split()]\n\n            serialize_image_data = serialize_image(image_encoded, image_name, target, species, box)\n            writer.write(serialize_image_data)\n\ntest_image_to_tfrec()","metadata":{"execution":{"iopub.status.busy":"2022-03-21T11:17:54.098444Z","iopub.execute_input":"2022-03-21T11:17:54.098727Z","iopub.status.idle":"2022-03-21T11:17:54.186345Z","shell.execute_reply.started":"2022-03-21T11:17:54.098697Z","shell.execute_reply":"2022-03-21T11:17:54.185406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_master = pd.read_csv(f'{OUTPUT_DIR_PATH}{LABEL_MASTER_CSV}')\nlabel_master = label_master['index'].values","metadata":{"execution":{"iopub.status.busy":"2022-03-21T11:20:05.15786Z","iopub.execute_input":"2022-03-21T11:20:05.158208Z","iopub.status.idle":"2022-03-21T11:20:05.175719Z","shell.execute_reply.started":"2022-03-21T11:20:05.158175Z","shell.execute_reply":"2022-03-21T11:20:05.174468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 画像をdecode(前処理：サイズ変更＋正規化)\ndef decode_image(image_data, box):\n    # boxで指定された枠を画像からトリミング\n    if box is not None and box[0] != -1:\n        left, top, right, bottom = box[0], box[1], box[2], box[3]\n        bbs = tf.convert_to_tensor([top, left, bottom - top, right - left])\n        image = tf.io.decode_and_crop_jpeg(image_data, bbs, channels=3)\n    else:\n        image = tf.image.decode_jpeg(image_data, channels=3)\n\n    # サイズを変更\n    image = tf.image.resize(image, [IMAGE_SIZE, IMAGE_SIZE])\n    # 正規化\n    image = tf.cast(image, tf.float32) / 255.0\n    return image\n\n\n# tfrecファイルを読み込んで中身を取得\ndef read_labeled_tfrecord(data):\n    LABELED_TFREC_FORMAT = {\n        \"image_name\": tf.io.FixedLenFeature([], tf.string),\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64),\n        'box': tf.io.FixedLenFeature([4], tf.int64)\n#         'detic_box': tf.io.FixedLenFeature([4], tf.int64)\n    }\n    \n    data = tf.io.parse_single_example(data, LABELED_TFREC_FORMAT)\n\n    box = tf.cast(data['box'], tf.int32)\n    image = decode_image(data['image'], box)\n\n    #label_group = tf.cast(data['target'], tf.int32)\n    target = tf.cast(data['target'], tf.int32)\n    label_group = (label_master == target)\n\n    posting_id = data['image_name']\n    return posting_id, image, label_group\n\n\n# 訓練用の画像ファイル(tfrecファイル)をload\ndef load_dataset(filenames, ordered = False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False \n        \n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(read_labeled_tfrecord, num_parallel_calls = AUTO) \n    return dataset\n\n\n# 訓練用の画像ファイル(tfrecファイル)を読み込み、必要な前処理を実施してtensorflow-datasetに展開する。\ndef get_training_dataset(filenames, train_flg=True):\n    dataset = load_dataset(filenames, ordered = False)\n#     dataset = dataset.map(data_augment, num_parallel_calls = AUTO)\n#     dataset = dataset.map(arcface_format, num_parallel_calls = AUTO)\n    dataset = dataset.map(lambda posting_id, image, label_group: (image, label_group))\n    if train_flg:\n        dataset = dataset.repeat()\n#     dataset = dataset.shuffle(2048)\n#     dataset = dataset.shuffle(20)\n    dataset = dataset.batch(2)\n    dataset = dataset.prefetch(buffer_size=AUTO)\n    return dataset","metadata":{"execution":{"iopub.status.busy":"2022-03-21T11:20:05.870201Z","iopub.execute_input":"2022-03-21T11:20:05.870686Z","iopub.status.idle":"2022-03-21T11:20:05.887819Z","shell.execute_reply.started":"2022-03-21T11:20:05.870633Z","shell.execute_reply":"2022-03-21T11:20:05.88673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 訓練画像(tfrecファイル)を読み込んで、tensorflow-datasetに展開\ntrain_ds = get_training_dataset(f'{OUTPUT_DIR_PATH}{TRAIN_IMAGES_TFREC}')","metadata":{"execution":{"iopub.status.busy":"2022-03-21T11:20:07.056359Z","iopub.execute_input":"2022-03-21T11:20:07.056856Z","iopub.status.idle":"2022-03-21T11:20:07.438908Z","shell.execute_reply.started":"2022-03-21T11:20:07.056802Z","shell.execute_reply":"2022-03-21T11:20:07.43789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for index, image in enumerate(train_ds.take(5)):\n    print(image[1])","metadata":{"execution":{"iopub.status.busy":"2022-03-21T11:20:08.517351Z","iopub.execute_input":"2022-03-21T11:20:08.517832Z","iopub.status.idle":"2022-03-21T11:20:08.842057Z","shell.execute_reply.started":"2022-03-21T11:20:08.51778Z","shell.execute_reply":"2022-03-21T11:20:08.841368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# VGG16のモデルを定義\nIMG_SHAPE = (IMAGE_SIZE, IMAGE_SIZE, 3)\nVGG16_MODEL = tf.keras.applications.VGG16(\n    input_shape=IMG_SHAPE,\n    include_top=False,\n    weights='imagenet'\n)","metadata":{"execution":{"iopub.status.busy":"2022-03-21T11:20:18.168879Z","iopub.execute_input":"2022-03-21T11:20:18.169384Z","iopub.status.idle":"2022-03-21T11:20:21.301954Z","shell.execute_reply.started":"2022-03-21T11:20:18.169331Z","shell.execute_reply":"2022-03-21T11:20:21.300997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 正解ラベルの数を取得\ndf_label_master = pd.read_csv(f'{OUTPUT_DIR_PATH}{LABEL_MASTER_CSV}')\nlabel_count = len(df_label_master)\n\n# 出力層を追加\nVGG16_MODEL.trainable = False\nglobal_average_layer = tf.keras.layers.GlobalAveragePooling2D()\nprediction_layer = tf.keras.layers.Dense(label_count, activation='softmax')","metadata":{"execution":{"iopub.status.busy":"2022-03-21T11:20:28.56299Z","iopub.execute_input":"2022-03-21T11:20:28.563294Z","iopub.status.idle":"2022-03-21T11:20:28.5843Z","shell.execute_reply.started":"2022-03-21T11:20:28.563263Z","shell.execute_reply":"2022-03-21T11:20:28.583333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# VGG16を元に、学習用のモデルを定義\nmodel = tf.keras.Sequential([\n    VGG16_MODEL,\n    global_average_layer,\n    prediction_layer\n])","metadata":{"execution":{"iopub.status.busy":"2022-03-21T11:20:29.617057Z","iopub.execute_input":"2022-03-21T11:20:29.617354Z","iopub.status.idle":"2022-03-21T11:20:29.76731Z","shell.execute_reply.started":"2022-03-21T11:20:29.617323Z","shell.execute_reply":"2022-03-21T11:20:29.766317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(\n    optimizer=tf.keras.optimizers.Adam(),\n    loss='categorical_crossentropy',\n    metrics=['acc']\n)","metadata":{"execution":{"iopub.status.busy":"2022-03-21T11:20:30.585937Z","iopub.execute_input":"2022-03-21T11:20:30.586253Z","iopub.status.idle":"2022-03-21T11:20:30.603624Z","shell.execute_reply.started":"2022-03-21T11:20:30.586219Z","shell.execute_reply":"2022-03-21T11:20:30.602601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2022-03-21T11:20:31.276702Z","iopub.execute_input":"2022-03-21T11:20:31.277365Z","iopub.status.idle":"2022-03-21T11:20:31.287384Z","shell.execute_reply.started":"2022-03-21T11:20:31.277315Z","shell.execute_reply":"2022-03-21T11:20:31.286391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 訓練画像の学習\nhistory = model.fit(\n    train_ds,\n    epochs=5,\n    steps_per_epoch=5\n#     validation_steps=validation_steps,\n#     validation_data=test_ds\n)","metadata":{"execution":{"iopub.status.busy":"2022-03-21T11:20:31.876132Z","iopub.execute_input":"2022-03-21T11:20:31.876444Z","iopub.status.idle":"2022-03-21T11:21:56.427405Z","shell.execute_reply.started":"2022-03-21T11:20:31.876411Z","shell.execute_reply":"2022-03-21T11:21:56.426362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# モデルを保存\nmodel.save(f'{OUTPUT_DIR_PATH}model_vgg16')","metadata":{"execution":{"iopub.status.busy":"2022-03-21T11:21:56.429968Z","iopub.execute_input":"2022-03-21T11:21:56.430313Z","iopub.status.idle":"2022-03-21T11:22:00.97665Z","shell.execute_reply.started":"2022-03-21T11:21:56.430275Z","shell.execute_reply":"2022-03-21T11:22:00.975506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 訓練画像(tfrecファイル)を読み込んで、tensorflow-datasetに展開\ntest_ds = get_training_dataset(f'{OUTPUT_DIR_PATH}{TEST_IMAGES_TFREC}', train_flg=False)","metadata":{"execution":{"iopub.status.busy":"2022-03-21T11:22:01.656655Z","iopub.execute_input":"2022-03-21T11:22:01.656971Z","iopub.status.idle":"2022-03-21T11:22:01.730219Z","shell.execute_reply.started":"2022-03-21T11:22:01.656937Z","shell.execute_reply":"2022-03-21T11:22:01.72915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = model.predict(test_ds)","metadata":{"execution":{"iopub.status.busy":"2022-03-21T11:22:02.355951Z","iopub.execute_input":"2022-03-21T11:22:02.356277Z","iopub.status.idle":"2022-03-21T11:22:23.216995Z","shell.execute_reply.started":"2022-03-21T11:22:02.356244Z","shell.execute_reply":"2022-03-21T11:22:23.215659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = np.argpartition(pred[0], 5)[:5]\nprint(preds[0].shape)\n\nlabel_master2 = pd.read_csv(f'{OUTPUT_DIR_PATH}{LABEL_MASTER_CSV}')\nlabel_master2.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-21T11:22:26.096738Z","iopub.execute_input":"2022-03-21T11:22:26.097059Z","iopub.status.idle":"2022-03-21T11:22:26.13174Z","shell.execute_reply.started":"2022-03-21T11:22:26.097024Z","shell.execute_reply":"2022-03-21T11:22:26.130546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv(f'{INPUT_DATA_PATH}sample_submission.csv')\ntest_df = test_df.head(10)\ntest_df","metadata":{"execution":{"iopub.status.busy":"2022-03-21T11:22:28.056291Z","iopub.execute_input":"2022-03-21T11:22:28.056954Z","iopub.status.idle":"2022-03-21T11:22:28.103706Z","shell.execute_reply.started":"2022-03-21T11:22:28.056909Z","shell.execute_reply":"2022-03-21T11:22:28.10296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for index in range(0, pred.shape[0]):\n    rank = np.argpartition(pred[index], 5)[:5]\n    rank = ' '.join([label_master2['individual_id'].iloc[index] for index in rank])\n    test_df['predictions'].iloc[index] = rank\n#     print(test_df['predictions'].iloc[index])\n\ntest_df.to_csv(f'{OUTPUT_DIR_PATH}submmission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-03-21T11:22:29.077027Z","iopub.execute_input":"2022-03-21T11:22:29.077509Z","iopub.status.idle":"2022-03-21T11:22:29.091555Z","shell.execute_reply.started":"2022-03-21T11:22:29.077474Z","shell.execute_reply":"2022-03-21T11:22:29.090349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}