{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":5048,"databundleVersionId":868335,"sourceType":"competition"},{"sourceId":7290419,"sourceType":"datasetVersion","datasetId":4228157},{"sourceId":7291688,"sourceType":"datasetVersion","datasetId":4229050}],"dockerImageVersionId":30627,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport shutil\n\nfrom keras.models import *\nfrom keras.layers import *\nfrom keras.applications import *\nfrom keras.preprocessing.image import *\n\nimport h5py\nimport math\n\n# dir = \"/ext/Data/distracted_driver_detection/\"\ndir = \"/kaggle/input/state-farm-distracted-driver-detection/imgs/\"\ndir_valid = \"/kaggle/input/driver-pre/\"\n\n\n\nresnet50_weight_file = \"/kaggle/input/driver-model/resnet50-imagenet-finetune152.h5\" #\"resnet50-imagenet-finetune152.h5\"\nxception_weight_file = \"/kaggle/input/driver-model/xception-imagenet-finetune116.h5\"  #\"xception-imagenet-finetune116.h5\"\ninceptionV3_weight_file = \"/kaggle/input/driver-model/inceptionV3-imagenet-finetune172.h5\"  #\"inceptionV3-imagenet-finetune172.h5\"\n\ndef write_gap(tag, MODEL, weight_file, image_size, lambda_func=None, featurewise_std_normalization=True):\n    input_tensor = Input((*image_size, 3))\n    x = input_tensor\n    if lambda_func:\n        x = Lambda(lambda_func)(x)\n    base_model = MODEL(input_tensor=x, weights=None, include_top=False)\n\n    model = Model(base_model.input, GlobalAveragePooling2D()(base_model.output))\n    model.load_weights(weight_file, by_name=True)\n\n    print(MODEL.__name__)\n    train_gen = ImageDataGenerator(\n        featurewise_std_normalization=featurewise_std_normalization,\n        samplewise_std_normalization=False,\n        rotation_range=10.,\n        width_shift_range=0.05,\n        height_shift_range=0.05,\n        shear_range=0.1,\n        zoom_range=0.1,\n    )\n    gen = ImageDataGenerator(\n        featurewise_std_normalization=featurewise_std_normalization,\n        samplewise_std_normalization=False,\n    )\n\n    batch_size = 64\n    train_generator = train_gen.flow_from_directory(os.path.join(dir, 'train'), image_size, shuffle=False, batch_size=batch_size)\n    print(\"subdior to train type {}\".format(train_generator.class_indices))\n    valid_generator = gen.flow_from_directory(os.path.join(dir_valid, 'valid'), image_size, shuffle=False, batch_size=batch_size)\n    print(\"subdior to valid type {}\".format(valid_generator.class_indices))\n\n    print(\"predict_generator train {}\".format(math.ceil(train_generator.samples//batch_size+1)))\n    \n    train = model.predict(train_generator, steps=math.ceil(train_generator.samples//batch_size+1))\n#     train = model.predict_generator(train_generator, math.ceil(train_generator.samples//batch_size+1))\n    \n    print(\"train: {}\".format(train.shape))\n    print(\"predict_generator valid {}\".format(math.ceil(valid_generator.samples//batch_size+1)))\n    \n    valid = model.predict(vaild_generator, steps=math.ceil(valid_generator.samples//batch_size+1))\n#     valid = model.predict_generator(valid_generator, math.ceil(valid_generator.samples//batch_size+1))\n    \n    print(\"valid: {}\".format(valid.shape))\n    print(\"train label: {}\".format(train_generator.classes.shape))\n    print(\"valid label: {}\".format(valid_generator.classes.shape))\n\n    print(\"begin create database {}\".format(Model.__name__))\n    with h5py.File(os.path.join(\"models\", tag, \"bottleneck_%s.h5\") % MODEL.__name__) as h:\n        h.create_dataset(\"train\", data=train)\n        h.create_dataset(\"valid\", data=valid)\n        h.create_dataset(\"label\", data=train_generator.classes)\n        h.create_dataset(\"valid_label\", data=valid_generator.classes)\n    print(\"write_gap {} successed\".format(Model.__name__))\n\ndef write_gap_test(tag, MODEL, weight_file, image_size, lambda_func=None, featurewise_std_normalization=True):\n    input_tensor = Input((*image_size, 3))\n    x = input_tensor\n    if lambda_func:\n        x = Lambda(lambda_func)(x)\n    base_model = MODEL(input_tensor=x, weights=None, include_top=False)\n    model = Model(base_model.input, GlobalAveragePooling2D()(base_model.output))\n    model.load_weights(weight_file, by_name=True)\n\n    print(MODEL.__name__)\n    gen = ImageDataGenerator(\n        featurewise_std_normalization=featurewise_std_normalization,\n        samplewise_std_normalization=False,\n    )\n    batch_size = 64\n    test_generator = gen.flow_from_directory(os.path.join(dir, 'test'), image_size, shuffle=False, batch_size=batch_size, class_mode=None)\n    print(\"predict_generator test {}\".format(math.ceil(test_generator.samples//batch_size+1)))\n    \n    test = model.predict(test_generator, steps=math.ceil(test_generator.samples//batch_size+1))\n#     test = model.predict_generator(test_generator, math.ceil(test_generator.samples//batch_size+1))\n    print(\"test: {}\".format(test.shape))\n\n    print(\"begin create database {}\".format(Model.__name__))\n    with h5py.File(os.path.join(\"models\", tag, \"bottleneck_%s_test.h5\") % MODEL.__name__) as h:\n        h.create_dataset(\"test\", data=test)\n    print(\"write_gap {} successed\".format(Model.__name__))\n\ndef normal_preprocess_input(x):\n    x /= 255.\n    x -= 0.5\n    x *= 2\n    return x\n\n###\n### subdir = noscale\n###\n","metadata":{"_uuid":"93c8bda7-efd7-42b5-81d2-d321f8093995","_cell_guid":"9f69fb94-0a49-4f7d-bae0-189e6f090b82","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-12-27T16:21:09.762195Z","iopub.execute_input":"2023-12-27T16:21:09.762697Z","iopub.status.idle":"2023-12-27T16:21:09.796200Z","shell.execute_reply.started":"2023-12-27T16:21:09.762657Z","shell.execute_reply":"2023-12-27T16:21:09.795143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"main\"\"\"\n# 这段代码的作用是使用已经训练好的 ResNet50、Xception 和 InceptionV3 模型在给定图像上提取特征，然后将这些特征保存为 .npy文件\n# 这些 .npy文件在以后的模型训练中被用作输入数据\nprint(\"===== Train & Valid =====\")\nwrite_gap(\"finetune\", ResNet50, resnet50_weight_file, (240, 320))\nwrite_gap(\"finetune\", Xception, xception_weight_file, (320, 480), xception.preprocess_input)\nwrite_gap(\"finetune\", InceptionV3, inceptionV3_weight_file, (320, 480), inception_v3.preprocess_input)\n\n\n","metadata":{"execution":{"iopub.status.busy":"2023-12-27T16:21:18.732864Z","iopub.execute_input":"2023-12-27T16:21:18.733781Z","iopub.status.idle":"2023-12-27T16:27:13.772458Z","shell.execute_reply.started":"2023-12-27T16:21:18.733745Z","shell.execute_reply":"2023-12-27T16:27:13.770675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"===== Test =====\")\nwrite_gap_test(\"finetune\", ResNet50, resnet50_weight_file, (240, 320))\nwrite_gap_test(\"finetune\", Xception, xception_weight_file, (320, 480), xception.preprocess_input)\nwrite_gap_test(\"finetune\", InceptionV3, inceptionV3_weight_file, (320, 480), inception_v3.preprocess_input)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train = model.predict(train_generator, steps=math.ceil(train_generator.samples//batch_size+1))","metadata":{},"execution_count":null,"outputs":[]}]}