{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install image-classifiers","metadata":{"_cell_guid":"c51e1290-06c0-45ab-8265-ea43c74ee111","_uuid":"2b6fe59a-2c14-48fb-83d7-21dd292d3d41","execution":{"iopub.execute_input":"2021-12-03T19:05:56.241685Z","iopub.status.busy":"2021-12-03T19:05:56.240583Z","iopub.status.idle":"2021-12-03T19:06:05.37208Z","shell.execute_reply":"2021-12-03T19:06:05.371289Z","shell.execute_reply.started":"2021-11-22T21:27:35.813732Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":9.150129,"end_time":"2021-12-03T19:06:05.372269","exception":false,"start_time":"2021-12-03T19:05:56.22214","status":"completed"},"tags":[],"collapsed":false},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport tensorflow as tf\nimport tensorflow.keras as keras\nfrom classification_models.tfkeras import Classifiers\nimport matplotlib.pyplot as plt\nimport os\nimport time\n\nfrom modelcompressionutils import eval_tflite_model, get_gzipped_model_size","metadata":{"_cell_guid":"b0c14741-7d1d-4cd7-a94b-3429d8e88632","_uuid":"c862c429-af8c-4850-bdd0-bda5ec884089","execution":{"iopub.execute_input":"2021-12-03T19:06:05.412092Z","iopub.status.busy":"2021-12-03T19:06:05.411446Z","iopub.status.idle":"2021-12-03T19:06:11.395692Z","shell.execute_reply":"2021-12-03T19:06:11.396912Z","shell.execute_reply.started":"2021-11-22T21:27:46.199202Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":6.007065,"end_time":"2021-12-03T19:06:11.397137","exception":false,"start_time":"2021-12-03T19:06:05.390072","status":"completed"},"tags":[],"collapsed":false},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from collections import namedtuple\nfrom typing import NamedTuple, List","metadata":{"_cell_guid":"8844fb92-eb33-4132-9aff-f44db7c80d6d","_uuid":"07c7fbc4-d3d4-4ed4-add3-d4583b4d28ae","execution":{"iopub.execute_input":"2021-12-03T19:06:11.442983Z","iopub.status.busy":"2021-12-03T19:06:11.441591Z","iopub.status.idle":"2021-12-03T19:06:11.443919Z","shell.execute_reply":"2021-12-03T19:06:11.444612Z","shell.execute_reply.started":"2021-11-22T21:27:51.944999Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.022947,"end_time":"2021-12-03T19:06:11.444834","exception":false,"start_time":"2021-12-03T19:06:11.421887","status":"completed"},"tags":[],"collapsed":false},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ModelDescriptor:\n    def __init__(self, name: str, path: str):\n        if not os.path.exists(path):\n            raise Exception('Path for model {name} is not valid ({path})'\n                            .format(name=name, path=path))\n        self.name = name\n        self.path = path\n    \n    def is_tflite_model(self):\n        return self.path.endswith('.tflite')\n    \n    def evalute_model(self, generator) -> float:\n        accuracy = -1.0\n        if self.is_tflite_model():\n            accuracy = eval_tflite_model(self.path, generator)\n        else:\n            model = keras.models.load_model(self.path)\n            model.compile(\n                optimizer='adam',\n                loss='categorical_crossentropy',\n                metrics=['accuracy']\n            )\n            res = model.evaluate(val_dg, verbose=1)\n            accuracy = res[1]\n        return accuracy","metadata":{"_cell_guid":"1c2d299a-76cc-4034-815c-08dda73c831d","_uuid":"0cefb5ae-83aa-406c-920f-b795588222a1","execution":{"iopub.execute_input":"2021-12-03T19:06:11.485695Z","iopub.status.busy":"2021-12-03T19:06:11.484956Z","iopub.status.idle":"2021-12-03T19:06:11.487667Z","shell.execute_reply":"2021-12-03T19:06:11.488238Z","shell.execute_reply.started":"2021-11-22T21:27:51.950004Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.025956,"end_time":"2021-12-03T19:06:11.488433","exception":false,"start_time":"2021-12-03T19:06:11.462477","status":"completed"},"tags":[],"collapsed":false},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def validate_model_paths(model_descriptors: List[ModelDescriptor]):\n    for model_desc in model_descriptors:\n        if not os.path.exists(model_desc.path):\n            raise Exception('Path for model {name} is not valid ({path})'\n                            .format(name=model_desc.name, path=model_desc.path))","metadata":{"_cell_guid":"79fe6473-c8ba-4c35-8064-9da116465993","_uuid":"d01beb11-5fe2-4f18-84e0-b6b02390d719","execution":{"iopub.execute_input":"2021-12-03T19:06:11.520956Z","iopub.status.busy":"2021-12-03T19:06:11.520129Z","iopub.status.idle":"2021-12-03T19:06:11.52528Z","shell.execute_reply":"2021-12-03T19:06:11.525808Z","shell.execute_reply.started":"2021-11-22T21:27:51.971817Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.023261,"end_time":"2021-12-03T19:06:11.525971","exception":false,"start_time":"2021-12-03T19:06:11.50271","status":"completed"},"tags":[],"collapsed":false},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ModelPerformance(NamedTuple):\n    name: str\n    evaluation_time: float # not comparable between lite and regurlar,\n    accuracy: float\n    size_on_disk: int\n    zipped_size: int","metadata":{"_cell_guid":"5e2906a8-6358-4101-9d20-e7bdf3ace7bf","_uuid":"9302ebb8-0af5-4583-aa2d-988f59d25534","execution":{"iopub.execute_input":"2021-12-03T19:06:11.559235Z","iopub.status.busy":"2021-12-03T19:06:11.558548Z","iopub.status.idle":"2021-12-03T19:06:11.563447Z","shell.execute_reply":"2021-12-03T19:06:11.564033Z","shell.execute_reply.started":"2021-11-22T21:27:51.982087Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.023589,"end_time":"2021-12-03T19:06:11.564197","exception":false,"start_time":"2021-12-03T19:06:11.540608","status":"completed"},"tags":[],"collapsed":false},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ModelDescriptorFactory = namedtuple('ModelDescriptor', ['name', 'path'])\nPerformanceFactory = namedtuple('ModelPerformance', ['name', 'evaluation_time', 'accuracy', 'size_on_disk', 'zipped_size'])","metadata":{"_cell_guid":"6f5158b5-71a3-4121-8fe1-aeda4f4ca40b","_uuid":"a1470331-a9bf-4867-84d6-c4cdd1a6a447","execution":{"iopub.execute_input":"2021-12-03T19:06:11.596666Z","iopub.status.busy":"2021-12-03T19:06:11.595829Z","iopub.status.idle":"2021-12-03T19:06:11.600173Z","shell.execute_reply":"2021-12-03T19:06:11.60088Z","shell.execute_reply.started":"2021-11-22T21:27:51.993069Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.022417,"end_time":"2021-12-03T19:06:11.601076","exception":false,"start_time":"2021-12-03T19:06:11.578659","status":"completed"},"tags":[],"collapsed":false},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_list = [\n    ModelDescriptor('ResNet18', '/media/pi/KINGSTON/models/base_modelv6.h5'),\n    ModelDescriptor('Base_TL', '/media/pi/KINGSTON/models/Quantization/base_tflite_model.tflite'),\n    # Distillation\n    ModelDescriptor('MobNetv2', '/media/pi/KINGSTON/models/Distillation/best_mobilenetv2.h5'),\n    ModelDescriptor('Dist_MobNetv2', '/media/pi/KINGSTON/models/Distillation/best_distilled_mobilenetv2.h5'),\n    # Pruning\n    ModelDescriptor('PR', '/media/pi/KINGSTON/models/Pruning/pruned_model.h5'),\n    ModelDescriptor('PR_TL_Fp16', '/media/pi/KINGSTON/models/Pruning/fp16_quant_pruned.tflite'),\n    ModelDescriptor('PR_TL_Int8', '/media/pi/KINGSTON/models/Pruning/int_quant_pruned.tflite'),\n    # Quantization\n    ModelDescriptor('QU_TL_Fp16', '/media/pi/KINGSTON/models/Quantization/quantized_fp16.tflite'),\n    ModelDescriptor('QU_TL_Int8', '/media/pi/KINGSTON/models/Quantization/quantized_int8.tflite'),\n    # QAT\n    ModelDescriptor('QAT_Fp16', '/media/pi/KINGSTON/models/QAT/qat_fp16.tflite'),\n    ModelDescriptor('QAT_Int8', '/media/pi/KINGSTON/models/QAT/qat_in8.tflite'),\n    # Weight clustering\n    ModelDescriptor('CL_KM32', '/media/pi/KINGSTON/models/Weight clustering/clustered_model_kpp32.h5'),\n    ModelDescriptor('CL_KM32_TL', '/media/pi/KINGSTON/models/Weight clustering/clustered_model_kpp32_tflite.tflite'),\n    ModelDescriptor('CL_KM256', '/media/pi/KINGSTON/models/Weight clustering/clustered_model_kpp256.h5'),\n    ModelDescriptor('CL_KM256_TL', '/media/pi/KINGSTON/models/Weight clustering/clustered_model_kpp256_tflite.tflite'),\n    ModelDescriptor('CL_Lin32', '/media/pi/KINGSTON/models/Weight clustering/clustered_model_lin32.h5'),\n    ModelDescriptor('CL_Lin32_TL', '/media/pi/KINGSTON/models/Weight clustering/clustered_model_lin32_tflite.tflite'),\n    # Combined Pruning, Weight clustering and QAT\n    ModelDescriptor('PCQ_TL_Fp16', '/media/pi/KINGSTON/models/Combined/quantized_fp16.tflite'),\n    ModelDescriptor('PCQ_TL_Int8', '/media/pi/KINGSTON/models/Combined/int8_quantized_model.tflite'),\n]","metadata":{"_cell_guid":"e4e21611-5fdc-4401-95e7-9b3ac03dd39c","_uuid":"0722a4dd-84f6-4372-8ff9-03c792e5e982","execution":{"iopub.execute_input":"2021-12-03T19:06:11.634667Z","iopub.status.busy":"2021-12-03T19:06:11.63396Z","iopub.status.idle":"2021-12-03T19:06:11.673978Z","shell.execute_reply":"2021-12-03T19:06:11.675663Z","shell.execute_reply.started":"2021-11-22T21:27:52.003148Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.060301,"end_time":"2021-12-03T19:06:11.675921","exception":false,"start_time":"2021-12-03T19:06:11.61562","status":"completed"},"tags":[],"collapsed":false},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validate_model_paths(model_list)","metadata":{"_cell_guid":"c8e719e8-1255-4892-a9bc-b23fdf49b380","_uuid":"825fe521-a3d0-4d33-8fce-c2337c87c6f5","execution":{"iopub.execute_input":"2021-12-03T19:06:11.716887Z","iopub.status.busy":"2021-12-03T19:06:11.716189Z","iopub.status.idle":"2021-12-03T19:06:11.718281Z","shell.execute_reply":"2021-12-03T19:06:11.719042Z","shell.execute_reply.started":"2021-11-22T21:27:52.041796Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.02389,"end_time":"2021-12-03T19:06:11.719235","exception":false,"start_time":"2021-12-03T19:06:11.695345","status":"completed"},"tags":[],"collapsed":false},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model testing","metadata":{"_cell_guid":"d576a726-aa96-4b1d-ad0c-3c6b97a1d6b1","_uuid":"743d83c8-b356-45d2-82ca-ee805fb5e249","papermill":{"duration":0.014851,"end_time":"2021-12-03T19:06:11.749591","exception":false,"start_time":"2021-12-03T19:06:11.73474","status":"completed"},"tags":[]}},{"cell_type":"code","source":"_, preprocess_input = Classifiers.get('resnet18')","metadata":{"_cell_guid":"32feca27-d67f-4ddc-8090-bb54e3a5567f","_uuid":"4a927908-2982-4241-84cc-01cb1ab1693d","execution":{"iopub.execute_input":"2021-12-03T19:06:11.782492Z","iopub.status.busy":"2021-12-03T19:06:11.781791Z","iopub.status.idle":"2021-12-03T19:06:11.785682Z","shell.execute_reply":"2021-12-03T19:06:11.78619Z","shell.execute_reply.started":"2021-11-22T21:27:52.048161Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.022336,"end_time":"2021-12-03T19:06:11.786398","exception":false,"start_time":"2021-12-03T19:06:11.764062","status":"completed"},"tags":[],"collapsed":false},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"VALIDATION_DS_PATH = '/media/pi/KINGSTON/CRC-VAL-HE-7K'\nSEED = 1\nBATCH_SIZE = 16","metadata":{"_cell_guid":"da29b673-7de7-4180-9c2e-eefed167a39f","_uuid":"cbc71ed3-9d14-4fe3-b9b5-a4ac7a9caad3","execution":{"iopub.execute_input":"2021-12-03T19:06:11.821018Z","iopub.status.busy":"2021-12-03T19:06:11.820346Z","iopub.status.idle":"2021-12-03T19:06:11.822214Z","shell.execute_reply":"2021-12-03T19:06:11.822742Z","shell.execute_reply.started":"2021-11-18T21:42:13.737548Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.022362,"end_time":"2021-12-03T19:06:11.822914","exception":false,"start_time":"2021-12-03T19:06:11.800552","status":"completed"},"tags":[],"collapsed":false},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_img_gen = keras.preprocessing.image.ImageDataGenerator(\n    preprocessing_function=preprocess_input\n)\nval_dg = val_img_gen.flow_from_directory(\n    VALIDATION_DS_PATH,\n    target_size=(224, 224),\n    class_mode='categorical',\n    batch_size=BATCH_SIZE,\n    shuffle=False, \n    seed=SEED\n)","metadata":{"_cell_guid":"7cb9d6b0-c23c-4ca1-97e3-3e724b4f12bd","_uuid":"203d0bd3-4234-4ead-9560-bbd971d859e8","execution":{"iopub.execute_input":"2021-12-03T19:06:11.854082Z","iopub.status.busy":"2021-12-03T19:06:11.853406Z","iopub.status.idle":"2021-12-03T19:06:16.338861Z","shell.execute_reply":"2021-12-03T19:06:16.339345Z","shell.execute_reply.started":"2021-11-18T21:42:13.753443Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":4.502651,"end_time":"2021-12-03T19:06:16.339528","exception":false,"start_time":"2021-12-03T19:06:11.836877","status":"completed"},"tags":[],"collapsed":false},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = []\nfor model_desc in model_list:\n    \n    print(f'Processing {model_desc.name}')\n    start_time = time.perf_counter()\n    accuracy = model_desc.evalute_model(val_dg)\n    end_time = time.perf_counter()\n    \n    size_on_disk = os.path.getsize(model_desc.path) / pow(10, 6)\n    zipped_size = get_gzipped_model_size(model_desc.path) / pow(10, 6)\n    evaluation_time = end_time - start_time # time in seconds\n    \n    model_perf = PerformanceFactory(\n        name=model_desc.name, \n        evaluation_time=evaluation_time,\n        accuracy=accuracy,\n        size_on_disk=size_on_disk,\n        zipped_size=zipped_size\n    )\n    results.append(model_perf)","metadata":{"_cell_guid":"b9bbb130-d3f5-4f65-8025-9d32b1993e6e","_uuid":"c063a991-d34b-4546-a3b0-fb7b0a46633d","execution":{"iopub.execute_input":"2021-12-03T19:06:16.377701Z","iopub.status.busy":"2021-12-03T19:06:16.376981Z","iopub.status.idle":"2021-12-04T02:20:37.745426Z","shell.execute_reply":"2021-12-04T02:20:37.746712Z","shell.execute_reply.started":"2021-11-18T21:42:14.447876Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":26061.392414,"end_time":"2021-12-04T02:20:37.747974","exception":false,"start_time":"2021-12-03T19:06:16.35556","status":"completed"},"tags":[],"collapsed":false},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fields = list(filter(lambda x: x != 'name', results[0]._fields))","metadata":{"_cell_guid":"9fdd597b-21f6-4f42-804c-52d4e52a3ff1","_uuid":"b3c0c910-2363-414c-9ac2-e4fa0229a07b","execution":{"iopub.execute_input":"2021-12-04T02:20:37.89758Z","iopub.status.busy":"2021-12-04T02:20:37.896638Z","iopub.status.idle":"2021-12-04T02:20:37.906661Z","shell.execute_reply":"2021-12-04T02:20:37.907106Z","shell.execute_reply.started":"2021-11-18T21:44:10.383155Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.087447,"end_time":"2021-12-04T02:20:37.907286","exception":false,"start_time":"2021-12-04T02:20:37.819839","status":"completed"},"tags":[],"collapsed":false},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def model_color_code(model_name: str):\n    if model_name.startswith(\"PR\"):\n        return 'green'\n    elif model_name.startswith(\"QU\"):\n        return 'orangered'\n    elif model_name.startswith(\"QAT\"):\n        return 'darkviolet'\n    elif model_name.startswith(\"CL\"):\n        return 'gold'\n    elif model_name.startswith(\"PCQ\"):\n        return 'purple'\n    elif model_name.startswith(\"Dist\"):\n        return \"cyan\"\n    else:\n        return 'blue'\n    \ndef plot_method_color_legend():\n    import matplotlib\n    # example taken from https://stackoverflow.com/a/53615732\n    # Create a color palette\n    palette = dict(zip(\n        ['Plain Model', 'Distillation', 'Pruning', 'Post Training Quantization', \n         'Quantization Aware Training (QAT)', 'Weight Clustering', 'Pruning + Weight Clustering + QAT (PCQAT)'], \n        ['blue', 'cyan', 'green', 'orangered', 'darkviolet', 'gold', 'purple'])\n    )\n    # Create legend handles manually\n    handles = [matplotlib.patches.Patch(color=palette[x], label=x) for x in palette.keys()]\n    # Create legend\n    plt.legend(handles=handles, loc='center', markerscale=2.0, fontsize='xx-large')\n    # Get current axes object and turn off axis\n    plt.gca().set_axis_off()\n    plt.show()","metadata":{"_cell_guid":"bc324389-7fc6-472a-87b0-1310e93ce5e3","_uuid":"03f4a13a-529d-4984-bdee-8085c8c60fc2","execution":{"iopub.execute_input":"2021-12-04T02:20:38.047351Z","iopub.status.busy":"2021-12-04T02:20:38.046198Z","iopub.status.idle":"2021-12-04T02:20:38.053089Z","shell.execute_reply":"2021-12-04T02:20:38.053555Z","shell.execute_reply.started":"2021-11-18T21:48:17.42915Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":0.078936,"end_time":"2021-12-04T02:20:38.053747","exception":false,"start_time":"2021-12-04T02:20:37.974811","status":"completed"},"tags":[],"collapsed":false},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_method_color_legend()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for field in fields:\n    fig = plt.figure(figsize=(20, 5))\n    ax = fig.add_axes([0,0,1,1])\n    names = list(map(lambda x: x.name, results))\n    values = list(map(lambda x: getattr(x, field), results))\n    colors = list(map(lambda x: model_color_code(x.name), results))\n    bars = ax.bar(names, values, color=colors)\n    for rect, val in zip(bars, values):\n        height = float(rect.get_height())\n        plt.text(rect.get_x() + rect.get_width()/2.0, height, '%.2f' % val,\n                 ha='center', va='bottom',fontsize=10)\n    ax.set_xlabel('Model name')\n    ax.set_ylabel(field)\n    plt.show()","metadata":{"_cell_guid":"47ad86e4-99ea-4e8a-9e87-b8383e85a96a","_uuid":"4f4603de-40ce-4cb6-81bb-78df37882c73","execution":{"iopub.execute_input":"2021-12-04T02:20:38.186219Z","iopub.status.busy":"2021-12-04T02:20:38.185203Z","iopub.status.idle":"2021-12-04T02:20:40.603951Z","shell.execute_reply":"2021-12-04T02:20:40.604406Z","shell.execute_reply.started":"2021-11-18T21:48:19.36139Z"},"jupyter":{"outputs_hidden":false},"papermill":{"duration":2.486952,"end_time":"2021-12-04T02:20:40.604587","exception":false,"start_time":"2021-12-04T02:20:38.117635","status":"completed"},"tags":[],"collapsed":false},"execution_count":null,"outputs":[]}]}