{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install neptune-client","metadata":{"execution":{"iopub.status.busy":"2022-05-19T10:40:16.294424Z","iopub.execute_input":"2022-05-19T10:40:16.294980Z","iopub.status.idle":"2022-05-19T10:40:33.153728Z","shell.execute_reply.started":"2022-05-19T10:40:16.294879Z","shell.execute_reply":"2022-05-19T10:40:33.152791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nimport os\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n\nimport neptune.new as neptune\n\nfrom sklearn.model_selection import train_test_split\n\nfrom tensorflow import keras\n\nRANDOM_STATE = 126","metadata":{"execution":{"iopub.status.busy":"2022-05-19T10:40:33.157446Z","iopub.execute_input":"2022-05-19T10:40:33.157704Z","iopub.status.idle":"2022-05-19T10:40:39.756069Z","shell.execute_reply.started":"2022-05-19T10:40:33.157672Z","shell.execute_reply":"2022-05-19T10:40:39.755149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Network framework","metadata":{}},{"cell_type":"code","source":"# model json structure\n_description = \"description\"\n_layers = \"layers\"\n_name = \"name\"\n_params = \"params\"\n_optimizer = \"optimizer\"\n_loss = \"loss\"\n_metrics = \"metrics\"\n\n# nn layers\n_Dense = \"Dense\"\n_Dropout = \"Dropout\"\n_Conv2d = \"Conv2d\"\n_BatchNormalization = \"BatchNormalization\"\n_MaxPooling2D = \"MaxPooling2D\"\n_Flatten = \"Flatten\"\n_BatchNormalization = \"BatchNormalization\"\n_Dropout = \"Dropout\"\n_GlobalAveragePooling2D = \"GlobalAveragePooling2D\"\n\n# nn layers params\n_units = \"units\"\n_activation = \"activation\"\n_rate = \"rate\"\n_filters = \"filters\"\n_kernel_size = \"kernel_size\"\n_strides = \"strides\"\n_padding = \"padding\"\n_input_shape = \"input_shape\"\n_pool_size = \"pool_size\"\n_rate = \"rate\"\n\n# activation functions\n_relu = \"relu\"\n_softmax = \"softmax\"\n\n# optimizers\n_SGD = \"SGD\"\n_Adam = \"Adam\"\n_RMSprop = \"RMSprop\"\n\n# optimizers params\n_learning_rate = \"learning_rate\"\n_momentum = \"momentum\"\n_rho = \"rho\"\n\n# losses\n_CategoricalCrossentropy = \"CategoricalCrossentropy\"\n_from_logits = \"from_logits\"\n\n# metrics\n_accuracy = \"accuracy\"\n\n# fitting params\n_epochs = \"epochs\"\n_batch_size = \"batch_size\"\n","metadata":{"execution":{"iopub.status.busy":"2022-05-19T10:40:39.757703Z","iopub.execute_input":"2022-05-19T10:40:39.758025Z","iopub.status.idle":"2022-05-19T10:40:39.769439Z","shell.execute_reply.started":"2022-05-19T10:40:39.757983Z","shell.execute_reply":"2022-05-19T10:40:39.768402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nn_layers = {\n    _Dense: keras.layers.Dense,\n    _Dropout: keras.layers.Dropout,\n    _Conv2d: keras.layers.Conv2D,\n    _BatchNormalization: keras.layers.BatchNormalization,\n    _MaxPooling2D: keras.layers.MaxPooling2D,\n    _Flatten: keras.layers.Flatten,\n    _BatchNormalization: keras.layers.BatchNormalization,\n    _Dropout: keras.layers.Dropout,\n    _GlobalAveragePooling2D: keras.layers.GlobalAveragePooling2D\n}\n\nnn_optimizers = {\n    _SGD: keras.optimizers.SGD,\n    _Adam: keras.optimizers.Adam,\n    _RMSprop: keras.optimizers.RMSprop,\n}\n\nnn_losses = {\n    _CategoricalCrossentropy: keras.losses.CategoricalCrossentropy,\n}\n\ndef make_nn(run_json, run=None):\n    \"\"\"\n    Создание модели нейросети из json описания эксперимента\n    \"\"\"\n    # выделяем слои сети и создаём пустой массив для объектов слоёв\n    model_layers_json = run_json[_layers]\n    model_layers = []\n    \n    model_layers.append(keras.layers.Lambda(lambda x: x/255))\n\n    # поочерёдно добавляем слои в сеть\n    for layer_json in model_layers_json:\n        if _name in layer_json and _params in layer_json:\n            layer_type = nn_layers[layer_json[_name]]\n            layer_params = layer_json[_params]\n\n            layer = layer_type(**layer_params)\n            model_layers.append(layer)\n\n    # создаём объект НС\n    model = keras.Sequential(model_layers)\n\n    # записываем параметры\n    if run is not None:\n        run[\"model\"] = model_layers_json\n\n    return model\n\n\ndef make_optimizer(run_json, run=None):\n    \"\"\"\n    Создание модели оптимизатора из json описания эксперимента\n    \"\"\"\n    optimizer_json = run_json[_optimizer]\n    optimizer_type = nn_optimizers[optimizer_json[_name]]\n    optimizer_params = optimizer_json[_params]\n\n    optimizer = optimizer_type(**optimizer_params)\n\n    if run is not None:\n        run[\"optimizer\"] = optimizer_json[_name]\n        run[\"learning_rate\"] = optimizer_params[_learning_rate]\n\n    return optimizer\n\n\ndef make_loss(run_json, run=None):\n    \"\"\"\n    Создание объекта функции потерь\n    \"\"\"\n    loss = nn_losses[run_json[_loss]]()\n\n    if run is not None:\n        run[\"loss\"] = _loss\n\n    return loss\n\n\ndef compile_nn(run_json, model, optimizer, loss, run=None):\n    \"\"\"\n    Компиляция модели\n    \"\"\"\n    model.compile(loss=loss, optimizer=optimizer, metrics=[run_json[_metrics]])\n\n    if run is not None:\n        run[\"metrics\"] = run_json[_metrics]\n\n\ndef fit_nn(run_json, model, x_train, y_train, run=None, lr_schedule_on=False):\n    \"\"\"\n    Обучение модели\n    \"\"\"\n    epochs = run_json[_epochs]\n    batch_size = run_json[_batch_size]\n\n    if run is not None:\n        run[\"epochs\"] = epochs\n        run[\"batch_size\"] = batch_size\n        \n    if lr_schedule_on:\n        optimizer_lr = run_json[_optimizer][_params][_learning_rate]\n        lr_schedule = keras.callbacks.LearningRateScheduler(\n            lambda epoch: optimizer_lr / 10**(epoch / 20))\n        callbacks = [lr_schedule]\n    else:\n        callbacks = []\n\n    history = model.fit(x_train, y_train, epochs=epochs, batch_size=batch_size, callbacks=callbacks)\n\n    if run is not None:\n        run[\"train/loss\"].log(history.history[\"loss\"])\n\n        metrics = run_json[_metrics]\n        for metric in metrics:\n            run[\"train/\" + metric].log(history.history[metric])\n            \n    return history\n\ndef evalueate_nn(model, x_test, y_test, run=None):\n    eval_accuracy = model.evaluate(x_test, y_test)[1]\n    \n    if run is not None:\n        run[\"eval/accuracy\"] = eval_accuracy\n        \n    return eval_accuracy\n","metadata":{"execution":{"iopub.status.busy":"2022-05-19T10:40:39.772799Z","iopub.execute_input":"2022-05-19T10:40:39.773231Z","iopub.status.idle":"2022-05-19T10:40:40.923946Z","shell.execute_reply.started":"2022-05-19T10:40:39.773176Z","shell.execute_reply":"2022-05-19T10:40:40.923060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def Dense(units, activation=_relu):\n    return {\n      _name: _Dense,\n      _params: {\n          _units: units,\n          _activation: activation\n      }\n    }\n\ndef Conv(filters, kernel_size):\n    return {\n      _name: _Conv2d,\n      _params: {\n          _filters: filters,\n          _kernel_size: kernel_size,\n          _activation: _relu\n      }\n    }\n\ndef MaxPooling(pool_size):\n    return {\n      _name: _MaxPooling2D,\n      _params: {\n          _pool_size: pool_size\n      }\n    }\n\ndef AvgPooling():\n    return {\n        _name: _GlobalAveragePooling2D,\n        _params: {}\n    }\n\n\n\ndef Flatten():\n    return {\n        _name: _Flatten,\n        _params: {}\n    }\n\ndef BatchNormalization():\n    return {\n        _name: _BatchNormalization,\n        _params: {}\n    }\n\ndef Dropout(rate):\n     return {\n        _name: _Dropout,\n        _params: {\n            _rate: rate\n        }\n    }\n\n\ndef SGD(lr, momentum=0.9):\n    return {\n      _name: _SGD,\n      _params: {\n          _learning_rate: lr,\n          _momentum: momentum\n      }\n    }\n\ndef Adam(lr):\n    return {\n      _name: _Adam,\n      _params: {\n          _learning_rate: lr\n      }\n    }\n\ndef RMSProp(lr):\n    return {\n      _name: _RMSprop,\n      _params: {\n          _learning_rate: lr\n      }\n    }","metadata":{"execution":{"iopub.status.busy":"2022-05-19T10:40:40.925464Z","iopub.execute_input":"2022-05-19T10:40:40.925767Z","iopub.status.idle":"2022-05-19T10:40:40.940265Z","shell.execute_reply.started":"2022-05-19T10:40:40.925724Z","shell.execute_reply":"2022-05-19T10:40:40.938876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data preparing","metadata":{}},{"cell_type":"code","source":"data_root_path = \"../input/cassava-leaf-disease-classification\"\ntrain_path = data_root_path + \"/train_images/\"\ntrain_df_path = data_root_path + \"/train.csv\"","metadata":{"execution":{"iopub.status.busy":"2022-05-19T10:40:40.942852Z","iopub.execute_input":"2022-05-19T10:40:40.943489Z","iopub.status.idle":"2022-05-19T10:40:40.963462Z","shell.execute_reply.started":"2022-05-19T10:40:40.943444Z","shell.execute_reply":"2022-05-19T10:40:40.962375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Selecting images to train and evaluate","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(train_df_path)\n\ndata_len = len(train_df)\nclasses_num = len(set(train_df[\"label\"]))\n\nprint(f\"Data lenght: {data_len}\")\nprint(f\"Classes: {classes_num}\")\n\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-19T10:40:40.965295Z","iopub.execute_input":"2022-05-19T10:40:40.965651Z","iopub.status.idle":"2022-05-19T10:40:41.026692Z","shell.execute_reply.started":"2022-05-19T10:40:40.965555Z","shell.execute_reply":"2022-05-19T10:40:41.025868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_one_class_obj_num = 600\nimgs_info = list()\n\nfor class_label in set(train_df[\"label\"]):\n    class_data = train_df.loc[train_df[\"label\"] == class_label]\n    \n    sampled_class_data = class_data.sample(n=total_one_class_obj_num, random_state=RANDOM_STATE)\n    imgs_info.extend(list(zip(sampled_class_data[\"image_id\"].values, sampled_class_data[\"label\"].values)))\n\nprint(f\"Images sampled: {len(imgs_info)}\")","metadata":{"execution":{"iopub.status.busy":"2022-05-19T10:40:41.028264Z","iopub.execute_input":"2022-05-19T10:40:41.028747Z","iopub.status.idle":"2022-05-19T10:40:41.056245Z","shell.execute_reply.started":"2022-05-19T10:40:41.028703Z","shell.execute_reply":"2022-05-19T10:40:41.055317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Reading selected images","metadata":{}},{"cell_type":"code","source":"def read_img(img_name):\n    img_bgr = cv2.imread(train_path + img_name)\n    img_rgb = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2RGB)\n    return img_rgb\n\nimgs = [ (read_img(im_name), im_class) for im_name, im_class in imgs_info ]\n\nimages_shape = np.mean([ img.shape for img, _ in imgs ], axis=0).astype(int)\n\nprint(f\"Images read: {len(imgs)}\")\nprint(f\"Images shape: {images_shape}\")","metadata":{"execution":{"iopub.status.busy":"2022-05-19T10:40:41.057568Z","iopub.execute_input":"2022-05-19T10:40:41.058253Z","iopub.status.idle":"2022-05-19T10:41:33.399195Z","shell.execute_reply.started":"2022-05-19T10:40:41.058207Z","shell.execute_reply":"2022-05-19T10:41:33.397205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Resizing","metadata":{}},{"cell_type":"code","source":"target_size = (256, 256)\nimgs_reszd = [ (cv2.resize(im, target_size), im_class) for im, im_class in imgs ]\n\nax = plt.subplot(121)\nax.imshow(imgs[0][0])\nax.set_title(\"Original\")\n\nax = plt.subplot(122)\nax.imshow(imgs_reszd[0][0])\nax.set_title(\"Resized\");","metadata":{"execution":{"iopub.status.busy":"2022-05-19T10:41:33.401847Z","iopub.execute_input":"2022-05-19T10:41:33.402175Z","iopub.status.idle":"2022-05-19T10:41:35.187673Z","shell.execute_reply.started":"2022-05-19T10:41:33.402137Z","shell.execute_reply":"2022-05-19T10:41:35.186892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data splitting","metadata":{}},{"cell_type":"code","source":"x = []\ny = []\n\nfor img_data, img_class in imgs_reszd:\n    x.append(img_data)\n    \n    # one-hot encoding for y\n    one_hot_y = np.zeros(classes_num)\n    one_hot_y[img_class] = 1\n    \n    y.append(one_hot_y)","metadata":{"execution":{"iopub.status.busy":"2022-05-19T10:41:35.188689Z","iopub.execute_input":"2022-05-19T10:41:35.188913Z","iopub.status.idle":"2022-05-19T10:41:35.203423Z","shell.execute_reply.started":"2022-05-19T10:41:35.188884Z","shell.execute_reply":"2022-05-19T10:41:35.202570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train, x_test, y_train, y_test = train_test_split(\n    x,\n    y,\n    test_size=200 * classes_num,\n    stratify=y,\n    random_state=RANDOM_STATE\n)\n\nx_train = np.array(x_train, dtype=np.float32 )\nx_test = np.array(x_test, dtype=np.float32 )\ny_train = np.array(y_train, dtype=np.float32)\ny_test = np.array(y_test, dtype=np.float32 )\n\nprint(f\"Train size: {len(x_train)}\")\nprint(f\"Test size: {len(x_test)}\")","metadata":{"execution":{"iopub.status.busy":"2022-05-19T10:41:35.205033Z","iopub.execute_input":"2022-05-19T10:41:35.205495Z","iopub.status.idle":"2022-05-19T10:41:35.863913Z","shell.execute_reply.started":"2022-05-19T10:41:35.205456Z","shell.execute_reply":"2022-05-19T10:41:35.863083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model fitting","metadata":{}},{"cell_type":"markdown","source":"## Simple CNN","metadata":{}},{"cell_type":"code","source":"import gc\n\ndef clean_memory():\n    model = None\n    optimizer = None\n    history = None\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-05-19T10:41:35.865102Z","iopub.execute_input":"2022-05-19T10:41:35.865991Z","iopub.status.idle":"2022-05-19T10:41:35.870401Z","shell.execute_reply.started":"2022-05-19T10:41:35.865929Z","shell.execute_reply":"2022-05-19T10:41:35.869626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# просто заглушка, чтобы работало\nclass Run:\n    history = None\n    \n    def __init__(self):\n        history = None\n    \n    def __setitem__(self, key, value):\n        self.__dict__[key] = value\n    \n    def __getitem__(self, key):\n        return Run()\n    \n    def log(self, values):\n        self.history = values\n        \n    def __enter__(self):\n        return Run()\n    \n    def __exit__(self, exception_type, exception_value, traceback):\n        ...","metadata":{"execution":{"iopub.status.busy":"2022-05-19T11:23:38.854941Z","iopub.execute_input":"2022-05-19T11:23:38.855208Z","iopub.status.idle":"2022-05-19T11:23:38.862583Z","shell.execute_reply.started":"2022-05-19T11:23:38.855171Z","shell.execute_reply":"2022-05-19T11:23:38.861802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if True:\n    with Run() as run:\n        clean_memory()\n\n        run_json = {\n          _description: \"Avg poling with dropouts\",\n          _layers: [\n            Conv(32, (3, 3)),\n            BatchNormalization(),\n            MaxPooling((2, 2)),\n            Conv(64, (3, 3)),\n            BatchNormalization(),\n            MaxPooling((2, 2)),\n            Conv(128, (3, 3)),\n            BatchNormalization(),\n            MaxPooling((2, 2)),\n            AvgPooling(),\n            Flatten(),\n            Dense(classes_num, activation=\"softmax\")\n          ],\n          _optimizer: Adam(5e-4),\n          _loss: _CategoricalCrossentropy,\n          _metrics: [_accuracy],\n          _epochs: 50,\n          _batch_size: 128\n\n        }\n\n        run[\"description\"] = run_json[_description]\n\n        # создаём модель\n        model = make_nn(run_json, run)\n\n        # создаём оптимизатор\n        optimizer = make_optimizer(run_json, run)\n\n        # инициализируем функцию потерь\n        loss = make_loss(run_json, run)\n\n        # компилируем модель\n        compile_nn(run_json, model, optimizer, loss, run)\n\n        # тренируем модель\n        history = fit_nn(run_json, model, x_train, y_train, run, lr_schedule_on=False)\n\n        # проверяем модель\n        evalueate_nn(model, x_test, y_test, run)","metadata":{"execution":{"iopub.status.busy":"2022-05-19T11:23:42.116931Z","iopub.execute_input":"2022-05-19T11:23:42.117492Z","iopub.status.idle":"2022-05-19T11:26:10.617737Z","shell.execute_reply.started":"2022-05-19T11:23:42.117454Z","shell.execute_reply":"2022-05-19T11:26:10.614304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Transfer learning","metadata":{}},{"cell_type":"code","source":"if True:\n    with neptune.init(\n        project=\"dimyakovenko/CompVision-lab6\",\n        api_token=\"eyJhcGlfYWRkcmVzcyI6Imh0dHBzOi8vYXBwLm5lcHR1bmUuYWkiLCJhcGlfdXJsIjoiaHR0cHM6Ly9hcHAubmVwdHVuZS5haSIsImFwaV9rZXkiOiIxYjNjMzBmYS0yNDM3LTRmMTctOGI0My0xNzVkZWM2ZTQ5MTAifQ==\",\n    ) as run:\n        clean_memory()\n        \n        fitted_model = keras.applications.ResNet50(weights='imagenet', \n                                 input_shape=(256, 256, 3),\n                                 include_top=False)\n\n        fitted_model.trainable = False\n\n        input_layer = keras.Input(shape=(256, 256, 3))\n        fitted_layer = fitted_model(input_layer, training=False)\n\n        run_json = {\n          _description: \"ResNet50 transfer learning\",\n          _layers: [\n            AvgPooling(),\n            Dense(100),\n            Dense(20),\n            Dense(classes_num, activation=\"softmax\")\n          ],\n          _optimizer: RMSProp(1e-2),\n          _loss: _CategoricalCrossentropy,\n          _metrics: [_accuracy],\n          _epochs: 50,\n          _batch_size: 128\n\n        }\n\n        run[\"description\"] = run_json[_description]\n\n        # создаём модель\n        ceil_model = make_nn(run_json, run)(fitted_layer)\n        model = keras.Model(input_layer, ceil_model)\n\n        # создаём оптимизатор\n        optimizer = make_optimizer(run_json, run)\n\n        # инициализируем функцию потерь\n        loss = make_loss(run_json, run)\n\n        # компилируем модель\n        compile_nn(run_json, model, optimizer, loss, run)\n\n        # тренируем модель\n        history = fit_nn(run_json, model, x_train, y_train, run, lr_schedule_on=False)\n\n        # проверяем модель\n        evalueate_nn(model, x_test, y_test, run)","metadata":{"execution":{"iopub.status.busy":"2022-05-19T02:33:00.326001Z","iopub.execute_input":"2022-05-19T02:33:00.326254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}