{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install tensorflow==1.15.0\n!pip install keras==2.2.5","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"%matplotlib inline\nimport os\nimport gc\nimport sys\nimport json\nimport glob\nimport random \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport cv2\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom pathlib import Path\n\nimport itertools\nfrom tqdm import tqdm\n\nfrom imgaug import augmenters as iaa\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Используем mask_rcnn и гугловский pix2pix для сегментации**"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"!git clone https://www.github.com/matterport/Mask_RCNN.git\n!pip install -q git+https://github.com/tensorflow/examples.git","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.chdir('Mask_RCNN')\nDATA_DIR = Path('/kaggle/input/imaterialist-fashion-2019-FGVC6/')\nROOT_DIR = Path('/kaggle/working')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Импортируем модули из репо сетки"},{"metadata":{"trusted":true},"cell_type":"code","source":"sys.path.append(str(ROOT_DIR/'Mask_RCNN'))\n\nfrom mrcnn.config import Config\nfrom mrcnn import utils\nimport mrcnn.model as modellib\nfrom mrcnn import visualize\nfrom mrcnn.model import log\n\n!wget --quiet https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5\n!ls -lh mask_rcnn_coco.h5\n\nCOCO_WEIGHTS_PATH = 'mask_rcnn_coco.h5'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Задаем конфигурацию сети, то есть ее тип, архитектуру, размер входщего изображеня, количество классов"},{"metadata":{"trusted":true},"cell_type":"code","source":"# config: either 'resnet50' or 'resnet101' is supported\n\nNUM_CATS = 46\nIMAGE_SIZE = 512\n\nclass Config_(Config):\n    def __init__(self, bbone):\n        super().__init__()\n        self._bone = bbone\n        \n    NAME = \"fashion\"\n    NUM_CLASSES = NUM_CATS + 1 # backgroun\n    \n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 2 \n    \n    BACKBONE = 'resnet50'\n    \n    IMAGE_MIN_DIM = IMAGE_SIZE\n    IMAGE_MAX_DIM = IMAGE_SIZE    \n    IMAGE_RESIZE_MODE = 'none'\n    \n    RPN_ANCHOR_SCALES = (16, 32, 64, 128, 256)\n    \n    # STEPS_PER_EPOCH should be the number of instances \n    # divided by (GPU_COUNT*IMAGES_PER_GPU), and so should VALIDATION_STEPS;\n    # however, due to the time limit, I set them so that this kernel can be run in 9 hours\n    STEPS_PER_EPOCH = 1000\n    VALIDATION_STEPS = 200\n    \nconfig = Config_('resnet50')\nconfig.display()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(DATA_DIR/\"label_descriptions.json\") as f:\n    label_descriptions = json.load(f)\n\nlabel_names = [x['name'] for x in label_descriptions['categories']]\n\nsegment_df = pd.read_csv(DATA_DIR/\"train.csv\")\nsegment_df['CategoryId'] = segment_df['ClassId'].str.split('_').str[0]\n\nmultilabel_percent = len(segment_df[segment_df['CategoryId'].str.contains('_')])/len(segment_df)*100","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"segment_df.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_df = segment_df.groupby('ImageId')['EncodedPixels', 'CategoryId'].agg(lambda x: list(x))\nsize_df = segment_df.groupby('ImageId')['Height', 'Width'].mean()\nimage_df = image_df.join(size_df, on='ImageId')\n\nprint(\"Total images: \", len(image_df))\nimage_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def resize_image(image_path):\n    img = cv2.imread(image_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    img = cv2.resize(img, (IMAGE_SIZE, IMAGE_SIZE), interpolation=cv2.INTER_AREA)  \n    return img","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Утилиты для работы с датасетом. Прогрузка каждого изображения, маски для сегментации и пр."},{"metadata":{"trusted":true},"cell_type":"code","source":"class Dataset(utils.Dataset):\n\n    def __init__(self, df):\n        super().__init__(self)\n        \n        # Add classes\n        for i, name in enumerate(label_names):\n            self.add_class(\"fashion\", i+1, name)\n        \n        # Add images \n        for i, row in df.iterrows():\n            self.add_image(\"fashion\", \n                           image_id=row.name, \n                           path=str(DATA_DIR/'train'/row.name), \n                           labels=row['CategoryId'],\n                           annotations=row['EncodedPixels'], \n                           height=row['Height'], width=row['Width'])\n\n    def image_reference(self, image_id):\n        info = self.image_info[image_id]\n        return info['path'], [label_names[int(x)] for x in info['labels']]\n    \n    def load_image(self, image_id):\n        return resize_image(self.image_info[image_id]['path'])\n\n    def load_mask(self, image_id):\n        info = self.image_info[image_id]\n                \n        mask = np.zeros((IMAGE_SIZE, IMAGE_SIZE, len(info['annotations'])), dtype=np.uint8)\n        labels = []\n        \n        for m, (annotation, label) in enumerate(zip(info['annotations'], info['labels'])):\n            sub_mask = np.full(info['height']*info['width'], 0, dtype=np.uint8)\n            annotation = [int(x) for x in annotation.split(' ')]\n            \n            for i, start_pixel in enumerate(annotation[::2]):\n                sub_mask[start_pixel: start_pixel+annotation[2*i+1]] = 1\n\n            sub_mask = sub_mask.reshape((info['height'], info['width']), order='F')\n            sub_mask = cv2.resize(sub_mask, (IMAGE_SIZE, IMAGE_SIZE), interpolation=cv2.INTER_NEAREST)\n            \n            mask[:, :, m] = sub_mask\n            labels.append(int(label)+1)\n            \n        return mask, np.array(labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = Dataset(image_df)\ndataset.prepare()\n\n\nimage_id = random.choice(dataset.image_ids)\nprint(dataset.image_reference(image_id))\n    \nimage = dataset.load_image(image_id)\nmask, class_ids = dataset.load_mask(image_id)\nvisualize.display_top_masks(image, mask, class_ids, dataset.class_names, limit=4)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Тут будет кроссвалидация"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import KFold\n\nFOLD = 0\nN_FOLDS = 10\n\nkf = KFold(n_splits=N_FOLDS, random_state=42, shuffle=True)\nsplits = kf.split(image_df) # ideally, this should be multilabel stratification\n\ndef get_fold():    \n    for i, (train_index, valid_index) in enumerate(splits):\n        if i == FOLD:\n            return image_df.iloc[train_index], image_df.iloc[valid_index]\n        \ntrain_df, valid_df = get_fold()\n\ntrain_dataset = Dataset(train_df)\ntrain_dataset.prepare()\n\nvalid_dataset = Dataset(valid_df)\nvalid_dataset.prepare()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_segments = np.concatenate(train_df['CategoryId'].values)\nprint(\"Total train images: \", len(train_df))\nprint(\"Total train segments: \", len(train_segments))\n\nplt.figure(figsize=(12, 3))\nvalues, counts = np.unique(train_segments, return_counts=True)\nplt.bar(values, counts)\nplt.xticks(values, label_names, rotation='vertical')\nplt.show()\n\nvalid_segments = np.concatenate(valid_df['CategoryId'].values).astype(int)\nprint(\"Total train images: \", len(valid_df))\nprint(\"Total validation segments: \", len(valid_segments))\n\nplt.figure(figsize=(12, 3))\nvalues, counts = np.unique(valid_segments, return_counts=True)\nplt.bar(values, counts)\nplt.xticks(values, label_names, rotation='vertical')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"LR = 5e-3\nEPOCHS = [2, 6, 8]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Обучаем первую сетку (резнет 50). Выбираем наилучшие показатели, чтобы откатиться потом к тем наборам весов"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_1 = modellib.MaskRCNN(mode='training', config=config, model_dir=ROOT_DIR)\n\nmodel_1.load_weights(COCO_WEIGHTS_PATH, by_name=True, exclude=[\n    'mrcnn_class_logits', 'mrcnn_bbox_fc', 'mrcnn_bbox', 'mrcnn_mask'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"augmentation = iaa.Sequential([\n    iaa.Fliplr(0.5) \n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_1.train(train_dataset, valid_dataset,\n            learning_rate=LR*2, # train heads with higher lr to speedup learning\n            epochs=EPOCHS[0],\n            layers='heads',\n            augmentation=None)\n\nhistory = model_1.keras_model.history.history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_1.train(train_dataset, valid_dataset,\n            learning_rate=LR,\n            epochs=EPOCHS[1],\n            layers='all',\n            augmentation=augmentation)\n\nnew_history = model_1.keras_model.history.history\nfor k in new_history: \n    history[k] = history[k] + new_history[k]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = range(EPOCHS[-1])\n\nplt.figure(figsize=(18, 6))\n\nplt.subplot(131)\nplt.plot(epochs, history['loss'], label=\"train loss\")\nplt.plot(epochs, history['val_loss'], label=\"valid loss\")\nplt.legend()\nplt.subplot(132)\nplt.plot(epochs, history['mrcnn_class_loss'], label=\"train class loss\")\nplt.plot(epochs, history['val_mrcnn_class_loss'], label=\"valid class loss\")\nplt.legend()\nplt.subplot(133)\nplt.plot(epochs, history['mrcnn_mask_loss'], label=\"train mask loss\")\nplt.plot(epochs, history['val_mrcnn_mask_loss'], label=\"valid mask loss\")\nplt.legend()\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_epoch = np.argmin(history[\"val_loss\"]) + 1\nprint(\"Best epoch: \", best_epoch)\nprint(\"Valid loss: \", history[\"val_loss\"][best_epoch-1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"config = Config_('resnet101')\nconfig.display","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Теперь проделываем все то де с архитектурой на 101 слой"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_2 = modellib.MaskRCNN(mode='training', config=config, model_dir=ROOT_DIR)\n\nmodel_2.load_weights(COCO_WEIGHTS_PATH, by_name=True, exclude=[\n    'mrcnn_class_logits', 'mrcnn_bbox_fc', 'mrcnn_bbox', 'mrcnn_mask'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_2.train(train_dataset, valid_dataset,\n            learning_rate=LR*2, # train heads with higher lr to speedup learning\n            epochs=EPOCHS[0],\n            layers='heads',\n            augmentation=None)\n\nhistory = model_2.keras_model.history.history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_2.train(train_dataset, valid_dataset,\n            learning_rate=LR,\n            epochs=EPOCHS[1],\n            layers='all',\n            augmentation=augmentation)\n\nnew_history = model_1.keras_model.history.history\nfor k in new_history: \n    history[k] = history[k] + new_history[k]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = range(EPOCHS[-1])\n\nplt.figure(figsize=(18, 6))\n\nplt.subplot(131)\nplt.plot(epochs, history['loss'], label=\"train loss\")\nplt.plot(epochs, history['val_loss'], label=\"valid loss\")\nplt.legend()\nplt.subplot(132)\nplt.plot(epochs, history['mrcnn_class_loss'], label=\"train class loss\")\nplt.plot(epochs, history['val_mrcnn_class_loss'], label=\"valid class loss\")\nplt.legend()\nplt.subplot(133)\nplt.plot(epochs, history['mrcnn_mask_loss'], label=\"train mask loss\")\nplt.plot(epochs, history['val_mrcnn_mask_loss'], label=\"valid mask loss\")\nplt.legend()\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_epoch = np.argmin(history[\"val_loss\"]) + 1\nprint(\"Best epoch: \", best_epoch)\nprint(\"Valid loss: \", history[\"val_loss\"][best_epoch-1])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}