{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"# consts\npath_1 = \"../input/kuzushiji-recognition/train.csv\"\npath_2 = \"../input/kuzushiji-recognition/train_images/\"\npath_3 = \"../input/kuzushiji-recognition/test_images/\"\npath_4 = \"../input/kuzushiji-recognition/sample_submission.csv\"\ninput_width, input_height = 512, 512\nbase_detect_num_h, base_detect_num_w = 25, 25\ncategory_n = 1\noutput_layer_n = category_n + 4\noutput_height, output_width = 128, 128","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# utils\n\nimport numpy as np\nimport json\nimport pandas as pd\nfrom PIL import Image, ImageDraw\nimport matplotlib.pyplot as plt\nfrom pandas.io.json import json_normalize\nimport random\nimport tensorflow as tf\nfrom sklearn.utils import shuffle\nfrom sklearn.model_selection import KFold,train_test_split\nimport glob\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.layers import Dense,Dropout, Conv2D,Conv2DTranspose, BatchNormalization, Activation,AveragePooling2D,GlobalAveragePooling2D, Input, Concatenate, MaxPool2D, Add, UpSampling2D, LeakyReLU,ZeroPadding2D\nfrom keras.models import Model\nfrom keras.objectives import mean_squared_error\nfrom keras import backend as K\nfrom keras.losses import binary_crossentropy\nfrom keras.callbacks import ModelCheckpoint, EarlyStopping, TensorBoard, ReduceLROnPlateau,LearningRateScheduler\nimport os\nfrom keras.optimizers import Adam, RMSprop, SGD\nimport cv2\n\ndef _get_df_train(train_csv_path):\n    \"\"\"csv_pathを読み込む．Noneデータの除去\n    Parameters\n    ----------\n    train_csv_path: str\n        train.csvまでのpath\n    \n    Returns\n    -------\n    df_train: Dataframe\n        Noneデータ除去した\n    \"\"\"\n    # csvの読み込み\n    df_train = pd.read_csv(train_csv_path)\n    # Noneデータの除去\n    df_train = df_train.dropna(axis=0, how='any')#you can use nan data(page with no letter)\n    # indexのリセット\n    df_train = df_train.reset_index(drop=True)\n    return df_train\n\ndef _get_category_names(df_train):\n    \"\"\"画像ファイル内のカテゴリを抽出\n    Parameters\n    ----------\n    df_train: DataFrame\n        train.csvをpandasで読み込んだ奴\n    \n    Returns\n    -----\n    cotegory_names: set\n        一意なカテゴリ名だけを抽出したもの\n    \"\"\"\n    category_names = set()\n    for i in range(len(df_train)):\n        ann = np.array(df_train.loc[i,\"labels\"].split(\" \")).reshape(-1,5)#cat,x,y,width,height for each picture\n        # 一意なカテゴリ名だけを抽出．\n        category_names = category_names.union({i for i in ann[:,0]})\n    return sorted(category_names)\n\n\ndef _make_category_dict(category_names):\n    \"\"\"カテゴリ名と数字を対応付ける．\n    Parameters\n    ---------\n    category_names: set\n        一意なカテゴリ名のみが入った集合\n    \n    Returns\n    -------\n    dict_cat: dict\n        キーがカテゴリ名，要素に数字の辞書\n    inv_dict_cat: dict\n        キーが数字，要素にカテゴリ名の辞書\n    \"\"\"\n    dict_cat = {list(category_names)[j]:str(j) for j in range(len(category_names))}\n    inv_dict_cat = {str(j): list(category_names)[j] for j in range(len(category_names))}\n    return dict_cat, inv_dict_cat\n\n\n\n\n\ndef calc_aspect_ration(annotation_list_train):\n    aspect_ratio_pic_all=[]\n    average_letter_size_all=[]\n    train_input_for_size_estimate=[]\n    resize_dir=\"resized/\"\n    if os.path.exists(resize_dir) == False:\n        os.mkdir(resize_dir)\n    for i in range(len(annotation_list_train)):\n        with Image.open(annotation_list_train[i][0]) as f:\n            width, height = f.size\n            area = width*height\n            aspect_ratio_pic = height/width\n            aspect_ratio_pic_all.append(aspect_ratio_pic)\n            letter_size = annotation_list_train[i][1][:,3]*annotation_list_train[i][1][:,4]\n            letter_size_ratio = letter_size/area\n        \n            average_letter_size = np.mean(letter_size_ratio)\n            average_letter_size_all.append(average_letter_size)\n            train_input_for_size_estimate.append([annotation_list_train[i][0],np.log(average_letter_size)])#logにしとく\n    return aspect_ratio_pic_all, average_letter_size_all, train_input_for_size_estimate\n\n\ndef calc_aspect_ration_test(id_test):\n    aspect_ratio_pic_all_test = []\n    for i in range(len(id_test)):\n        with Image.open(id_test[i]) as f:\n            width, height=f.size\n            aspect_ratio_pic = height/width\n            aspect_ratio_pic_all_test.append(aspect_ratio_pic)\n    return aspect_ratio_pic_all_test\n\n\ndef aggregation_block(x_shallow, x_deep, deep_ch, out_ch):\n    x_deep = Conv2DTranspose(deep_ch, kernel_size=2, strides=2, padding='same', use_bias=False)(x_deep)\n    x_deep = BatchNormalization()(x_deep)   \n    x_deep = LeakyReLU(alpha=0.1)(x_deep)\n    x = Concatenate()([x_shallow, x_deep])\n    x = Conv2D(out_ch, kernel_size=1, strides=1, padding=\"same\")(x)\n    x = BatchNormalization()(x)   \n    x = LeakyReLU(alpha=0.1)(x)\n    return x\n\n\ndef cbr(x, out_layer, kernel, stride):\n    x = Conv2D(out_layer, kernel_size=kernel, strides=stride, padding=\"same\")(x)\n    x = BatchNormalization()(x)\n    x = LeakyReLU(alpha=0.1)(x)\n    return x\n\n\ndef resblock(x_in,layer_n):\n    x=cbr(x_in,layer_n,3,1)\n    x=cbr(x,layer_n,3,1)\n    x=Add()([x,x_in])\n    return x\n\n\ndef create_model(input_shape, size_detection_mode=True, aggregation=True):\n\n    input_layer = Input(input_shape)\n    \n    #resized input\n    input_layer_1=AveragePooling2D(2)(input_layer)\n    input_layer_2=AveragePooling2D(2)(input_layer_1)\n\n    #### ENCODER ####\n    x_0 = cbr(input_layer, 16, 3, 2)#512->256\n    concat_1 = Concatenate()([x_0, input_layer_1])\n\n    x_1 = cbr(concat_1, 32, 3, 2)#256->128\n    concat_2 = Concatenate()([x_1, input_layer_2])\n\n    x_2 = cbr(concat_2, 64, 3, 2)#128->64\n    x = cbr(x_2,64,3,1)\n    x = resblock(x,64)\n    x = resblock(x,64)\n    \n    x_3 = cbr(x, 128, 3, 2)#64->32\n    x = cbr(x_3, 128, 3, 1)\n    x = resblock(x,128)\n    x = resblock(x,128)\n    x = resblock(x,128)\n    \n    x_4 = cbr(x, 256, 3, 2)#32->16\n    x = cbr(x_4, 256, 3, 1)\n    x = resblock(x,256)\n    x = resblock(x,256)\n    x = resblock(x,256)\n    x = resblock(x,256)\n    x = resblock(x,256)\n \n    x_5 = cbr(x, 512, 3, 2)#16->8\n    x = cbr(x_5, 512, 3, 1)\n    x = resblock(x,512)\n    x = resblock(x,512)\n    x = resblock(x,512)\n    \n    if size_detection_mode:\n        x = GlobalAveragePooling2D()(x)\n        x = Dropout(0.2)(x)\n        out = Dense(1,activation=\"linear\")(x)\n      \n    else:#centernet mode\n    #### DECODER ####\n        x_1 = cbr(x_1, output_layer_n, 1, 1)\n        x_1 = aggregation_block(x_1, x_2, output_layer_n, output_layer_n)\n        x_2 = cbr(x_2, output_layer_n, 1, 1)\n        x_2 = aggregation_block(x_2, x_3, output_layer_n, output_layer_n)\n        x_1 = aggregation_block(x_1, x_2, output_layer_n, output_layer_n)\n        x_3 = cbr(x_3, output_layer_n, 1, 1)\n        x_3 = aggregation_block(x_3, x_4, output_layer_n, output_layer_n) \n        x_2 = aggregation_block(x_2, x_3, output_layer_n, output_layer_n)\n        x_1 = aggregation_block(x_1, x_2, output_layer_n, output_layer_n)\n\n        x_4 = cbr(x_4, output_layer_n, 1, 1)\n\n        x = cbr(x, output_layer_n, 1, 1)\n        x = UpSampling2D(size=(2, 2))(x)#8->16 tconvのがいいか\n\n        x = Concatenate()([x, x_4])\n        x = cbr(x, output_layer_n, 3, 1)\n        x = UpSampling2D(size=(2, 2))(x)#16->32\n\n        x = Concatenate()([x, x_3])\n        x = cbr(x, output_layer_n, 3, 1)\n        x = UpSampling2D(size=(2, 2))(x)#32->64   128のがいいかも？ \n\n        x = Concatenate()([x, x_2])\n        x = cbr(x, output_layer_n, 3, 1)\n        x = UpSampling2D(size=(2, 2))(x)#64->128 \n\n        x = Concatenate()([x, x_1])\n        x = Conv2D(output_layer_n, kernel_size=3, strides=1, padding=\"same\")(x)\n        out = Activation(\"sigmoid\")(x)\n    \n    model = Model(input_layer, out)\n    \n    return model\n\ndef create_classification_model(input_shape, n_category):\n    input_layer = Input(input_shape)#32\n    x = cbr(input_layer,64,3,1)\n    x = resblock(x,64)\n    x = resblock(x,64)\n    x = cbr(input_layer,128,3,2)#16\n    x = resblock(x,128)\n    x = resblock(x,128)\n    x = cbr(input_layer,256,3,2)#8\n    x = resblock(x,256)\n    x = resblock(x,256)\n    x = GlobalAveragePooling2D()(x)\n    x = Dropout(0.2)(x)\n    out = Dense(n_category, activation=\"softmax\")(x)#sigmoid???catcrossていぎ\n    \n    classification_model = Model(input_layer, out)\n    \n    return classification_model","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# step1 function\n\ndef _make_annotation_list_train(df_train, dict_cat):\n    \"\"\"画像ファイル名と，アノテーション情報を一つにしたlistを作成する\n    Parameters\n    ----------\n    df_train: DataFrame\n        train.csvを読み込んだやつ\n    dict_cat: dict\n    Returns\n    ------\n    annotation_list_train: list\n        画像ファイル名とアノテーション情報を一つにしたリスト\n    \"\"\"\n    annotation_list_train = []\n    for i in range(len(df_train)):\n        # category, x, y, width, heightの順で配列を作成．\n        ann = np.array(df_train.loc[i, \"labels\"].split(\" \")).reshape(-1, 5)#cat,left,top,width,height for each picture\n        for j,category_name in enumerate(ann[:, 0]):\n            ann[j, 0] = int(dict_cat[category_name])  \n        ann=ann.astype('int32')\n        ann[:, 1] += ann[:, 3]//2 # center_x\n        ann[:, 2] += ann[:, 4]//2 # center_y\n        annotation_list_train.append([\"{}{}.jpg\".format(path_2, df_train.loc[i, \"image_id\"]), ann])\n    return annotation_list_train\n\n\ndef lrs(epoch):\n    lr = 0.0005\n    if epoch > 10:\n        lr = 0.0001\n    return lr\n\n\ndef Datagen_sizecheck_model(filenames, batch_size, size_detection_mode=True, is_train=True,random_crop=True):\n    x = []\n    y = []\n    count = 0\n\n    while True:\n        for i in range(len(filenames)):\n            if random_crop:\n                crop_ratio=np.random.uniform(0.7,1)\n            else:\n                crop_ratio=1\n            with Image.open(filenames[i][0]) as f:\n               #random crop \n                if random_crop and is_train:\n                    pic_width,pic_height=f.size\n                    f=np.asarray(f.convert('RGB'),dtype=np.uint8)\n                    top_offset=np.random.randint(0,pic_height-int(crop_ratio*pic_height))\n                    left_offset=np.random.randint(0,pic_width-int(crop_ratio*pic_width))\n                    bottom_offset=top_offset+int(crop_ratio*pic_height)\n                    right_offset=left_offset+int(crop_ratio*pic_width)\n                    f=cv2.resize(f[top_offset:bottom_offset,left_offset:right_offset,:],(input_height, input_width))\n                else:\n                    f=f.resize((input_width, input_height))\n                    f=np.asarray(f.convert('RGB'),dtype=np.uint8)          \n                x.append(f)\n\n      \n            if random_crop and is_train:\n                y.append(filenames[i][1]-np.log(crop_ratio))\n            else:\n                y.append(filenames[i][1])\n\n            count+=1\n            if count==batch_size:\n                x=np.array(x, dtype=np.float32)\n                y=np.array(y, dtype=np.float32)\n\n                inputs=x/255\n                targets=y       \n                x=[]\n                y=[]\n                count=0\n                yield inputs, targets\n\n\ndef model_fit_sizecheck_model(model, train_list, cv_list, n_epoch, batch_size=32):\n    hist = model.fit_generator(\n        Datagen_sizecheck_model(train_list, batch_size, is_train=True, random_crop=True),\n        steps_per_epoch = len(train_list) // batch_size,\n        epochs = n_epoch,\n        validation_data = Datagen_sizecheck_model(cv_list, batch_size, is_train=False,random_crop=False),\n        validation_steps = len(cv_list) // batch_size,\n        callbacks = [lr_schedule, model_checkpoint], #[early_stopping, reduce_lr, model_checkpoint],\n        shuffle = True,\n        verbose = 1\n    )\n    return hist\n\n\ndef calc_annotation_list_train_w_split(model, train_input_for_size_estimate, aspect_ratio_pic_all, annotation_list_train, batch_size=1):\n    predict_train = model.predict_generator(Datagen_sizecheck_model(train_input_for_size_estimate,batch_size, \n                                                                    is_train=False, random_crop=False, ), steps=len(train_input_for_size_estimate)//batch_size)\n\n    annotation_list_train_w_split = []\n\n    for i, predicted_size in enumerate(predict_train):\n        detect_num_h = aspect_ratio_pic_all[i]*np.exp(-predicted_size/2)\n        detect_num_w = detect_num_h/aspect_ratio_pic_all[i]\n        h_split_recommend = np.maximum(1, detect_num_h/base_detect_num_h)\n        w_split_recommend = np.maximum(1, detect_num_w/base_detect_num_w)\n        annotation_list_train_w_split.append([annotation_list_train[i][0], annotation_list_train[i][1], h_split_recommend, w_split_recommend])\n    return annotation_list_train_w_split","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# step1 main\ndf_train = _get_df_train(path_1)\ncategory_names = _get_category_names(df_train)\ndict_cat, inv_dict_cat = _make_category_dict(category_names)\nannotation_list_train = _make_annotation_list_train(df_train, dict_cat)\naspect_ratio_pic_all, average_letter_size_all, train_input_for_size_estimate = calc_aspect_ration(annotation_list_train)\n\nK.clear_session()\nmodel = create_model(input_shape=(input_height, input_width, 3), size_detection_mode=True)\n\nlr_schedule = LearningRateScheduler(lrs)\nmodel_checkpoint = ModelCheckpoint(\"final_weights_step1.h5\", monitor = 'val_loss', verbose = 1,\n                                   save_best_only = True, save_weights_only = True, period = 1)\nprint(model.summary())\n\n\ntrain_list, cv_list = train_test_split(train_input_for_size_estimate, random_state = 111, test_size = 0.2)\nlearning_rate = 0.0005\nn_epoch = 1\nbatch_size = 32\nmodel.compile(loss=mean_squared_error, optimizer=Adam(lr=learning_rate))\nhist = model_fit_sizecheck_model(model, train_list, cv_list, n_epoch, batch_size)\n\nmodel.save_weights('final_weights_step1.h5')\n\n\n# predict = model.predict_generator(Datagen_sizecheck_model(cv_list, batch_size, is_train=False, random_crop=False), steps=len(cv_list) // batch_size)\n# target = [cv[1] for cv in cv_list]\n# plt.scatter(predict, target[:len(predict)])\n# plt.title('---letter_size/picture_size--- estimated vs target ', loc='center', fontsize=10)\n# plt.show()\n\n\nannotation_list_train_w_split = calc_annotation_list_train_w_split(model, train_input_for_size_estimate, \n                                                                   aspect_ratio_pic_all, annotation_list_train)\n\nprint(\"recommended height split:{}, recommended width_split:{}\".format(annotation_list_train_w_split[0][2], annotation_list_train_w_split[0][3]))\nimg = np.asarray(Image.open(annotation_list_train_w_split[0][0]).convert('RGB'))\nplt.imshow(img)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# step2 function\ndef Datagen_centernet(filenames, batch_size):\n    x=[]\n    y=[]\n\n    count=0\n\n    while True:\n        for i in range(len(filenames)):\n            h_split=filenames[i][2]\n            w_split=filenames[i][3]\n            max_crop_ratio_h=1/h_split\n            max_crop_ratio_w=1/w_split\n            crop_ratio=np.random.uniform(0.5,1)\n            crop_ratio_h=max_crop_ratio_h*crop_ratio\n            crop_ratio_w=max_crop_ratio_w*crop_ratio\n            \n            with Image.open(filenames[i][0]) as f:\n        \n        #random crop\n        \n                pic_width,pic_height=f.size\n                f=np.asarray(f.convert('RGB'),dtype=np.uint8)\n                top_offset=np.random.randint(0,pic_height-int(crop_ratio_h*pic_height))\n                left_offset=np.random.randint(0,pic_width-int(crop_ratio_w*pic_width))\n                bottom_offset=top_offset+int(crop_ratio_h*pic_height)\n                right_offset=left_offset+int(crop_ratio_w*pic_width)\n                f=cv2.resize(f[top_offset:bottom_offset,left_offset:right_offset,:],(input_height, input_width))\n                x.append(f)      \n\n            output_layer=np.zeros((output_height, output_width,(output_layer_n+category_n)))\n            for annotation in filenames[i][1]:\n                x_c=(annotation[1]-left_offset)*(output_width/int(crop_ratio_w*pic_width))\n                y_c=(annotation[2]-top_offset)*(output_height/int(crop_ratio_h*pic_height))\n                width=annotation[3]*(output_width/int(crop_ratio_w*pic_width))\n                height=annotation[4]*(output_height/int(crop_ratio_h*pic_height))\n                top=np.maximum(0,y_c-height/2)\n                left=np.maximum(0,x_c-width/2)\n                bottom=np.minimum(output_height,y_c+height/2)\n                right=np.minimum(output_width,x_c+width/2)\n          \n                if top>=(output_height-0.1) or left>=(output_width-0.1) or bottom<=0.1 or right<=0.1:#random crop(out of picture)\n                    continue\n                width=right-left\n                height=bottom-top\n                x_c=(right+left)/2\n                y_c=(top+bottom)/2\n\n        \n                category = 0#not classify, just detect\n                heatmap=((np.exp(-(((np.arange(output_width)-x_c)/(width/10))**2)/2)).reshape(1,-1)\n                                    *(np.exp(-(((np.arange(output_height)-y_c)/(height/10))**2)/2)).reshape(-1,1))\n                output_layer[:, :, category]=np.maximum(output_layer[:, :, category], heatmap[:, :])\n                output_layer[int(y_c//1), int(x_c//1), category_n+category]=1\n                output_layer[int(y_c//1), int(x_c//1), 2*category_n]=y_c%1#height offset\n                output_layer[int(y_c//1), int(x_c//1), 2*category_n+1]=x_c%1\n                output_layer[int(y_c//1), int(x_c//1), 2*category_n+2]=height/output_height\n                output_layer[int(y_c//1), int(x_c//1), 2*category_n+3]=width/output_width\n            y.append(output_layer)\n    \n            count += 1\n            if count == batch_size:\n                x=np.array(x, dtype=np.float32)\n                y=np.array(y, dtype=np.float32)\n\n                inputs=x/255\n                targets=y       \n                x=[]\n                y=[]\n                count=0\n                yield inputs, targets\n\ndef all_loss(y_true, y_pred):\n    mask=K.sign(y_true[..., 2*category_n+2])\n    N=K.sum(mask)\n    alpha=2.\n    beta=4.\n\n    heatmap_true_rate = K.flatten(y_true[..., :category_n])\n    heatmap_true = K.flatten(y_true[..., category_n:(2*category_n)])\n    heatmap_pred = K.flatten(y_pred[..., :category_n])\n    heatloss=-K.sum(heatmap_true*((1-heatmap_pred)**alpha)*K.log(heatmap_pred+1e-6)+(1-heatmap_true)*((1-heatmap_true_rate)**beta)*(heatmap_pred**alpha)*K.log(1-heatmap_pred+1e-6))\n    offsetloss=K.sum(K.abs(y_true[...,2*category_n]-y_pred[...,category_n]*mask)+K.abs(y_true[...,2*category_n+1]-y_pred[...,category_n+1]*mask))\n    sizeloss=K.sum(K.abs(y_true[...,2*category_n+2]-y_pred[...,category_n+2]*mask)+K.abs(y_true[...,2*category_n+3]-y_pred[...,category_n+3]*mask))\n    \n    all_loss=(heatloss+1.0*offsetloss+5.0*sizeloss)/N\n    return all_loss\n\ndef size_loss(y_true, y_pred):\n    mask=K.sign(y_true[...,2*category_n+2])\n    N=K.sum(mask)\n    sizeloss=K.sum(K.abs(y_true[...,2*category_n+2]-y_pred[..., category_n+2]*mask)+K.abs(y_true[...,2*category_n+3]-y_pred[...,category_n+3]*mask))\n    return (5*sizeloss)/N\n\ndef offset_loss(y_true, y_pred):\n    mask=K.sign(y_true[...,2*category_n+2])\n    N=K.sum(mask)\n    offsetloss=K.sum(K.abs(y_true[...,2*category_n]-y_pred[...,category_n]*mask)+K.abs(y_true[...,2*category_n+1]-y_pred[...,category_n+1]*mask))\n    return (offsetloss)/N\n  \ndef heatmap_loss(y_true, y_pred):\n    mask=K.sign(y_true[...,2*category_n+2])\n    N=K.sum(mask)\n    alpha=2.\n    beta=4.\n\n    heatmap_true_rate = K.flatten(y_true[..., :category_n])\n    heatmap_true = K.flatten(y_true[..., category_n:(2*category_n)])\n    heatmap_pred = K.flatten(y_pred[..., :category_n])\n    heatloss=-K.sum(heatmap_true*((1-heatmap_pred)**alpha)*K.log(heatmap_pred+1e-6)+(1-heatmap_true)*((1-heatmap_true_rate)**beta)*(heatmap_pred**alpha)*K.log(1-heatmap_pred+1e-6))\n    return heatloss/N\n\n  \ndef model_fit_centernet(model,train_list,cv_list,n_epoch,batch_size=32):\n    hist = model.fit_generator(\n        Datagen_centernet(train_list,batch_size),\n        steps_per_epoch = len(train_list) // batch_size,\n        epochs = n_epoch,\n        validation_data=Datagen_centernet(cv_list,batch_size),\n        validation_steps = len(cv_list) // batch_size,\n        callbacks = [lr_schedule],#early_stopping, reduce_lr, model_checkpoint],\n        shuffle = True,\n        verbose = 1\n    )\n    return hist\n\n\ndef NMS_all(predicts, category_n, score_thresh, iou_thresh):\n    y_c = predicts[..., category_n]+np.arange(pred_out_h).reshape(-1,1)\n    x_c = predicts[..., category_n+1]+np.arange(pred_out_w).reshape(1,-1)\n    height = predicts[..., category_n+2]*pred_out_h\n    width = predicts[..., category_n+3]*pred_out_w\n\n    count = 0\n    for category in range(category_n):\n        predict = predicts[..., category]\n        mask = (predict>score_thresh)\n        #print(\"box_num\",np.sum(mask))\n        if mask.all == False:\n            continue\n        box_and_score = NMS(predict[mask], y_c[mask], x_c[mask], height[mask], width[mask], iou_thresh)\n        box_and_score = np.insert(box_and_score, 0, category, axis=1)#category,score,top,left,bottom,right\n        if count == 0:\n            box_and_score_all = box_and_score\n        else:\n            box_and_score_all = np.concatenate((box_and_score_all, box_and_score), axis=0)\n        count += 1\n    score_sort = np.argsort(box_and_score_all[:, 1])[::-1]\n    box_and_score_all = box_and_score_all[score_sort]\n    #print(box_and_score_all)\n\n    _, unique_idx = np.unique(box_and_score_all[:, 2], return_index=True)\n    #print(unique_idx)\n    return box_and_score_all[sorted(unique_idx)]\n  \ndef NMS(score, y_c, x_c, height, width, iou_thresh, merge_mode=False):\n    if merge_mode:\n        score = score\n        top = y_c\n        left = x_c\n        bottom = height\n        right = width\n    else:\n        #flatten\n        score = score.reshape(-1)\n        y_c = y_c.reshape(-1)\n        x_c = x_c.reshape(-1)\n        height = height.reshape(-1)\n        width = width.reshape(-1)\n        size = height*width\n        \n        top = y_c - height/2\n        left = x_c - width/2\n        bottom = y_c + height/2\n        right = x_c + width/2\n        \n        inside_pic = (top>0)*(left>0)*(bottom<pred_out_h)*(right<pred_out_w)\n        outside_pic = len(inside_pic) - np.sum(inside_pic)\n        #if outside_pic>0:\n        #  print(\"{} boxes are out of picture\".format(outside_pic))\n        normal_size = (size<(np.mean(size)*10))*(size>(np.mean(size)/10))\n        score = score[inside_pic*normal_size]\n        top = top[inside_pic*normal_size]\n        left = left[inside_pic*normal_size]\n        bottom = bottom[inside_pic*normal_size]\n        right = right[inside_pic*normal_size]\n  \n  #sort  \n    score_sort = np.argsort(score)[::-1]\n    score = score[score_sort]  \n    top = top[score_sort]\n    left = left[score_sort]\n    bottom = bottom[score_sort]\n    right = right[score_sort]\n    \n    area = ((bottom-top)*(right-left))\n    \n    boxes = np.concatenate((score.reshape(-1, 1), top.reshape(-1, 1), left.reshape(-1, 1), bottom.reshape(-1, 1), right.reshape(-1, 1)), axis=1)\n    \n    box_idx = np.arange(len(top))\n    alive_box = []\n    while len(box_idx)>0:\n  \n        alive_box.append(box_idx[0])\n        \n        y1 = np.maximum(top[0], top)\n        x1 = np.maximum(left[0], left)\n        y2 = np.minimum(bottom[0], bottom)\n        x2 = np.minimum(right[0], right)\n        \n        cross_h = np.maximum(0, y2-y1)\n        cross_w = np.maximum(0, x2-x1)\n        still_alive = (((cross_h*cross_w)/area[0])<iou_thresh)\n        if np.sum(still_alive) == len(box_idx):\n            print(\"error\")\n            print(np.max((cross_h*cross_w)), area[0])\n        top = top[still_alive]\n        left = left[still_alive]\n        bottom = bottom[still_alive]\n        right = right[still_alive]\n        area = area[still_alive]\n        box_idx = box_idx[still_alive]\n    return boxes[alive_box] #score, top, left, bottom, right\n\n\ndef draw_rectangle(box_and_score, img, color):\n    number_of_rect=np.minimum(500, len(box_and_score))\n  \n    for i in reversed(list(range(number_of_rect))):\n        top, left, bottom, right = box_and_score[i, :]\n    \n        top = np.floor(top + 0.5).astype('int32')\n        left = np.floor(left + 0.5).astype('int32')\n        bottom = np.floor(bottom + 0.5).astype('int32')\n        right = np.floor(right + 0.5).astype('int32')\n        #label = '{} {:.2f}'.format(predicted_class, score)\n        #print(label)\n        #rectangle=np.array([[left, top], [left, bottom], [right, bottom], [right, top]])\n\n        draw = ImageDraw.Draw(img)\n        #label_size = draw.textsize(label)\n        #print(label_size)\n\n        #if top - label_size[1] >= 0:\n        #  text_origin = np.array([left, top - label_size[1]])\n        #else:\n        #  text_origin = np.array([left, top + 1])\n\n        thickness = 4\n        if color == \"red\":\n            rect_color = (255, 0, 0)\n        elif color == \"blue\":\n            rect_color = (0, 0, 255)\n        else:\n            rect_color = (0, 0, 0)\n\n\n        if i == 0:\n            thickness = 4\n        for j in range(2*thickness):#薄いから何重にか描く\n            draw.rectangle([left + j, top + j, right - j, bottom - j], outline = rect_color)\n        #draw.rectangle(\n        #            [tuple(text_origin), tuple(text_origin + label_size)],\n        #            fill=(0, 0, 255))\n        #draw.text(text_origin, label, fill=(0, 0, 0))\n\n    del draw\n    return img\n\n\ndef check_iou_score(true_boxes, detected_boxes, iou_thresh):\n    iou_all = []\n    for detected_box in detected_boxes:\n        y1 = np.maximum(detected_box[0], true_boxes[:, 0])\n        x1 = np.maximum(detected_box[1], true_boxes[:, 1])\n        y2 = np.minimum(detected_box[2], true_boxes[:, 2])\n        x2 = np.minimum(detected_box[3], true_boxes[:, 3])\n        \n        cross_section = np.maximum(0, y2-y1)*np.maximum(0, x2-x1)\n        all_area = (detected_box[2]-detected_box[0])*(detected_box[3]-detected_box[1])+(true_boxes[:, 2]-true_boxes[:, 0])*(true_boxes[:, 3]-true_boxes[:, 1])\n        iou = np.max(cross_section/(all_area-cross_section))\n        #argmax=np.argmax(cross_section/(all_area-cross_section))\n        iou_all.append(iou)\n    score = 2*np.sum(iou_all)/(len(detected_boxes)+len(true_boxes))\n    print(\"score:{}\".format(np.round(score, 3)))\n    return score\n\n\ndef split_and_detect(model,img,height_split_recommended,width_split_recommended,score_thresh=0.3,iou_thresh=0.4):\n    width,height=img.size\n    pred_in_w,pred_in_h=512,512\n    pred_out_w,pred_out_h=128,128\n    maxlap=0.5\n    height_split=int(-(-height_split_recommended//1)+1)\n    width_split=int(-(-width_split_recommended//1)+1)\n    height_lap=(height_split-height_split_recommended)/(height_split-1)\n    height_lap=np.minimum(maxlap,height_lap)\n    width_lap=(width_split-width_split_recommended)/(width_split-1)\n    width_lap=np.minimum(maxlap,width_lap)\n\n    if height>width:\n        crop_size=int((height)/(height_split-(height_split-1)*height_lap))#crop_height and width\n        if crop_size>=width:\n            crop_size=width\n            stride=int((crop_size*height_split-height)/(height_split-1))\n            top_list=[i*stride for i in range(height_split-1)]+[height-crop_size]\n            left_list=[0]\n        else:\n            stride=int((crop_size*height_split-height)/(height_split-1))\n            top_list=[i*stride for i in range(height_split-1)]+[height-crop_size]\n            width_split=-(-width//crop_size)\n            stride=int((crop_size*width_split-width)/(width_split-1))\n            left_list=[i*stride for i in range(width_split-1)]+[width-crop_size]\n        \n    else:\n        crop_size=int((width)/(width_split-(width_split-1)*width_lap))#crop_height and width\n        if crop_size>=height:\n            crop_size=height\n            stride=int((crop_size*width_split-width)/(width_split-1))\n            left_list=[i*stride for i in range(width_split-1)]+[width-crop_size]\n            top_list=[0]\n        else:\n            stride=int((crop_size*width_split-width)/(width_split-1))\n            left_list=[i*stride for i in range(width_split-1)]+[width-crop_size]\n            height_split=-(-height//crop_size)\n            stride=int((crop_size*height_split-height)/(height_split-1))\n            top_list=[i*stride for i in range(height_split-1)]+[height-crop_size]\n\n    count=0\n\n    for top_offset in top_list:\n        for left_offset in left_list:\n            img_crop = img.crop((left_offset, top_offset, left_offset+crop_size, top_offset+crop_size))\n            predict=model.predict((np.asarray(img_crop.resize((pred_in_w,pred_in_h))).reshape(1,pred_in_h,pred_in_w,3))/255).reshape(pred_out_h,pred_out_w,(category_n+4))\n            \n            box_and_score=NMS_all(predict,category_n,score_thresh,iou_thresh)#category,score,top,left,bottom,right\n            \n            #print(\"after NMS\",len(box_and_score))\n            if len(box_and_score)==0:\n                continue\n            #reshape and offset\n            box_and_score=box_and_score*[1,1,crop_size/pred_out_h,crop_size/pred_out_w,crop_size/pred_out_h,crop_size/pred_out_w]+np.array([0,0,top_offset,left_offset,top_offset,left_offset])\n        \n            if count==0:\n                box_and_score_all=box_and_score\n            else:\n                box_and_score_all=np.concatenate((box_and_score_all,box_and_score),axis=0)\n        count+=1\n        #print(\"all_box_num:\",len(box_and_score_all))\n        #print(box_and_score_all[:10,:],np.min(box_and_score_all[:,2:]))\n    if count==0:\n        box_and_score_all=[]\n    else:\n        score=box_and_score_all[:,1]\n        y_c=(box_and_score_all[:,2]+box_and_score_all[:,4])/2\n        x_c=(box_and_score_all[:,3]+box_and_score_all[:,5])/2\n        height=-box_and_score_all[:,2]+box_and_score_all[:,4]\n        width=-box_and_score_all[:,3]+box_and_score_all[:,5]\n        #print(np.min(height),np.min(width))\n        box_and_score_all=NMS(box_and_score_all[:,1],box_and_score_all[:,2],box_and_score_all[:,3],box_and_score_all[:,4],box_and_score_all[:,5],iou_thresh=0.5,merge_mode=True)\n    return box_and_score_all\n\ndef lrs(epoch):\n    lr = 0.001\n    if epoch >= 20: lr = 0.0002\n    return lr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = _get_df_train(path_1)\ncategory_names = _get_category_names(df_train)\ndict_cat, inv_dict_cat = _make_category_dict(category_names)\nannotation_list_train = _make_annotation_list_train(df_train, dict_cat)\naspect_ratio_pic_all, average_letter_size_all, train_input_for_size_estimate = calc_aspect_ration(annotation_list_train)\n\nK.clear_session()\nmodel_1 = create_model(input_shape=(input_height, input_width, 3), size_detection_mode=True)\nmodel_1.load_weights('final_weights_step1.h5', by_name=True, skip_mismatch=True)\nlr_schedule = LearningRateScheduler(lrs)\n\"\"\"    h_split = annotation_list_train_w_split[0][2]\nw_split = annotation_list_train_w_split[0][3]\nmax_crop_ratio_h = 1 / h_split\nmax_crop_ratio_w = 1 / w_split\ncrop_ratio = np.random.uniform(0.5, 1)\ncrop_ratio_h = max_crop_ratio_h * crop_ratio\ncrop_ratio_w = max_crop_ratio_w * crop_ratio\"\"\"\n\nannotation_list_train_w_split = calc_annotation_list_train_w_split(model_1, train_input_for_size_estimate, aspect_ratio_pic_all, annotation_list_train)\nprint(annotation_list_train_w_split[0][2])\nprint(annotation_list_train_w_split[0][3])\n\ntrain_list, cv_list = train_test_split(annotation_list_train_w_split, random_state = 111,test_size = 0.2)#stratified split is better\n\nmodel_2 = create_model(input_shape=(input_height, input_width, 3), size_detection_mode=False)\nmodel_2.load_weights('final_weights_step1.h5', by_name=True, skip_mismatch=True)\n\nlearning_rate = 0.001\nn_epoch = 1\nbatch_size = 32\nmodel_2.compile(loss=all_loss, optimizer=Adam(lr=learning_rate), metrics=[heatmap_loss, size_loss, offset_loss])\nhist = model_fit_centernet(model_2, train_list, cv_list, n_epoch, batch_size)\n\nmodel_2.save_weights('final_weights_step2.h5')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_in_h = 512\npred_in_w = 512\npred_out_h = int(pred_in_h / 4)\npred_out_w = int(pred_in_w / 4)\n\nfor i in np.arange(0, 5):\n    # print(cv_list[i][2:])\n    img = Image.open(cv_list[i][0]).convert(\"RGB\")\n    width, height = img.size\n    predict = model_2.predict((np.asarray(img.resize((pred_in_w, pred_in_h))).reshape(1, pred_in_h, pred_in_w, 3))/255).reshape(pred_out_h, pred_out_w, (category_n+4))\n\n    box_and_score = NMS_all(predict, category_n, score_thresh=0.3, iou_thresh=0.4)\n\n    # print(\"after NMS\", len(box_and_score))\n    if len(box_and_score)==0:\n        continue\n\n    true_boxes = cv_list[i][1][:, 1:] # c_x, c_y, width_height\n    top = true_boxes[:, 1:2]-true_boxes[:, 3:4]/2\n    left = true_boxes[:, 0:1]-true_boxes[:, 2:3]/2\n    bottom = top+true_boxes[:, 3:4]\n    right = left+true_boxes[:, 2:3]\n    true_boxes = np.concatenate((top, left, bottom, right), axis=1)\n\n    heatmap = predict[:, :, 0]\n\n    print_w, print_h = img.size\n    #resize predocted box to original size\n    box_and_score = box_and_score*[1, 1, print_h/pred_out_h, print_w/pred_out_w, print_h/pred_out_h, print_w/pred_out_w]\n    print(box_and_score)\n    check_iou_score(true_boxes, box_and_score[:, 2:], iou_thresh=0.5)\n    img = draw_rectangle(box_and_score[:, 2:], img, \"red\")\n    img = draw_rectangle(true_boxes, img, \"blue\")\n\n    fig, axes = plt.subplots(1,  2, figsize=(15, 15))\n    #axes[0].set_axis_off()\n    axes[0].imshow(img)\n    #axes[1].set_axis_off()\n    axes[1].imshow(heatmap)#, cmap='gray')\n    #axes[2].set_axis_off()\n    #axes[2].imshow(heatmap_1)#, cmap='gray')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"test run. 5 image\")\nall_iou_score = []\nfor i in np.arange(0, 5):\n    img = Image.open(cv_list[i][0]).convert(\"RGB\")\n    box_and_score_all = split_and_detect(model_2, img, cv_list[i][2], cv_list[i][3], score_thresh=0.3, iou_thresh=0.4)\n    if len(box_and_score_all) == 0:\n        print(\"no box found\")\n        continue\n    true_boxes = cv_list[i][1][:, 1:] # c_x, c_y, width_height\n    top = true_boxes[:, 1:2] - true_boxes[:, 3:4]/2\n    left = true_boxes[:, 0:1] - true_boxes[:, 2:3]/2\n    bottom = top + true_boxes[:, 3:4]\n    right = left + true_boxes[:, 2:3]\n    true_boxes = np.concatenate((top, left, bottom, right), axis=1)\n\n\n    print_w, print_h = img.size\n    iou_score = check_iou_score(true_boxes, box_and_score_all[:, 1:], iou_thresh=0.5)\n    all_iou_score.append(iou_score)\n#   \"\"\"\n#   img=draw_rectangle(box_and_score_all[:,1:],img,\"red\")\n#   img=draw_rectangle(true_boxes,img,\"blue\")\n\n#   fig, axes = plt.subplots(1, 2,figsize=(15,15))\n#   #axes[0].set_axis_off()\n#   axes[0].imshow(img)\n#   #axes[1].set_axis_off()\n#   axes[1].imshow(heatmap)#, cmap='gray')\n\n#   plt.show()\n#   \"\"\"\nprint(\"average_score:\", np.mean(all_iou_score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submission = pd.read_csv(path_4)\nid_test=path_3 + df_submission[\"image_id\"]+\".jpg\"\n\n\ndef pipeline(i, print_img=False):\n    # model1: determine how to split image\n    if print_img: print(\"model 1\")\n    img = np.asarray(Image.open(id_test[i]).resize((512,512)).convert('RGB'))\n    predicted_size = model_1.predict(img.reshape(1,512,512,3)/255)\n    detect_num_h = aspect_ratio_pic_all_test[i]*np.exp(-predicted_size/2)\n    detect_num_w = detect_num_h/aspect_ratio_pic_all_test[i]\n    h_split_recommend = np.maximum(1, detect_num_h/base_detect_num_h)\n    w_split_recommend = np.maximum(1, detect_num_w/base_detect_num_w)\n    if print_img: print(\"recommended split_h:{}, split_w:{}\".format(h_split_recommend,w_split_recommend))\n\n    # model2: detection\n    if print_img: print(\"model 2\")\n    img = Image.open(id_test[i]).convert(\"RGB\")\n    box_and_score_all = split_and_detect(model_2,img,h_split_recommend, w_split_recommend, score_thresh=0.3, iou_thresh=0.4)# output:score, top, left, bottom, right\n    if print_img: print(\"find {} boxes\".format(len(box_and_score_all)))\n    print_w, print_h = img.size\n    if (len(box_and_score_all) > 0) and print_img: \n        img = draw_rectangle(box_and_score_all[:, 1:], img,\"red\")\n        plt.imshow(img)\n        plt.savefig('bounding_box_test_images/' + id_test[i] + '.jpg')\n        \n    if (len(box_and_score_all) > 0):\n        box_all = box_and_score_all[:, 1:]\n    else:\n        box_all = []\n\n    return box_all","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"K.clear_session()\nprint(\"loading models...\")\nmodel_1 = create_model(input_shape=(512,512,3),size_detection_mode=True)\nmodel_1.load_weights('final_weights_step1.h5')\n\nmodel_2 = create_model(input_shape=(512,512,3),size_detection_mode=False)\nmodel_2.load_weights('final_weights_step2.h5')\n\nsample_boxes = pipeline(0, print_img=True)\n\n\nfilenames_and_boxes = np.array()\nfor i in tqdm(range(len(id_test))):\n    boxes = pipeline(i,print_img=False)\n    np.append(filenames_and_boxes, [id_test[i], boxes])\nans = []\nfor filename, boxes in filenames_and_boxes:\n    cnt = 0\n    for box in boxes:\n        upper_l_y, upper_l_x, bottom_r_y, bottom_r_x = box\n        ans.append([filename, cnt, upper_l_y, upper_l_x, bottom_r_y, bottom_r_x])\n        cnt+=1\nprint(ans)\n\n\nwith open(\"test_crop_cordinate.csv\", \"w\") as f:\n    writer = csv.writer(f)\n    writer.writerrow(['filename','img_id','upper_left_x','upper_left_y','bottom_right_x'])\n    for row in ans:\n        writer.writerrow(row)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}