{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"markdown","source":"## はじめに\nくずし字コンペは、古書？のくずし字の位置を推定し、それが何の文字であるかを推測するコンペです。  \nどのようなtrain_dataかは目次の「train_data可視化」へ飛んでください。\n\n大きく分けて物体検出と文字認識の２段階に分けて推測します。  \n\n1. 物体検出では、くずし字の位置を推測します。\n使用したアーキテクチャは「centernet」です。  \n次のカーネルをもとに作成しています。 https://www.kaggle.com/kmat2019/centernet-keypoint-detector  \n\n 以下のStepにわかれているようです。  \nStep1. 全体の画像と文字の大きさの比率を予測  \nStep2. step1を元にcropしたあと、centernetで物体検出  \n\n 予測結果は目次の「物体検出予測可視化」から飛んでください\n\n2. 文字認識では、学習済みモデルを使用して約4000ラベルの文字をクラス分類します。  \n使用したアーキテクチャはpnasnet5-leargeです。  \ntransformで正方形にしたり、grayscaleにしたりしています。目次の「transform可視化」から飛んでください。\n予測結果は目次の「文字認識予測結果」から飛んでください。\n\n  ","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## 可視化","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install japanize-matplotlib\n!pip install cnn_finetune","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import matplotlib\nimport pandas as pd\nimport numpy as np\nfrom PIL import Image\nfrom pylab import rcParams\nimport os\nimport json\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport glob\nimport japanize_matplotlib\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!unzip ../input/kuzushiji-recognition/train_images.zip -d train_images >/dev/null\n!unzip ../input/kuzushiji-recognition/test_images.zip -d test_images >/dev/null","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ut=pd.read_csv(\"../input/kuzushiji-recognition/unicode_translation.csv\")\nut_dict=ut.set_index(\"Unicode\")[\"char\"].to_dict()\ntrain=pd.read_csv(\"../input/kuzushiji-recognition/train.csv\")\ntrain_image_id=[os.path.basename(p).split(\".\")[0] for p in glob.glob(\"./train_images/*.jpg\")]\ntrain=train[train[\"image_id\"].isin(train_image_id)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lists=[]\nfor image_id,labels in train.values:\n    if labels == labels:\n        df=pd.DataFrame([],columns=[\"image_id\",\"label\"])\n        df[\"label\"]=[label for i,label in enumerate(labels.split(\" \")) if i%5==0]\n        df[\"X\"]=[int(label) for i,label in enumerate(labels.split(\" \")) if i%5==1]\n        df[\"Y\"]=[int(label) for i,label in enumerate(labels.split(\" \")) if i%5==2]\n        df[\"width\"]=[int(label) for i,label in enumerate(labels.split(\" \")) if i%5==3]\n        df[\"height\"]=[int(label) for i,label in enumerate(labels.split(\" \")) if i%5==4]\n        df[\"image_id\"]=image_id\n        lists.append(df)\ntrain_labels=pd.concat(lists,ignore_index=True) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_labels.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 出現文字数\ntrain_labels[\"label\"].map(ut_dict).value_counts()[:20]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"categories = [str(i) for i in train_labels[\"label\"]]\nunicode_categories = [ut_dict[i] if i in ut_dict.keys() else \"-\" for i in categories]\nlabel2id={l:i for i,l in enumerate(train_labels[\"label\"])}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def image_write(i,bboxes_df,folder=\"train_images\",label_show=True):\n    image_id=bboxes_df[\"image_id\"].unique()[i]\n    image_name=\"./\"+folder+\"/\"+image_id+\".jpg\"\n    img=Image.open(image_name)\n    num_img=np.array(img)\n    fig = plt.figure()\n    ax = fig.add_subplot(111)\n    ax.imshow(img)\n    for image_id,label,X,Y,width,height in bboxes_df[[\"image_id\",\"label\",\"X\",\"Y\",\"width\",\"height\"]].query(\"image_id=='{j}'\".format(j=image_id)).values:\n        rect = plt.Rectangle((X,Y),width,height,color=\"red\",fill=False)\n        ax.add_patch(rect)\n        if label_show:\n            ax.text(X+width, Y+height/2, ut_dict[label], size = 16, color = \"blue\")\n    plt.figure(figsize=(50,50))\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### train_data可視化\nimg_numを変えることで、training_dataの画像を見ることができます。","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"img_num=1\n\nrcParams['figure.figsize']=[10,10]\nimage_write(img_num,train_labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## 物体検出","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"物体検出はFaster RCNN、SSD等も試しましたが、kaggleカーネルのCenterNetが最強だったのでそれを流用。  \n*-----This kernel is written in both English and Japanese.------*\n\n初心者カーネルですが、日本語でも並記します。皆様のご参考になれば幸いです。\n\n本カーネルでは、最近話題になっているキーポイントベースの検出器を試してみました。CornerNet派生の『CenterNet』と呼ばれるもので、YOLOなどのようにアンカーを使用せず、セグメンテーション(U-Net)のようなヒートマップで対象物の中心点を検出する手法です。(シングルアンカーのような雰囲気ですが、ヒートマップだけでいいので実装しやすい印象です)\n\nDeNAさんはじめとした、様々な日本語の記事でも勉強させていただいておりますので、この場を借りてお礼申し上げます。","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport json\nimport pandas as pd\nfrom PIL import Image, ImageDraw\nimport matplotlib.pyplot as plt\nfrom pandas.io.json import json_normalize\nimport random\nimport tensorflow as tf\nfrom sklearn.utils import shuffle\nfrom sklearn.model_selection import KFold,train_test_split\nimport matplotlib.pyplot as plt\nimport glob\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.layers import Dense,Dropout, Conv2D,Conv2DTranspose, BatchNormalization, Activation,AveragePooling2D,GlobalAveragePooling2D, Input, Concatenate, MaxPool2D, Add, UpSampling2D, LeakyReLU,ZeroPadding2D\nfrom keras.models import Model\nfrom keras.objectives import mean_squared_error\nfrom keras import backend as K\nfrom keras.losses import binary_crossentropy\nfrom keras.callbacks import ModelCheckpoint, EarlyStopping, TensorBoard, ReduceLROnPlateau,LearningRateScheduler\nimport os  \nimport keras\nfrom keras.optimizers import Adam, RMSprop, SGD\nfrom tensorflow.compat.v1 import ConfigProto\nfrom tensorflow.compat.v1 import InteractiveSession","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"メモリ制限","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"physical_devices = tf.config.experimental.list_physical_devices('GPU')\nif len(physical_devices) > 0:\n    for k in range(len(physical_devices)):\n        tf.config.experimental.set_memory_growth(physical_devices[k], True)\n        print('memory growth:', tf.config.experimental.get_memory_growth(physical_devices[k]))\nelse:\n    print(\"Not enough GPU hardware devices available\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path_1=\"../input/kuzushiji-recognition/train.csv\"\npath_2=\"./train_images/\"\npath_3=\"./test_images/\"\npath_4=\"../input/kuzushiji-recognition/sample_submission.csv\"\ndf_train=pd.read_csv(path_1)\n\ntrain_image_id=[os.path.basename(p).split(\".\")[0] for p in glob.glob(path_2+\"*.jpg\")]\ndf_train=df_train[df_train[\"image_id\"].isin(train_image_id)]\n\n#print(df_train.head())\n#print(df_train.shape)\ndf_train=df_train.dropna(axis=0, how='any')#you can use nan data(page with no letter)\ndf_train=df_train.reset_index(drop=True)\n#print(df_train.shape)\n\nannotation_list_train=[]\ncategory_names=set()\n\nfor i in range(len(df_train)):\n    ann=np.array(df_train.loc[i,\"labels\"].split(\" \")).reshape(-1,5)#cat,x,y,width,height for each picture\n    category_names=category_names.union({i for i in ann[:,0]})\n\ncategory_names=sorted(category_names)\ndict_cat={list(category_names)[j]:str(j) for j in range(len(category_names))}\ninv_dict_cat={str(j):list(category_names)[j] for j in range(len(category_names))}\n#print(dict_cat)\n  \nfor i in range(len(df_train)):\n    ann=np.array(df_train.loc[i,\"labels\"].split(\" \")).reshape(-1,5)#cat,left,top,width,height for each picture\n    for j,category_name in enumerate(ann[:,0]):\n        ann[j,0]=int(dict_cat[category_name])  \n    ann=ann.astype('int32')\n    ann[:,1]+=ann[:,3]//2#center_x\n    ann[:,2]+=ann[:,4]//2#center_y\n    annotation_list_train.append([\"{}{}.jpg\".format(path_2,df_train.loc[i,\"image_id\"]),ann])\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### STEP 1: Preprocessing (Check Object Size)","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"まず、検出モデルを作る前に、文字サイズをチェックしておきます。CenterNetの出力方式に対して過少に小さい文字は、検出できませんので。","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# get directory of test images\ndf_submission=pd.read_csv(path_4).reset_index(drop=True)\n\ntest_image_id=[os.path.basename(p).split(\".\")[0] for p in glob.glob(path_3+\"*.jpg\")]\ndf_submission=df_submission[df_submission[\"image_id\"].isin(test_image_id)]\n\nid_test=path_3+df_submission[\"image_id\"].values+\".jpg\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"aspect_ratio_pic_all=[]\naspect_ratio_pic_all_test=[]\naverage_letter_size_all=[]\ntrain_input_for_size_estimate=[]\nfor i in range(len(annotation_list_train)):\n    with Image.open(annotation_list_train[i][0]) as f:\n        width,height=f.size\n        area=width*height\n        aspect_ratio_pic=height/width\n        aspect_ratio_pic_all.append(aspect_ratio_pic)\n        letter_size=annotation_list_train[i][1][:,3]*annotation_list_train[i][1][:,4]\n        letter_size_ratio=letter_size/area\n    \n        average_letter_size=np.mean(letter_size_ratio)\n        average_letter_size_all.append(average_letter_size)\n        train_input_for_size_estimate.append([annotation_list_train[i][0],np.log(average_letter_size)])#logにしとく\n    \n\nfor i in range(len(id_test)):\n    with Image.open(id_test[i]) as f:\n        width,height=f.size\n        aspect_ratio_pic=height/width\n        aspect_ratio_pic_all_test.append(aspect_ratio_pic)\n\nrcParams['figure.figsize']=[6,6]\nplt.hist(np.log(average_letter_size_all),bins=100)\nplt.title('log(ratio of letter_size to picture_size))',loc='center',fontsize=12)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ncategory_n=1\nimport cv2\ninput_width,input_height=512, 512\n\ndef Datagen_sizecheck_model(filenames, batch_size, size_detection_mode=True, is_train=True,random_crop=True):\n    x=[]\n    y=[]\n    \n    count=0\n\n    while True:\n        for i in range(len(filenames)):\n            if random_crop:\n                crop_ratio=np.random.uniform(0.7,1)\n            else:\n                crop_ratio=1\n            with Image.open(filenames[i][0]) as f:\n                #random crop\n                if random_crop and is_train:\n                    pic_width,pic_height=f.size\n                    f=np.asarray(f.convert('RGB'),dtype=np.uint8)\n                    top_offset=np.random.randint(0,pic_height-int(crop_ratio*pic_height))\n                    left_offset=np.random.randint(0,pic_width-int(crop_ratio*pic_width))\n                    bottom_offset=top_offset+int(crop_ratio*pic_height)\n                    right_offset=left_offset+int(crop_ratio*pic_width)\n                    f=cv2.resize(f[top_offset:bottom_offset,left_offset:right_offset,:],(input_height,input_width))\n                else:\n                    f=f.resize((input_width, input_height))\n                    f=np.asarray(f.convert('RGB'),dtype=np.uint8)                    \n                x.append(f)\n            \n            \n            if random_crop and is_train:\n                y.append(filenames[i][1]-np.log(crop_ratio))\n            else:\n                y.append(filenames[i][1])\n            \n            count+=1\n            if count==batch_size:\n                x=np.array(x, dtype=np.float32)\n                y=np.array(y, dtype=np.float32)\n\n                inputs=x/255\n                targets=y             \n                x=[]\n                y=[]\n                count=0\n                yield inputs, targets\n\n\n\ndef aggregation_block(x_shallow, x_deep, deep_ch, out_ch):\n    x_deep= Conv2DTranspose(deep_ch, kernel_size=2, strides=2, padding='same', use_bias=False)(x_deep)\n    x_deep = BatchNormalization()(x_deep)     \n    x_deep = LeakyReLU(alpha=0.1)(x_deep)\n    x = Concatenate()([x_shallow, x_deep])\n    x=Conv2D(out_ch, kernel_size=1, strides=1, padding=\"same\")(x)\n    x = BatchNormalization()(x)     \n    x = LeakyReLU(alpha=0.1)(x)\n    return x\n    \n\n\ndef cbr(x, out_layer, kernel, stride):\n    x=Conv2D(out_layer, kernel_size=kernel, strides=stride, padding=\"same\")(x)\n    x = BatchNormalization()(x)\n    x = LeakyReLU(alpha=0.1)(x)\n    return x\n\ndef resblock(x_in,layer_n):\n    x=cbr(x_in,layer_n,3,1)\n    x=cbr(x,layer_n,3,1)\n    x=Add()([x,x_in])\n    return x    \n\n\n#I use the same network at CenterNet\ndef create_model(input_shape, size_detection_mode=True, aggregation=True):\n        input_layer = Input(input_shape)\n        \n        #resized input\n        input_layer_1=AveragePooling2D(2)(input_layer)\n        input_layer_2=AveragePooling2D(2)(input_layer_1)\n\n        #### ENCODER ####\n\n        x_0= cbr(input_layer, 16, 3, 2)#512->256\n        concat_1 = Concatenate()([x_0, input_layer_1])\n\n        x_1= cbr(concat_1, 32, 3, 2)#256->128\n        concat_2 = Concatenate()([x_1, input_layer_2])\n\n        x_2= cbr(concat_2, 64, 3, 2)#128->64\n        \n        x=cbr(x_2,64,3,1)\n        x=resblock(x,64)\n        x=resblock(x,64)\n        \n        x_3= cbr(x, 128, 3, 2)#64->32\n        x= cbr(x_3, 128, 3, 1)\n        x=resblock(x,128)\n        x=resblock(x,128)\n        x=resblock(x,128)\n        \n        x_4= cbr(x, 256, 3, 2)#32->16\n        x= cbr(x_4, 256, 3, 1)\n        x=resblock(x,256)\n        x=resblock(x,256)\n        x=resblock(x,256)\n        x=resblock(x,256)\n        x=resblock(x,256)\n \n        x_5= cbr(x, 512, 3, 2)#16->8\n        x= cbr(x_5, 512, 3, 1)\n        \n        x=resblock(x,512)\n        x=resblock(x,512)\n        x=resblock(x,512)\n        \n        if size_detection_mode:\n            x=GlobalAveragePooling2D()(x)\n            x=Dropout(0.2)(x)\n            out=Dense(1,activation=\"linear\")(x)\n        \n        else:#centernet mode\n        #### DECODER ####\n            x_1= cbr(x_1, output_layer_n, 1, 1)\n            x_1 = aggregation_block(x_1, x_2, output_layer_n, output_layer_n)\n            x_2= cbr(x_2, output_layer_n, 1, 1)\n            x_2 = aggregation_block(x_2, x_3, output_layer_n, output_layer_n)\n            x_1 = aggregation_block(x_1, x_2, output_layer_n, output_layer_n)\n            x_3= cbr(x_3, output_layer_n, 1, 1)\n            x_3 = aggregation_block(x_3, x_4, output_layer_n, output_layer_n) \n            x_2 = aggregation_block(x_2, x_3, output_layer_n, output_layer_n)\n            x_1 = aggregation_block(x_1, x_2, output_layer_n, output_layer_n)\n            \n            x_4= cbr(x_4, output_layer_n, 1, 1)\n\n            x=cbr(x, output_layer_n, 1, 1)\n            x= UpSampling2D(size=(2, 2))(x)#8->16 tconvのがいいか\n\n            x = Concatenate()([x, x_4])\n            x=cbr(x, output_layer_n, 3, 1)\n            x= UpSampling2D(size=(2, 2))(x)#16->32\n        \n            x = Concatenate()([x, x_3])\n            x=cbr(x, output_layer_n, 3, 1)\n            x= UpSampling2D(size=(2, 2))(x)#32->64     128のがいいかも？ \n        \n            x = Concatenate()([x, x_2])\n            x=cbr(x, output_layer_n, 3, 1)\n            x= UpSampling2D(size=(2, 2))(x)#64->128 \n            \n            x = Concatenate()([x, x_1])\n            x=Conv2D(output_layer_n, kernel_size=3, strides=1, padding=\"same\")(x)\n            out = Activation(\"sigmoid\")(x)\n        \n        model=Model(input_layer, out)\n        \n        return model\n    \n        \n\n\ndef model_fit_sizecheck_model(model,train_list,cv_list,n_epoch,batch_size=32):\n        hist = model.fit_generator(\n                Datagen_sizecheck_model(train_list,batch_size, is_train=True,random_crop=True),\n                steps_per_epoch = len(train_list) // batch_size,\n                epochs = n_epoch,\n                validation_data=Datagen_sizecheck_model(cv_list,batch_size, is_train=False,random_crop=False),\n                validation_steps = len(cv_list) // batch_size,\n                callbacks = [lr_schedule, model_checkpoint],#[early_stopping, reduce_lr, model_checkpoint],\n                shuffle = True,\n                verbose = 1\n        )\n        return hist\n\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not os.path.exists(\"./models\"):\n    os.makedirs(\"./models\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"K.clear_session()\nmodel=create_model(input_shape=(input_height,input_width,3))\n\n\"\"\"\n# EarlyStopping\nearly_stopping = EarlyStopping(monitor = 'val_loss', min_delta=0, patience = 10, verbose = 1)\n# ModelCheckpoint\nweights_dir = '/model_1/'\nif os.path.exists(weights_dir) == False:os.mkdir(weights_dir)\nmodel_checkpoint = ModelCheckpoint(weights_dir + \"val_loss{val_loss:.3f}.hdf5\", monitor = 'val_loss', verbose = 1,\n                                      save_best_only = True, save_weights_only = True, period = 1)\n# reduce learning rate\nreduce_lr = ReduceLROnPlateau(monitor = 'val_loss', factor = 0.5, patience = 10, verbose = 1)\n\"\"\"\ndef lrs(epoch):\n    lr = 0.0001\n    return lr\n\nlr_schedule = LearningRateScheduler(lrs)\nmodel_checkpoint = ModelCheckpoint(\"./models/centernet_step1.hdf5\", monitor = 'val_loss', verbose = 1,\n                                      save_best_only = True, save_weights_only = True, period = 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"training n_epochを変えてください。saveのコメントアウトも外してください","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"## step1 training\ntrain_list, cv_list = train_test_split(train_input_for_size_estimate, random_state = 111,test_size = 0.2)\n# for layer in model.layers:\n#     layer.trainable = False\n\nlearning_rate=0.0005\nn_epoch=0\nbatch_size=4\n\nmodel.compile(loss=mean_squared_error, optimizer=Adam(lr=learning_rate))\nhist = model_fit_sizecheck_model(model,train_list,cv_list,n_epoch,batch_size)\n\n#model.save_weights('./centernet_step1.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## step1 predict\n#model.load_weights('./models/centernet_step1.h5')\nmodel.load_weights('../input/models/centernet_step1.h5')\npredict = model.predict_generator(Datagen_sizecheck_model(cv_list,batch_size, is_train=False,random_crop=False),\n                                  steps=len(cv_list) // batch_size)\ntarget=[cv[1] for cv in cv_list]\nplt.scatter(predict,target[:len(predict)])\nplt.title('---letter_size/picture_size--- estimated vs target ',loc='center',fontsize=10)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size=1\npredict_train = model.predict_generator(Datagen_sizecheck_model(train_input_for_size_estimate,batch_size, is_train=False,random_crop=False, ),\n                                  steps=len(train_input_for_size_estimate)//batch_size,verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base_detect_num_h,base_detect_num_w=25,25\nannotation_list_train_w_split=[]\nfor i, predicted_size in enumerate(predict_train):\n    detect_num_h=aspect_ratio_pic_all[i]*np.exp(-predicted_size/2)\n    detect_num_w=detect_num_h/aspect_ratio_pic_all[i]\n    h_split_recommend=np.maximum(1,detect_num_h/base_detect_num_h)\n    w_split_recommend=np.maximum(1,detect_num_w/base_detect_num_w)\n    annotation_list_train_w_split.append([annotation_list_train[i][0],annotation_list_train[i][1],h_split_recommend,w_split_recommend])\nfor i in np.arange(0,3):\n    print(\"recommended height split:{}, recommended width_split:{}\".format(annotation_list_train_w_split[i][2],annotation_list_train_w_split[i][3]))\n    img = np.asarray(Image.open(annotation_list_train_w_split[i][0]).convert('RGB'))\n    plt.imshow(img)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"予測結果は、log(文字のサイズ/全体の画像のサイズ)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"predict_train","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### STEP 2:Center net","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"category_n=1\noutput_layer_n=category_n+4\noutput_height,output_width=128,128\n\ndef Datagen_centernet(filenames, batch_size):\n    x=[]\n    y=[]\n    \n    count=0\n\n    while True:\n        for i in range(len(filenames)):\n            h_split=filenames[i][2]\n            w_split=filenames[i][3]\n            max_crop_ratio_h=1/h_split\n            max_crop_ratio_w=1/w_split\n            crop_ratio=np.random.uniform(0.5,1)\n            crop_ratio_h=max_crop_ratio_h*crop_ratio\n            crop_ratio_w=max_crop_ratio_w*crop_ratio\n            \n            with Image.open(filenames[i][0]) as f:\n                \n                #random crop\n                \n                pic_width,pic_height=f.size\n                f=np.asarray(f.convert('RGB'),dtype=np.uint8)\n                top_offset=np.random.randint(0,pic_height-int(crop_ratio_h*pic_height))\n                left_offset=np.random.randint(0,pic_width-int(crop_ratio_w*pic_width))\n                bottom_offset=top_offset+int(crop_ratio_h*pic_height)\n                right_offset=left_offset+int(crop_ratio_w*pic_width)\n                f=cv2.resize(f[top_offset:bottom_offset,left_offset:right_offset,:],(input_height,input_width))\n                x.append(f)            \n\n            output_layer=np.zeros((output_height,output_width,(output_layer_n+category_n)))\n            for annotation in filenames[i][1]:\n                x_c=(annotation[1]-left_offset)*(output_width/int(crop_ratio_w*pic_width))\n                y_c=(annotation[2]-top_offset)*(output_height/int(crop_ratio_h*pic_height))\n                width=annotation[3]*(output_width/int(crop_ratio_w*pic_width))\n                height=annotation[4]*(output_height/int(crop_ratio_h*pic_height))\n                top=np.maximum(0,y_c-height/2)\n                left=np.maximum(0,x_c-width/2)\n                bottom=np.minimum(output_height,y_c+height/2)\n                right=np.minimum(output_width,x_c+width/2)\n                    \n                if top>=(output_height-0.1) or left>=(output_width-0.1) or bottom<=0.1 or right<=0.1:#random crop(out of picture)\n                    continue\n                width=right-left\n                height=bottom-top\n                x_c=(right+left)/2\n                y_c=(top+bottom)/2\n\n                \n                category=0#not classify, just detect\n                heatmap=((np.exp(-(((np.arange(output_width)-x_c)/(width/10))**2)/2)).reshape(1,-1)\n                                                        *(np.exp(-(((np.arange(output_height)-y_c)/(height/10))**2)/2)).reshape(-1,1))\n                output_layer[:,:,category]=np.maximum(output_layer[:,:,category],heatmap[:,:])\n                output_layer[int(y_c//1),int(x_c//1),category_n+category]=1\n                output_layer[int(y_c//1),int(x_c//1),2*category_n]=y_c%1#height offset\n                output_layer[int(y_c//1),int(x_c//1),2*category_n+1]=x_c%1\n                output_layer[int(y_c//1),int(x_c//1),2*category_n+2]=height/output_height\n                output_layer[int(y_c//1),int(x_c//1),2*category_n+3]=width/output_width\n            y.append(output_layer)    \n        \n            count+=1\n            if count==batch_size:\n                x=np.array(x, dtype=np.float32)\n                y=np.array(y, dtype=np.float32)\n\n                inputs=x/255\n                targets=y             \n                x=[]\n                y=[]\n                count=0\n                yield inputs, targets\n\ndef all_loss(y_true, y_pred):\n        mask=K.sign(y_true[...,2*category_n+2])\n        N=K.sum(mask)\n        alpha=2.\n        beta=4.\n\n        heatmap_true_rate = K.flatten(y_true[...,:category_n])\n        heatmap_true = K.flatten(y_true[...,category_n:(2*category_n)])\n        heatmap_pred = K.flatten(y_pred[...,:category_n])\n        heatloss=-K.sum(heatmap_true*((1-heatmap_pred)**alpha)*K.log(heatmap_pred+1e-6)+(1-heatmap_true)*((1-heatmap_true_rate)**beta)*(heatmap_pred**alpha)*K.log(1-heatmap_pred+1e-6))\n        offsetloss=K.sum(K.abs(y_true[...,2*category_n]-y_pred[...,category_n]*mask)+K.abs(y_true[...,2*category_n+1]-y_pred[...,category_n+1]*mask))\n        sizeloss=K.sum(K.abs(y_true[...,2*category_n+2]-y_pred[...,category_n+2]*mask)+K.abs(y_true[...,2*category_n+3]-y_pred[...,category_n+3]*mask))\n        \n        all_loss=(heatloss+1.0*offsetloss+5.0*sizeloss)/N\n        return all_loss\n\ndef size_loss(y_true, y_pred):\n        mask=K.sign(y_true[...,2*category_n+2])\n        N=K.sum(mask)\n        sizeloss=K.sum(K.abs(y_true[...,2*category_n+2]-y_pred[...,category_n+2]*mask)+K.abs(y_true[...,2*category_n+3]-y_pred[...,category_n+3]*mask))\n        return (5*sizeloss)/N\n\ndef offset_loss(y_true, y_pred):\n        mask=K.sign(y_true[...,2*category_n+2])\n        N=K.sum(mask)\n        offsetloss=K.sum(K.abs(y_true[...,2*category_n]-y_pred[...,category_n]*mask)+K.abs(y_true[...,2*category_n+1]-y_pred[...,category_n+1]*mask))\n        return (offsetloss)/N\n    \ndef heatmap_loss(y_true, y_pred):\n        mask=K.sign(y_true[...,2*category_n+2])\n        N=K.sum(mask)\n        alpha=2.\n        beta=4.\n\n        heatmap_true_rate = K.flatten(y_true[...,:category_n])\n        heatmap_true = K.flatten(y_true[...,category_n:(2*category_n)])\n        heatmap_pred = K.flatten(y_pred[...,:category_n])\n        heatloss=-K.sum(heatmap_true*((1-heatmap_pred)**alpha)*K.log(heatmap_pred+1e-6)+(1-heatmap_true)*((1-heatmap_true_rate)**beta)*(heatmap_pred**alpha)*K.log(1-heatmap_pred+1e-6))\n        return heatloss/N\n\n    \ndef model_fit_centernet(model,train_list,cv_list,n_epoch,batch_size=32):\n        hist = model.fit_generator(\n                Datagen_centernet(train_list,batch_size),\n                steps_per_epoch = len(train_list) // batch_size,\n                epochs = n_epoch,\n                validation_data=Datagen_centernet(cv_list,batch_size),\n                validation_steps = len(cv_list) // batch_size,\n                callbacks = [lr_schedule],#early_stopping, reduce_lr, model_checkpoint],\n                shuffle = True,\n                verbose = 1\n        )\n        return hist","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import keras\nK.clear_session()\nmodel=create_model(input_shape=(input_height,input_width,3),size_detection_mode=False)\n\ndef lrs(epoch):\n    lr = 0.0005\n    if epoch >= 20: lr = 0.0002\n    return lr\n\nlr_schedule = LearningRateScheduler(lrs)\n\n# EarlyStopping\nearly_stopping = EarlyStopping(monitor = 'val_loss', min_delta=0, patience = 60, verbose = 1)\nmodel_checkpoint = ModelCheckpoint(\"./models/val_loss{val_loss:.3f}.hdf5\", monitor = 'val_loss', verbose = 1,\n                                      save_best_only = True, save_weights_only = True, period = 3)\n# reduce learning rate\nreduce_lr = ReduceLROnPlateau(monitor = 'val_loss', factor = 0.5, patience = 10, verbose = 1)\nmodel.load_weights('../input/models/centernet_step1.h5',by_name=True, skip_mismatch=True)\n# model.load_weights('./models/centernet_step1.h5',by_name=True, skip_mismatch=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"training(学習時、n_epochの変更とsaveを忘れずに)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_list, cv_list = train_test_split(annotation_list_train_w_split, random_state = 111,test_size = 0.2)#stratified split is better\nn_epoch=0\nbatch_size=1\nmodel.compile(loss=all_loss, optimizer=Adam(lr=learning_rate), metrics=[heatmap_loss,size_loss,offset_loss])\nhist = model_fit_centernet(model,train_list,cv_list,n_epoch,batch_size)\n\n#model.save_weights('./models/centernet_step2.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# predict(一部)\n\n#model.load_weights('./models/centernet_step2.h5')\nmodel.load_weights('../input/models/centernet_step2.h5')\npred_in_h=512\npred_in_w=512\npred_out_h=int(pred_in_h/4)\npred_out_w=int(pred_in_w/4)\n\nfor i in np.arange(0,1):\n    img = np.asarray(Image.open(cv_list[i][0]).resize((pred_in_w,pred_in_h)).convert('RGB'))\n    predict=model.predict((img.reshape(1,pred_in_h,pred_in_w,3))/255).reshape(pred_out_h,pred_out_w,(category_n+4))\n    heatmap=predict[:,:,0]\n\n    fig, axes = plt.subplots(1, 2,figsize=(15,15))\n    axes[0].set_axis_off()\n    axes[0].imshow(img)\n    axes[1].set_axis_off()\n    axes[1].imshow(heatmap)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## NMSとかの関数定義(多くかぶったところを取り除く)\n\nfrom PIL import Image, ImageDraw\n\ndef NMS_all(predicts,category_n,score_thresh,iou_thresh):\n    y_c=predicts[...,category_n]+np.arange(pred_out_h).reshape(-1,1)\n    x_c=predicts[...,category_n+1]+np.arange(pred_out_w).reshape(1,-1)\n    height=predicts[...,category_n+2]*pred_out_h\n    width=predicts[...,category_n+3]*pred_out_w\n\n    count=0\n    for category in range(category_n):\n        predict=predicts[...,category]\n        mask=(predict>score_thresh)\n        #print(\"box_num\",np.sum(mask))\n        if mask.all==False:\n            continue\n        box_and_score=NMS(predict[mask],y_c[mask],x_c[mask],height[mask],width[mask],iou_thresh)\n        box_and_score=np.insert(box_and_score,0,category,axis=1)#category,score,top,left,bottom,right\n        if count==0:\n            box_and_score_all=box_and_score\n        else:\n            box_and_score_all=np.concatenate((box_and_score_all,box_and_score),axis=0)\n        count+=1\n        score_sort=np.argsort(box_and_score_all[:,1])[::-1]\n        box_and_score_all=box_and_score_all[score_sort]\n        #print(box_and_score_all)\n\n    _,unique_idx=np.unique(box_and_score_all[:,2],return_index=True)\n    #print(unique_idx)\n    return box_and_score_all[sorted(unique_idx)]\n  \ndef NMS(score,y_c,x_c,height,width,iou_thresh,merge_mode=False):\n    if merge_mode:\n        score=score\n        top=y_c\n        left=x_c\n        bottom=height\n        right=width\n    else:\n        #flatten\n        score=score.reshape(-1)\n        y_c=y_c.reshape(-1)\n        x_c=x_c.reshape(-1)\n        height=height.reshape(-1)\n        width=width.reshape(-1)\n        size=height*width\n\n\n        top=y_c-height/2\n        left=x_c-width/2\n        bottom=y_c+height/2\n        right=x_c+width/2\n\n        inside_pic=(top>0)*(left>0)*(bottom<pred_out_h)*(right<pred_out_w)\n        outside_pic=len(inside_pic)-np.sum(inside_pic)\n        #if outside_pic>0:\n        #  print(\"{} boxes are out of picture\".format(outside_pic))\n        normal_size=(size<(np.mean(size)*10))*(size>(np.mean(size)/10))\n        score=score[inside_pic*normal_size]\n        top=top[inside_pic*normal_size]\n        left=left[inside_pic*normal_size]\n        bottom=bottom[inside_pic*normal_size]\n        right=right[inside_pic*normal_size]\n\n\n\n\n    #sort  \n    score_sort=np.argsort(score)[::-1]\n    score=score[score_sort]  \n    top=top[score_sort]\n    left=left[score_sort]\n    bottom=bottom[score_sort]\n    right=right[score_sort]\n\n    area=((bottom-top)*(right-left))\n\n    boxes=np.concatenate((score.reshape(-1,1),top.reshape(-1,1),left.reshape(-1,1),bottom.reshape(-1,1),right.reshape(-1,1)),axis=1)\n\n    box_idx=np.arange(len(top))\n    alive_box=[]\n    while len(box_idx)>0:\n\n        alive_box.append(box_idx[0])\n\n        y1=np.maximum(top[0],top)\n        x1=np.maximum(left[0],left)\n        y2=np.minimum(bottom[0],bottom)\n        x2=np.minimum(right[0],right)\n\n        cross_h=np.maximum(0,y2-y1)\n        cross_w=np.maximum(0,x2-x1)\n        still_alive=(((cross_h*cross_w)/area[0])<iou_thresh)\n        if np.sum(still_alive)==len(box_idx):\n            print(\"error\")\n            print(np.max((cross_h*cross_w)),area[0])\n        top=top[still_alive]\n        left=left[still_alive]\n        bottom=bottom[still_alive]\n        right=right[still_alive]\n        area=area[still_alive]\n        box_idx=box_idx[still_alive]\n    return boxes[alive_box]#score,top,left,bottom,right\n\n\n\ndef draw_rectangle(box_and_score,img,color):\n    number_of_rect=np.minimum(500,len(box_and_score))\n  \n    for i in reversed(list(range(number_of_rect))):\n        top, left, bottom, right = box_and_score[i,:]\n\n\n        top = np.floor(top + 0.5).astype('int32')\n        left = np.floor(left + 0.5).astype('int32')\n        bottom = np.floor(bottom + 0.5).astype('int32')\n        right = np.floor(right + 0.5).astype('int32')\n        #label = '{} {:.2f}'.format(predicted_class, score)\n        #print(label)\n        #rectangle=np.array([[left,top],[left,bottom],[right,bottom],[right,top]])\n\n        draw = ImageDraw.Draw(img)\n        #label_size = draw.textsize(label)\n        #print(label_size)\n\n        #if top - label_size[1] >= 0:\n        #  text_origin = np.array([left, top - label_size[1]])\n        #else:\n        #  text_origin = np.array([left, top + 1])\n\n        thickness=4\n        if color==\"red\":\n            rect_color=(255, 0, 0)\n        elif color==\"blue\":\n            rect_color=(0, 0, 255)\n        else:\n            rect_color=(0, 0, 0)\n      \n    \n        if i==0:\n            thickness=4\n        for j in range(2*thickness):#薄いから何重にか描く\n            draw.rectangle([left + j, top + j, right - j, bottom - j],\n                        outline=rect_color)\n            #draw.rectangle(\n            #            [tuple(text_origin), tuple(text_origin + label_size)],\n            #            fill=(0, 0, 255))\n            #draw.text(text_origin, label, fill=(0, 0, 0))\n\n        del draw\n        return img\n\ndef check_iou_score(true_boxes,detected_boxes,iou_thresh):\n    iou_all=[]\n    for detected_box in detected_boxes:\n        y1=np.maximum(detected_box[0],true_boxes[:,0])\n        x1=np.maximum(detected_box[1],true_boxes[:,1])\n        y2=np.minimum(detected_box[2],true_boxes[:,2])\n        x2=np.minimum(detected_box[3],true_boxes[:,3])\n\n        cross_section=np.maximum(0,y2-y1)*np.maximum(0,x2-x1)\n        all_area=(detected_box[2]-detected_box[0])*(detected_box[3]-detected_box[1])+(true_boxes[:,2]-true_boxes[:,0])*(true_boxes[:,3]-true_boxes[:,1])\n        iou=np.max(cross_section/(all_area-cross_section))\n        #argmax=np.argmax(cross_section/(all_area-cross_section))\n    iou_all.append(iou)\n    score=2*np.sum(iou_all)/(len(detected_boxes)+len(true_boxes))\n    print(\"score:{}\".format(np.round(score,3)))\n    return score\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def split_and_detect(model,img,height_split_recommended,width_split_recommended,score_thresh=0.3,iou_thresh=0.4):\n    width,height=img.size\n    pred_in_w,pred_in_h=512,512\n    pred_out_w,pred_out_h=128,128\n    category_n=1\n    maxlap=0.5\n    height_split=int(-(-height_split_recommended//1)+1)\n    width_split=int(-(-width_split_recommended//1)+1)\n    height_lap=(height_split-height_split_recommended)/(height_split-1)\n    height_lap=np.minimum(maxlap,height_lap)\n    width_lap=(width_split-width_split_recommended)/(width_split-1)\n    width_lap=np.minimum(maxlap,width_lap)\n\n    if height>width:\n        crop_size=int((height)/(height_split-(height_split-1)*height_lap))#crop_height and width\n        if crop_size>=width:\n            crop_size=width\n            stride=int((crop_size*height_split-height)/(height_split-1))\n            top_list=[i*stride for i in range(height_split-1)]+[height-crop_size]\n            left_list=[0]\n        else:\n            stride=int((crop_size*height_split-height)/(height_split-1))\n            top_list=[i*stride for i in range(height_split-1)]+[height-crop_size]\n            width_split=-(-width//crop_size)\n            stride=int((crop_size*width_split-width)/(width_split-1))\n            left_list=[i*stride for i in range(width_split-1)]+[width-crop_size]\n\n    else:\n        crop_size=int((width)/(width_split-(width_split-1)*width_lap))#crop_height and width\n        if crop_size>=height:\n            crop_size=height\n            stride=int((crop_size*width_split-width)/(width_split-1))\n            left_list=[i*stride for i in range(width_split-1)]+[width-crop_size]\n            top_list=[0]\n        else:\n            stride=int((crop_size*width_split-width)/(width_split-1))\n            left_list=[i*stride for i in range(width_split-1)]+[width-crop_size]\n            height_split=-(-height//crop_size)\n            stride=int((crop_size*height_split-height)/(height_split-1))\n            top_list=[i*stride for i in range(height_split-1)]+[height-crop_size]\n    \n    count=0\n\n    for top_offset in top_list:\n        for left_offset in left_list:\n            img_crop = img.crop((left_offset, top_offset, left_offset+crop_size, top_offset+crop_size))\n            predict=model.predict((np.asarray(img_crop.resize((pred_in_w,pred_in_h))).reshape(1,pred_in_h,pred_in_w,3))/255).reshape(pred_out_h,pred_out_w,(category_n+4))\n    \n            box_and_score=NMS_all(predict,category_n,score_thresh,iou_thresh)#category,score,top,left,bottom,right\n            \n            #print(\"after NMS\",len(box_and_score))\n            if len(box_and_score)==0:\n                continue\n            #reshape and offset\n            box_and_score=box_and_score*[1,1,crop_size/pred_out_h,crop_size/pred_out_w,crop_size/pred_out_h,crop_size/pred_out_w]+np.array([0,0,top_offset,left_offset,top_offset,left_offset])\n            \n            if count==0:\n                box_and_score_all=box_and_score\n            else:\n                box_and_score_all=np.concatenate((box_and_score_all,box_and_score),axis=0)\n            count+=1\n    #print(\"all_box_num:\",len(box_and_score_all))\n    #print(box_and_score_all[:10,:],np.min(box_and_score_all[:,2:]))\n    if count==0:\n        box_and_score_all=[]\n    else:\n        score=box_and_score_all[:,1]\n        y_c=(box_and_score_all[:,2]+box_and_score_all[:,4])/2\n        x_c=(box_and_score_all[:,3]+box_and_score_all[:,5])/2\n        height=-box_and_score_all[:,2]+box_and_score_all[:,4]\n        width=-box_and_score_all[:,3]+box_and_score_all[:,5]\n        #print(np.min(height),np.min(width))\n        box_and_score_all=NMS(box_and_score_all[:,1],box_and_score_all[:,2],box_and_score_all[:,3],box_and_score_all[:,4],box_and_score_all[:,5],iou_thresh=0.5,merge_mode=True)\n    return box_and_score_all\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm_notebook\nfrom joblib import Parallel,delayed\n\nK.clear_session()\nprint(\"loading models...\")\nmodel_1=create_model(input_shape=(512,512,3),size_detection_mode=True)\nmodel_1.load_weights('../input/models/centernet_step1.h5')\n#model_1.load_weights('./input/models/centernet_step1.h5')\n\nmodel_2=create_model(input_shape=(512,512,3),size_detection_mode=False)\nmodel_2.load_weights('../input/models/centernet_step2.h5')\n#model_1.load_weights('./models/centernet_step2.h5')\n\ndef pipeline(i):\n    # model1: determine how to split image\n    img = np.asarray(Image.open(id_test[i]).resize((512,512)).convert('RGB'))\n    predicted_size=model_1.predict(img.reshape(1,512,512,3)/255)\n    detect_num_h=aspect_ratio_pic_all_test[i]*np.exp(-predicted_size/2)\n    detect_num_w=detect_num_h/aspect_ratio_pic_all_test[i]\n    h_split_recommend=np.maximum(1,detect_num_h/base_detect_num_h)\n    w_split_recommend=np.maximum(1,detect_num_w/base_detect_num_w)\n\n    # model2: detection\n    img=Image.open(id_test[i]).convert(\"RGB\")\n    box_and_score_all=split_and_detect(model_2,img,h_split_recommend,w_split_recommend,score_thresh=0.3,iou_thresh=0.4)#output:score,top,left,bottom,right\n    lists=[]\n    if (len(box_and_score_all)>0):\n        for box in box_and_score_all[:,1:]:\n            top,left,bottom,right=box\n            lists.append([left,top,right-left,bottom-top])\n    df=pd.DataFrame(lists,columns=[\"X\",\"Y\",\"width\",\"height\"])\n    df[\"image_id\"]=os.path.basename(id_test[i]).split(\".\")[0]\n    df[\"label\"]=\"\"\n    df=df[[\"image_id\",\"label\",\"X\",\"Y\",\"width\",\"height\"]]\n    return df\nprint(\"predicts...\")\n#I'm sorry. Not nice coding. Time consuming.\nbboxes_df=pd.concat([pipeline(i) for i in tqdm_notebook(range(len(id_test)))])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bboxes_df.to_pickle(\"./test_centernet_p3.pkl\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 物体検出予測可視化\n数字の部分を変えてください","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"rcParams['figure.figsize'] = 10,10\nimage_write(1,bboxes_df,folder=\"test_images\",label_show=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\nK.clear_session()\ndel model\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 文字認識","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"以下でCUDA out of memoryが出たら、リスタートして物体検出の章を飛ばして実行してみてください","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib\nimport pandas as pd\nimport numpy as np\nfrom PIL import Image\nfrom pylab import rcParams\nimport os\nimport json\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport glob\n\n%matplotlib inline\n\nimport torch\nimport os\nimport pandas as pd\nimport pickle\nimport torchvision.transforms as transforms\nfrom torch.utils.data import Dataset\nfrom PIL import Image\nfrom tqdm import tqdm_notebook\nfrom cnn_finetune import make_model\nimport torch.nn as nn\nfrom torch.autograd import Variable\nimport torch.nn as nn\nimport torch.optim as optim\n\nut=pd.read_csv(\"../input/kuzushiji-recognition/unicode_translation.csv\")\nut_dict=ut.set_index(\"Unicode\")[\"char\"].to_dict()\ntrain=pd.read_csv(\"../input/kuzushiji-recognition/train.csv\")\ntrain_labels[\"id\"]=np.arange(len(train_labels))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 同じラベルが5以下のものは5個に、400以上のものは400にする\nid_list=[]\nfor l,sdf in train_labels.groupby(\"label\"):\n    image_names=sdf[\"id\"].values\n    if len(sdf)<5:\n        image_names=list(image_names)+list(np.random.choice(sdf[\"id\"].values, 5-len(sdf)))\n    elif len(sdf)>400:\n        image_names=np.random.choice(sdf[\"id\"].values, 400,replace=False)\n    id_list+=list(image_names)\ntrain_labels_m=train_labels.set_index(\"id\").loc[id_list].reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# trainとvalに分割\nfrom sklearn.model_selection import train_test_split\ntrain_labels_train, train_labels_val, _, _ = train_test_split(train_labels_m,\\\n                                                    train_labels_m[\"label\"],\\\n                                                    test_size=0.2,\\\n                                                    random_state=100,\\\n                                                    stratify=train_labels_m[\"label\"])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# labelを数値に変換(一応保存)\nlabel_id={label:i for i,label in enumerate(train_labels[\"label\"].unique())}\nwith open(\"./label_id.pkl\",\"wb\") as f:\n    pickle.dump(label_id,f)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(\"./label_id.pkl\",\"rb\") as f:\n    label_id=pickle.load(f)\nlabel_id_r={v:k for k,v in label_id.items()}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"pytorch用データセット作成。  \ntransformでは、グレーにして少し回転をかける  \n画像は正方形で切り取る(文字を広げない)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"resize = (256, 256)  # 入力画像サイズ\ntrain_dir=\"./train_images\"\ntrans= [transforms.Resize(resize),\n#            transforms.ColorJitter(brightness=0.1, contrast=0.1, saturation=0, hue=0),\n            transforms.Grayscale(num_output_channels=3),\n            transforms.RandomAffine(5,translate=(0.1,0.1),fillcolor=\"white\"),\n            transforms.RandomCrop((224,224),fill=\"white\"),\n#            transforms.RandomRotation(degrees=5,fill=\"white\"),\n            transforms.Resize(resize),\n            transforms.ToTensor(),\n            transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))]\n\n\nclass MyDataSet(Dataset):\n    def __init__(self,img_dir,train_labels):\n        self.train_labels = train_labels\n        self.transform = transforms.Compose(trans)\n        self.img_dir=img_dir\n        self.images = list(self.train_labels[\"id\"].unique())\n        self.labels = list(self.train_labels[\"label\"].unique())\n      \n    def __len__(self):\n        return len(self.train_labels)\n    \n    def image_open(self,t):\n        image = Image.open(os.path.join(self.img_dir, t+\".jpg\"))\n        return image.convert('RGB')\n\n    def __getitem__(self, idx):\n        image_id,X,Y,width,height,label = self.train_labels[[\"image_id\",\"X\",\"Y\",\"width\",\"height\",\"label\"]].iloc[idx]\n        img = Image.open( os.path.join(self.img_dir, image_id+\".jpg\") )\n        if width < height:\n            img_crop = img.crop((X+(width-height)//2, Y, X+(width+height)//2, Y+height))\n        else:\n            img_crop = img.crop((X, Y+(height-width)/2, X+width, Y+(height+width)/2))\n        \n        return self.transform(img_crop),label_id[label]\n\nkwargs = {'num_workers': 1, 'pin_memory': True} \ntrain_set = MyDataSet(train_dir,train_labels_train)\ntrain_loader = torch.utils.data.DataLoader(train_set, batch_size=5, shuffle=True)\nval_set = MyDataSet(train_dir,train_labels_val)\nval_loader = torch.utils.data.DataLoader(val_set, batch_size=5, shuffle=True)\ndataloaders_dict={\"train\":train_loader,\"val\":val_loader}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# transform可視化用関数\nrcParams['figure.figsize'] = 4,4\ndef tran_picture(idx):\n    print(\"words:\",ut_dict[train_labels.iloc[idx][\"label\"]])\n    image_id,X,Y,width,height = train_labels[[\"image_id\",\"X\",\"Y\",\"width\",\"height\"]].iloc[idx]\n    img = Image.open( os.path.join(train_dir, image_id+\".jpg\") )\n    if width < height:\n            img_crop = img.crop((X+(width-height)//2, Y, X+(width+height)//2, Y+height))\n    else:\n            img_crop = img.crop((X, Y+(height-width)/2, X+width, Y+(height+width)/2))\n    p=img_crop\n    plt.imshow(p)\n    plt.show()\n    img_transformed=transforms.Compose(trans)(p)\n    img_transformed = img_transformed.numpy().transpose((1, 2, 0))\n    img_transformed = np.clip(img_transformed, 0, 1)\n    plt.imshow(img_transformed)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## transform可視化\n数字を変えてみてください","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"rcParams['figure.figsize']=[4,4]\nfor i in range(10):\n    print(tran_picture(i))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"resize = (256, 256)  # 入力画像サイズ\n\nclass Identity(nn.Module):\n    def __init__(self):\n        super(Identity, self).__init__()        \n\n    def forward(self, x):\n        return x\n\n\ndef make_pnas():\n# 実際はpnasnet5leargeにしましたが、遅くなるため今回はresnetで\n#    model = make_model('pnasnet5large', pretrained=True, input_size=resize,num_classes=4212)\n    model = make_model('resnet101', pretrained=False, input_size=resize,num_classes=4212)\n    return model\nmodel = make_pnas()\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9)\nmodel.cuda()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"実際の学習時はnum_epochsを増やしてください","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# epochのループ\nnum_epochs=0\nstart_num=0\nmodel_path=0\n#model.load_state_dict(torch.load('./models/model7/model-epoch-3.pth'))\nif not os.path.exists(\"./models/model{}\".format(model_path)):\n    os.makedirs(\"./models/model{}\".format(model_path))\n\nnet, dataloaders_dict, criterion, optimizer=model, dataloaders_dict, criterion, optimizer\nfor epoch in range(num_epochs):\n    print('Epoch {}/{}'.format(epoch+1, num_epochs))\n    print('-------------')\n\n    # epochごとの学習と検証のループ\n    for phase in ['train', 'val']:\n        if phase == 'train':\n            net.train()  # モデルを訓練モードに\n        else:\n            net.eval()   # モデルを検証モードに\n\n        epoch_loss = 0.0  # epochの損失和\n        epoch_corrects = 0  # epochの正解数\n\n        # データローダーからミニバッチを取り出すループ\n        for inputs, labels in tqdm_notebook(dataloaders_dict[phase]):\n            inputs,labels = inputs.cuda(),labels.cuda()\n            # optimizerを初期化\n            optimizer.zero_grad()\n\n            # 順伝搬（forward）計算\n            with torch.set_grad_enabled(phase == 'train'):\n                outputs = net(inputs)\n                outputs=outputs\n                loss = criterion(outputs, labels)  # 損失を計算\n                _, preds = torch.max(outputs, 1)  # ラベルを予測\n\n\n                # 訓練時はバックプロパゲーション\n                if phase == 'train':\n                    loss.backward()\n                    optimizer.step()\n\n                # イタレーション結果の計算\n                # lossの合計を更新\n                epoch_loss += loss.item() * inputs.size(0)  \n                # 正解数の合計を更新\n                epoch_corrects += torch.sum(preds == labels.data)\n\n        # epochごとのlossと正解率を表示\n        epoch_loss = epoch_loss / len(dataloaders_dict[phase].dataset)\n        epoch_acc = epoch_corrects.double(\n        ) / len(dataloaders_dict[phase].dataset)\n\n        print('{} Loss: {:.4f} Acc: {:.4f}'.format(\n            phase, epoch_loss, epoch_acc))\n            \n    torch.save(model.state_dict(), './models/model{}/model-epoch-{}.pth'.format(model_path,epoch))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"resize=(256,256)\ntest_dir=\"./test_images\"\ntest_bboxes_df = pd.read_pickle(\"./test_centernet_p3.pkl\")\nmodel.load_state_dict(torch.load('../input/models/resnet-trained.pth'))\nmodel.cuda()\n\ntrans= [transforms.Resize(resize),\n#            transforms.ColorJitter(brightness=0.1, contrast=0.1, saturation=0, hue=0),\n            transforms.Grayscale(num_output_channels=3),\n            transforms.Resize(resize),\n            transforms.ToTensor(),\n            transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))]\nclass MyDataSet_test(Dataset):\n    def __init__(self,img_dir,train_labels,groupbysample_n=False):\n        if groupbysample_n:\n            self.train_labels = train_labels.groupby(\"label\").apply(lambda x: x.sample(n=groupbysample_n,replace=True))\n        else:\n            self.train_labels = train_labels\n        self.transform = transforms.Compose(trans)\n        self.img_dir=img_dir\n        \n    def __len__(self):\n        return len(self.train_labels)\n    \n    def __getitem__(self, idx):\n        image_id,X,Y,width,height = self.train_labels[[\"image_id\",\"X\",\"Y\",\"width\",\"height\"]].iloc[idx]\n        img = Image.open( os.path.join(self.img_dir, image_id+\".jpg\") )\n        if width < height:\n            img_crop = img.crop((X+(width-height)//2, Y, X+(width+height)//2, Y+height))\n        else:\n            img_crop = img.crop((X, Y+(height-width)/2, X+width, Y+(height+width)/2))\n        \n        return self.transform(img_crop), image_id,X,Y,width,height\n    \ntest_set = MyDataSet_test(test_dir, test_bboxes_df)\ntest_loader = torch.utils.data.DataLoader(test_set,batch_size=3, shuffle=False)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lists=[]\nfor i,(x,image_ids,Xs,Ys,widths,heights) in tqdm_notebook(enumerate(test_loader),total=len(test_loader)):\n    a=model(x.cuda())\n    a=torch.max(a,1)\n    df=pd.DataFrame(a[1].cpu().detach().numpy(),columns=[\"labels_id\"])\n    df[\"label\"]=df[\"labels_id\"].map(label_id_r)\n    df[\"image_id\"]=image_ids\n    df[\"X\"]=Xs\n    df[\"Y\"]=Ys\n    df[\"width\"]=widths\n    df[\"height\"]=heights\n    lists.append(df[[\"image_id\",\"X\",\"Y\",\"width\",\"height\",\"label\"]])\n# 予測時間長いので、今回は1000でストップ\n    if i>=1000:\n        break\ntest_labels_p=pd.concat(lists)\ntest_labels_p.to_pickle(\"./test_labels_p4.pkl\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## 文字認識予測結果","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"数字を動かしてみてください","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"rcParams['figure.figsize']=[10,10]\nimage_write(0,test_labels_p,folder=\"test_images\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## 提出用データ\nimport glob\ndata=[]\nfor image,sdf in test_labels_p[[\"image_id\",\"X\",\"Y\",\"width\",\"height\",\"label\"]].groupby(\"image_id\"):\n    labels=\" \".join([\"{} {} {}\".format(l,int(X+1/2*w),int(Y+1/2*h)) for image,X,Y,w,h,l in sdf.values])\n    data.append([image,labels])\ndf=pd.DataFrame(data,columns=[\"image_id\",\"labels\"])\ntest_imgs=glob.glob(\"./test_images/*\")\ntest_img_df=pd.DataFrame([test_img.split(\"/\")[-1].split(\".\")[0] for test_img in test_imgs],columns=[\"image_id\"])\ndf=pd.merge(test_img_df,df,how=\"left\",on=\"image_id\").fillna(\"\")\ndf.to_csv(\"./prediction.csv\",index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm -R train_images\n!rm -R test_images\n!rm -R models","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}