{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nfrom skimage.io import imread\nimport matplotlib.pyplot as plt\nimport gc; gc.enable() \nprint(os.listdir(\"../input/airbus-ship-detection\"))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"masks = pd.read_csv(os.path.join('../input/airbus-ship-detection', 'train_ship_segmentations_v2.csv'))\nnot_empty = pd.notna(masks.EncodedPixels)\nprint(not_empty.sum(), 'masks in', masks[not_empty].ImageId.nunique(), 'images')#非空图片中的mask数量\nprint((~not_empty).sum(), 'empty images in', masks.ImageId.nunique(), 'total images')#所有图片中非空图片\nmasks.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"masks['ships'] = masks['EncodedPixels'].map(lambda c_row: 1 if isinstance(c_row, str) else 0)\nmasks.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unique_img_ids = masks.groupby('ImageId').agg({'ships': 'sum'}).reset_index()\nunique_img_ids.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unique_img_ids['has_ship'] = unique_img_ids['ships'].map(lambda x: 1.0 if x>0 else 0.0)\n\nunique_img_ids.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ship_dir = '../input/airbus-ship-detection'\ntrain_image_dir = os.path.join(ship_dir, 'train_v2')\ntest_image_dir = os.path.join(ship_dir, 'test_v2')\nunique_img_ids['has_ship_vec'] = unique_img_ids['has_ship'].map(lambda x: [x])\nunique_img_ids['file_size_kb'] = unique_img_ids['ImageId'].map(lambda c_img_id: \n                                                               os.stat(os.path.join(train_image_dir, \n                                                                                    c_img_id)).st_size/1024)\nunique_img_ids.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unique_img_ids = unique_img_ids[unique_img_ids['file_size_kb'] > 50] # keep only +50kb files\nplt.hist(x = unique_img_ids['file_size_kb'], # 指定绘图数据\n           bins = 6, # 指定直方图中条块的个数\n           color = 'steelblue', # 指定直方图的填充色\n           edgecolor = 'black' # 指定直方图的边框色\n          )\nplt.xticks([50,100,150,200,250,300,350,400,450,500])\nplt.ylabel(\"number\")\nplt.xlabel('file_size_kb')\n#unique_img_ids['file_size_kb'].hist()#绘制直方图\nmasks.drop(['ships'], axis=1, inplace=True)\nunique_img_ids.sample(7)\nplt.title(\"Number of images of each size\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(unique_img_ids['ships'].value_counts())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_0 = unique_img_ids[unique_img_ids['ships']==1].sample(1800)\ntrain_1 = unique_img_ids[unique_img_ids['ships']==2].sample(1800)\ntrain_2 = unique_img_ids[unique_img_ids['ships']==3].sample(1800)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_3 = unique_img_ids[unique_img_ids['ships']!=3]\ntrain_3 = train_3[unique_img_ids['ships']!=2]\ntrain_3 = train_3[unique_img_ids['ships']!=1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unique_img_ids=pd.concat([train_0,train_1,train_2,train_3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SAMPLES_PER_GROUP = 10000\nbalanced_train_df = unique_img_ids.groupby('ships').apply(lambda x: x.sample(SAMPLES_PER_GROUP) if len(x) > SAMPLES_PER_GROUP else x)\n#图片有相同船舶数量，但超出2000的不要\nrect=plt.hist(x = balanced_train_df['ships'], # 指定绘图数据\n           bins = 16, # 指定直方图中条块的个数\n           color = 'steelblue', # 指定直方图的填充色\n           edgecolor = 'black' # 指定直方图的边框色\n          )\nplt.yticks(range(0,1800,300))\nplt.xticks(range(0,15))\nplt.ylabel(\"Number of images\")\nplt.xlabel('Number of ships')\nplt.title(\"Number of images containing different number of vessels\")\n#balanced_train_df['ships'].hist(bins=balanced_train_df['ships'].max()+1)\nprint(balanced_train_df.shape[0], 'images',balanced_train_df.shape)#取出1万张图片\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"balanced_train_df=balanced_train_df.reset_index(drop = True)#删除原来的索引。\nbalanced_train_df=balanced_train_df.sample(frac=1.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from PIL import Image\nx = np.empty(shape=(20188, 256,256,3),dtype=np.uint8)\ny = np.empty(shape=20188,dtype=np.uint8)\nfor index, image in enumerate(balanced_train_df['ImageId']):\n    image_array= Image.open('../input/airbus-ship-detection/train_v2/' + image).resize((256,256)).convert('RGB')\n    x[index] = image_array\n    y[index]=balanced_train_df[balanced_train_df['ImageId']==image]['has_ship'].iloc[0]\n\nprint(x.shape)\nprint(y.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Set target to one hot target for classification problem\n#为分类问题将目标设置为一个热目标\nfrom sklearn.preprocessing import OneHotEncoder\ny_targets =y.reshape(len(y),-1)\nenc = OneHotEncoder()\nenc.fit(y_targets)\ny = enc.transform(y_targets).toarray()\nprint(y.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nx_train, x_val, y_train, y_val  = train_test_split(x,y,test_size = 0.2,random_state=1,stratify=y)\nx_train.shape, x_val.shape, y_train.shape, y_val.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n \nimport os\n \nfrom keras import backend as K\nfrom keras.models import Model\nfrom keras.layers import Activation\nfrom keras.layers import AveragePooling2D\nfrom keras.layers import BatchNormalization\nfrom keras.layers import Concatenate\nfrom keras.layers import Conv2D\nfrom keras.layers import Dense\nfrom keras.layers import GlobalAveragePooling2D\nfrom keras.layers import GlobalMaxPooling2D\nfrom keras.layers import Input\nfrom keras.layers import MaxPooling2D\nfrom keras.layers import ZeroPadding2D\nfrom keras.utils.data_utils import get_file\nfrom keras.engine.topology import get_source_inputs\nfrom keras.applications import imagenet_utils\nfrom keras.applications.imagenet_utils import decode_predictions\nfrom keras_applications.imagenet_utils import _obtain_input_shape \n \n\n \ndef dense_block(x, blocks, name):\n    \"\"\"A dense block.\n    密集的模块\n    # Arguments\n    参数\n        x: input tensor.\n        x: 输入参数\n        blocks: integer, the number of building blocks.\n        blocks: 整型，生成块的个数。\n        name: string, block label.\n        name: 字符串，块的标签\n    # Returns\n    返回\n        output tensor for the block.\n        为块输出张量\n    \"\"\"\n    for i in range(blocks):\n        x = conv_block(x, 32, name=name + '_block' + str(i + 1))\n    return x\n \n \ndef transition_block(x, reduction, name):\n    \"\"\"A transition block.\n    转换块\n    # Arguments\n    参数\n        x: input tensor.\n        x: 输入参数\n        reduction: float, compression rate at transition layers.\n        reduction: 浮点数，转换层的压缩率\n        name: string, block label.\n        name: 字符串，块标签\n    # Returns\n    返回\n        output tensor for the block.\n        块输出张量\n    \"\"\"\n    bn_axis = 3 if K.image_data_format() == 'channels_last' else 1\n    x = BatchNormalization(axis=bn_axis, epsilon=1.001e-5,\n                           name=name + '_bn')(x)\n    x = Activation('relu', name=name + '_relu')(x)\n    x = Conv2D(int(K.int_shape(x)[bn_axis] * reduction), 1, use_bias=False,\n               name=name + '_conv')(x)\n    x = AveragePooling2D(2, strides=2, name=name + '_pool')(x)\n    return x\n \n \ndef conv_block(x, growth_rate, name):\n    \"\"\"A building block for a dense block.\n    密集块正在建立的块\n    # Arguments\n    参数\n        x: input tensor.\n        x: 输入张量\n        growth_rate: float, growth rate at dense layers.\n        growth_rate:浮点数，密集层的增长率。\n        name: string, block label.\n        name: 字符串，块标签\n    # Returns\n    返回\n        output tensor for the block.\n        块输出张量\n    \"\"\"\n    bn_axis = 3 if K.image_data_format() == 'channels_last' else 1\n    x1 = BatchNormalization(axis=bn_axis, epsilon=1.001e-5,\n                            name=name + '_0_bn')(x)\n    x1 = Activation('relu', name=name + '_0_relu')(x1)\n    x1 = Conv2D(4 * growth_rate, 1, use_bias=False,\n                name=name + '_1_conv')(x1)\n    x1 = BatchNormalization(axis=bn_axis, epsilon=1.001e-5,\n                            name=name + '_1_bn')(x1)\n    x1 = Activation('relu', name=name + '_1_relu')(x1)\n    x1 = Conv2D(growth_rate, 3, padding='same', use_bias=False,\n                name=name + '_2_conv')(x1)\n    x = Concatenate(axis=bn_axis, name=name + '_concat')([x, x1])\n    return x\n \n \ndef DenseNet(blocks,\n             include_top=True,\n             weights='imagenet',\n             input_tensor=None,\n             input_shape=None,\n             pooling=None,\n             classes=2):\n    \"\"\"Instantiates the DenseNet architecture.\n    实例化DenseNet结构\n    Optionally loads weights pre-trained\n    on ImageNet. Note that when using TensorFlow,\n    for best performance you should set\n    `image_data_format='channels_last'` in your Keras config\n    at ~/.keras/keras.json.\n    可选择加载预训练的ImageNet权重。注意，如果是Tensorflow，最好在Keras配置中设置`image_data_format='channels_last'\n    The model and the weights are compatible with\n    TensorFlow, Theano, and CNTK. The data format\n    convention used by the model is the one\n    specified in your Keras config file.\n    模型和权重兼容TensorFlow, Theano, and CNTK.模型使用的数据格式约定是Keras配置文件中指定的一种格式。\n    # Arguments\n    参数\n        blocks: numbers of building blocks for the four dense layers.\n        blocks: （构建）4个密集层需要块数量\n        include_top: whether to include the fully-connected\n            layer at the top of the network.\n        include_top: 在网络的顶层（一般指最后一层）师傅包含全连接层\n        weights: one of `None` (random initialization),\n              'imagenet' (pre-training on ImageNet),\n              or the path to the weights file to be loaded.\n              以下的一个：`None` (随机初始化),'imagenet' (ImageNet预训练),或者下载权重文件的路径。\n        input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)\n            to use as image input for the model.\n        input_tensor: 可选的Keras张量（即，`layers.Input()`的输出），用作模型的图像输入。\n        input_shape: optional shape tuple, only to be specified\n            if `include_top` is False (otherwise the input shape\n            has to be `(224, 224, 3)` (with `channels_last` data format)\n            or `(3, 224, 224)` (with `channels_first` data format).\n            It should have exactly 3 inputs channels.\n        input_shape: 可选的形状元组，只有`include_top`是False（否则，输入形状必须\n        是“（224, 224, 3）”（带有`channels_first` 数据格式。））时需要确认，它应该有3个输入通道。\n        pooling: optional pooling mode for feature extraction\n            when `include_top` is `False`.\n            可选，当 `include_top`是FALSE，特征提取的池化模式。\n            - `None` means that the output of the model will be\n                the 4D tensor output of the\n                last convolutional layer.\n                `None` 表示，模型输出层是4维张量，从上一个的卷积层输出。\n            - `avg` means that global average pooling\n                will be applied to the output of the\n                last convolutional layer, and thus\n                the output of the model will be a 2D tensor.\n              `avg`表示全局平均池化被应用到上一个的卷积层输出，所以模型输出是2维张量。\n            - `max` means that global max pooling will\n                be applied.\n              `max`  表示全局最大池化被应用\n        classes: optional number of classes to classify images\n            into, only to be specified if `include_top` is True, and\n            if no `weights` argument is specified.\n        classes: 可选的类数分类的图像，只有指定，如果'include_top'是真的，如果没有'weights'参数被指定。\n    # Returns\n    返回\n        A Keras model instance.\n        一个Keras模型实例\n    # Raises\n    补充\n        ValueError: in case of invalid argument for `weights`,\n            or invalid input shape.\n        ValueError: weights`无效的参数，或者无效的输入形状\n    \"\"\"\n    if not (weights in {'imagenet', None} or os.path.exists(weights)):\n        raise ValueError('The `weights` argument should be either '\n                         '`None` (random initialization), `imagenet` '\n                         '(pre-training on ImageNet), '\n                         'or the path to the weights file to be loaded.')\n \n    if weights == 'imagenet' and include_top and classes != 1000:\n        raise ValueError('If using `weights` as imagenet with `include_top`'\n                         ' as true, `classes` should be 1000')\n \n    # Determine proper input shape\n    input_shape = _obtain_input_shape(input_shape,\n                                      default_size=256,\n                                      min_size=221,\n                                      data_format=K.image_data_format(),\n                                      require_flatten=include_top,\n                                      weights=weights)\n \n    if input_tensor is None:\n        img_input = Input(shape=input_shape)\n    else:\n        if not K.is_keras_tensor(input_tensor):\n            img_input = Input(tensor=input_tensor, shape=input_shape)\n        else:\n            img_input = input_tensor\n \n    bn_axis = 3 if K.image_data_format() == 'channels_last' else 1\n \n    x = ZeroPadding2D(padding=((3, 3), (3, 3)))(img_input)\n    x = Conv2D(64, 7, strides=2, use_bias=False, name='conv1/conv')(x)\n    x = BatchNormalization(axis=bn_axis, epsilon=1.001e-5,\n                           name='conv1/bn')(x)\n    x = Activation('relu', name='conv1/relu')(x)\n    x = ZeroPadding2D(padding=((1, 1), (1, 1)))(x)\n    x = MaxPooling2D(3, strides=2, name='pool1')(x)\n \n    x = dense_block(x, blocks[0], name='conv2')\n    x = transition_block(x, 0.5, name='pool2')\n    x = dense_block(x, blocks[1], name='conv3')\n    x = transition_block(x, 0.5, name='pool3')\n    x = dense_block(x, blocks[2], name='conv4')\n    x = transition_block(x, 0.5, name='pool4')\n    x = dense_block(x, blocks[3], name='conv5')\n \n    x = BatchNormalization(axis=bn_axis, epsilon=1.001e-5,\n                           name='bn')(x)\n \n    if include_top:\n        x = GlobalAveragePooling2D(name='avg_pool')(x)\n        x = Dense(classes, activation='sigmoid', name='fc1000')(x)\n    else:\n        if pooling == 'avg':\n            x = GlobalAveragePooling2D(name='avg_pool')(x)\n        elif pooling == 'max':\n            x = GlobalMaxPooling2D(name='max_pool')(x)\n \n    # Ensure that the model takes into account\n    # any potential predecessors of `input_tensor`.\n    # 确保模型考虑到任何潜在的前缀“input_tensor”。\n    if input_tensor is not None:\n        inputs = get_source_inputs(input_tensor)\n    else:\n        inputs = img_input\n \n    # Create model.\n    # 建立模型\n    if blocks == [6, 12, 24, 16]:\n        model = Model(inputs, x, name='densenet121')\n    elif blocks == [6, 12, 32, 32]:\n        model = Model(inputs, x, name='densenet169')\n    elif blocks == [6, 12, 48, 32]:\n        model = Model(inputs, x, name='densenet201')\n    else:\n        model = Model(inputs, x, name='densenet')\n \n \n    return model\n \n \ndef DenseNet121(include_top=True,\n                weights=None,\n                input_tensor=None,\n                input_shape=None,\n                pooling=None,\n                classes=2):\n    return DenseNet([6, 12, 24, 16],\n                    include_top, weights,\n                    input_tensor, input_shape,\n                    pooling, classes)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_final =DenseNet121()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import Callback\nfrom sklearn.metrics import confusion_matrix, f1_score, precision_score, recall_score\nclass Metrics(Callback):\n    def on_train_begin(self, logs={}):\n        self.val_f1s = []\n        self.val_recalls = []\n        self.val_precisions = []\n\n    def on_epoch_end(self, epoch, logs={}):\n#         val_predict = (np.asarray(self.model.predict(self.validation_data[0]))).round()\n        val_predict = np.argmax(np.asarray(self.model.predict(self.validation_data[0])), axis=1)\n#         val_targ = self.validation_data[1]\n        val_targ = np.argmax(self.validation_data[1], axis=1)\n        _val_f1 = f1_score(val_targ, val_predict, average='macro')\n        _val_recall = recall_score(val_targ, val_predict)\n        _val_precision = precision_score(val_targ, val_predict)\n        self.val_f1s.append(_val_f1)\n        self.val_recalls.append(_val_recall)\n        self.val_precisions.append(_val_precision)\n        print('— val_f1: %f — val_precision: %f — val_recall %f' %(_val_f1, _val_precision, _val_recall))\n#         print(' — val_f1:' ,_val_f1)\n        return\n\nmetrics1 = Metrics()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import ModelCheckpoint, LearningRateScheduler, EarlyStopping, ReduceLROnPlateau\nweight_path=\"{}_weights.best.hdf5\".format('boat_detector')\n\ncheckpoint = ModelCheckpoint(weight_path, monitor='val_loss', verbose=1, \n                             save_best_only=True, mode='min', save_weights_only = True)\n\nreduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=2, verbose=1, mode='auto', epsilon=0.01, cooldown=0, min_lr=0.0001)\nearly = EarlyStopping(monitor=\"val_loss\", \n                      mode=\"min\", \n                      patience=3) # probably needs to be more patient, but kaggle time is limited\ncallbacks_list = [checkpoint, early, reduceLROnPlat,metrics1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras import optimizers\ndef fit():\n    epochs = 40\n    lrate = 0.01\n    decay = lrate/epochs\n    #adam = optimizers.Adam(lr=lrate,beta_1=0.9, beta_2=0.999, decay=decay)\n    sgd = optimizers.SGD(lr=lrate, momentum=0.9, decay=decay, nesterov=False)\n    model_final.compile(loss='binary_crossentropy', optimizer=sgd, metrics=['binary_accuracy'])\n    loss_history=[model_final.fit(x_train, y_train, validation_data=(x_val, y_val),epochs=40, batch_size=50,callbacks=callbacks_list)]\n    \n    return loss_history\nnum=0\n\nwhile True:\n    num=num+1\n#     prefix='%d'%(num)\n    loss_history = fit()\n    model_final.save_weights('my_model_weights%d.h5'% num)\n    if np.min([mh.history['val_loss'] for mh in loss_history]) < 0.1:\n        break\n    if num==1:\n        break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_loss(loss_history):\n    epochs = np.concatenate([mh.epoch for mh in loss_history])\n    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(22, 10))\n    \n    _ = ax1.plot(epochs, np.concatenate([mh.history['loss'] for mh in loss_history]), 'b-',\n                 epochs, np.concatenate([mh.history['val_loss'] for mh in loss_history]), 'r-')\n    ax1.legend(['Training', 'Validation'])#图表，损失函数（训练和验证）的迭代图表\n    ax1.set_title('Loss')\n    \n    _ = ax2.plot(epochs, np.concatenate([mh.history['binary_accuracy'] for mh in loss_history]), 'b-',\n                 epochs, np.concatenate([mh.history['val_binary_accuracy'] for mh in loss_history]), 'r-')\n    ax2.legend(['Training', 'Validation'])#准确率，（训练和迭代的）\n    ax2.set_title('Binary Accuracy (%)')\n\nshow_loss(loss_history)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unique_img_ids1 = unique_img_ids[20000:30000]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_test = np.empty(shape=(10000, 256,256,3),dtype=np.uint8)#10680 256\ny_test = np.empty(shape=10000,dtype=np.uint8)\nfor index, image in enumerate(unique_img_ids1['ImageId']):\n    image_array= Image.open('../input/airbus-ship-detection/train_v2/' + image).resize((256,256)).convert('RGB') #256\n    x_test[index] = image_array\n    y_test[index]=unique_img_ids1[unique_img_ids1['ImageId']==image]['has_ship'].iloc[0]\n\nprint(x_test.shape)\nprint(y_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_test_targets =y_test.reshape(len(y_test),-1)\nenc = OneHotEncoder()\nenc.fit(y_test_targets)\ny_test = enc.transform(y_test_targets).toarray()\nprint(y_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predict_ship = model_final.evaluate( x_test,y_test)\nacc=predict_ship[1]*100","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print ('Accuracy of random data = '+ str(acc) + \"%\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}