{"cells":[{"cell_type":"markdown","metadata":{"_cell_guid":"270eaaee-1b9a-54e7-2fb1-8c63faf265f1"},"source":"This was put together in a couple of hours using code that I used to participate in other Kaggle events. This is a basic startup for those that are interested in using residual networks using Keras. With the code below, the hardware that I was using is:\n\n1.    Ubuntu 14.04\n2.    x64 I7 processor\n3.    Python 2.7\n4.    Used PIP installer for all python packages (and apt-get for specialized code such as OpenCV)\n5.    SSD Harddrive with 1TB storage (I don't think you need more than 100GB though)\n6.    NVidia 1080 GTX Founders Edition graphics card\n\nThis was meant as a starting point to build upon for those that are interested.\n\nEnjoy!\n\nRodney Thomas"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"9fe89c7b-c5be-5618-b7a6-32814c9826cd"},"outputs":[],"source":"from __future__ import division\n\nimport six\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport glob\nimport random\n\nnp.random.seed(2016)\nrandom.seed(2016)\n\nfrom keras.models import Model\nfrom keras.layers import Input, Activation, merge, Dense, Flatten\nfrom keras.layers.convolutional import Convolution2D, MaxPooling2D, AveragePooling2D\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.regularizers import l2\nfrom keras import backend as K\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint"},{"cell_type":"markdown","metadata":{"_cell_guid":"b897fe1a-0d94-6dd4-9ebf-aa41e9493496"},"source":"## Removes autoscroll throughout process"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"5b73b6ca-6267-a157-51de-0a24c590ae8e"},"outputs":[],"source":"%%javascript\nIPython.OutputArea.prototype._should_scroll = function(lines) {\n    return false;\n}"},{"cell_type":"markdown","metadata":{"_cell_guid":"3bfe3c4e-340f-56eb-37bc-7ae2d70bf43a"},"source":"## Global Declarations"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"149372a9-9346-c815-5b5c-a9e5129588cf"},"outputs":[],"source":"conf = dict()\n\n# How many patients will be in train and validation set during training. Range: (0; 1)\nconf['train_valid_fraction'] = 0.8\n\n# Batch size for CNN [Depends on GPU and memory available]\nconf['batch_size'] = 1\n\n# Number of epochs for CNN training\n#conf['nb_epoch'] = 200\nconf['nb_epoch'] = 20\n\n# Early stopping. Stop training after epochs without improving on validation\nconf['patience'] = 3\n\n# Shape of image for CNN (Larger the better, but you need to increase CNN as well)\nconf['image_shape'] = (4160,4128)\n#conf['image_shape'] = (2080,2064)\n#conf['image_shape'] = (1024,1024)\n#conf['image_shape'] = (64,64)"},{"cell_type":"markdown","metadata":{"_cell_guid":"299102ae-9b9d-2ddb-b25f-23ec4952062c"},"source":"## Residual Network Class"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"133e1380-e935-f8ed-4992-681f4bf226a5"},"outputs":[],"source":"def _bn_relu(input):\n    \"\"\"Helper to build a BN -> relu block\n    \"\"\"\n    norm = BatchNormalization(axis=CHANNEL_AXIS)(input)\n    return Activation(\"relu\")(norm)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"d0bac076-0e43-33ac-f774-b71885d7e539"},"outputs":[],"source":"def _conv_bn_relu(**conv_params):\n    \"\"\"Helper to build a conv -> BN -> relu block\n    \"\"\"\n    nb_filter = conv_params[\"nb_filter\"]\n    nb_row = conv_params[\"nb_row\"]\n    nb_col = conv_params[\"nb_col\"]\n    subsample = conv_params.setdefault(\"subsample\", (1, 1))\n    init = conv_params.setdefault(\"init\", \"he_normal\")\n    border_mode = conv_params.setdefault(\"border_mode\", \"same\")\n    W_regularizer = conv_params.setdefault(\"W_regularizer\", l2(1.e-4))\n\n    def f(input):\n        conv = Convolution2D(nb_filter=nb_filter, nb_row=nb_row, nb_col=nb_col, subsample=subsample,\n                             init=init, border_mode=border_mode, W_regularizer=W_regularizer)(input)\n        return _bn_relu(conv)\n\n    return f"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"57acac11-f948-9acf-2aca-0bf25003f93e"},"outputs":[],"source":"def _bn_relu_conv(**conv_params):\n    \"\"\"Helper to build a BN -> relu -> conv block.\n    This is an improved scheme proposed in http://arxiv.org/pdf/1603.05027v2.pdf\n    \"\"\"\n    nb_filter = conv_params[\"nb_filter\"]\n    nb_row = conv_params[\"nb_row\"]\n    nb_col = conv_params[\"nb_col\"]\n    subsample = conv_params.setdefault(\"subsample\", (1,1))\n    init = conv_params.setdefault(\"init\", \"he_normal\")\n    border_mode = conv_params.setdefault(\"border_mode\", \"same\")\n    W_regularizer = conv_params.setdefault(\"W_regularizer\", l2(1.e-4))\n\n    def f(input):\n        activation = _bn_relu(input)\n        return Convolution2D(nb_filter=nb_filter, nb_row=nb_row, nb_col=nb_col, subsample=subsample,\n                             init=init, border_mode=border_mode, W_regularizer=W_regularizer)(activation)\n\n    return f"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"0df72c82-0b89-466b-c558-5e3dfa8cfdc7"},"outputs":[],"source":"def _shortcut(input, residual):\n    \"\"\"Adds a shortcut between input and residual block and merges them with \"sum\"\n    \"\"\"\n    # Expand channels of shortcut to match residual.\n    # Stride appropriately to match residual (width, height)\n    # Should be int if network architecture is correctly configured.\n    input_shape = K.int_shape(input)\n    residual_shape = K.int_shape(residual)\n    stride_width = int(round(input_shape[ROW_AXIS] / residual_shape[ROW_AXIS]))\n    stride_height = int(round(input_shape[COL_AXIS] / residual_shape[COL_AXIS]))\n    equal_channels = input_shape[CHANNEL_AXIS] == residual_shape[CHANNEL_AXIS]\n\n    shortcut = input\n    # 1 X 1 conv if shape is different. Else identity.\n    if stride_width > 1 or stride_height > 1 or not equal_channels:\n        shortcut = Convolution2D(nb_filter=residual_shape[CHANNEL_AXIS],\n                                 nb_row=1, nb_col=1,\n                                 subsample=(stride_width, stride_height),\n                                 init=\"he_normal\", border_mode=\"valid\",\n                                 W_regularizer=l2(0.0001))(input)\n\n    return merge([shortcut, residual], mode=\"sum\")"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"d95bf31e-6a35-7c75-05a6-66a7653638ed"},"outputs":[],"source":"def _residual_block(block_function, nb_filter, repetitions, is_first_layer=False):\n    \"\"\"Builds a residual block with repeating bottleneck blocks.\n    \"\"\"\n    def f(input):\n        for i in range(repetitions):\n            init_subsample = (1, 1)\n            if i == 0 and not is_first_layer:\n                init_subsample = (2, 2)\n            input = block_function(nb_filter=nb_filter, init_subsample=init_subsample,\n                                   is_first_block_of_first_layer=(is_first_layer and i == 0))(input)\n        return input\n\n    return f"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"0cd79778-7888-cf7f-070f-2bbfd71e118a"},"outputs":[],"source":"def basic_block(nb_filter, init_subsample=(1, 1), is_first_block_of_first_layer=False):\n    \"\"\"Basic 3 X 3 convolution blocks for use on resnets with layers <= 34.\n    Follows improved proposed scheme in http://arxiv.org/pdf/1603.05027v2.pdf\n    \"\"\"\n    def f(input):\n\n        if is_first_block_of_first_layer:\n            # don't repeat bn->relu since we just did bn->relu->maxpool\n            conv1 = Convolution2D(nb_filter=nb_filter,\n                                 nb_row=3, nb_col=3,\n                                 subsample=init_subsample,\n                                 init=\"he_normal\", border_mode=\"same\",\n                                 W_regularizer=l2(0.0001))(input)\n        else:\n            conv1 = _bn_relu_conv(nb_filter=nb_filter, nb_row=3, nb_col=3,\n                                  subsample=init_subsample)(input)\n\n        residual = _bn_relu_conv(nb_filter=nb_filter, nb_row=3, nb_col=3)(conv1)\n        return _shortcut(input, residual)\n\n    return f"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"2be51fe6-118c-9af9-ce6e-053621cba09c"},"outputs":[],"source":"def bottleneck(nb_filter, init_subsample=(1, 1), is_first_block_of_first_layer=False):\n    \"\"\"Bottleneck architecture for > 34 layer resnet.\n    Follows improved proposed scheme in http://arxiv.org/pdf/1603.05027v2.pdf\n\n    Returns:\n        A final conv layer of nb_filter * 4\n    \"\"\"\n    def f(input):\n\n        if is_first_block_of_first_layer:\n            # don't repeat bn->relu since we just did bn->relu->maxpool\n            conv_1_1 = Convolution2D(nb_filter=nb_filter,\n                                 nb_row=1, nb_col=1,\n                                 subsample=init_subsample,\n                                 init=\"he_normal\", border_mode=\"same\",\n                                 W_regularizer=l2(0.0001))(input)\n        else:\n            conv_1_1 = _bn_relu_conv(nb_filter=nb_filter, nb_row=1, nb_col=1,\n                                     subsample=init_subsample)(input)\n\n        conv_3_3 = _bn_relu_conv(nb_filter=nb_filter, nb_row=3, nb_col=3)(conv_1_1)\n        residual = _bn_relu_conv(nb_filter=nb_filter * 4, nb_row=1, nb_col=1)(conv_3_3)\n        return _shortcut(input, residual)\n\n    return f"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"539a2e25-ebd8-3660-1064-9cbb6e43cc8b"},"outputs":[],"source":"def _handle_dim_ordering():\n    global ROW_AXIS\n    global COL_AXIS\n    global CHANNEL_AXIS\n    if K.image_dim_ordering() == 'tf':\n        ROW_AXIS = 1\n        COL_AXIS = 2\n        CHANNEL_AXIS = 3\n    else:\n        CHANNEL_AXIS = 1\n        ROW_AXIS = 2\n        COL_AXIS = 3"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"3c1820a8-9f4d-c080-09ad-acc923acb1dd"},"outputs":[],"source":"def _get_block(identifier):\n    if isinstance(identifier, six.string_types):\n        res = globals().get(identifier)\n        if not res:\n            raise ValueError('Invalid {}'.format(identifier))\n        return res\n    return identifier"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"0e8dc80f-f617-09e9-a94b-a19c8e43220c"},"outputs":[],"source":"class ResnetBuilder(object):\n    @staticmethod\n    def build(input_shape, num_outputs, block_fn, repetitions):\n        \"\"\"Builds a custom ResNet like architecture.\n\n        Args:\n            input_shape: The input shape in the form (nb_channels, nb_rows, nb_cols)\n            num_outputs: The number of outputs at final softmax layer\n            block_fn: The block function to use. This is either `basic_block` or `bottleneck`.\n                The original paper used basic_block for layers < 50\n            repetitions: Number of repetitions of various block units.\n                At each block unit, the number of filters are doubled and the input size is halved\n\n        Returns:\n            The keras `Model`.\n        \"\"\"\n        _handle_dim_ordering()\n        if len(input_shape) != 3:\n            raise Exception(\"Input shape should be a tuple (nb_channels, nb_rows, nb_cols)\")\n\n        # Permute dimension order if necessary\n        if K.image_dim_ordering() == 'tf':\n            input_shape = (input_shape[1], input_shape[2], input_shape[0])\n\n        # Load function from str if needed.\n        block_fn = _get_block(block_fn)\n\n        input = Input(shape=input_shape)\n        conv1 = _conv_bn_relu(nb_filter=64, nb_row=7, nb_col=7, subsample=(2, 2))(input)\n        pool1 = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), border_mode=\"same\")(conv1)\n\n        block = pool1\n        nb_filter = 64\n        for i, r in enumerate(repetitions):\n            block = _residual_block(block_fn, nb_filter=nb_filter, repetitions=r, is_first_layer=(i == 0))(block)\n            nb_filter *= 2\n\n        # Last activation\n        block = _bn_relu(block)\n\n        block_norm = BatchNormalization(mode=0, axis=CHANNEL_AXIS)(block)\n        block_output = Activation(\"relu\")(block_norm)\n\n        # Classifier block\n        block_shape = K.int_shape(block)\n        pool2 = AveragePooling2D(pool_size=(block_shape[ROW_AXIS], block_shape[COL_AXIS]),\n                                 strides=(1, 1))(block_output)\n        flatten1 = Flatten()(pool2)\n        dense = Dense(output_dim=num_outputs, init=\"he_normal\", activation=\"softmax\")(flatten1)\n        #dense = Dense(output_dim=num_outputs, W_regularizer=l2(0.01), init=\"he_normal\", activation=\"linear\")(flatten1)\n\n        model = Model(input=input, output=dense)\n        return model\n\n    @staticmethod\n    def build_resnet_test(input_shape, num_outputs):\n        return ResnetBuilder.build(input_shape, num_outputs, basic_block, [1, 1, 1, 1])\n\n    @staticmethod\n    def build_resnet_18(input_shape, num_outputs):\n        return ResnetBuilder.build(input_shape, num_outputs, basic_block, [2, 2, 2, 2])\n\n    @staticmethod\n    def build_resnet_34(input_shape, num_outputs):\n        return ResnetBuilder.build(input_shape, num_outputs, basic_block, [3, 4, 6, 3])\n\n    @staticmethod\n    def build_resnet_50(input_shape, num_outputs):\n        return ResnetBuilder.build(input_shape, num_outputs, bottleneck, [3, 4, 6, 3])\n\n    @staticmethod\n    def build_resnet_101(input_shape, num_outputs):\n        return ResnetBuilder.build(input_shape, num_outputs, bottleneck, [3, 4, 23, 3])\n\n    @staticmethod\n    def build_resnet_152(input_shape, num_outputs):\n        return ResnetBuilder.build(input_shape, num_outputs, bottleneck, [3, 8, 36, 3])"},{"cell_type":"markdown","metadata":{"_cell_guid":"90acd634-8ebd-a939-0122-7e3c38e4f70d"},"source":"## Batch Generator for model fit_generator"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"ec47837a-9512-bb0b-c3be-b497f72e313c"},"outputs":[],"source":"def batch_generator_train(files, batch_size):\n    number_of_batches = np.ceil(len(files)/batch_size)\n    counter = 0\n    random.shuffle(files)\n    while True:\n        batch_files = files[batch_size*counter:batch_size*(counter+1)]\n        image_list = []\n        mask_list = []\n        for f in batch_files:\n            image = cv2.imread(f)\n            image = cv2.resize(image, conf['image_shape'])\n\n            cancer_type = f[20:21] # relies on path lengths that is hard coded below\n            if cancer_type == '1':\n                mask = [1, 0, 0]\n            elif cancer_type == '2':\n                mask = [0, 1, 0]\n            else:\n                mask = [0, 0, 1]\n\n            image_list.append(image)\n            mask_list.append(mask)\n        counter += 1\n        image_list = np.array(image_list)\n        mask_list = np.array(mask_list)\n\n        yield image_list, mask_list\n\n        if counter == number_of_batches:\n            random.shuffle(files)\n            counter = 0"},{"cell_type":"markdown","metadata":{"_cell_guid":"cdd1e9e7-a7eb-1f90-5b0c-30bcb18d8a74"},"source":"## Hardcoded paths to training files. Note that \"additional\" has been renamed to \"add01\" since the path lengths must be the same for substring extraction."},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"be957ce1-5948-1aa3-ecb2-640e5d913b88"},"outputs":[],"source":"# file paths to training and additional samples\nfilepaths = []\nfilepaths.append('../input/train/Type_1/')\nfilepaths.append('../input/train/Type_2/')\nfilepaths.append('../input/train/Type_3/')\n"},{"cell_type":"markdown","metadata":{"_cell_guid":"22ad65a0-53fb-a3b0-8af8-95bf0c4e2b81"},"source":"## Get a list of all training files"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"e6ee8bfd-cc38-e02c-bef9-5eccf83aaa77"},"outputs":[],"source":"allFiles = []\n\nfor i, filepath in enumerate(filepaths):\n    files = glob.glob(filepath + '*.jpg')\n    allFiles = allFiles + files"},{"cell_type":"markdown","metadata":{"_cell_guid":"9473e7d8-9ce1-667c-d45f-248ee4127d21"},"source":"## Split data into training and validation sets"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"f4938ca0-aef9-26a3-8167-5fed28371cc6"},"outputs":[],"source":"split_point = int(round(conf['train_valid_fraction']*len(allFiles)))\n\nrandom.shuffle(allFiles)\n\ntrain_list = allFiles[:split_point]\nvalid_list = allFiles[split_point:]\nprint('Train patients: {}'.format(len(train_list)))\nprint('Valid patients: {}'.format(len(valid_list)))"},{"cell_type":"markdown","metadata":{"_cell_guid":"6e56eec6-a8b6-9423-cb3b-433c2e1c8c54"},"source":"## Testing model generator"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"d76b0978-28b8-8b12-850b-3c26e8797d2d"},"outputs":[],"source":"print('Create and compile model...')\n\nnb_classes = 3\nimg_rows, img_cols = conf['image_shape'][1], conf['image_shape'][0]\nimg_channels = 3\n\nmodel = ResnetBuilder.build_resnet_34((img_channels, img_rows, img_cols), nb_classes)\nmodel.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n#model.compile(loss='hinge',optimizer='adadelta',metrics=['accuracy'])\n\ncallbacks = [\n    EarlyStopping(monitor='val_loss', patience=conf['patience'], verbose=0),\n    ModelCheckpoint('cervical_best.hdf5', monitor='val_loss', save_best_only=True, verbose=0),\n]\n\nprint('Fit model...')\nfit = model.fit_generator(generator=batch_generator_train(train_list, conf['batch_size']),\n                      nb_epoch=conf['nb_epoch'],\n                      #samples_per_epoch=len(train_list),\n                      samples_per_epoch=3,\n                      validation_data=batch_generator_train(valid_list, conf['batch_size']),\n                      #nb_val_samples=len(valid_list),\n                      nb_val_samples=1,\n                      verbose=1,\n                      callbacks=callbacks)"},{"cell_type":"markdown","metadata":{"_cell_guid":"df7b3fb4-c63b-8feb-9e17-2e316b4f60d2"},"source":"## Create submission files with prediction for submission"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"d6c1c116-2829-3da6-841c-3917e3700d80"},"outputs":[],"source":"#from keras.models import load_model\n#model = load_model('cervical_best.hdf5')\n\nsample_subm = pd.read_csv(\"../input/sample_submission.csv\")\nids = sample_subm['image_name'].values\n\nfor id in ids:\n    print('Predict for image {}'.format(id))\n    files = glob.glob(\"../input/test/\" + id)\n    image_list = []\n    for f in files:\n        image = cv2.imread(f)\n        image = cv2.resize(image, conf['image_shape'])\n        image_list.append(image)\n        \n    image_list = np.array(image_list)\n\n    predictions = model.predict(image_list, verbose=1, batch_size=1)\n\n    sample_subm.loc[sample_subm['image_name'] == id, 'Type_1'] = predictions[0,0]\n    sample_subm.loc[sample_subm['image_name'] == id, 'Type_2'] = predictions[0,1]\n    sample_subm.loc[sample_subm['image_name'] == id, 'Type_3'] = predictions[0,2]\n    \nsample_subm.to_csv(\"subm.csv\", index=False)"}],"metadata":{"_change_revision":0,"_is_fork":false,"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.0"}},"nbformat":4,"nbformat_minor":0}