{"cells":[
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# -*- coding: utf-8 -*-\r\n\r\nimport numpy as np\r\n\r\nimport os\r\nimport glob\r\nimport cv2\r\nimport math\r\nimport pickle\r\nimport datetime\r\nimport pandas as pd\r\n\r\nfrom sklearn.cross_validation import train_test_split\r\nfrom sklearn.cross_validation import KFold\r\nfrom keras.models import Sequential\r\nfrom keras.layers.core import Dense, Dropout, Flatten\r\nfrom keras.layers.convolutional import Convolution2D, MaxPooling2D, \\\r\n                                       ZeroPadding2D\r\n\r\n# from keras.layers.normalization import BatchNormalization\r\n# from keras.optimizers import Adam\r\nfrom keras.optimizers import SGD\r\nfrom keras.utils import np_utils\r\nfrom keras.models import model_from_json\r\n# from sklearn.metrics import log_loss\r\nfrom numpy.random import permutation\r\n\r\n\r\nnp.random.seed(2016)\r\nuse_cache = 1\r\n# color type: 1 - grey, 3 - rgb\r\ncolor_type_global = 3\r\n\r\n# color_type = 1 - gray\r\n# color_type = 3 - RGB\r\n\r\n\r\ndef get_im(path, img_rows, img_cols, color_type=1):\r\n    # Load as grayscale\r\n    if color_type == 1:\r\n        img = cv2.imread(path, 0)\r\n    elif color_type == 3:\r\n        img = cv2.imread(path)\r\n    # Reduce size\r\n    resized = cv2.resize(img, (img_cols, img_rows))\r\n    # mean_pixel = [103.939, 116.799, 123.68]\r\n    # resized = resized.astype(np.float32, copy=False)\r\n\r\n    # for c in range(3):\r\n    #    resized[:, :, c] = resized[:, :, c] - mean_pixel[c]\r\n    # resized = resized.transpose((2, 0, 1))\r\n    # resized = np.expand_dims(img, axis=0)\r\n    return resized\r\n\r\n\r\ndef get_driver_data():\r\n    dr = dict()\r\n    path = os.path.join('..', 'input', 'driver_imgs_list.csv')\r\n    print('Read drivers data')\r\n    f = open(path, 'r')\r\n    line = f.readline()\r\n    while (1):\r\n        line = f.readline()\r\n        if line == '':\r\n            break\r\n        arr = line.strip().split(',')\r\n        dr[arr[2]] = arr[0]\r\n    f.close()\r\n    return dr\r\n\r\n\r\ndef load_train(img_rows, img_cols, color_type=1):\r\n    X_train = []\r\n    y_train = []\r\n    driver_id = []\r\n\r\n    driver_data = get_driver_data()\r\n\r\n    print('Read train images')\r\n    for j in range(10):\r\n        print('Load folder c{}'.format(j))\r\n        path = os.path.join('..', 'input', 'imgs', 'train',\r\n                            'c' + str(j), '*.jpg')\r\n        files = glob.glob(path)\r\n        for fl in files:\r\n            flbase = os.path.basename(fl)\r\n            img = get_im(fl, img_rows, img_cols, color_type)\r\n            X_train.append(img)\r\n            y_train.append(j)\r\n            driver_id.append(driver_data[flbase])\r\n\r\n    unique_drivers = sorted(list(set(driver_id)))\r\n    print('Unique drivers: {}'.format(len(unique_drivers)))\r\n    print(unique_drivers)\r\n    return X_train, y_train, driver_id, unique_drivers\r\n\r\n\r\ndef load_test(img_rows, img_cols, color_type=1):\r\n    print('Read test images')\r\n    path = os.path.join('..', 'input', 'imgs', 'test', '*.jpg')\r\n    files = glob.glob(path)\r\n    X_test = []\r\n    X_test_id = []\r\n    total = 0\r\n    thr = math.floor(len(files)/10)\r\n    for fl in files:\r\n        flbase = os.path.basename(fl)\r\n        img = get_im(fl, img_rows, img_cols, color_type)\r\n        X_test.append(img)\r\n        X_test_id.append(flbase)\r\n        total += 1\r\n        if total % thr == 0:\r\n            print('Read {} images from {}'.format(total, len(files)))\r\n\r\n    return X_test, X_test_id\r\n\r\n\r\ndef cache_data(data, path):\r\n    if not os.path.isdir('cache'):\r\n        os.mkdir('cache')\r\n    if os.path.isdir(os.path.dirname(path)):\r\n        file = open(path, 'wb')\r\n        pickle.dump(data, file)\r\n        file.close()\r\n    else:\r\n        print('Directory doesnt exists')\r\n\r\n\r\ndef restore_data(path):\r\n    data = dict()\r\n    if os.path.isfile(path):\r\n        print('Restore data from pickle........')\r\n        file = open(path, 'rb')\r\n        data = pickle.load(file)\r\n    return data\r\n\r\n\r\ndef save_model(model, index, cross=''):\r\n    json_string = model.to_json()\r\n    if not os.path.isdir('cache'):\r\n        os.mkdir('cache')\r\n    json_name = 'architecture' + str(index) + cross + '.json'\r\n    weight_name = 'model_weights' + str(index) + cross + '.h5'\r\n    open(os.path.join('cache', json_name), 'w').write(json_string)\r\n    model.save_weights(os.path.join('cache', weight_name), overwrite=True)\r\n\r\n\r\ndef read_model(index, cross=''):\r\n    json_name = 'architecture' + str(index) + cross + '.json'\r\n    weight_name = 'model_weights' + str(index) + cross + '.h5'\r\n    model = model_from_json(open(os.path.join('cache', json_name)).read())\r\n    model.load_weights(os.path.join('cache', weight_name))\r\n    return model\r\n\r\n\r\ndef split_validation_set(train, target, test_size):\r\n    random_state = 51\r\n    X_train, X_test, y_train, y_test = \\\r\n        train_test_split(train, target,\r\n                         test_size=test_size,\r\n                         random_state=random_state)\r\n    return X_train, X_test, y_train, y_test\r\n\r\n\r\ndef create_submission(predictions, test_id, info):\r\n    result1 = pd.DataFrame(predictions, columns=['c0', 'c1', 'c2', 'c3',\r\n                                                 'c4', 'c5', 'c6', 'c7',\r\n                                                 'c8', 'c9'])\r\n    result1.loc[:, 'img'] = pd.Series(test_id, index=result1.index)\r\n    now = datetime.datetime.now()\r\n    if not os.path.isdir('subm'):\r\n        os.mkdir('subm')\r\n    suffix = info + '_' + str(now.strftime(\"%Y-%m-%d-%H-%M\"))\r\n    sub_file = os.path.join('subm', 'submission_' + suffix + '.csv')\r\n    result1.to_csv(sub_file, index=False)\r\n\r\n\r\ndef read_and_normalize_and_shuffle_train_data(img_rows, img_cols,\r\n                                              color_type=1):\r\n\r\n    cache_path = os.path.join('cache', 'train_r_' + str(img_rows) +\r\n                              '_c_' + str(img_cols) + '_t_' +\r\n                              str(color_type) + '.dat')\r\n\r\n    if not os.path.isfile(cache_path) or use_cache == 0:\r\n        train_data, train_target, driver_id, unique_drivers = \\\r\n            load_train(img_rows, img_cols, color_type)\r\n        cache_data((train_data, train_target, driver_id, unique_drivers),\r\n                   cache_path)\r\n    else:\r\n        print('Restore train from cache!')\r\n        (train_data, train_target, driver_id, unique_drivers) = \\\r\n            restore_data(cache_path)\r\n\r\n    train_data = np.array(train_data, dtype=np.uint8)\r\n    train_target = np.array(train_target, dtype=np.uint8)\r\n\r\n    if color_type == 1:\r\n        train_data = train_data.reshape(train_data.shape[0], color_type,\r\n                                        img_rows, img_cols)\r\n    else:\r\n        train_data = train_data.transpose((0, 3, 1, 2))\r\n\r\n    train_target = np_utils.to_categorical(train_target, 10)\r\n    train_data = train_data.astype('float32')\r\n    mean_pixel = [103.939, 116.779, 123.68]\r\n    for c in range(3):\r\n        train_data[:, c, :, :] = train_data[:, c, :, :] - mean_pixel[c]\r\n    # train_data /= 255\r\n    perm = permutation(len(train_target))\r\n    train_data = train_data[perm]\r\n    train_target = train_target[perm]\r\n    print('Train shape:', train_data.shape)\r\n    print(train_data.shape[0], 'train samples')\r\n    return train_data, train_target, driver_id, unique_drivers\r\n\r\n\r\ndef read_and_normalize_test_data(img_rows=224, img_cols=224, color_type=1):\r\n    cache_path = os.path.join('cache', 'test_r_' + str(img_rows) +\r\n                              '_c_' + str(img_cols) + '_t_' +\r\n                              str(color_type) + '.dat')\r\n    if not os.path.isfile(cache_path) or use_cache == 0:\r\n        test_data, test_id = load_test(img_rows, img_cols, color_type)\r\n        cache_data((test_data, test_id), cache_path)\r\n    else:\r\n        print('Restore test from cache!')\r\n        (test_data, test_id) = restore_data(cache_path)\r\n\r\n    test_data = np.array(test_data, dtype=np.uint8)\r\n\r\n    if color_type == 1:\r\n        test_data = test_data.reshape(test_data.shape[0], color_type,\r\n                                      img_rows, img_cols)\r\n    else:\r\n        test_data = test_data.transpose((0, 3, 1, 2))\r\n\r\n    test_data = test_data.astype('float32')\r\n    mean_pixel = [103.939, 116.779, 123.68]\r\n    for c in range(3):\r\n        test_data[:, c, :, :] = test_data[:, c, :, :] - mean_pixel[c]\r\n    # test_data /= 255\r\n    print('Test shape:', test_data.shape)\r\n    print(test_data.shape[0], 'test samples')\r\n    return test_data, test_id\r\n\r\n\r\ndef dict_to_list(d):\r\n    ret = []\r\n    for i in d.items():\r\n        ret.append(i[1])\r\n    return ret\r\n\r\n\r\ndef merge_several_folds_mean(data, nfolds):\r\n    a = np.array(data[0])\r\n    for i in range(1, nfolds):\r\n        a += np.array(data[i])\r\n    a /= nfolds\r\n    return a.tolist()\r\n\r\n\r\ndef merge_several_folds_geom(data, nfolds):\r\n    a = np.array(data[0])\r\n    for i in range(1, nfolds):\r\n        a *= np.array(data[i])\r\n    a = np.power(a, 1/nfolds)\r\n    return a.tolist()\r\n\r\n\r\ndef copy_selected_drivers(train_data, train_target, driver_id, driver_list):\r\n    data = []\r\n    target = []\r\n    index = []\r\n    for i in range(len(driver_id)):\r\n        if driver_id[i] in driver_list:\r\n            data.append(train_data[i])\r\n            target.append(train_target[i])\r\n            index.append(i)\r\n    data = np.array(data, dtype=np.float32)\r\n    target = np.array(target, dtype=np.float32)\r\n    index = np.array(index, dtype=np.uint32)\r\n    return data, target, index\r\n\r\n\r\ndef vgg_std16_model(img_rows, img_cols, color_type=1):\r\n    model = Sequential()\r\n    model.add(ZeroPadding2D((1, 1), input_shape=(color_type,\r\n                                                 img_rows, img_cols)))\r\n    model.add(Convolution2D(64, 3, 3, activation='relu'))\r\n    model.add(ZeroPadding2D((1, 1)))\r\n    model.add(Convolution2D(64, 3, 3, activation='relu'))\r\n    model.add(MaxPooling2D((2, 2), strides=(2, 2)))\r\n\r\n    model.add(ZeroPadding2D((1, 1)))\r\n    model.add(Convolution2D(128, 3, 3, activation='relu'))\r\n    model.add(ZeroPadding2D((1, 1)))\r\n    model.add(Convolution2D(128, 3, 3, activation='relu'))\r\n    model.add(MaxPooling2D((2, 2), strides=(2, 2)))\r\n\r\n    model.add(ZeroPadding2D((1, 1)))\r\n    model.add(Convolution2D(256, 3, 3, activation='relu'))\r\n    model.add(ZeroPadding2D((1, 1)))\r\n    model.add(Convolution2D(256, 3, 3, activation='relu'))\r\n    model.add(ZeroPadding2D((1, 1)))\r\n    model.add(Convolution2D(256, 3, 3, activation='relu'))\r\n    model.add(MaxPooling2D((2, 2), strides=(2, 2)))\r\n\r\n    model.add(ZeroPadding2D((1, 1)))\r\n    model.add(Convolution2D(512, 3, 3, activation='relu'))\r\n    model.add(ZeroPadding2D((1, 1)))\r\n    model.add(Convolution2D(512, 3, 3, activation='relu'))\r\n    model.add(ZeroPadding2D((1, 1)))\r\n    model.add(Convolution2D(512, 3, 3, activation='relu'))\r\n    model.add(MaxPooling2D((2, 2), strides=(2, 2)))\r\n\r\n    model.add(ZeroPadding2D((1, 1)))\r\n    model.add(Convolution2D(512, 3, 3, activation='relu'))\r\n    model.add(ZeroPadding2D((1, 1)))\r\n    model.add(Convolution2D(512, 3, 3, activation='relu'))\r\n    model.add(ZeroPadding2D((1, 1)))\r\n    model.add(Convolution2D(512, 3, 3, activation='relu'))\r\n    model.add(MaxPooling2D((2, 2), strides=(2, 2)))\r\n\r\n    model.add(Flatten())\r\n    model.add(Dense(4096, activation='relu'))\r\n    model.add(Dropout(0.5))\r\n    model.add(Dense(4096, activation='relu'))\r\n    model.add(Dropout(0.5))\r\n    model.add(Dense(1000, activation='softmax'))\r\n\r\n    model.load_weights('../input/vgg16_weights.h5')\r\n\r\n    # Code above loads pre-trained data and\r\n    model.layers.pop()\r\n    model.add(Dense(10, activation='softmax'))\r\n    # Learning rate is changed to 0.001\r\n    sgd = SGD(lr=1e-3, decay=1e-6, momentum=0.9, nesterov=True)\r\n    model.compile(optimizer=sgd, loss='categorical_crossentropy')\r\n    return model\r\n\r\n\r\ndef run_cross_validation(nfolds=10, nb_epoch=10, split=0.2, modelStr=''):\r\n\r\n    # Now it loads color image\r\n    # input image dimensions\r\n    img_rows, img_cols = 224, 224\r\n    batch_size = 64\r\n    random_state = 20\r\n\r\n    train_data, train_target, driver_id, unique_drivers = \\\r\n        read_and_normalize_and_shuffle_train_data(img_rows, img_cols,\r\n                                                  color_type_global)\r\n\r\n    # ishuf_train_data = []\r\n    # shuf_train_target = []\r\n    # index_shuf = range(len(train_target))\r\n    # shuffle(index_shuf)\r\n    # for i in index_shuf:\r\n    #     shuf_train_data.append(train_data[i])\r\n    #     shuf_train_target.append(train_target[i])\r\n\r\n    # yfull_train = dict()\r\n    # yfull_test = []\r\n    num_fold = 0\r\n    kf = KFold(len(unique_drivers), n_folds=nfolds,\r\n               shuffle=True, random_state=random_state)\r\n    for train_drivers, test_drivers in kf:\r\n        num_fold += 1\r\n        print('Start KFold number {} from {}'.format(num_fold, nfolds))\r\n        # print('Split train: ', len(X_train), len(Y_train))\r\n        # print('Split valid: ', len(X_valid), len(Y_valid))\r\n        # print('Train drivers: ', unique_list_train)\r\n        # print('Test drivers: ', unique_list_valid)\r\n        # model = create_model_v1(img_rows, img_cols, color_type_global)\r\n        # model = vgg_bn_model(img_rows, img_cols, color_type_global)\r\n        model = vgg_std16_model(img_rows, img_cols, color_type_global)\r\n\r\n        model.fit(train_data, train_target, batch_size=batch_size,\r\n                  nb_epoch=nb_epoch,\r\n                  show_accuracy=True, verbose=1,\r\n                  validation_split=split, shuffle=True)\r\n\r\n        # print('losses: ' + hist.history.losses[-1])\r\n\r\n        # print('Score log_loss: ', score[0])\r\n\r\n        save_model(model, num_fold, modelStr)\r\n\r\n        # predictions_valid = model.predict(X_valid, batch_size=128, verbose=1)\r\n        # score = log_loss(Y_valid, predictions_valid)\r\n        # print('Score log_loss: ', score)\r\n        # Store valid predictions\r\n        # for i in range(len(test_index)):\r\n        #    yfull_train[test_index[i]] = predictions_valid[i]\r\n\r\n    print('Start testing............')\r\n    test_data, test_id = read_and_normalize_test_data(img_rows, img_cols,\r\n                                                      color_type_global)\r\n    yfull_test = []\r\n\r\n    for index in range(1, num_fold + 1):\r\n        # 1,2,3,4,5\r\n        # Store test predictions\r\n        model = read_model(index, modelStr)\r\n        test_prediction = model.predict(test_data, batch_size=128, verbose=1)\r\n        yfull_test.append(test_prediction)\r\n\r\n    info_string = 'loss_' + modelStr \\\r\n                  + '_r_' + str(img_rows) \\\r\n                  + '_c_' + str(img_cols) \\\r\n                  + '_folds_' + str(nfolds) \\\r\n                  + '_ep_' + str(nb_epoch)\r\n\r\n    test_res = merge_several_folds_mean(yfull_test, nfolds)\r\n    create_submission(test_res, test_id, info_string)\r\n\r\n\r\ndef test_model_and_submit(start=1, end=1, modelStr=''):\r\n    img_rows, img_cols = 224, 224\r\n    # batch_size = 64\r\n    # random_state = 51\r\n    nb_epoch = 15\r\n\r\n    print('Start testing............')\r\n    test_data, test_id = read_and_normalize_test_data(img_rows, img_cols,\r\n                                                      color_type_global)\r\n    yfull_test = []\r\n\r\n    for index in range(start, end + 1):\r\n        # Store test predictions\r\n        model = read_model(index, modelStr)\r\n        test_prediction = model.predict(test_data, batch_size=128, verbose=1)\r\n        yfull_test.append(test_prediction)\r\n\r\n    info_string = 'loss_' + modelStr \\\r\n                  + '_r_' + str(img_rows) \\\r\n                  + '_c_' + str(img_cols) \\\r\n                  + '_folds_' + str(end - start + 1) \\\r\n                  + '_ep_' + str(nb_epoch)\r\n\r\n    test_res = merge_several_folds_mean(yfull_test, end - start + 1)\r\n    create_submission(test_res, test_id, info_string)\r\n\r\n# nfolds, nb_epoch, split\r\nrun_cross_validation(2, 20, 0.15, '_vgg_16_2x20')\r\n\r\n# nb_epoch, split\r\n# run_one_fold_cross_validation(10, 0.1)\r\n\r\n# test_model_and_submit(1, 10, 'high_epoch')\r\n"
 }
],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}}, "nbformat": 4, "nbformat_minor": 0}