{"cells":[
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n\n# Any results you write to the current directory are saved as output."
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "import numpy as np\nnp.random.seed(2016)\n\nimport os\nimport glob\nimport cv2\nimport math\nimport pickle\nimport datetime, sklearn, keras, theano, scipy\nimport pandas as pd\nimport statistics\nimport time\nfrom shutil import copy2\nimport warnings\nimport random\nwarnings.filterwarnings(\"ignore\")\n\nfrom sklearn.cross_validation import train_test_split\nfrom sklearn.cross_validation import KFold\nfrom keras.models import Sequential\nfrom keras.layers.core import Dense, Dropout, Activation, Flatten\nfrom keras.layers.convolutional import Convolution2D, MaxPooling2D, ZeroPadding2D\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.optimizers import SGD\nfrom keras.optimizers import Adam\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint\nfrom keras.utils import np_utils\nfrom keras.models import model_from_json\nfrom sklearn.metrics import log_loss\nfrom scipy.misc import imread, imresize, imshow\n\nuse_cache = 1\n\n\ndef show_image(im, name='image'):\n    cv2.imshow(name, im)\n    cv2.waitKey(0)\n    cv2.destroyAllWindows()\n\n# color_type = 1 - gray\n# color_type = 3 - RGB\ndef get_im_cv2(path, img_rows, img_cols, color_type=1):\n    # Load as grayscale\n    if color_type == 1:\n        img = cv2.imread(path, 0)\n    elif color_type == 3:\n        img = cv2.imread(path)\n    # Reduce size\n    resized = cv2.resize(img, (img_cols, img_rows), cv2.INTER_LINEAR)\n    return resized\n\n\ndef get_im_cv2_mod(path, img_rows, img_cols, color_type=1):\n    # Load as grayscale\n    if color_type == 1:\n        img = cv2.imread(path, 0)\n    else:\n        img = cv2.imread(path)\n    # Reduce size\n    rotate = random.uniform(-10, 10)\n    M = cv2.getRotationMatrix2D((img.shape[1]/2, img.shape[0]/2), rotate, 1)\n    img = cv2.warpAffine(img, M, (img.shape[1], img.shape[0]))\n    resized = cv2.resize(img, (img_cols, img_rows), cv2.INTER_LINEAR)\n    return resized\n\n\ndef get_driver_data():\n    dr = dict()\n    clss = dict()\n    path = os.path.join('..', 'input', 'driver_imgs_list.csv')\n    print('Read drivers data')\n    f = open(path, 'r')\n    line = f.readline()\n    while (1):\n        line = f.readline()\n        if line == '':\n            break\n        arr = line.strip().split(',')\n        dr[arr[2]] = arr[0]\n        if arr[0] not in clss.keys():\n            clss[arr[0]] = [(arr[1], arr[2])]\n        else:\n            clss[arr[0]].append((arr[1], arr[2]))\n    f.close()\n    return dr, clss\n\n\ndef load_train(img_rows, img_cols, color_type=1):\n    X_train = []\n    X_train_id = []\n    y_train = []\n    driver_id = []\n    start_time = time.time()\n    driver_data, dr_class = get_driver_data()\n\n    print('Read train images')\n    for j in range(10):\n        print('Load folder c{}'.format(j))\n        path = os.path.join('..', 'input', 'train', 'c' + str(j), '*.jpg')\n        files = glob.glob(path)\n        for fl in files:\n            flbase = os.path.basename(fl)\n            # img = get_im_cv2(fl, img_rows, img_cols, color_type)\n            img = get_im_cv2_mod(fl, img_rows, img_cols, color_type)\n            X_train.append(img)\n            X_train_id.append(flbase)\n            y_train.append(j)\n            driver_id.append(driver_data[flbase])\n\n    print('Read train data time: {} seconds'.format(round(time.time() - start_time, 2)))\n    unique_drivers = sorted(list(set(driver_id)))\n    print('Unique drivers: {}'.format(len(unique_drivers)))\n    print(unique_drivers)\n    return X_train, y_train, X_train_id, driver_id, unique_drivers\n\n\ndef load_test(img_rows, img_cols, color_type=1):\n    print('Read test images')\n    path = os.path.join('..', 'input', 'test', '*.jpg')\n    files = glob.glob(path)\n    X_test = []\n    X_test_id = []\n    total = 0\n    start_time = time.time()\n    thr = math.floor(len(files)/10)\n    for fl in files:\n        flbase = os.path.basename(fl)\n        # img = get_im_cv2(fl, img_rows, img_cols, color_type)\n        img = get_im_cv2_mod(fl, img_rows, img_cols, color_type)\n        X_test.append(img)\n        X_test_id.append(flbase)\n        total += 1\n        if total%thr == 0:\n            print('Read {} images from {}'.format(total, len(files)))\n\n    print('Read test data time: {} seconds'.format(round(time.time() - start_time, 2)))\n    return X_test, X_test_id\n\n\ndef cache_data(data, path):\n    if os.path.isdir(os.path.dirname(path)):\n        file = open(path, 'wb')\n        pickle.dump(data, file)\n        file.close()\n    else:\n        print('Directory doesnt exists')\n\n\ndef restore_data(path):\n    data = dict()\n    if os.path.isfile(path):\n        file = open(path, 'rb')\n        data = pickle.load(file)\n    return data\n\n\ndef save_model(model, arch_path, weights_path):\n    json_string = model.to_json()\n    if not os.path.isdir('cache'):\n        os.mkdir('cache')\n    open(arch_path, 'w').write(json_string)\n    model.save_weights(weights_path, overwrite=True)\n\n\ndef read_model(arch_path, weights_path):\n    model = model_from_json(open(arch_path).read())\n    model.load_weights(weights_path)\n    return model\n\n\ndef split_validation_set(train, target, test_size):\n    random_state = 51\n    X_train, X_test, y_train, y_test = train_test_split(train, target, test_size=test_size, random_state=random_state)\n    return X_train, X_test, y_train, y_test\n\n\ndef create_submission(predictions, test_id, info):\n    result1 = pd.DataFrame(predictions, columns=['c0', 'c1', 'c2', 'c3', 'c4', 'c5', 'c6', 'c7', 'c8', 'c9'])\n    result1.loc[:, 'img'] = pd.Series(test_id, index=result1.index)\n    now = datetime.datetime.now()\n    if not os.path.isdir('subm'):\n        os.mkdir('subm')\n    suffix = info + '_' + str(now.strftime(\"%Y-%m-%d-%H-%M\"))\n    sub_file = os.path.join('subm', 'submission_' + suffix + '.csv')\n    result1.to_csv(sub_file, index=False)\n\n\ndef save_useful_data(predictions_valid, valid_ids, model, info):\n    result1 = pd.DataFrame(predictions_valid, columns=['c0', 'c1', 'c2', 'c3', 'c4', 'c5', 'c6', 'c7', 'c8', 'c9'])\n    result1.loc[:, 'img'] = pd.Series(valid_ids, index=result1.index)\n    now = datetime.datetime.now()\n    if not os.path.isdir(os.path.join('subm', 'data')):\n        os.mkdir(os.path.join('subm', 'data'))\n    suffix = info + '_' + str(now.strftime(\"%Y-%m-%d-%H-%M\"))\n    # Save predictions\n    pred_file = os.path.join('subm', 'data', 's_' + suffix + '_train_predictions.csv')\n    result1.to_csv(pred_file, index=False)\n    # Save model\n    json_string = model.to_json()\n    model_file = os.path.join('subm', 'data', 's_' + suffix + '_model.json')\n    open(model_file, 'w').write(json_string)\n    # Save code\n    cur_code = os.path.realpath(__file__)\n    code_file = os.path.join('subm', 'data', 's_' + suffix + '_code.py')\n    copy2(cur_code, code_file)\n\n\ndef read_and_normalize_train_data(img_rows, img_cols, color_type=1):\n    cache_path = os.path.join('cache', 'train_r_' + str(img_rows) + '_c_' + str(img_cols) + '_t_' + str(color_type) + '_rotated.dat')\n    if not os.path.isfile(cache_path) or use_cache == 0:\n        train_data, train_target, train_id, driver_id, unique_drivers = load_train(img_rows, img_cols, color_type)\n        cache_data((train_data, train_target, train_id, driver_id, unique_drivers), cache_path)\n    else:\n        print('Restore train from cache!')\n        (train_data, train_target, train_id, driver_id, unique_drivers) = restore_data(cache_path)\n\n    train_data = np.array(train_data, dtype=np.uint8)\n    train_target = np.array(train_target, dtype=np.uint8)\n\n    if color_type == 1:\n        train_data = train_data.reshape(train_data.shape[0], 1, img_rows, img_cols)\n    else:\n        train_data = train_data.transpose((0, 3, 1, 2))\n\n    train_target = np_utils.to_categorical(train_target, 10)\n    train_data = train_data.astype('float32')\n    train_data /= 255\n    print('Train shape:', train_data.shape)\n    print(train_data.shape[0], 'train samples')\n    return train_data, train_target, train_id, driver_id, unique_drivers\n\n\ndef read_and_normalize_test_data(img_rows, img_cols, color_type=1):\n    cache_path = os.path.join('cache', 'test_r_' + str(img_rows) + '_c_' + str(img_cols) + '_t_' + str(color_type) + '_rotated.dat')\n    if not os.path.isfile(cache_path) or use_cache == 0:\n        test_data, test_id = load_test(img_rows, img_cols, color_type)\n        cache_data((test_data, test_id), cache_path)\n    else:\n        print('Restore test from cache!')\n        (test_data, test_id) = restore_data(cache_path)\n\n    test_data = np.array(test_data, dtype=np.uint8)\n    if color_type == 1:\n        test_data = test_data.reshape(test_data.shape[0], 1, img_rows, img_cols)\n    else:\n        test_data = test_data.transpose((0, 3, 1, 2))\n    # test_data = test_data.swapaxes(3, 1)\n    test_data = test_data.astype('float32')\n    test_data /= 255\n    print('Test shape:', test_data.shape)\n    print(test_data.shape[0], 'test samples')\n    return test_data, test_id\n\n\ndef merge_several_folds_mean(data, nfolds):\n    a = np.array(data[0])\n    for i in range(1, nfolds):\n        a += np.array(data[i])\n    a /= nfolds\n    return a.tolist()\n\n\ndef merge_several_folds_geom(data, nfolds):\n    a = np.array(data[0])\n    for i in range(1, nfolds):\n        a *= np.array(data[i])\n    a = np.power(a, 1/nfolds)\n    return a.tolist()\n\n\ndef copy_selected_drivers(train_data, train_target, driver_id, driver_list):\n    data = []\n    target = []\n    index = []\n    for i in range(len(driver_id)):\n        if driver_id[i] in driver_list:\n            data.append(train_data[i])\n            target.append(train_target[i])\n            index.append(i)\n    data = np.array(data)\n    target = np.array(target)\n    index = np.array(index)\n    return data, target, index\n\n\ndef create_model_v1(img_rows, img_cols, color_type=1):\n    model = Sequential()\n    model.add(Convolution2D(32, 3, 3, border_mode='same', init='he_normal',\n                            input_shape=(color_type, img_rows, img_cols)))\n    model.add(MaxPooling2D(pool_size=(2, 2)))\n    model.add(Dropout(0.5))\n\n    model.add(Convolution2D(64, 3, 3, border_mode='same', init='he_normal'))\n    model.add(MaxPooling2D(pool_size=(2, 2)))\n    model.add(Dropout(0.5))\n\n    model.add(Convolution2D(128, 3, 3, border_mode='same', init='he_normal'))\n    model.add(MaxPooling2D(pool_size=(8, 8)))\n    model.add(Dropout(0.5))\n\n    model.add(Flatten())\n    model.add(Dense(10))\n    model.add(Activation('softmax'))\n\n    model.compile(Adam(lr=1e-3), loss='categorical_crossentropy')\n    return model\n\n\ndef get_validation_predictions(train_data, predictions_valid):\n    pv = []\n    for i in range(len(train_data)):\n        pv.append(predictions_valid[i])\n    return pv\n\n\ndef run_cross_validation(nfolds=10):\n    # input image dimensions\n    img_rows, img_cols = 64, 64\n    # color type: 1 - grey, 3 - rgb\n    color_type_global = 1\n    batch_size = 16\n    nb_epoch = 50\n    random_state = 51\n    restore_from_last_checkpoint = 0\n\n    train_data, train_target, train_id, driver_id, unique_drivers = read_and_normalize_train_data(img_rows, img_cols, color_type_global)\n    test_data, test_id = read_and_normalize_test_data(img_rows, img_cols, color_type_global)\n    model = create_model_v1(img_rows, img_cols, color_type_global)\n\n    yfull_train = dict()\n    yfull_test = []\n    kf = KFold(len(unique_drivers), n_folds=nfolds, shuffle=True, random_state=random_state)\n    num_fold = 0\n    sum_score = 0\n    for train_drivers, test_drivers in kf:\n        unique_list_train = [unique_drivers[i] for i in train_drivers]\n        X_train, Y_train, train_index = copy_selected_drivers(train_data, train_target, driver_id, unique_list_train)\n        unique_list_valid = [unique_drivers[i] for i in test_drivers]\n        X_valid, Y_valid, test_index = copy_selected_drivers(train_data, train_target, driver_id, unique_list_valid)\n\n        num_fold += 1\n        print('Start KFold number {} from {}'.format(num_fold, nfolds))\n        print('Split train: ', len(X_train), len(Y_train))\n        print('Split valid: ', len(X_valid), len(Y_valid))\n        print('Train drivers: ', unique_list_train)\n        print('Test drivers: ', unique_list_valid)\n\n        kfold_weights_path = os.path.join('cache', 'weights_kfold_' + str(num_fold) + '.h5')\n        if not os.path.isfile(kfold_weights_path) or restore_from_last_checkpoint == 0:\n            callbacks = [\n                EarlyStopping(monitor='val_loss', patience=1, verbose=0),\n                ModelCheckpoint(kfold_weights_path, monitor='val_loss', save_best_only=True, verbose=0),\n            ]\n            model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch,\n                  shuffle=True, verbose=1, validation_data=(X_valid, Y_valid),\n                  callbacks=callbacks)\n        if os.path.isfile(kfold_weights_path):\n            model.load_weights(kfold_weights_path)\n\n        # score = model.evaluate(X_valid, Y_valid, show_accuracy=True, verbose=0)\n        # print('Score log_loss: ', score[0])\n\n        predictions_valid = model.predict(X_valid, batch_size=batch_size, verbose=1)\n        score = log_loss(Y_valid, predictions_valid)\n        print('Score log_loss: ', score)\n        sum_score += score*len(test_index)\n\n        # Store valid predictions\n        for i in range(len(test_index)):\n            yfull_train[test_index[i]] = predictions_valid[i]\n\n        # Store test predictions\n        test_prediction = model.predict(test_data, batch_size=batch_size, verbose=1)\n        yfull_test.append(test_prediction)\n\n    score = sum_score/len(train_data)\n    print(\"Log_loss train independent avg: \", score)\n\n    predictions_valid = get_validation_predictions(train_data, yfull_train)\n    score1 = log_loss(train_target, predictions_valid)\n    if abs(score1 - score) > 0.0001:\n        print('Check error: {} != {}'.format(score, score1))\n\n    print('Final log_loss: {}, rows: {} cols: {} nfolds: {} epoch: {}'.format(score, img_rows, img_cols, nfolds, nb_epoch))\n    info_string = 'loss_' + str(score) \\\n                    + '_r_' + str(img_rows) \\\n                    + '_c_' + str(img_cols) \\\n                    + '_folds_' + str(nfolds) \\\n                    + '_ep_' + str(nb_epoch)\n\n    test_res = merge_several_folds_mean(yfull_test, nfolds)\n    # test_res = merge_several_folds_geom(yfull_test, nfolds)\n    create_submission(test_res, test_id, info_string)\n    save_useful_data(predictions_valid, train_id, model, info_string)\n\n\ndef run_single():\n    # input image dimensions\n    img_rows, img_cols = 64, 64\n    color_type_global = 1\n    batch_size = 32\n    nb_epoch = 50\n    random_state = 51\n\n    train_data, train_target, train_id, driver_id, unique_drivers = read_and_normalize_train_data(img_rows, img_cols, color_type_global)\n    test_data, test_id = read_and_normalize_test_data(img_rows, img_cols, color_type_global)\n\n    yfull_test = []\n    unique_list_train = ['p002', 'p012', 'p014', 'p015', 'p016', 'p021', 'p022', 'p035', 'p041', 'p042', 'p045', 'p047', 'p049', 'p050', 'p051', 'p052', 'p056', 'p061', 'p064', 'p066', 'p075', 'p081']\n    X_train, Y_train, train_index = copy_selected_drivers(train_data, train_target, driver_id, unique_list_train)\n    unique_list_valid = ['p024', 'p026', 'p039', 'p072']\n    X_valid, Y_valid, test_index = copy_selected_drivers(train_data, train_target, driver_id, unique_list_valid)\n\n    print('Start Single Run')\n    print('Split train: ', len(X_train))\n    print('Split valid: ', len(X_valid))\n    print('Train drivers: ', unique_list_train)\n    print('Valid drivers: ', unique_list_valid)\n\n    callbacks = [\n        EarlyStopping(monitor='val_loss', patience=2, verbose=0),\n    ]\n    model = create_model_v1(img_rows, img_cols, color_type_global)\n    model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch,\n              shuffle=True, verbose=1, validation_data=(X_valid, Y_valid),\n              callbacks=callbacks)\n\n    # score = model.evaluate(X_valid, Y_valid, show_accuracy=True, verbose=0)\n    # print('Score log_loss: ', score[0])\n\n    predictions_valid = model.predict(X_valid, batch_size=batch_size, verbose=1)\n    score = log_loss(Y_valid, predictions_valid)\n    print('Score log_loss: ', score)\n\n    # Store test predictions\n    test_prediction = model.predict(test_data, batch_size=batch_size, verbose=1)\n    yfull_test.append(test_prediction)\n\n    print('Final log_loss: {}, rows: {} cols: {} epoch: {}'.format(score, img_rows, img_cols, nb_epoch))\n    info_string = 'loss_' + str(score) \\\n                    + '_r_' + str(img_rows) \\\n                    + '_c_' + str(img_cols) \\\n                    + '_ep_' + str(nb_epoch)\n\n    full_pred = model.predict(train_data, batch_size=batch_size, verbose=1)\n    score = log_loss(train_target, full_pred)\n    print('Full score log_loss: ', score)\n\n    test_res = merge_several_folds_mean(yfull_test, 1)\n    create_submission(test_res, test_id, info_string)\n    save_useful_data(full_pred, train_id, model, info_string)\n\n\nif __name__ == '__main__':\n    run_cross_validation(13)\n    # run_single()"
 }
],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}}, "nbformat": 4, "nbformat_minor": 0}