{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import os\nimport glob\nimport cv2\nimport math\nimport pickle\nimport datetime\n\nfrom sklearn.model_selection import train_test_split\n# from sklearn.cross_validation import train_test_split\n# from sklearn.cross_validation import KFold\nfrom keras.models import Sequential\nfrom keras.layers.core import Dense, Dropout, Flatten\nfrom keras.layers.convolutional import Convolution2D, MaxPooling2D, \\\n                                       ZeroPadding2D\n\n# from keras.layers.normalization import BatchNormalization\n# from keras.optimizers import Adam\nfrom keras.optimizers import SGD\nfrom keras.utils import np_utils\nfrom keras.models import model_from_json\n# from sklearn.metrics import log_loss\nfrom numpy.random import permutation","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.random.seed(2016)\nuse_cache = 1\n# color type: 1 - grey, 3 - rgb\ncolor_type_global = 3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef get_im(path, img_rows, img_cols, color_type=1):\n    # Load as grayscale\n    if color_type == 1:\n        img = cv2.imread(path, 0)\n    elif color_type == 3:\n        img = cv2.imread(path)\n    # Reduce size\n    resized = cv2.resize(img, (img_cols, img_rows))\n    # mean_pixel = [103.939, 116.799, 123.68]\n    # resized = resized.astype(np.float32, copy=False)\n\n    # for c in range(3):\n    #    resized[:, :, c] = resized[:, :, c] - mean_pixel[c]\n    # resized = resized.transpose((2, 0, 1))\n    # resized = np.expand_dims(img, axis=0)\n    return resized\n\n\ndef get_driver_data():\n    dr = dict()\n    path = os.path.join('..', 'input', 'driver_imgs_list.csv')\n    print('Read drivers data')\n    f = open(path, 'r')\n    line = f.readline()\n    while (1):\n        line = f.readline()\n        if line == '':\n            break\n        arr = line.strip().split(',')\n        dr[arr[2]] = arr[0]\n    f.close()\n    return dr\n\n\ndef load_train(img_rows, img_cols, color_type=1):\n    X_train = []\n    y_train = []\n\n    driver_data = get_driver_data()\n    print('Read train images')\n    for j in range(10):\n        print('Load folder c{}'.format(j))\n#         path = os.path.join('..', 'input', 'imgs', 'train',\n#                             'c' + str(j), '*.jpg')\n        path = os.path.join('..', 'input', 'train',\n                            'c' + str(j), '*.jpg')\n        files = glob.glob(path)\n        for fl in files:\n            flbase = os.path.basename(fl)\n            img = get_im(fl, img_rows, img_cols, color_type)\n            X_train.append(img)\n            y_train.append(j)\n\n    return X_train, y_train\n\n\ndef load_test(img_rows, img_cols, color_type=1):\n    print('Read test images')\n    path = os.path.join('..', 'input', 'imgs', 'test', '*.jpg')\n    files = glob.glob(path)\n    X_test = []\n    X_test_id = []\n    total = 0\n    thr = math.floor(len(files)/10)\n    for fl in files:\n        flbase = os.path.basename(fl)\n        img = get_im(fl, img_rows, img_cols, color_type)\n        X_test.append(img)\n        X_test_id.append(flbase)\n        total += 1\n        if total % thr == 0:\n            print('Read {} images from {}'.format(total, len(files)))\n\n    return X_test, X_test_id\n\n\ndef cache_data(data, path):\n    if not os.path.isdir('cache'):\n        os.mkdir('cache')\n    if os.path.isdir(os.path.dirname(path)):\n        file = open(path, 'wb')\n        pickle.dump(data, file)\n        file.close()\n    else:\n        print('Directory doesnt exists')\n\n\ndef restore_data(path):\n    data = dict()\n    if os.path.isfile(path):\n        print('Restore data from pickle........')\n        file = open(path, 'rb')\n        data = pickle.load(file)\n    return data\n\n\ndef save_model(model, index, cross=''):\n    json_string = model.to_json()\n    if not os.path.isdir('cache'):\n        os.mkdir('cache')\n    json_name = 'architecture' + str(index) + cross + '.json'\n    weight_name = 'model_weights' + str(index) + cross + '.h5'\n    open(os.path.join('cache', json_name), 'w').write(json_string)\n    model.save_weights(os.path.join('cache', weight_name), overwrite=True)\n\n\ndef read_model(index, cross=''):\n    json_name = 'architecture' + str(index) + cross + '.json'\n    weight_name = 'model_weights' + str(index) + cross + '.h5'\n    model = model_from_json(open(os.path.join('cache', json_name)).read())\n    model.load_weights(os.path.join('cache', weight_name))\n    return model\n\n\ndef split_validation_set(train, target, test_size):\n    random_state = 51\n    X_train, X_test, y_train, y_test = \\\n        train_test_split(train, target,\n                         test_size=test_size,\n                         random_state=random_state)\n    return X_train, X_test, y_train, y_test\n\n\ndef create_submission(predictions, test_id, info):\n    result1 = pd.DataFrame(predictions, columns=['c0', 'c1', 'c2', 'c3',\n                                                 'c4', 'c5', 'c6', 'c7',\n                                                 'c8', 'c9'])\n    result1.loc[:, 'img'] = pd.Series(test_id, index=result1.index)\n    now = datetime.datetime.now()\n    if not os.path.isdir('subm'):\n        os.mkdir('subm')\n    suffix = info + '_' + str(now.strftime(\"%Y-%m-%d-%H-%M\"))\n    sub_file = os.path.join('subm', 'submission_' + suffix + '.csv')\n    result1.to_csv(sub_file, index=False)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def dict_to_list(d):\n    ret = []\n    for i in d.items():\n        ret.append(i[1])\n    return ret\n\n\ndef merge_several_folds_mean(data, nfolds):\n    a = np.array(data[0])\n    for i in range(1, nfolds):\n        a += np.array(data[i])\n    a /= nfolds\n    return a.tolist()\n\n\ndef merge_several_folds_geom(data, nfolds):\n    a = np.array(data[0])\n    for i in range(1, nfolds):\n        a *= np.array(data[i])\n    a = np.power(a, 1/nfolds)\n    return a.tolist()\n\n\ndef copy_selected_drivers(train_data, train_target, driver_id, driver_list):\n    data = []\n    target = []\n    index = []\n    for i in range(len(driver_id)):\n        if driver_id[i] in driver_list:\n            data.append(train_data[i])\n            target.append(train_target[i])\n            index.append(i)\n    data = np.array(data, dtype=np.float32)\n    target = np.array(target, dtype=np.float32)\n    index = np.array(index, dtype=np.uint32)\n    return data, target, index\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef read_and_normalize_and_shuffle_train_data(img_rows, img_cols,\n                                              color_type=1):\n\n    cache_path = os.path.join('cache', 'train_r_' + str(img_rows) +\n                              '_c_' + str(img_cols) + '_t_' +\n                              str(color_type) + '.dat')\n\n    if not os.path.isfile(cache_path) or use_cache == 0:\n        train_data, train_target = load_train(img_rows, img_cols, color_type)\n        cache_data((train_data, train_target),cache_path)\n    else:\n        print('Restore train from cache!')\n        (train_data, train_target) = \\\n            restore_data(cache_path)\n    \n    train_data = np.array(train_data, dtype=np.uint8)\n    train_target = np.array(train_target, dtype=np.uint8)\n\n    if color_type == 1:\n        train_data = train_data.reshape(train_data.shape[0], color_type,\n                                        img_rows, img_cols)\n    else:\n        print(train_data.shape)\n        train_data = train_data.transpose((0, 3, 1, 2))\n\n    train_target = np_utils.to_categorical(train_target, 10)\n    train_data = train_data.astype('float32')\n    mean_pixel = [103.939, 116.779, 123.68]\n    for c in range(3):\n        train_data[:, c, :, :] = train_data[:, c, :, :] - mean_pixel[c]\n    # train_data /= 255\n    perm = permutation(len(train_target))\n    train_data = train_data[perm]\n    train_target = train_target[perm]\n    print('Train shape:', train_data.shape)\n    print(train_data.shape[0], 'train samples')\n    return train_data, train_target\n\n\ndef read_and_normalize_test_data(img_rows=224, img_cols=224, color_type=1):\n    cache_path = os.path.join('cache', 'test_r_' + str(img_rows) +\n                              '_c_' + str(img_cols) + '_t_' +\n                              str(color_type) + '.dat')\n    if not os.path.isfile(cache_path) or use_cache == 0:\n        test_data, test_id = load_test(img_rows, img_cols, color_type)\n        cache_data((test_data, test_id), cache_path)\n    else:\n        print('Restore test from cache!')\n        (test_data, test_id) = restore_data(cache_path)\n\n    test_data = np.array(test_data, dtype=np.uint8)\n\n    if color_type == 1:\n        test_data = test_data.reshape(test_data.shape[0], color_type,\n                                      img_rows, img_cols)\n    else:\n        test_data = test_data.transpose((0, 3, 1, 2))\n\n    test_data = test_data.astype('float32')\n    mean_pixel = [103.939, 116.779, 123.68]\n    for c in range(3):\n        test_data[:, c, :, :] = test_data[:, c, :, :] - mean_pixel[c]\n    # test_data /= 255\n    print('Test shape:', test_data.shape)\n    print(test_data.shape[0], 'test samples')\n    return test_data, test_id\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_rows, img_cols = 64, 64\nbatch_size = 64\nrandom_state = 20\n    \ntrain_data, train_target = \\\n        read_and_normalize_and_shuffle_train_data(img_rows, img_cols, color_type_global)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_data.shape)\nprint(train_target.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}