{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os, sys, math\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt, cv2\nimport tensorflow as tf\nimport albumentations as A\nfrom sklearn.utils import shuffle\nfrom sklearn.model_selection import GroupKFold\nfrom glob import glob\nimport random\n# import cv2\nprint(\"Tensorflow version \" + tf.__version__)\nAUTO = tf.data.experimental.AUTOTUNE # used in tf.data.Dataset API","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"SEED = 42\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n# define the class that we need\nclass_1 = 'Cover'\nclass_2 = 'UERD'\n\ndataset = []\n\nfor label, kind in enumerate([class_1, class_2]):\n    for path in glob('../input/alaska2-image-steganalysis/Cover/*.jpg'):\n        dataset.append({\n            'kind': kind,\n            'image_name': path.split('/')[-1],\n            'label': label\n        })\n\nrandom.shuffle(dataset)\ndataset = pd.DataFrame(dataset)\n\ngkf = GroupKFold(n_splits=5)\n\ndataset.loc[:, 'fold'] = 0\nfor fold_number, (train_index, val_index) in enumerate(gkf.split(X=dataset.index, y=dataset['label'], groups=dataset['image_name'])):\n    dataset.loc[dataset.iloc[val_index].index, 'fold'] = fold_number","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(dataset.shape)\ndataset.kind.unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fold_number = 3\n\n# df[(df['Salary_in_1000']>=100) & (df['Age']<60) & df['FT_Team'].str.startswith('S')][['Name','Age','Salary_in_1000']]\n\n# train_data = dataset[dataset['fold'] != 4].reset_index(drop=True)\nvalid_data = dataset[(dataset['fold'] == fold_number)].reset_index(drop=True)\nvalid_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# valid_data = valid_data[(valid_data['kind'] == class_1)]\nvalid_data.kind.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _bytes_feature(value):\n    \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n    #value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _float_feature(value):\n    \"\"\"Returns a float_list from a float / double.\"\"\"\n    return tf.train.Feature(float_list=tf.train.FloatList(value=value))\n\ndef _int64_feature(value):\n    \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_train_transforms():\n    return A.Compose([\n            A.HorizontalFlip(p=0.5),\n            A.VerticalFlip(p=0.5),\n            A.Resize(height=512, width=512, p=1.0)\n        ], p=1.0)\n\ndef get_valid_transforms():\n    return A.Compose([\n            A.Resize(height=512, width=512, p=1.0),\n        ], p=1.0)\n\ntrain_transform = get_train_transforms()\nvalid_transform = get_valid_transforms()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def onehot(size, target):\n    vec = np.zeros(size, dtype=np.float32)\n    vec[target] = 1.\n    return vec","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def serialize_example(image, label):\n    feature = {\n        'image': _bytes_feature(image),\n        'label': _int64_feature(label),\n    }\n    example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n    return example_proto.SerializeToString()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_paths = []\n\nfor i in range(len(valid_data)):\n    kind = valid_data['kind'].loc[i]\n    img_id = valid_data['image_name'].loc[i]\n    valid_paths.append(f'../input/alaska2-image-steganalysis/{kind}/{img_id}')\nvalid_paths[0:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_data['path'] = valid_paths\nvalid_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_data.loc[1]['label'], valid_data.loc[1]['kind'], valid_data.loc[1]['image_name'], valid_data.loc[1]['path']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n# !mkdir valid\n# !mkdir valid/fold_0_1\n\nSIZE = 2000\nCT = valid_data.shape[0]//SIZE + int(valid_data.shape[0]%SIZE!=0)\nfor j in range(CT):\n    print()\n    print('Writing TFRecord %i of %i...'%(j,CT))\n    CT2 = min(SIZE, valid_data.shape[0]-j*SIZE)\n    with tf.io.TFRecordWriter('coverUERDValid%.2i-%i.tfrec'%(j,CT2)) as writer:\n        for k in range(CT2):\n            img = cv2.imread(valid_data.loc[k]['path'])\n            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB).astype(np.uint8)\n            img = cv2.imencode('.jpg', img, (cv2.IMWRITE_JPEG_QUALITY, 94))[1].tostring()\n            \n            one_hot_class = onehot(4, valid_data.loc[k]['label'])\n            example = serialize_example(\n                img,\n                valid_data.loc[k]['label'],\n            )\n            writer.write(example)\n            if k%1000==0: print(k,', ',end='')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = 'coverUERDValid00-2000.tfrec'\n\ndef read_tfrecord(example):\n    features = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"label\": tf.io.FixedLenFeature([], tf.int64),  \n    }\n    # decode the TFRecord\n    example = tf.io.parse_single_example(example, features)\n\n    image = tf.image.decode_jpeg(example['image'], channels=3)\n    image = tf.reshape(image, [512, 512, 3])\n    class_num = example['label']\n    \n    return image, class_num","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset4 = tf.data.TFRecordDataset(path)\ndataset4 = dataset4.map(read_tfrecord)\n# dataset4 = dataset4.shuffle(300)\na = None\ncount = 0\nfor tensor in dataset4:\n    count += 1\n    print(count)\n    if count == 2:\n        a = tensor[0]\n        print(tensor)\n        break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(a.numpy())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}