{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport cv2\nimport pandas as pd\nfrom tqdm.notebook import tqdm\n\n\nDATA_PATH = '/kaggle/input/cassava-leaf-disease-classification'\n\nJPEG_PATH = os.path.join(DATA_PATH, 'train_images')\nJPEG_SAVE_PATH = '/kaggle/train_images_jpeg'\n\nCSV_PATH = os.path.join(DATA_PATH, 'train.csv')\n\nRESIZE = 227\nNUM_TFREDORDS = 1338\nIMG_QUALITY = 95\nDEBUG = False\n\n\n#os.makedirs(JPEG_SAVE_PATH, exist_ok=True)\ntrain_df = pd.read_csv(CSV_PATH)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## TFRecords"},{"metadata":{"trusted":true},"cell_type":"code","source":"import math\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\n\nfiles = sorted(os.listdir(JPEG_PATH))\n\nnum_iter = math.ceil(len(files) / NUM_TFREDORDS)\n\n\ndef _bytes_feature(value):\n  \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n  if isinstance(value, type(tf.constant(0))):\n    value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\n\ndef _int64_feature(value):\n  \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n  return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\n\ndef serialize_example(feature0, feature1, feature2):\n  feature = {\n      'image': _bytes_feature(feature0),\n      'image_name': _bytes_feature(feature1),\n      'target': _int64_feature(feature2)\n  }\n  example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n  return example_proto.SerializeToString()\n\n\n\nfor i in range(num_iter):\n    cnt = min(NUM_TFREDORDS, len(files) - i*NUM_TFREDORDS)\n    tf_filename = f'ld_train{str(i).zfill(2)}-{cnt}.tfrec'    \n    print(f'Writing TFRecord: {i} {tf_filename}')\n\n    \n    with tf.io.TFRecordWriter(tf_filename) as wf:\n        for j in range(cnt):\n            img_id = files[NUM_TFREDORDS*i + j]\n            img = cv2.imread(os.path.join(JPEG_PATH, img_id))\n            img = cv2.resize(img, (RESIZE, RESIZE))\n            # img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)  -> Fix:20201121\n            \n            img = cv2.imencode('.jpg', img, (cv2.IMWRITE_JPEG_QUALITY, IMG_QUALITY))[1].tostring()\n            img_id = str.encode(img_id)\n            target = train_df['label'][NUM_TFREDORDS*i + j]\n            \n            example = serialize_example(img, img_id, target)\n            \n            wf.write(example)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### image checking"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\n\n\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n    return image\n\n\ndef parse_example(example):\n    LABELED_TFREC_FORMAT = {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'image_name': tf.io.FixedLenFeature([], tf.string),\n        'target': tf.io.FixedLenFeature([], tf.int64)\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = example['image_name']\n    target = example['target']\n    return image, label, target\n\n\ndef display_one(image, title, target, subplot, red=False, titlesize=16):\n    plt.subplot(*subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    plt.title(f'{title}: {target}')\n    return (subplot[0], subplot[1], subplot[2]+1)\n\n\ndef display_batch_of_images(databatch):\n    images, labels, targets = databatch\n    images = images.numpy()\n    labels = labels.numpy()\n    targets = targets.numpy()\n    if labels is None:\n        labels = [None for _ in enumerate(images)]\n    if targets is None:\n        targets = [None for _ in enumerate(targets)]\n        \n    # auto-squaring: this will drop data that does not fit into square or square-ish rectangle\n    rows = int(math.sqrt(len(images)))\n    cols = len(images)//rows\n        \n    # size and spacing\n    FIGSIZE = 13.0\n    SPACING = 0.2\n    subplot=(rows, cols, 1)\n    if rows < cols:\n        plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))\n    else:\n        plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))\n    \n    # display\n    for i, (image, label, target) in enumerate(zip(images[:rows*cols], labels[:rows*cols], targets[:rows*cols])):\n        title = label\n        title = title.decode('utf-8')\n        correct = True\n        dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3 # magic formula tested to work from 1x1 to 10x10 images\n        subplot = display_one(image, title, target, subplot, not correct, titlesize=dynamic_titlesize)\n    \n    #layout\n    plt.tight_layout()\n    if label is None and predictions is None:\n        plt.subplots_adjust(wspace=0.2, hspace=0.2)\n    else:\n        plt.subplots_adjust(wspace=SPACING, hspace=SPACING)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model = tf.keras.Sequential([\n#     tf.keras.layers.Flatten(input_shape=(128,128,3)),\n#     tf.keras.layers.Dense(128, activation='relu'),\n#     tf.keras.layers.Dense(5)\n# ])\n\n# model.compile(optimizer='adam',\n#               loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n#               metrics=['accuracy'])\n\ndef alexnet(in_shape=(227,227,3), n_classes=5, opt='sgd'):\n    in_layer = tf.keras.layers.Input(in_shape)\n    conv1 = tf.keras.layers.Conv2D(96, 11, strides=4, activation='relu')(in_layer)\n    pool1 = tf.keras.layers.MaxPool2D(3, 2)(conv1)\n    conv2 = tf.keras.layers.Conv2D(256, 5, strides=1, padding='same', activation='relu')(pool1)\n    pool2 = tf.keras.layers.MaxPool2D(3, 2)(conv2)\n    conv3 = tf.keras.layers.Conv2D(384, 3, strides=1, padding='same', activation='relu')(pool2)\n    conv4 = tf.keras.layers.Conv2D(256, 3, strides=1, padding='same', activation='relu')(conv3)\n    pool3 = tf.keras.layers.MaxPool2D(3, 2)(conv4)\n    flattened = tf.keras.layers.Flatten()(pool3)\n    dense1 = tf.keras.layers.Dense(4096, activation='relu')(flattened)\n    drop1 = tf.keras.layers.Dropout(0.5)(dense1)\n    dense2 = tf.keras.layers.Dense(4096, activation='relu')(drop1)\n    drop2 = tf.keras.layers.Dropout(0.5)(dense2)\n    preds = tf.keras.layers.Dense(n_classes, activation='softmax')(drop2)\n\n    model = tf.keras.models.Model(in_layer, preds)\n    model.compile(loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), optimizer=opt, metrics=[\"accuracy\"])\n    return model\n\nmodel = alexnet()\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# Writing TFRecord: 0 ld_train00-1338.tfrec\n# Writing TFRecord: 1 ld_train01-1338.tfrec\n# Writing TFRecord: 2 ld_train02-1338.tfrec\n# Writing TFRecord: 3 ld_train03-1338.tfrec\n# Writing TFRecord: 4 ld_train04-1338.tfrec\n# Writing TFRecord: 5 ld_train05-1338.tfrec\n# Writing TFRecord: 6 ld_train06-1338.tfrec\n# Writing TFRecord: 7 ld_train07-1338.tfrec\n# Writing TFRecord: 8 ld_train08-1338.tfrec\n# Writing TFRecord: 9 ld_train09-1338.tfrec\n# Writing TFRecord: 10 ld_train10-1338.tfrec\n# Writing TFRecord: 11 ld_train11-1338.tfrec\n# Writing TFRecord: 12 ld_train12-1338.tfrec\n# Writing TFRecord: 13 ld_train13-1338.tfrec\n# Writing TFRecord: 14 ld_train14-1338.tfrec\n# Writing TFRecord: 15 ld_train15-1327.tfrec\n\n# image_list=[]\n# target_list=[]\n# tfrec_list = [\n#                 \"ld_train00-1338.tfrec\",\n#                 \"ld_train01-1338.tfrec\",\n#                 \"ld_train02-1338.tfrec\",\n#                 \"ld_train03-1338.tfrec\",\n#                 \"ld_train04-1338.tfrec\",\n#                 \"ld_train05-1338.tfrec\",\n#                 \"ld_train06-1338.tfrec\",\n#                 \"ld_train07-1338.tfrec\",\n#                 \"ld_train08-1338.tfrec\",\n#                 \"ld_train09-1338.tfrec\",\n#                 \"ld_train10-1338.tfrec\",\n#                 \"ld_train11-1338.tfrec\",\n#                 \"ld_train12-1338.tfrec\",\n#                 \"ld_train13-1338.tfrec\",\n#                 \"ld_train14-1338.tfrec\",\n#                 \"ld_train15-1327.tfrec\"\n# ]\n\n\n    \ndataset = tf.data.TFRecordDataset([ \"ld_train00-1338.tfrec\",\n                \"ld_train01-1338.tfrec\",\n                \"ld_train02-1338.tfrec\",\n                \"ld_train03-1338.tfrec\",\n                \"ld_train04-1338.tfrec\",\n                \"ld_train05-1338.tfrec\",\n                \"ld_train06-1338.tfrec\",\n                \"ld_train07-1338.tfrec\",\n                \"ld_train08-1338.tfrec\",\n                \"ld_train09-1338.tfrec\",\n                \"ld_train10-1338.tfrec\",\n                \"ld_train11-1338.tfrec\",\n                \"ld_train12-1338.tfrec\",\n                \"ld_train13-1338.tfrec\",\n                \"ld_train14-1338.tfrec\",\n                \"ld_train15-1327.tfrec\"]).map(parse_example).batch(10000) #1338 #21397\n\ndata = iter(dataset)\nimages, labels, targets = next(data)\ndel data\ndel dataset\ndel labels\n\nprint(len(images))\n#images = images.numpy()\n#labels = labels.numpy()\n#targets = targets.numpy()\n\n\n#model.fit(images.numpy(), targets.numpy(), epochs=10)\n\n\n# for resize_file in tfrec_list:\n#     if resize_file == \"ld_train15-1327.tfrec\":\n#         dataset = tf.data.TFRecordDataset([resize_file]).map(parse_example).batch(1327) #1327\n#     else:\n#         dataset = tf.data.TFRecordDataset([resize_file]).map(parse_example).batch(1338) #1338\n\n#     data = iter(dataset)\n#     images, labels, targets = next(data)\n#     print(resize_file, len(images))\n#     images = images.numpy()\n#     labels = labels.numpy()\n#     targets = targets.numpy()\n\n#     for i in images:\n#         image_list.append(i)\n\n#     for i in targets:\n#         target_list.append(i)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(images, targets, epochs=1)\n#model.fit(images, targets, epochs=10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_path = DATA_PATH + \"/test_images/\" + \"2216849948.jpg\"\nimage = tf.keras.preprocessing.image.load_img(image_path, target_size=(227, 227, 3))\ninput_arr = tf.keras.preprocessing.image.img_to_array(image)\ninput_arr = np.array([input_arr])  # Convert single image to a batch.\npredictions = model.predict(input_arr)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(predictions)\nnp.argmax(predictions, axis=1)\n#model.predict_classes(input_arr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\npath = DATA_PATH+\"/label_num_to_disease_map.json\"\ndata = pd.read_csv(path)\ndata","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\n\ndata = pd.read_csv(DATA_PATH+\"/sample_submission.csv\")\ndata.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#df = pd.DataFrame({'image_id': ['2216849948.jpg'], 'label': [np.argmax(predictions, axis=1)[0]] })\ndf = pd.DataFrame({'image_id': ['2216849948.jpg'], 'label': 4 })\ndf.to_csv(\"submission.csv\",index=False)\ndf.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = pd.read_csv(\"submission.csv\")\ndata","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.to_csv(\"submission2.csv\")\ndata2 = pd.read_csv(\"submission2.csv\")\ndata2","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}