{"metadata":{"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"language":"python","name":"python3","display_name":"Python 3"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":8241658,"sourceType":"datasetVersion","datasetId":4863559},{"sourceId":8234277,"sourceType":"datasetVersion","datasetId":4883674}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!rm -rf /kaggle/working/*","metadata":{"execution":{"iopub.status.busy":"2024-04-27T06:25:01.598115Z","iopub.execute_input":"2024-04-27T06:25:01.598550Z","iopub.status.idle":"2024-04-27T06:25:02.727870Z","shell.execute_reply.started":"2024-04-27T06:25:01.598519Z","shell.execute_reply":"2024-04-27T06:25:02.726525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%load_ext Cython","metadata":{"execution":{"iopub.status.busy":"2024-04-27T06:24:28.890657Z","iopub.execute_input":"2024-04-27T06:24:28.891109Z","iopub.status.idle":"2024-04-27T06:24:30.054067Z","shell.execute_reply.started":"2024-04-27T06:24:28.891079Z","shell.execute_reply":"2024-04-27T06:24:30.052844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport tensorflow as tf","metadata":{"execution":{"iopub.status.busy":"2024-04-27T06:24:30.056393Z","iopub.execute_input":"2024-04-27T06:24:30.056872Z","iopub.status.idle":"2024-04-27T06:24:30.062470Z","shell.execute_reply.started":"2024-04-27T06:24:30.056835Z","shell.execute_reply":"2024-04-27T06:24:30.061278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n%%cython\n\nimport numpy as np\nimport tensorflow as tf\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.io import TFRecordWriter\nfrom libcpp cimport bool\nfrom cython cimport boundscheck, wraparound\nimport base64\ncimport tensorflow as tf\ncimport numpy as np\nimport os\n\ncdef struct Image:\n    int width\n    int height\n    unsigned char *data  # Assuming image data is stored as unsigned char (byte) values\n    \ncpdef bytes read_image(str file_path):\n    with open(file_path, \"rb\") as file:\n        return file.read()\n\ncdef parse_tfrecord_fn(example):\n    feature_description = {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'mask': tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, feature_description)\n    cdef Image image = tf.io.decode_image(example['image'], channels=1,)  # Adjust channels if necessary\n    cdef Image mask = tf.io.decode_image(example['mask'], channels=1)    # Adjust channels if necessary\n    \n    # You can perform additional preprocessing or checks here\n    return image, mask\n\n\ncdef int array_length(np.ndarray strc):\n    return <int>(sizeof(strc) / sizeof(strc[0]))\n\ncpdef list[str] getdirfrompath(str directory):\n    cdef list[str] listd = []\n    for filename in os.listdir(directory):\n        listd.append(directory + \"/\" +  filename)\n        \n    return listd\n\ncdef _bytes_feature(bytes value):\n    \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ncdef bytes image_to_tfexample(bytes frame, bytes mask):\n    return tf.train.Example(features=tf.train.Features(feature={\n        b'image': _bytes_feature(frame),\n        b'mask': _bytes_feature(mask)\n    })).SerializeToString()\n\ncdef save_as_tfrec(list[str] frames_list, list[str] masks_list):\n    cdef int total_size_limit = 157286400 #150megabytes\n    cdef int current_size = 0\n    cdef int j = 0\n    cdef bint write_new = True\n    cdef object writer = None\n    cdef bytes byteframe\n    cdef bytes bytemask\n    cdef bytes serialized_example\n    cdef int serialized_example_size\n    \n    for i, (frame, mask) in enumerate(zip(frames_list, masks_list)):\n        if current_size > total_size_limit:\n            print(\"Reached size limit\")\n            return\n        \n        if write_new:\n            write_new = False\n            writer = TFRecordWriter(f\"Curvelanes{j}.tfrec\")\n            current_size = 0\n\n        byteframe = read_image(frame)\n        bytemask = read_image(mask)\n        \n        serialized_example = image_to_tfexample(byteframe, bytemask)\n        serialized_example_size = len(serialized_example)  # Assuming serialized_example is a bytes object\n        if current_size + serialized_example_size > total_size_limit:\n            write_new = True\n            writer.close()\n            j += 1\n            print(j)\n            continue\n\n        writer.write(serialized_example)\n        current_size += serialized_example_size\n\n    if writer is not None:\n        writer.close()\n            \nmasks_direct_culane = getdirfrompath(\"/kaggle/input/culane-preprocessed/temp/masks\")\nimages_direct_culane = getdirfrompath(\"/kaggle/input/culane-preprocessed/temp/frames\")            \nsave_as_tfrec(images_direct_culane,masks_direct_culane)","metadata":{"_uuid":"127ad4b7c19e87ff2fb180d1030dbbe4ea430a94","_kg_hide-input":false,"_kg_hide-output":true,"_cell_guid":"519501a2-749b-4240-94e2-957f055d2cc9","execution":{"iopub.status.busy":"2024-04-27T06:24:31.352093Z","iopub.execute_input":"2024-04-27T06:24:31.352538Z","iopub.status.idle":"2024-04-27T06:24:55.741402Z","shell.execute_reply.started":"2024-04-27T06:24:31.352504Z","shell.execute_reply":"2024-04-27T06:24:55.739083Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import necessary libraries\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\n\n# Define a function to parse TFRecord example\ndef parse_tfrecord_example(example):\n    feature_description = {\n        'image': tf.io.VarLenFeature(tf.string),\n        'mask': tf.io.VarLenFeature(tf.string)\n    }\n    example = tf.io.parse_single_example(example, feature_description)\n    image = tf.io.decode_jpeg(example['image'].values[0], channels=1)  # Assuming JPEG images with 3 channels\n    mask = tf.io.decode_jpeg(example['mask'].values[0], channels=1)  # Assuming grayscale masks\n    return image, mask\n\n\n# Define the path to your TFRecord file\ntfrecord_file = '/kaggle/input/culane-tfrec/0.tfrec'\n\n# Create a TFRecordDataset from the TFRecord file\ndataset = tf.data.TFRecordDataset(tfrecord_file)\n\n# Parse the TFRecord examples\nparsed_dataset = dataset.map(parse_tfrecord_example)\n\n# Iterate over the dataset to view the images\nfor image, mask in parsed_dataset.take(1):\n\n    # Display the images using Matplotlib\n    plt.subplot(1, 2, 1)\n    plt.imshow(image, cmap = 'gray')\n    plt.title('Image')\n    plt.axis('off')\n\n    plt.subplot(1, 2, 2)\n    plt.imshow(mask, cmap='gray')  # Assuming mask is grayscale\n    plt.title('Mask')\n    plt.axis('off')\n\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-27T07:19:30.190618Z","iopub.execute_input":"2024-04-27T07:19:30.191105Z","iopub.status.idle":"2024-04-27T07:19:30.636862Z","shell.execute_reply.started":"2024-04-27T07:19:30.191070Z","shell.execute_reply":"2024-04-27T07:19:30.635411Z"},"trusted":true},"execution_count":null,"outputs":[]}]}