{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":20270,"databundleVersionId":1222630,"sourceType":"competition"}],"dockerImageVersionId":30840,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport math\nimport tensorflow as tf\nimport pandas as pd\n\ndef process_single_tfrecord(input_file, output_file, metadata_df, training):\n    \"\"\"\n    Reads one TFRecord (input_file), parses it, merges metadata from metadata_df,\n    then writes an enhanced TFRecord (output_file).\n    \"\"\"\n    # --- 1) Parse function ---\n    def parse_tfrecord(example):\n        features = {\n            'image': tf.io.FixedLenFeature([], tf.string),\n            'image_name': tf.io.FixedLenFeature([], tf.string),\n        }\n        if training:\n            features['target'] = tf.io.FixedLenFeature([], tf.int64)\n        return tf.io.parse_single_example(example, features)\n\n    # --- 2) Function to add metadata via py_function ---\n    # def add_metadata(image, image_name, target):\n    #     \"\"\"Python-side function to look up the row in metadata_df.\"\"\"\n    #     image_name_str = image_name.numpy().decode('utf-8')\n    #     row = metadata_df[metadata_df['image_name'] == image_name_str]\n    #     if not row.empty:\n    #         sex_val = row['sex'].values[0]\n    #         sex_val = 'unknown' if pd.isna(sex_val) else sex_val\n\n    #         age_val = row['age_approx'].values[0]\n    #         age_val = -1 if pd.isna(age_val) else age_val\n\n    #         patient_val = row['patient_id'].values[0]\n    #         patient_val = 'unknown' if pd.isna(patient_val) else patient_val\n\n    #         anatom_val = row['anatom_site_general_challenge'].values[0]\n    #         anatom_val = 'unknown' if pd.isna(anatom_val) else anatom_val\n    #     else:\n    #         sex_val = 'unknown'\n    #         age_val = -1\n    #         patient_val = 'unknown'\n    #         anatom_val = 'unknown'\n        \n    #     # Convert to tf Tensors\n    #     sex_t = tf.convert_to_tensor(sex_val, dtype=tf.string)\n    #     age_t = tf.convert_to_tensor(age_val, dtype=tf.int64)\n    #     pid_t = tf.convert_to_tensor(patient_val, dtype=tf.string)\n    #     site_t = tf.convert_to_tensor(anatom_val, dtype=tf.string)\n    #     return sex_t, age_t, pid_t, site_t\n    def add_metadata_py(image, image_name):\n        image_name_str = image_name.numpy().decode('utf-8')\n        row = metadata_df[metadata_df['image_name'] == image_name_str]\n        if not row.empty:\n            sex_val = row['sex'].values[0]\n            sex_val = 'unknown' if pd.isna(sex_val) else sex_val\n            age_val = row['age_approx'].values[0]\n            age_val = -1 if pd.isna(age_val) else age_val\n            patient_val = row['patient_id'].values[0]\n            patient_val = 'unknown' if pd.isna(patient_val) else patient_val\n            anatom_val = row['anatom_site_general_challenge'].values[0]\n            anatom_val = 'unknown' if pd.isna(anatom_val) else anatom_val\n        else:\n            sex_val = 'unknown'\n            age_val = -1\n            patient_val = 'unknown'\n            anatom_val = 'unknown'\n\n        return (tf.convert_to_tensor(sex_val, tf.string),\n                tf.convert_to_tensor(age_val, tf.int64),\n                tf.convert_to_tensor(patient_val, tf.string),\n                tf.convert_to_tensor(anatom_val, tf.string))\n\n    # --- 3) Combine original record with new metadata ---\n    def combine_records(record, metadata):\n        sex, age_approx, patient_id, anatom_site = metadata\n        record['sex'] = sex\n        record['age_approx'] = age_approx\n        record['patient_id'] = patient_id\n        record['anatom_site_general_challenge'] = anatom_site\n        return record\n\n    # --- 4) Serialize combined record back into TFRecord ---\n    def serialize_example(record):\n        feature = {\n            # 'target': tf.train.Feature(\n            #     int64_list=tf.train.Int64List(value=[record['target'].numpy()])\n            # ),\n            'image': tf.train.Feature(\n                bytes_list=tf.train.BytesList(value=[record['image'].numpy()])\n            ),\n            'image_name': tf.train.Feature(\n                bytes_list=tf.train.BytesList(value=[record['image_name'].numpy()])\n            ),\n            'sex': tf.train.Feature(\n                bytes_list=tf.train.BytesList(value=[record['sex'].numpy()])\n            ),\n            'age_approx': tf.train.Feature(\n                int64_list=tf.train.Int64List(value=[record['age_approx'].numpy()])\n            ),\n            'patient_id': tf.train.Feature(\n                bytes_list=tf.train.BytesList(value=[record['patient_id'].numpy()])\n            ),\n            'anatom_site_general_challenge': tf.train.Feature(\n                bytes_list=tf.train.BytesList(value=[record['anatom_site_general_challenge'].numpy()])\n            ),\n        }\n        if training:\n            feature['target'] = tf.train.Feature(int64_list=tf.train.Int64List(value=[record['target'].numpy()]))\n\n        example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n        return example_proto.SerializeToString()\n\n    # # --- 5) Build the dataset pipeline for this file ---\n    raw_dataset = tf.data.TFRecordDataset([input_file])\n    parsed_dataset = raw_dataset.map(parse_tfrecord, num_parallel_calls=tf.data.AUTOTUNE)\n\n    # # Add metadata via py_function\n    # tfrecords_with_metadata = parsed_dataset.map(\n    #     lambda rec: tf.py_function(\n    #         add_metadata,\n    #         [rec['image'], rec['image_name'], rec['target']],\n    #         Tout=(tf.string, tf.int64, tf.string, tf.string)\n    #     )\n    # )\n    # final_dataset = tf.data.Dataset.zip((parsed_dataset, tfrecords_with_metadata)) \\\n    #                                .map(combine_records)\n\n    # # --- 6) Materialize and write out ---\n    # with tf.io.TFRecordWriter(output_file) as writer:\n    #     for record in final_dataset:\n    #         example = serialize_example(record)\n    #         writer.write(example)\n\n    # Dynamically call py_function with or without target\n    if training:\n        def pyfn(record):\n            return tf.py_function(add_metadata_py, [record['image'], record['image_name']],\n                                  [tf.string, tf.int64, tf.string, tf.string])\n    else:\n        def pyfn(record):\n            return tf.py_function(add_metadata_py, [record['image'], record['image_name']],\n                                  [tf.string, tf.int64, tf.string, tf.string])\n\n    metadata_dataset = parsed_dataset.map(pyfn, num_parallel_calls=tf.data.AUTOTUNE)\n    final_dataset = tf.data.Dataset.zip((parsed_dataset, metadata_dataset)) \\\n                                   .map(combine_records, num_parallel_calls=tf.data.AUTOTUNE)\n\n    with tf.io.TFRecordWriter(output_file) as writer:\n        for record in final_dataset:\n            serialized = serialize_example(record)\n            writer.write(serialized)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-23T16:32:29.022994Z","iopub.execute_input":"2025-05-23T16:32:29.023546Z","iopub.status.idle":"2025-05-23T16:32:29.042551Z","shell.execute_reply.started":"2025-05-23T16:32:29.023503Z","shell.execute_reply":"2025-05-23T16:32:29.041197Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"INPUT_DIR = '/kaggle/input/siim-isic-melanoma-classification/'\n\nprint('Training dataset:')\nmetadata_df = pd.read_csv(INPUT_DIR + \"train.csv\")\n\n# All input TFRecord files\ntfrecord_files = tf.io.gfile.glob(INPUT_DIR + 'tfrecords/train*.tfrec')\nprint(f\"Found {len(tfrecord_files)} tfrecords:\", tfrecord_files)\n\n# Output directory for enhanced files\noutput_dir = './enhanced_dataset'\ntf.io.gfile.makedirs(output_dir)\n\n# Loop over each input file => produce 1 output file\nfor infile in sorted(tfrecord_files):\n    # e.g. infile = \"/kaggle/input/.../train00-2071.tfrec\"\n    base_name = os.path.basename(infile)\n    # e.g. \"train00-2071.tfrec\" => \"train00-2071-enhanced.tfrecord\"\n    out_name = base_name.replace('.tfrec', '-enhanced.tfrec')\n    outfile = os.path.join(output_dir, out_name)\n    \n    print(f\"Processing: {infile} => {outfile}\")\n    process_single_tfrecord(infile, outfile, metadata_df, training=True)\n\nprint('Finished Training dataset.')\n\nprint('Test dataset:')\nmetadata_df = pd.read_csv(INPUT_DIR + \"test.csv\")\n# All input TFRecord files\ntfrecord_files = tf.io.gfile.glob(INPUT_DIR + 'tfrecords/test*.tfrec')\nprint(f\"Found {len(tfrecord_files)} tfrecords:\", tfrecord_files)\n\n# Loop over each input file => produce 1 output file\nfor infile in sorted(tfrecord_files):\n    base_name = os.path.basename(infile)\n    # e.g. \"test00-687.tfrec\" => \"test00-687-enhanced.tfrecord\"\n    out_name = base_name.replace('.tfrec', '-enhanced.tfrec')\n    outfile = os.path.join(output_dir, out_name)\n    \n    print(f\"Processing: {infile} => {outfile}\")\n    process_single_tfrecord(infile, outfile, metadata_df, training=False)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-23T16:32:29.044237Z","iopub.execute_input":"2025-05-23T16:32:29.044702Z","iopub.status.idle":"2025-05-23T16:37:49.419900Z","shell.execute_reply.started":"2025-05-23T16:32:29.044661Z","shell.execute_reply":"2025-05-23T16:37:49.416130Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def parse_enhanced_tfrecord(example):\n    features = {\n        'target': tf.io.FixedLenFeature([], tf.int64),\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'image_name': tf.io.FixedLenFeature([], tf.string),\n        'sex': tf.io.FixedLenFeature([], tf.string),\n        'age_approx': tf.io.FixedLenFeature([], tf.int64),\n        'patient_id': tf.io.FixedLenFeature([], tf.string),\n        'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.string),\n    }\n    return tf.io.parse_single_example(example, features)\n\nenhanced_file = './enhanced_dataset/train01-2071-enhanced.tfrec'\ndebug_dataset = tf.data.TFRecordDataset(enhanced_file).map(parse_enhanced_tfrecord)\n\nimport numpy as np\n\n# Number of records to validate (to avoid printing everything for huge files)\nn_samples_to_check = 10\n\nfor i, record in enumerate(debug_dataset.take(n_samples_to_check)):\n    # Extract the Tensor values in Python\n    image_name_str = record['image_name'].numpy().decode('utf-8')\n    sex_str = record['sex'].numpy().decode('utf-8')\n    age_approx_val = record['age_approx'].numpy()\n    patient_id_str = record['patient_id'].numpy().decode('utf-8')\n    anatom_site_str = record['anatom_site_general_challenge'].numpy().decode('utf-8')\n\n    # Lookup the corresponding row in the dataframe\n    row = metadata_df[metadata_df['image_name'] == image_name_str]\n    \n    # Print a summary for cross-check\n    print(f\"--- Record #{i} ---\")\n    print(\"image_name:\", image_name_str)\n    \n    if row.empty:\n        print(\"  >> Not found in metadata_df. The TFRecord shows:\")\n        print(f\"     sex={sex_str}, age_approx={age_approx_val}, \"\n              f\"patient_id={patient_id_str}, anatom_site={anatom_site_str}\")\n    else:\n        # There's exactly one row or more, but we assume 1\n        row = row.iloc[0]\n        \n        # In your code, if 'sex' is NaN, you stored 'unknown'\n        df_sex = row['sex'] if not pd.isna(row['sex']) else 'unknown'\n        df_age = row['age_approx'] if not pd.isna(row['age_approx']) else -1\n        df_pid = row['patient_id'] if not pd.isna(row['patient_id']) else 'unknown'\n        df_site = row['anatom_site_general_challenge'] if not pd.isna(row['anatom_site_general_challenge']) else 'unknown'\n        \n        # Convert them all to strings for easy printing/comparison\n        df_sex = str(df_sex)\n        df_age = str(int(df_age))  # or just str(df_age)\n        df_pid = str(df_pid)\n        df_site = str(df_site)\n        \n        # Compare\n        match_sex = (sex_str == df_sex)\n        match_age = (str(age_approx_val) == df_age)\n        match_pid = (patient_id_str == df_pid)\n        match_site = (anatom_site_str == df_site)\n        \n        # Print out the comparisons\n        print(\"  TFRecord vs DataFrame:\")\n        print(f\"    sex: {sex_str} vs {df_sex}  -> {'OK' if match_sex else 'MISMATCH!'}\")\n        print(f\"    age_approx: {age_approx_val} vs {df_age}  -> {'OK' if match_age else 'MISMATCH!'}\")\n        print(f\"    patient_id: {patient_id_str} vs {df_pid}  -> {'OK' if match_pid else 'MISMATCH!'}\")\n        print(f\"    site: {anatom_site_str} vs {df_site}  -> {'OK' if match_site else 'MISMATCH!'}\")\n\n    print()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-23T16:37:49.426887Z","iopub.execute_input":"2025-05-23T16:37:49.427577Z","iopub.status.idle":"2025-05-23T16:37:50.275040Z","shell.execute_reply.started":"2025-05-23T16:37:49.427526Z","shell.execute_reply":"2025-05-23T16:37:50.272387Z"}},"outputs":[],"execution_count":null}]}