{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":21154,"databundleVersionId":1243559,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom kaggle_datasets import KaggleDatasets\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout\nfrom tensorflow.keras.optimizers import Adam\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2024-05-08T10:37:58.251501Z","iopub.execute_input":"2024-05-08T10:37:58.252590Z","iopub.status.idle":"2024-05-08T10:37:58.259524Z","shell.execute_reply.started":"2024-05-08T10:37:58.252541Z","shell.execute_reply":"2024-05-08T10:37:58.258593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Detect and initialize the TPU","metadata":{}},{"cell_type":"code","source":"\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()  # Default strategy that works on CPU and single GPU\n\nprint('Number of replicas:', strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2024-05-08T10:38:00.939495Z","iopub.execute_input":"2024-05-08T10:38:00.939964Z","iopub.status.idle":"2024-05-08T10:38:00.949861Z","shell.execute_reply.started":"2024-05-08T10:38:00.939923Z","shell.execute_reply":"2024-05-08T10:38:00.948349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Access the GCS path for the Kaggle dataset","metadata":{}},{"cell_type":"code","source":"\nGCS_DS_PATH = KaggleDatasets().get_gcs_path('tpu-getting-started')\ndata_dir = GCS_DS_PATH + \"/tfrecords-jpeg-192x192\"","metadata":{"execution":{"iopub.status.busy":"2024-05-08T10:38:05.571350Z","iopub.execute_input":"2024-05-08T10:38:05.571775Z","iopub.status.idle":"2024-05-08T10:38:06.028640Z","shell.execute_reply.started":"2024-05-08T10:38:05.571728Z","shell.execute_reply":"2024-05-08T10:38:06.027193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Set image height, width, and batch size","metadata":{}},{"cell_type":"code","source":"\nimg_height, img_width = 192, 192\nbatch_size = 16 * strategy.num_replicas_in_sync","metadata":{"execution":{"iopub.status.busy":"2024-05-08T10:38:08.163068Z","iopub.execute_input":"2024-05-08T10:38:08.163501Z","iopub.status.idle":"2024-05-08T10:38:08.171100Z","shell.execute_reply.started":"2024-05-08T10:38:08.163469Z","shell.execute_reply":"2024-05-08T10:38:08.169517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Set up data input functions","metadata":{}},{"cell_type":"code","source":"\ndef decode_image(image):\n    image = tf.image.decode_jpeg(image, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n    image = tf.reshape(image, [img_height, img_width, 3])\n    return image","metadata":{"execution":{"iopub.status.busy":"2024-05-08T10:38:10.483312Z","iopub.execute_input":"2024-05-08T10:38:10.483734Z","iopub.status.idle":"2024-05-08T10:38:10.490887Z","shell.execute_reply.started":"2024-05-08T10:38:10.483701Z","shell.execute_reply":"2024-05-08T10:38:10.489211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_tfrecord(example, labeled=True):\n    tfrecord_format = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"class\": tf.io.FixedLenFeature([], tf.int64),\n    } if labeled else {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"id\": tf.io.FixedLenFeature([], tf.string),\n    }\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example['image'])\n    if labeled:\n        label = tf.one_hot(example['class'], 104)\n        return image, label\n    return image, example['id']\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    dataset = tf.data.TFRecordDataset(filenames)\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False  # disable order, increase speed\n    dataset = dataset.with_options(ignore_order)\n    return dataset.map(lambda x: read_tfrecord(x, labeled), num_parallel_calls=tf.data.experimental.AUTOTUNE)","metadata":{"execution":{"iopub.status.busy":"2024-05-08T10:38:12.932282Z","iopub.execute_input":"2024-05-08T10:38:12.932702Z","iopub.status.idle":"2024-05-08T10:38:12.944917Z","shell.execute_reply.started":"2024-05-08T10:38:12.932667Z","shell.execute_reply":"2024-05-08T10:38:12.943490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load datasets","metadata":{}},{"cell_type":"code","source":"\ntrain_files = tf.io.gfile.glob(data_dir + '/train/*.tfrec')\nval_files = tf.io.gfile.glob(data_dir + '/val/*.tfrec')\ntest_files = tf.io.gfile.glob(data_dir + '/test/*.tfrec')","metadata":{"execution":{"iopub.status.busy":"2024-05-08T10:38:16.483517Z","iopub.execute_input":"2024-05-08T10:38:16.483944Z","iopub.status.idle":"2024-05-08T10:38:18.792582Z","shell.execute_reply.started":"2024-05-08T10:38:16.483911Z","shell.execute_reply":"2024-05-08T10:38:18.791163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Assuming the structure and size of the dataset are known:","metadata":{}},{"cell_type":"code","source":"\nnum_train_examples = 12753  # example value\nnum_val_examples = 3712     # example value\nnum_test_files = len(test_files)\nnum_test_examples = num_test_files * 2048  # Assuming batch size for test dataset is 2048\nTRAIN_STEPS = num_train_examples // batch_size\nVAL_STEPS = num_val_examples // batch_size\n\ntrain_dataset = load_dataset(train_files).repeat().shuffle(2048).batch(batch_size).prefetch(tf.data.experimental.AUTOTUNE)\nval_dataset = load_dataset(val_files).batch(batch_size).cache().prefetch(tf.data.experimental.AUTOTUNE)\ntest_dataset = load_dataset(test_files, labeled=False, ordered=True).batch(batch_size)","metadata":{"execution":{"iopub.status.busy":"2024-05-08T10:38:21.491804Z","iopub.execute_input":"2024-05-08T10:38:21.492270Z","iopub.status.idle":"2024-05-08T10:38:21.961124Z","shell.execute_reply.started":"2024-05-08T10:38:21.492232Z","shell.execute_reply":"2024-05-08T10:38:21.959643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define the model architecture","metadata":{}},{"cell_type":"code","source":"\nwith strategy.scope():\n    model = Sequential([\n        Conv2D(32, (3, 3), activation='relu', input_shape=(img_height, img_width, 3)),\n        MaxPooling2D((2, 2)),\n        Conv2D(64, (3, 3), activation='relu'),\n        MaxPooling2D((2, 2)),\n        Conv2D(128, (3, 3), activation='relu'),\n        MaxPooling2D((2, 2)),\n        Flatten(),\n        Dense(512, activation='relu'),\n        Dropout(0.5),\n        Dense(104, activation='softmax')\n    ])\n    model.compile(loss='categorical_crossentropy', optimizer=Adam(learning_rate=0.001), metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2024-05-08T10:38:24.723935Z","iopub.execute_input":"2024-05-08T10:38:24.724346Z","iopub.status.idle":"2024-05-08T10:38:25.038468Z","shell.execute_reply.started":"2024-05-08T10:38:24.724315Z","shell.execute_reply":"2024-05-08T10:38:25.036725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train the model","metadata":{}},{"cell_type":"code","source":"\nhistory = model.fit(train_dataset, steps_per_epoch=TRAIN_STEPS, validation_data=val_dataset, validation_steps=VAL_STEPS, epochs=10)","metadata":{"execution":{"iopub.status.busy":"2024-05-08T10:38:27.786875Z","iopub.execute_input":"2024-05-08T10:38:27.788236Z","iopub.status.idle":"2024-05-08T12:49:42.045988Z","shell.execute_reply.started":"2024-05-08T10:38:27.788191Z","shell.execute_reply":"2024-05-08T12:49:42.044461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction and submission preparation","metadata":{}},{"cell_type":"code","source":"\ndef predict_and_prepare_submission(model, test_dataset):\n    print('Predicting on test data...')\n    predictions = model.predict(test_dataset)\n    predicted_classes = np.argmax(predictions, axis=1)\n\n    print('Generating submission file...')\n    test_ids_ds = test_dataset.map(lambda image, idnum: idnum).unbatch()\n    test_ids = next(iter(test_ids_ds.batch(num_test_examples))).numpy().astype('U')  # Adjust dtype for submission\n\n    submission_df = pd.DataFrame({'id': test_ids, 'label': predicted_classes})\n    submission_df.to_csv('submission.csv', index=False)\n\n    print('Submission file created successfully!')\n\n    return predicted_classes  # Return predicted classes\n\n# Call the function to predict and prepare submission\npredicted_classes = predict_and_prepare_submission(model, test_dataset)\n","metadata":{"execution":{"iopub.status.busy":"2024-05-08T13:13:52.148590Z","iopub.execute_input":"2024-05-08T13:13:52.149068Z","iopub.status.idle":"2024-05-08T13:17:01.364869Z","shell.execute_reply.started":"2024-05-08T13:13:52.149030Z","shell.execute_reply":"2024-05-08T13:17:01.363418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict and prepare submission","metadata":{}},{"cell_type":"code","source":"\npredict_and_prepare_submission(model, test_dataset)","metadata":{"execution":{"iopub.status.busy":"2024-05-08T13:17:28.576973Z","iopub.execute_input":"2024-05-08T13:17:28.577467Z","iopub.status.idle":"2024-05-08T13:20:05.594110Z","shell.execute_reply.started":"2024-05-08T13:17:28.577428Z","shell.execute_reply":"2024-05-08T13:20:05.592852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualize training history","metadata":{}},{"cell_type":"code","source":"\nplt.figure(figsize=(12, 6))\nplt.subplot(1, 2, 1)\nplt.plot(history.history['accuracy'], label='Training Accuracy')\nplt.plot(history.history['val_accuracy'], label='Validation Accuracy')\nplt.title('Model Accuracy')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.legend()\n\nplt.subplot(1, 2, 2)\nplt.plot(history.history['loss'], label='Training Loss')\nplt.plot(history.history['val_loss'], label='Validation Loss')\nplt.title('Model Loss')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.legend()\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-08T13:20:27.869108Z","iopub.execute_input":"2024-05-08T13:20:27.869544Z","iopub.status.idle":"2024-05-08T13:20:28.857049Z","shell.execute_reply.started":"2024-05-08T13:20:27.869511Z","shell.execute_reply":"2024-05-08T13:20:28.855815Z"},"trusted":true},"execution_count":null,"outputs":[]}]}