{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Install and import package\n!pip install kaggledatasets\n!pip install tensorflow_datasets\n!pip install keras-tuner --upgrade\n# !pip install -q git+https://github.com/keras-team/keras-tuner@master # Use github head for newly added TPU support","metadata":{"execution":{"iopub.status.busy":"2021-07-26T02:48:30.468298Z","iopub.execute_input":"2021-07-26T02:48:30.468758Z","iopub.status.idle":"2021-07-26T02:48:54.571914Z","shell.execute_reply.started":"2021-07-26T02:48:30.468670Z","shell.execute_reply":"2021-07-26T02:48:54.570526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport seaborn as sns\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow_datasets as tfds\nimport numpy as np\nfrom tensorflow.keras.layers.experimental import preprocessing\nfrom kaggle_datasets import KaggleDatasets\nfrom kaggle_secrets import UserSecretsClient\nfrom tensorflow.keras import layers\nimport keras_tuner as kt","metadata":{"execution":{"iopub.status.busy":"2021-07-26T02:48:54.574129Z","iopub.execute_input":"2021-07-26T02:48:54.574650Z","iopub.status.idle":"2021-07-26T02:49:03.933033Z","shell.execute_reply.started":"2021-07-26T02:48:54.574589Z","shell.execute_reply":"2021-07-26T02:49:03.931620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Turn on tpu\n# Detect TPU, return appropriate distribution strategy\nstrategy = tf.distribute.get_strategy() \n\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver() \n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() \n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T02:49:03.941918Z","iopub.execute_input":"2021-07-26T02:49:03.942254Z","iopub.status.idle":"2021-07-26T02:49:09.258045Z","shell.execute_reply.started":"2021-07-26T02:49:03.942221Z","shell.execute_reply":"2021-07-26T02:49:09.256323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# If you use private dataset, uncomment it\n#user_secrets = UserSecretsClient()\n#user_credential = user_secrets.get_gcloud_credential()\n#user_secrets.set_tensorflow_credential(user_credential)\n\n\nds_name = KaggleDatasets().get_gcs_path(\"g2net-tfrecord-spectrogram\")\n\ntrain_filenames = tf.io.gfile.glob(ds_name + \"/train*.tfrec\")\n#val_filenames = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')\ntest_filenames = tf.io.gfile.glob(ds_name + \"/test*.tfrec\")\n\n# List dir with real regex\n# [x for x in os.listdir('.') if re.match('index_[0-9]*.csv', x)]","metadata":{"execution":{"iopub.status.busy":"2021-07-26T02:49:09.260170Z","iopub.execute_input":"2021-07-26T02:49:09.260652Z","iopub.status.idle":"2021-07-26T02:49:09.753484Z","shell.execute_reply.started":"2021-07-26T02:49:09.260602Z","shell.execute_reply":"2021-07-26T02:49:09.752264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read train data\ntrain_tfrec = tf.data.TFRecordDataset(train_filenames)\n\n# Read val data\n#val_tfrec = tf.data.TFRecordDataset(val_filenames)\n\n# Read test dataset\ntest_tfrec = tf.data.TFRecordDataset(test_filenames)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T02:49:09.754772Z","iopub.execute_input":"2021-07-26T02:49:09.755106Z","iopub.status.idle":"2021-07-26T02:49:09.783433Z","shell.execute_reply.started":"2021-07-26T02:49:09.755032Z","shell.execute_reply":"2021-07-26T02:49:09.782200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Parse an train example to get feature_description\nfor raw_record in train_tfrec.take(1):\n    example = tf.train.Example()\n    example.ParseFromString(raw_record.numpy())\n    # print(example)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T02:49:09.785859Z","iopub.execute_input":"2021-07-26T02:49:09.786188Z","iopub.status.idle":"2021-07-26T02:49:12.168069Z","shell.execute_reply.started":"2021-07-26T02:49:09.786158Z","shell.execute_reply":"2021-07-26T02:49:12.166828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# parse tfrecord to get feature and label\nfeature_description = {\n    \"image\": tf.io.FixedLenFeature([], tf.string, default_value=\"\"),\n    \"image_name\": tf.io.FixedLenFeature([], tf.string, default_value=\"\"),\n    \"target\": tf.io.FixedLenFeature([], tf.int64, default_value=0),\n}\n\ndef parse_labeled_data(example_proto):\n    # Parse the input `tf.train.Example` proto using the dictionary above.\n    parsed = tf.io.parse_single_example(example_proto, feature_description)\n    image = tf.image.decode_jpeg(parsed[\"image\"], channels=3)\n    image = tf.cast(image, tf.float32) \n    image = tf.reshape(image, [129, 65, 3])\n    return image, parsed[\"target\"]\n\ndef parse_unlabeled_data(example_proto):\n    # Parse the input `tf.train.Example` proto using the dictionary above.\n    parsed = tf.io.parse_single_example(example_proto, feature_description)\n    image = tf.image.decode_jpeg(parsed[\"image\"], channels=3)\n    image = tf.cast(image, tf.float32) \n    image = tf.reshape(image, [129, 65, 3])\n    return image, parsed[\"image_name\"]\n\ntrain_dataset = train_tfrec.map(parse_labeled_data, num_parallel_calls=10).shuffle(560000).batch(128)\ntrain_dataset = train_dataset.cache()\ntrain_dataset = train_dataset.prefetch(10)\ntest_dataset = test_tfrec.map(parse_unlabeled_data, num_parallel_calls=10).batch(128)\ntest_dataset = test_dataset.cache()\ntest_dataset = test_dataset.prefetch(10)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T02:49:12.170244Z","iopub.execute_input":"2021-07-26T02:49:12.170577Z","iopub.status.idle":"2021-07-26T02:49:12.376408Z","shell.execute_reply.started":"2021-07-26T02:49:12.170546Z","shell.execute_reply":"2021-07-26T02:49:12.375190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %% [code]\nimport collections\nimport copy\nfrom keras_tuner.engine import tuner_utils\nimport numpy as np\nimport keras_tuner as kt\n\n# Reimplement bayesianOptimization to run with tpu\nclass TpuRandomSearchOracle(kt.oracles.RandomSearchOracle):\n    def _save_trial(self, trial):\n        # Write trial status to trial directory\n        trial_id = trial.trial_id\n        # trial.save(os.path.join(self._get_trial_dir(trial_id), \"trial.json\"))\n\nclass TpuRandomSearch(kt.engine.multi_execution_tuner.MultiExecutionTuner):\n    def __init__(\n        self,\n        hypermodel,\n        objective,\n        max_trials,\n        seed=None,\n        hyperparameters=None,\n        tune_new_entries=True,\n        allow_new_entries=True,\n        **kwargs\n    ):\n        self.seed = seed\n        oracle = TpuRandomSearchOracle(\n            objective=objective,\n            max_trials=max_trials,\n            seed=seed,\n            hyperparameters=hyperparameters,\n            tune_new_entries=tune_new_entries,\n            allow_new_entries=allow_new_entries,\n        )\n        super(TpuRandomSearch, self).__init__(oracle, hypermodel, **kwargs)\n        \n    def run_trial(self, trial, *fit_args, **fit_kwargs):\n        original_callbacks = fit_kwargs.pop(\"callbacks\", [])\n        # Run the training process multiple times.\n        metrics = collections.defaultdict(list)\n        for execution in range(self.executions_per_trial):\n            copied_fit_kwargs = copy.copy(fit_kwargs)\n            callbacks = self._deepcopy_callbacks(original_callbacks)\n            self._configure_tensorboard_dir(callbacks, trial, execution)\n            callbacks.append(tuner_utils.TunerCallback(self, trial))\n            # Only checkpoint the best epoch across all executions.\n            copied_fit_kwargs[\"callbacks\"] = callbacks\n\n            history = self._build_and_fit_model(trial, fit_args, copied_fit_kwargs)\n            for metric, epoch_values in history.history.items():\n                if self.oracle.objective.direction == \"min\":\n                    best_value = np.min(epoch_values)\n                else:\n                    best_value = np.max(epoch_values)\n                metrics[metric].append(best_value)\n\n        # Average the results across executions and send to the Oracle.\n        averaged_metrics = {}\n        for metric, execution_values in metrics.items():\n            averaged_metrics[metric] = np.mean(execution_values)\n        self.oracle.update_trial(\n            trial.trial_id, metrics=averaged_metrics, step=self._reported_step\n        )","metadata":{"execution":{"iopub.status.busy":"2021-07-26T02:49:12.378207Z","iopub.execute_input":"2021-07-26T02:49:12.378658Z","iopub.status.idle":"2021-07-26T02:49:12.394381Z","shell.execute_reply.started":"2021-07-26T02:49:12.378613Z","shell.execute_reply":"2021-07-26T02:49:12.393238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create model\\\ndef create_model(hp):\n    dropout_1 = hp.Float(\"dropout_1\", 0.2, 0.5)\n    dropout_2 = hp.Float(\"dropout_2\", 0.2, 0.5)\n    lr = hp.Float(\"learning_rate\", 5e-4, 1e-7)\n    \n    pretrained_model = tf.keras.applications.efficientnet.EfficientNetB0(\n        include_top=False, weights='imagenet', pooling=\"avg\",\n        input_shape=(129, 65, 3)\n    )\n\n    model = tf.keras.Sequential([\n        pretrained_model,\n        layers.Dense(1024, activation=\"relu\", name=\"dense1\"),\n        layers.Dropout(dropout_1),\n        layers.Dense(1024, activation=\"relu\", name=\"dense2\"),\n        layers.Dropout(dropout_2),\n        layers.Dense(1 ,activation='sigmoid')\n    ])\n\n    model.compile(\n        optimizer=tf.keras.optimizers.Adam(learning_rate=lr),\n        loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),\n        metrics=[tf.keras.metrics.AUC()]\n    )\n    \n    return model\n\ntuner = TpuRandomSearch(\n    create_model,\n    objective=kt.Objective(\"auc\", direction=\"max\"),\n    max_trials=12,\n    distribution_strategy=strategy,\n    overwrite=True,\n    directory=\"tuner\",\n    project_name=\"g2net\",   \n)\ntuner.search_space_summary()\ntuner.search(train_dataset, epochs=3)\ntuner.results_summary(num_trials=12)\nbest_hp = tuner.get_best_hyperparameters()[0]\nmodel = tuner.hypermodel.build(best_hp)\n\nmodel.summary()\n","metadata":{"execution":{"iopub.status.busy":"2021-07-26T02:49:12.395900Z","iopub.execute_input":"2021-07-26T02:49:12.396222Z","iopub.status.idle":"2021-07-26T02:51:31.368438Z","shell.execute_reply.started":"2021-07-26T02:49:12.396180Z","shell.execute_reply":"2021-07-26T02:51:31.364901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# Train model\nmodel.fit(\n    train_dataset, \n    epochs=10, \n)\n\nsave_locally = tf.saved_model.SaveOptions(experimental_io_device='/job:localhost')\nmodel.save(\n    'saved-model', \n    options=save_locally\n)\n","metadata":{"execution":{"iopub.status.busy":"2021-07-26T02:51:31.369921Z","iopub.status.idle":"2021-07-26T02:51:31.370781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load model, predict and write submission file\nload_locally = tf.saved_model.LoadOptions(\n    experimental_io_device='/job:localhost'\n)\nmodel = tf.keras.models.load_model(\n    'saved-model',\n    options=load_locally,\n)\ntest_images_ds = test_dataset.map(lambda image, idnum: image)\npredictions = model.predict(test_images_ds)\nprint(predictions)\n\nprint('Generating submission.csv file...')\ntest_ids_ds = test_dataset.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(np.size(predictions)))).numpy().astype('U') # all in one batch\ndata = {\n    \"id\": test_ids\n}\nsubmission = pd.DataFrame(data)\nsubmission = submission.assign(target=predictions)\nsubmission.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T02:51:31.372162Z","iopub.status.idle":"2021-07-26T02:51:31.373010Z"},"trusted":true},"execution_count":null,"outputs":[]}]}